diff options
| author | Dana Robinson <43805+derobins@users.noreply.github.com> | 2023-04-28 20:37:04 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-04-28 20:37:04 (GMT) |
| commit | a150e5fc2567b9b0f2a6dafec1fd3f330fca2e55 (patch) | |
| tree | e300ebb600a0f29f83f27c12d2ecbb911fbbe350 | |
| parent | f82f056d7b4804f9f29abf0c3cac36fdd9e9d7a3 (diff) | |
| download | hdf5-a150e5fc2567b9b0f2a6dafec1fd3f330fca2e55.zip hdf5-a150e5fc2567b9b0f2a6dafec1fd3f330fca2e55.tar.gz hdf5-a150e5fc2567b9b0f2a6dafec1fd3f330fca2e55.tar.bz2 | |
Sync with develop (#2849)
Cherry pick of
ea7dfcd (Change Powershell to PowerShell in docs)
to
4497feb (Update H5Dget_space_status bug note to reference 1.14.0)
45 files changed, 2495 insertions, 1221 deletions
@@ -41,3 +41,4 @@ src/H5overflow.h src/H5version.h /.classpath +/CMakeUserPresets.json diff --git a/CMakeInstallation.cmake b/CMakeInstallation.cmake index 00ed5cd..cc7d219 100644 --- a/CMakeInstallation.cmake +++ b/CMakeInstallation.cmake @@ -150,18 +150,55 @@ if (HDF5_PACK_EXAMPLES) DESTINATION ${HDF5_INSTALL_DATA_DIR} COMPONENT hdfdocuments ) - if (EXISTS "${HDF5_EXAMPLES_COMPRESSED_DIR}/${HDF5_EXAMPLES_COMPRESSED}") + + option (EXAMPLES_USE_RELEASE_NAME "Use the released examples artifact name" OFF) + option (EXAMPLES_DOWNLOAD "Download to use released examples files" OFF) + if (EXAMPLES_DOWNLOAD) + if (NOT EXAMPLES_USE_LOCALCONTENT) + set (EXAMPLES_URL ${EXAMPLES_TGZ_ORIGPATH}/${EXAMPLES_TGZ_ORIGNAME}) + else () + set (EXAMPLES_URL ${TGZPATH}/${EXAMPLES_TGZ_ORIGNAME}) + endif () + message (VERBOSE "Examples file is ${EXAMPLES_URL}") + file (DOWNLOAD ${EXAMPLES_URL} ${HDF5_BINARY_DIR}/${HDF5_EXAMPLES_COMPRESSED}) + if (EXISTS "${HDF5_BINARY_DIR}/${HDF5_EXAMPLES_COMPRESSED}") + execute_process( + COMMAND ${CMAKE_COMMAND} -E tar xzf ${HDF5_EXAMPLES_COMPRESSED} + WORKING_DIRECTORY ${HDF5_BINARY_DIR} + COMMAND_ECHO STDOUT + ) + endif () + set (EXAMPLES_USE_RELEASE_NAME ON CACHE BOOL "" FORCE) + else () + if (EXISTS "${HDF5_EXAMPLES_COMPRESSED_DIR}/${HDF5_EXAMPLES_COMPRESSED}") + execute_process( + COMMAND ${CMAKE_COMMAND} -E tar xzf ${HDF5_EXAMPLES_COMPRESSED_DIR}/${HDF5_EXAMPLES_COMPRESSED} + WORKING_DIRECTORY ${HDF5_BINARY_DIR} + COMMAND_ECHO STDOUT + ) + endif () + endif () + if (EXAMPLES_USE_RELEASE_NAME) + get_filename_component (EX_LAST_EXT ${HDF5_EXAMPLES_COMPRESSED} LAST_EXT) + if (${EX_LAST_EXT} STREQUAL ".zip") + get_filename_component (EX_DIR_NAME ${HDF5_EXAMPLES_COMPRESSED} NAME_WLE) + else () + get_filename_component (EX_DIR_NAME ${HDF5_EXAMPLES_COMPRESSED} NAME_WLE) + get_filename_component (EX_DIR_NAME ${EX_DIR_NAME} NAME_WLE) + endif () execute_process( - COMMAND ${CMAKE_COMMAND} -E tar xzf ${HDF5_EXAMPLES_COMPRESSED_DIR}/${HDF5_EXAMPLES_COMPRESSED} - ) - install ( - DIRECTORY ${HDF5_BINARY_DIR}/HDF5Examples - DESTINATION ${HDF5_INSTALL_DATA_DIR} - USE_SOURCE_PERMISSIONS - COMPONENT hdfdocuments + COMMAND ${CMAKE_COMMAND} -E rename ${EX_DIR_NAME} HDF5Examples + WORKING_DIRECTORY ${HDF5_BINARY_DIR} + COMMAND_ECHO STDOUT ) endif () install ( + DIRECTORY ${HDF5_BINARY_DIR}/HDF5Examples + DESTINATION ${HDF5_INSTALL_DATA_DIR} + USE_SOURCE_PERMISSIONS + COMPONENT hdfdocuments + ) + install ( FILES ${HDF5_SOURCE_DIR}/release_docs/USING_CMake_Examples.txt DESTINATION ${HDF5_INSTALL_DATA_DIR} diff --git a/CMakePresets.json b/CMakePresets.json new file mode 100644 index 0000000..d861b44 --- /dev/null +++ b/CMakePresets.json @@ -0,0 +1,237 @@ +{ + "version": 6, + "include": [ + "config/cmake-presets/hidden-presets.json" + ], + "configurePresets": [ + { + "name": "ci-base-tgz", + "hidden": true, + "inherits": "ci-base", + "cacheVariables": { + "HDF5_ALLOW_EXTERNAL_SUPPORT": "NO", + "TGZPATH": {"type": "STRING", "value": "${sourceParentDir}/temp"} + } + }, + { + "name": "ci-StdCompression", + "hidden": true, + "inherits": "ci-base-tgz", + "cacheVariables": { + "HDF5_ENABLE_Z_LIB_SUPPORT": "ON", + "HDF5_ENABLE_SZIP_SUPPORT": "ON", + "HDF5_ENABLE_SZIP_ENCODING": "ON", + "BUILD_ZLIB_WITH_FETCHCONTENT": "ON", + "ZLIB_PACKAGE_NAME": {"type": "STRING", "value": "zlib"}, + "ZLIB_TGZ_ORIGPATH": {"type": "STRING", "value": "https://github.com/madler/zlib/releases/download/v1.2.13"}, + "ZLIB_TGZ_ORIGNAME": {"type": "STRING", "value": "zlib-1.2.13.tar.gz"}, + "ZLIB_USE_LOCALCONTENT": "OFF", + "BUILD_SZIP_WITH_FETCHCONTENT": "ON", + "LIBAEC_PACKAGE_NAME": {"type": "STRING", "value": "libaec"}, + "LIBAEC_TGZ_ORIGPATH": {"type": "STRING", "value": "https://github.com/MathisRosenhauer/libaec/releases/download/v1.0.6"}, + "LIBAEC_TGZ_ORIGNAME": {"type": "STRING", "value": "libaec-1.0.6.tar.gz"}, + "LIBAEC_USE_LOCALCONTENT": "OFF" + } + }, + { + "name": "ci-base-plugins", + "hidden": true, + "inherits": "ci-base-tgz", + "cacheVariables": { + "PLUGIN_TGZ_NAME": {"type": "STRING", "value": "hdf5_plugins-1.14.0.tar.gz"}, + "PLUGIN_PACKAGE_NAME": {"type": "STRING", "value": "pl"}, + "BSHUF_TGZ_NAME": {"type": "STRING", "value": "bitshuffle.tar.gz"}, + "BSHUF_PACKAGE_NAME": {"type": "STRING", "value": "bshuf"}, + "BLOSC_TGZ_NAME": {"type": "STRING", "value": "c-blosc.tar.gz"}, + "BLOSC_PACKAGE_NAME": {"type": "STRING", "value": "blosc"}, + "BLOSC_ZLIB_TGZ_NAME": {"type": "STRING", "value": "ZLib.tar.gz"}, + "BLOSC_ZLIB_PACKAGE_NAME": {"type": "STRING", "value": "zlib"}, + "BZ2_TGZ_NAME": {"type": "STRING", "value": "BZ2.tar.gz"}, + "BZ2_PACKAGE_NAME": {"type": "STRING", "value": "bz2"}, + "FPZIP_TGZ_NAME": {"type": "STRING", "value": "fpzip.tar.gz"}, + "FPZIP_PACKAGE_NAME": {"type": "STRING", "value": "fpzip"}, + "JPEG_TGZ_NAME": {"type": "STRING", "value": "JPEG.tar.gz"}, + "JPEG_PACKAGE_NAME": {"type": "STRING", "value": "jpeg"}, + "BUILD_LZ4_LIBRARY_SOURCE": "ON", + "LZ4_TGZ_NAME": {"type": "STRING", "value": "lz4.tar.gz"}, + "LZ4_PACKAGE_NAME": {"type": "STRING", "value": "lz4"}, + "LZF_TGZ_NAME": {"type": "STRING", "value": "lzf.tar.gz"}, + "LZF_PACKAGE_NAME": {"type": "STRING", "value": "lzf"}, + "SZ_TGZ_NAME": {"type": "STRING", "value": "szf.tar.gz"}, + "SZ_PACKAGE_NAME": {"type": "STRING", "value": "SZ"}, + "ZFP_TGZ_NAME": {"type": "STRING", "value": "zfp.tar.gz"}, + "ZFP_PACKAGE_NAME": {"type": "STRING", "value": "zfp"}, + "ZSTD_TGZ_NAME": {"type": "STRING", "value": "zstd.tar.gz"}, + "ZSTD_PACKAGE_NAME": {"type": "STRING", "value": "zstd"} + } + }, + { + "name": "ci-StdPlugins", + "hidden": true, + "inherits": ["ci-base-plugins", "ci-base-tgz"], + "cacheVariables": { + "HDF5_ENABLE_PLUGIN_SUPPORT": "ON", + "PLUGIN_TGZ_ORIGPATH": {"type": "STRING", "value": "https://github.com/HDFGroup/hdf5_plugins/archive/refs/tags"}, + "PLUGIN_TGZ_ORIGNAME": {"type": "STRING", "value": "hdf5_plugins-1.14.0.tar.gz"} + } + }, + { + "name": "ci-StdExamples", + "hidden": true, + "inherits": "ci-base", + "cacheVariables": { + "HDF5_PACK_EXAMPLES": "ON", + "HDF5_EXAMPLES_COMPRESSED": {"type": "STRING", "value": "hdf5-examples-2.0.3.tar.gz"}, + "HDF5_EXAMPLES_COMPRESSED_DIR": {"type": "STRING", "value": "${sourceParentDir}/temp"}, + "EXAMPLES_TGZ_ORIGPATH": {"type": "STRING", "value": "https://github.com/HDFGroup/hdf5-examples/archive/refs/tags/"}, + "EXAMPLES_TGZ_ORIGNAME": {"type": "STRING", "value": "2.0.3.tar.gz"}, + "EXAMPLES_DOWNLOAD": "ON" + } + }, + { + "name": "ci-StdShar", + "hidden": true, + "inherits": "ci-StdCompression", + "cacheVariables": { + "HDF_PACKAGE_NAMESPACE": {"type": "STRING", "value": "hdf5::"}, + "HDF5_INSTALL_MOD_FORTRAN": "NO", + "HDF5_BUILD_GENERATORS": "ON", + "HDF5_ENABLE_ALL_WARNINGS": "ON", + "HDF5_MINGW_STATIC_GCC_LIBS": "ON", + "HDF_TEST_EXPRESS": "2" + } + }, + { + "name": "ci-StdShar-MSVC", + "description": "MSVC Standard Config for x64 (Release)", + "inherits": [ + "ci-x64-Release-MSVC", + "ci-CPP", + "ci-Fortran", + "ci-Java", + "ci-StdShar", + "ci-StdExamples" + ] + }, + { + "name": "ci-StdShar-Clang", + "description": "Clang Standard Config for x64 (Release)", + "inherits": [ + "ci-x64-Release-Clang", + "ci-CPP", + "ci-Fortran", + "ci-Java", + "ci-StdShar", + "ci-StdExamples" + ] + }, + { + "name": "ci-StdShar-GNUC", + "description": "GNUC Standard Config for x64 (Release)", + "inherits": [ + "ci-x64-Release-GNUC", + "ci-CPP", + "ci-Fortran", + "ci-Java", + "ci-StdShar", + "ci-StdExamples" + ] + } + ], + "buildPresets": [ + { + "name": "ci-StdShar-MSVC", + "description": "MSVC Standard Build for x64 (Release)", + "configurePreset": "ci-StdShar-MSVC", + "inherits": [ + "ci-x64-Release-MSVC" + ] + }, + { + "name": "ci-StdShar-Clang", + "description": "Clang Standard Build for x64 (Release)", + "configurePreset": "ci-StdShar-Clang", + "inherits": [ + "ci-x64-Release-Clang" + ] + }, + { + "name": "ci-StdShar-GNUC", + "description": "GNUC Standard Build for x64 (Release)", + "configurePreset": "ci-StdShar-GNUC", + "verbose": false, + "inherits": [ + "ci-x64-Release-GNUC" + ] + } + ], + "testPresets": [ + { + "name": "ci-StdShar-MSVC", + "configurePreset": "ci-StdShar-MSVC", + "inherits": [ + "ci-x64-Release-MSVC" + ] + }, + { + "name": "ci-StdShar-Clang", + "configurePreset": "ci-StdShar-Clang", + "inherits": [ + "ci-x64-Release-Clang" + ] + }, + { + "name": "ci-StdShar-GNUC", + "configurePreset": "ci-StdShar-GNUC", + "inherits": [ + "ci-x64-Release-GNUC" + ] + } + ], + "packagePresets": [ + { + "name": "ci-StdShar-MSVC", + "configurePreset": "ci-StdShar-MSVC", + "inherits": "ci-x64-Release-MSVC" + }, + { + "name": "ci-StdShar-Clang", + "configurePreset": "ci-StdShar-Clang", + "inherits": "ci-x64-Release-Clang" + }, + { + "name": "ci-StdShar-GNUC", + "configurePreset": "ci-StdShar-GNUC", + "inherits": "ci-x64-Release-GNUC" + } + ], + "workflowPresets": [ + { + "name": "ci-StdShar-MSVC", + "steps": [ + {"type": "configure", "name": "ci-StdShar-MSVC"}, + {"type": "build", "name": "ci-StdShar-MSVC"}, + {"type": "test", "name": "ci-StdShar-MSVC"}, + {"type": "package", "name": "ci-StdShar-MSVC"} + ] + }, + { + "name": "ci-StdShar-Clang", + "steps": [ + {"type": "configure", "name": "ci-StdShar-Clang"}, + {"type": "build", "name": "ci-StdShar-Clang"}, + {"type": "test", "name": "ci-StdShar-Clang"}, + {"type": "package", "name": "ci-StdShar-Clang"} + ] + }, + { + "name": "ci-StdShar-GNUC", + "steps": [ + {"type": "configure", "name": "ci-StdShar-GNUC"}, + {"type": "build", "name": "ci-StdShar-GNUC"}, + {"type": "test", "name": "ci-StdShar-GNUC"}, + {"type": "package", "name": "ci-StdShar-GNUC"} + ] + } + ] +}
\ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dc1a200..687e981 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -86,7 +86,31 @@ application developers, library developers, and system administrators. # Release Note <A NAME="releasenote"></A> -The use and format of release notes is described in `doc/release_txt_notes.md` in this source code tree +* **Entry Syntax** +The release note entry syntax is shown below. + +``` + - Title/Problem + + Problem/Solution + +``` + +* **Entry Elements** - The elements of the entry - title, problem, solution, and signature - are described in more detail in the table +below. Descriptions of the problem and the solution should be clear without any ambiguities and should be short without losing clarity or specifics. + + * **Title** - The title or tag should identify one or more categories that will help readers decide if the entry is something they need to study. Can be combined with the `Problem` element + * **Problem** - Describe the problem and how users might see the problem in a paragraph. +You might also consider the following as you describe the problem: + * Under what specific conditions does this issue arise? + * Under what specific conditions are we sure this issue will not arise? + * For a performance issue, instead of saying something is a performance issue, describe what the performance impact of issue is? + * **Solution** - Describe the solution in another paragraph. +You might also consider the following as you describe the solution: + * What was done to resolve the issue? + * What is the functional impact? + * Is there a workaround – a way for users design their software so as not to encounter the issue? If so, what is the workaround? + * For a performance fix, how has the performance improved? Links to published documentation would be good. # Checklist <A NAME="checklist"></A> diff --git a/config/cmake-presets/hidden-presets.json b/config/cmake-presets/hidden-presets.json new file mode 100644 index 0000000..c616e7d --- /dev/null +++ b/config/cmake-presets/hidden-presets.json @@ -0,0 +1,491 @@ +{ + "version": 6, + "configurePresets": [ + { + "name": "ci-base", + "displayName": "Basic Config", + "description": "Basic build using Ninja generator", + "generator": "Ninja", + "hidden": true, + "binaryDir": "${sourceParentDir}/build/${presetName}", + "installDir": "${sourceParentDir}/install/${presetName}" + }, + { + "name": "ci-x64", + "architecture": { + "value": "x64", + "strategy": "external" + }, + "hidden": true + }, + { + "name": "ci-x86", + "architecture": { + "value": "x86", + "strategy": "external" + }, + "hidden": true + }, + { + "name": "ci-Debug", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + }, + "hidden": true + }, + { + "name": "ci-Release", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo", + "HDF5_BUILD_DOC": "ON" + }, + "hidden": true + }, + { + "name": "ci-MSVC", + "hidden": true, + "cacheVariables": { + "CMAKE_C_COMPILER": "cl", + "CMAKE_CXX_COMPILER": "cl" + }, + "toolset": { + "value": "host=x64", + "strategy": "external" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Windows" + } + }, + { + "name": "ci-Clang", + "hidden": true, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "config/toolchain/clang.cmake" + }, + "toolset": { + "value": "host=x64", + "strategy": "external" + } + }, + { + "name": "ci-GNUC", + "hidden": true, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "config/toolchain/gcc.cmake" + }, + "condition": { + "type": "equals", + "lhs": "${hostSystemName}", + "rhs": "Linux" + }, + "toolset": { + "value": "host=x64", + "strategy": "external" + } + }, + { + "name": "ci-Intel", + "hidden": true, + "cacheVariables": { + "CMAKE_TOOLCHAIN_FILE": "config/toolchain/intel.cmake" + }, + "toolset": { + "value": "host=x64", + "strategy": "external" + } + }, + { + "name": "ci-Fortran-Clang", + "hidden": true, + "cacheVariables": { + "CMAKE_Fortran_COMPILER": "gfortran" + }, + "condition": { + "type": "matches", + "string": "${presetName}", + "regex": ".*-Clang" + } + }, + { + "name": "ci-Fortran", + "hidden": true, + "inherits": "ci-Fortran-Clang", + "cacheVariables": { + "HDF5_BUILD_FORTRAN": "ON" + }, + "toolset": { + "value": "host=x64", + "strategy": "external" + } + }, + { + "name": "ci-CPP", + "hidden": true, + "cacheVariables": { + "HDF5_BUILD_CPP_LIB": "ON" + } + }, + { + "name": "ci-Java", + "hidden": true, + "cacheVariables": { + "HDF5_BUILD_JAVA": "ON" + }, + "toolset": { + "value": "host=x64", + "strategy": "external" + } + }, + { + "name": "ci-x64-Debug-MSVC", + "description": "MSVC for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Debug", + "ci-MSVC" + ] + }, + { + "name": "ci-x64-Release-MSVC", + "description": "MSVC for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Release", + "ci-MSVC" + ] + }, + { + "name": "ci-x64-Debug-Clang", + "description": "Clang/LLVM for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Debug", + "ci-Clang" + ] + }, + { + "name": "ci-x64-Release-Clang", + "description": "Clang/LLVM for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Release", + "ci-Clang" + ] + }, + { + "name": "ci-x64-Debug-GNUC", + "description": "GNUC for x64 (Debug)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Debug", + "ci-GNUC" + ] + }, + { + "name": "ci-x64-Release-GNUC", + "description": "GNUC for x64 (Release)", + "hidden": true, + "inherits": [ + "ci-base", + "ci-x64", + "ci-Release", + "ci-GNUC" + ] + }, + { + "name": "ci-x64-Debug-MSVC-asan", + "description": "x64-Debug-MSVC with /fsanitize=address", + "hidden": true, + "inherits": "ci-x64-Debug-MSVC", + "cacheVariables": { + "USE_SANITIZER": "Address", + "HDF5_ENABLE_SANITIZERS": "ON" + } + }, + { + "name": "ci-x64-Debug-GNUC-asan", + "hidden": true, + "inherits": "ci-x64-Debug-GNUC", + "cacheVariables": { + "USE_SANITIZER": "Address", + "HDF5_ENABLE_SANITIZERS": "ON" + } + }, + { + "name": "ci-x64-Debug-GNUC-tsan", + "hidden": true, + "inherits": "ci-x64-Debug-GNUC", + "cacheVariables": { + "USE_SANITIZER": "Thread", + "HDF5_ENABLE_SANITIZERS": "ON" + } + }, + { + "name": "ci-x64-Debug-GNUC-lsan", + "hidden": true, + "inherits": "ci-x64-Debug-GNUC", + "cacheVariables": { + "USE_SANITIZER": "Leak", + "HDF5_ENABLE_SANITIZERS": "ON" + } + }, + { + "name": "ci-x64-Debug-GNUC-ubsan", + "hidden": true, + "inherits": "ci-x64-Debug-GNUC", + "cacheVariables": { + "USE_SANITIZER": "Undefined", + "HDF5_ENABLE_SANITIZERS": "ON" + } + } + ], + "buildPresets": [ + { + "name": "ci-base", + "configurePreset": "ci-base", + "hidden": true, + "verbose": true, + "jobs": 8 + }, + { + "name": "ci-x64-Debug-MSVC", + "configurePreset": "ci-x64-Debug-MSVC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-MSVC", + "configurePreset": "ci-x64-Release-MSVC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-Clang", + "configurePreset": "ci-x64-Debug-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-Clang", + "configurePreset": "ci-x64-Release-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC", + "configurePreset": "ci-x64-Debug-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-GNUC", + "configurePreset": "ci-x64-Release-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-MSVC-asan", + "configurePreset": "ci-x64-Debug-MSVC-asan", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC-asan", + "configurePreset": "ci-x64-Debug-GNUC-asan", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC-tsan", + "configurePreset": "ci-x64-Debug-GNUC-tsan", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC-lsan", + "configurePreset": "ci-x64-Debug-GNUC-lsan", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC-ubsan", + "configurePreset": "ci-x64-Debug-GNUC-ubsan", + "hidden": true, + "inherits": [ + "ci-base" + ] + } + ], + "testPresets": [ + { + "name": "ci-base", + "configurePreset": "ci-base", + "output": { + "outputOnFailure": false, + "shortProgress": true, + "verbosity": "verbose" + }, + "hidden": true, + "execution": { + "noTestsAction": "error", + "timeout": 180, + "jobs": 8 + } + }, + { + "name": "ci-x64-Debug-MSVC", + "configurePreset": "ci-x64-Debug-MSVC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-MSVC", + "configurePreset": "ci-x64-Release-MSVC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-Clang", + "configurePreset": "ci-x64-Debug-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-Clang", + "configurePreset": "ci-x64-Release-Clang", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC", + "configurePreset": "ci-x64-Debug-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Release-GNUC", + "configurePreset": "ci-x64-Release-GNUC", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-MSVC-asan", + "configurePreset": "ci-x64-Debug-MSVC-asan", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC-asan", + "configurePreset": "ci-x64-Debug-GNUC-asan", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC-tsan", + "configurePreset": "ci-x64-Debug-GNUC-tsan", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC-lsan", + "configurePreset": "ci-x64-Debug-GNUC-lsan", + "hidden": true, + "inherits": [ + "ci-base" + ] + }, + { + "name": "ci-x64-Debug-GNUC-ubsan", + "configurePreset": "ci-x64-Debug-GNUC-ubsan", + "inherits": [ + "ci-base" + ] + } + ], + "packagePresets": [ + { + "name": "ci-base", + "hidden": true, + "output": { + "verbose": true + } + }, + { + "name": "ci-x64-Release-MSVC", + "configurePreset": "ci-x64-Release-MSVC", + "hidden": true, + "inherits": "ci-base", + "generators": [ + "ZIP" + ] + }, + { + "name": "ci-x64-Release-Clang", + "configurePreset": "ci-x64-Release-Clang", + "hidden": true, + "inherits": "ci-base", + "generators": [ + "TGZ" + ] + }, + { + "name": "ci-x64-Release-GNUC", + "configurePreset": "ci-x64-Release-GNUC", + "hidden": true, + "inherits": "ci-base", + "generators": [ + "TGZ" + ] + } + ] +}
\ No newline at end of file diff --git a/config/cmake/LIBAEC/CMakeLists.txt b/config/cmake/LIBAEC/CMakeLists.txt index 212c9bf..fb650ec 100644 --- a/config/cmake/LIBAEC/CMakeLists.txt +++ b/config/cmake/LIBAEC/CMakeLists.txt @@ -369,6 +369,10 @@ if (WIN32) find_program (WIX_EXECUTABLE candle PATHS "${CPACK_WIX_ROOT}/bin") endif () +configure_file (${LIBAEC_SOURCE_DIR}/LICENSE.txt ${LIBAEC_BINARY_DIR}/LIBAEC_LICENSE.txt @ONLY) +configure_file (${LIBAEC_SOURCE_DIR}/README.SZIP ${LIBAEC_BINARY_DIR}/LIBAEC_README.SZIP @ONLY) +configure_file (${LIBAEC_SOURCE_DIR}/README.md ${LIBAEC_BINARY_DIR}/LIBAEC_README.md @ONLY) + #----------------------------------------------------------------------------- # Set the cpack variables #----------------------------------------------------------------------------- @@ -383,9 +387,9 @@ if (NOT LIBAEC_EXTERNALLY_CONFIGURED) set (CPACK_PACKAGE_VERSION_MAJOR "${LIBAEC_PACKAGE_VERSION_MAJOR}") set (CPACK_PACKAGE_VERSION_MINOR "${LIBAEC_PACKAGE_VERSION_MINOR}") set (CPACK_PACKAGE_VERSION_PATCH "") - set (CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.txt") - set (CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.SZIP") - set (CPACK_RESOURCE_FILE_README "${CMAKE_CURRENT_SOURCE_DIR}/README.md") + set (CPACK_RESOURCE_FILE_LICENSE "${LIBAEC_BINARY_DIR}/LIBAEC_LICENSE.txt") + set (CPACK_PACKAGE_DESCRIPTION_FILE "${LIBAEC_BINARY_DIR}/LIBAEC_README.SZIP") + set (CPACK_RESOURCE_FILE_README "${LIBAEC_BINARY_DIR}/LIBAEC_README.md") set (CPACK_PACKAGE_RELOCATABLE TRUE) set (CPACK_PACKAGE_DESCRIPTION_SUMMARY "libaec - Adaptive Entropy Coding library by Deutsches Klimarechenzentrum GmbH") set (CPACK_PACKAGE_INSTALL_DIRECTORY "${CPACK_PACKAGE_VENDOR}/${CPACK_PACKAGE_NAME}/${CPACK_PACKAGE_VERSION}") @@ -419,7 +423,7 @@ if (NOT LIBAEC_EXTERNALLY_CONFIGURED) endif () #WiX variables set (CPACK_WIX_UNINSTALL "1") - set (CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE.txt") + set (CPACK_RESOURCE_FILE_LICENSE "${LIBAEC_BINARY_DIR}/LIBAEC_LICENSE.txt") elseif (APPLE) list (APPEND CPACK_GENERATOR "STGZ") list (APPEND CPACK_GENERATOR "DragNDrop") diff --git a/config/cmake/ZLIB/CMakeLists.txt b/config/cmake/ZLIB/CMakeLists.txt index c74ecea..5e42fb2 100644 --- a/config/cmake/ZLIB/CMakeLists.txt +++ b/config/cmake/ZLIB/CMakeLists.txt @@ -423,6 +423,16 @@ if (WIN32) endif () #----------------------------------------------------------------------------- +# Configure the LICENSE.txt file for the windows binary package +#----------------------------------------------------------------------------- +if (WIN32) + configure_file (${ZLIB_SOURCE_DIR}/LICENSE ${ZLIB_BINARY_DIR}/ZLIB_LICENSE.txt @ONLY) +else () + configure_file (${ZLIB_SOURCE_DIR}/LICENSE ${ZLIB_BINARY_DIR}/ZLIB_LICENSE @ONLY) +endif () +configure_file (${ZLIB_SOURCE_DIR}/README ${ZLIB_BINARY_DIR}/ZLIB_README @ONLY) + +#----------------------------------------------------------------------------- # Set the cpack variables #----------------------------------------------------------------------------- if (NOT ZLIB_EXTERNALLY_CONFIGURED) @@ -436,9 +446,9 @@ if (NOT ZLIB_EXTERNALLY_CONFIGURED) set (CPACK_PACKAGE_VERSION_MAJOR "${ZLIB_PACKAGE_VERSION_MAJOR}") set (CPACK_PACKAGE_VERSION_MINOR "${ZLIB_PACKAGE_VERSION_MINOR}") set (CPACK_PACKAGE_VERSION_PATCH "") - set (CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/README") - set (CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README") - set (CPACK_RESOURCE_FILE_README "${CMAKE_CURRENT_SOURCE_DIR}/README") + set (CPACK_RESOURCE_FILE_LICENSE "${ZLIB_BINARY_DIR}/ZLIB_LICENSE") + set (CPACK_PACKAGE_DESCRIPTION_FILE "${ZLIB_BINARY_DIR}/ZLIB_README") + set (CPACK_RESOURCE_FILE_README "${ZLIB_BINARY_DIR}/ZLIB_README") set (CPACK_PACKAGE_RELOCATABLE TRUE) set (CPACK_PACKAGE_DESCRIPTION_SUMMARY "zlib Installation") set (CPACK_PACKAGE_INSTALL_DIRECTORY "${CPACK_PACKAGE_VENDOR}/${CPACK_PACKAGE_NAME}/${CPACK_PACKAGE_VERSION}") @@ -472,7 +482,7 @@ if (NOT ZLIB_EXTERNALLY_CONFIGURED) endif () #WiX variables set (CPACK_WIX_UNINSTALL "1") - set (CPACK_RESOURCE_FILE_LICENSE "${JPEG_BINARY_DIR}/README") + set (CPACK_RESOURCE_FILE_LICENSE "${ZLIB_BINARY_DIR}/ZLIB_LICENSE.txt") elseif (APPLE) list (APPEND CPACK_GENERATOR "STGZ") list (APPEND CPACK_GENERATOR "DragNDrop") diff --git a/doc/getting-started-with-hdf5-development.md b/doc/getting-started-with-hdf5-development.md index 732d817..04c42dc 100644 --- a/doc/getting-started-with-hdf5-development.md +++ b/doc/getting-started-with-hdf5-development.md @@ -746,7 +746,7 @@ to do this via a script. These are normally named `test_<thing>.sh.in`. The step. In the past, we have tried to stick to POSIX Bourne shell scripts, but many scripts now require bash. -If you write a new test script, it is important to also add a Powershell +If you write a new test script, it is important to also add a PowerShell equivalent for testing on Windows. It's helpful to run any new shell scripts through `shellcheck` diff --git a/fortran/test/tH5T_F03.F90 b/fortran/test/tH5T_F03.F90 index 2256b50..86e49b6 100644 --- a/fortran/test/tH5T_F03.F90 +++ b/fortran/test/tH5T_F03.F90 @@ -1374,7 +1374,7 @@ SUBROUTINE t_enum(total_error) INTEGER(SIZE_T) , PARAMETER :: NAME_BUF_SIZE = 16 ! Enumerated type - INTEGER, PARAMETER :: SOLID=0, LIQUID=1, GAS=2, PLASMA=3 + INTEGER, PARAMETER :: SOLID=0, PLASMA=3 INTEGER(HID_T) :: file, filetype, memtype, space, dset ! Handles diff --git a/release_docs/INSTALL_CMake.txt b/release_docs/INSTALL_CMake.txt index f1f56d6..34ac936 100644 --- a/release_docs/INSTALL_CMake.txt +++ b/release_docs/INSTALL_CMake.txt @@ -14,6 +14,7 @@ Section VI: CMake option defaults for HDF5 Section VII: User Defined Options for HDF5 Libraries with CMake Section VIII: User Defined Compile Flags for HDF5 Libraries with CMake Section IX: Considerations for cross-compiling +Section X: Using CMakePresets.json for compiling ************************************************************************ @@ -209,10 +210,10 @@ Notes: This short set of instructions is written for users who want to 5. Configure the C library, tools and tests with one of the following commands: On Windows 32 bit - cmake -G "Visual Studio 12 2013" -DCMAKE_BUILD_TYPE:STRING=Release -DBUILD_SHARED_LIBS:BOOL=OFF -DBUILD_TESTING:BOOL=ON -DHDF5_BUILD_TOOLS:BOOL=ON ..\hdf5-1.14."X" + cmake -G "Visual Studio 16 2019" -A Win32 -DCMAKE_BUILD_TYPE:STRING=Release -DBUILD_SHARED_LIBS:BOOL=OFF -DBUILD_TESTING:BOOL=ON -DHDF5_BUILD_TOOLS:BOOL=ON ..\hdf5-1.15."X" On Windows 64 bit - cmake -G "Visual Studio 12 2013 Win64" -DCMAKE_BUILD_TYPE:STRING=Release -DBUILD_SHARED_LIBS:BOOL=OFF -DBUILD_TESTING:BOOL=ON -DHDF5_BUILD_TOOLS:BOOL=ON ..\hdf5-1.14."X" + cmake -G "Visual Studio 16 2019 Win64" -A x64 -DCMAKE_BUILD_TYPE:STRING=Release -DBUILD_SHARED_LIBS:BOOL=OFF -DBUILD_TESTING:BOOL=ON -DHDF5_BUILD_TOOLS:BOOL=ON ..\hdf5-1.15."X" On Linux and Mac cmake -G "Unix Makefiles" -DCMAKE_BUILD_TYPE:STRING=Release -DBUILD_SHARED_LIBS:BOOL=OFF -DBUILD_TESTING:BOOL=ON -DHDF5_BUILD_TOOLS:BOOL=ON ../hdf5-1.14."X" @@ -1042,6 +1043,99 @@ The HDF5 CMake variables; HDF5_USE_PREGEN: set this to true HDF5_USE_PREGEN_DIR: set this path to the preset H5Tinit.c file + +======================================================================== +X: Using CMakePresets.json for compiling +======================================================================== + +One problem that CMake users often face is sharing settings with other people for common +ways to configure a project. This may be done to support CI builds, or for users who +frequently use the same build. CMake supports two main files, CMakePresets.json and CMakeUserPresets.json, +that allow users to specify common configure options and share them with others. CMake also supports +files included with the include field. + +CMakePresets.json and CMakeUserPresets.json live in the project's root directory. They +both have exactly the same format, and both are optional (though at least one must be +present if --preset is specified). CMakePresets.json is meant to specify project-wide build +details, while CMakeUserPresets.json is meant for developers to specify their own local build details. + +See CMake documentation for details: https://cmake.org/cmake/help/latest/manual/cmake-presets.7.html + +HDF-provided CMakePresets.json +------------------------------- +The CMakePresets.json provided by HDF requires CMake version 3.25, which supports package +and workflow presets, and ninja build system. The top-level configuration group is intended to be +a standard set of options to produce a package of shared and staic libraries and tools. Other configurations +used for inheriting settings are in the included json file in "config/cmake-presets/hidden-presets.json". + +Available configurations presets can be displayed by executing: + cmake -S <path-to-source> --list-presets + +Using individual command presets (where <compiler-type> is GNUC or MSVC or Clang): + change directory to the hdf5 source folder + cmake --presets=ci-StdShar-<compiler-type> + cmake --build --presets=ci-StdShar-<compiler-type> + ctest --presets=ci-StdShar-<compiler-type> + cpack --presets=ci-StdShar-<compiler-type> + + +Using the workflow preset to configure, build, test and package the standard configuration is: + change directory to the hdf5 source folder + execute "cmake --workflow --presets=ci-StdShar-<compiler-type> --fresh" + where <compiler-type> is GNUC or MSVC or Clang + +Creating your own configurations +-------------------------------- +The quickest way is to copy CMakePresets.json to CMakeUserPresets.json and +edit CMakeUserPresets.json configuration names from ci-* to my-*. Change the +"configurePresets" section "inherits" field only for those that you have alternate +options. Then change the "configurePreset" field entries in the "buildPresets", +"testPresets", "packagePresets" sections to match your my-StdShar-<compiler-type>. +And finally the names settings in the "workflowPresets" steps will also need the ci-* to my-* change. + +For instance, to change the support files to use a local directory, edit CMakeUserPresets.json: +...... + { + "name": "my-base-tgz", + "hidden": true, + "inherits": "ci-base", + "cacheVariables": { + "HDF5_ALLOW_EXTERNAL_SUPPORT": {"type": "STRING", "value": "TGZ"}, + "TGZPATH": {"type": "STRING", "value": "${sourceParentDir}/temp"} + } + }, + { + "name": "my-StdCompression", + "hidden": true, + "inherits": "my-base-tgz", + "cacheVariables": { +...... + { + "name": "my-StdShar", + "hidden": true, + "inherits": "my-StdCompression", + "cacheVariables": { +...... + { + "name": "my-StdShar-GNUC", + "description": "GNUC Standard Config for x64 (Release)", + "inherits": [ + "ci-x64-Release-GNUC", + "ci-CPP", + "ci-Fortran", + "ci-Java", + "my-StdShar", + "my-StdExamples" + ] + } +...... + + +Then you can change or add options for your specific case. + + + + ======================================================================== For further assistance, send email to help@hdfgroup.org ======================================================================== diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 2d708b3..8220786 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -47,6 +47,15 @@ New Features Configuration: ------------- + - Added support for CMake presets file. + + CMake supports two main files, CMakePresets.json and CMakeUserPresets.json, + that allow users to specify common configure options and share them with others. + HDF added a CMakePresets.json file of a typical configuration and support + file, config/cmake-presets/hidden-presets.json. + Also added a section to INSTALL_CMake.txt with very basic explanation of the + process to use CMakePresets. + - Deprecated and removed old SZIP library in favor of LIBAEC library LIBAEC library has been used in HDF5 binaries as the szip library of choice @@ -154,6 +163,24 @@ Bug Fixes since HDF5-1.14.0 release =================================== Library ------- + - Fixed a bug in H5Ocopy that could generate invalid HDF5 files + + H5Ocopy was missing a check to determine whether the new object's + object header version is greater than version 1. Without this check, + copying of objects with object headers that are smaller than a + certain size would cause H5Ocopy to create an object header for the + new object that has a gap in the header data. According to the + HDF5 File Format Specification, this is not allowed for version + 1 of the object header format. + + Fixes GitHub issue #2653 + + - Fixed H5Pget_vol_cap_flags and H5Pget_vol_id to accept H5P_DEFAULT + + H5Pget_vol_cap_flags and H5Pget_vol_id were updated to correctly + accept H5P_DEFAULT for the 'plist_id' FAPL parameter. Previously, + they would fail if provided with H5P_DEFAULT as the FAPL. + - Fixed ROS3 VFD anonymous credential usage with h5dump and h5ls ROS3 VFD anonymous credential functionality became broken in h5dump diff --git a/src/H5Bcache.c b/src/H5Bcache.c index cd0a0ba..437bc1b 100644 --- a/src/H5Bcache.c +++ b/src/H5Bcache.c @@ -13,10 +13,8 @@ /*------------------------------------------------------------------------- * * Created: H5Bcache.c - * Oct 31 2005 - * Quincey Koziol * - * Purpose: Implement B-tree metadata cache methods. + * Purpose: Implement B-tree metadata cache methods * *------------------------------------------------------------------------- */ @@ -83,13 +81,9 @@ const H5AC_class_t H5AC_BT[1] = {{ /*------------------------------------------------------------------------- * Function: H5B__cache_get_initial_load_size * - * Purpose: Compute the size of the data structure on disk. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * May 18, 2010 + * Purpose: Compute the size of the data structure on disk * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -117,24 +111,20 @@ H5B__cache_get_initial_load_size(void *_udata, size_t *image_len) /*------------------------------------------------------------------------- * Function: H5B__cache_deserialize * - * Purpose: Deserialize the data structure from disk. - * - * Return: Success: Pointer to a new B-tree node. - * Failure: NULL - * - * Programmer: Quincey Koziol - * Mar 24, 2008 + * Purpose: Deserialize the data structure from disk * + * Return: Success: Pointer to a new B-tree node + * Failure: NULL *------------------------------------------------------------------------- */ static void * -H5B__cache_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata, - hbool_t H5_ATTR_UNUSED *dirty) +H5B__cache_deserialize(const void *_image, size_t len, void *_udata, hbool_t H5_ATTR_UNUSED *dirty) { H5B_t *bt = NULL; /* Pointer to the deserialized B-tree node */ H5B_cache_ud_t *udata = (H5B_cache_ud_t *)_udata; /* User data for callback */ H5B_shared_t *shared; /* Pointer to shared B-tree info */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into image buffer */ + const uint8_t *p_end = image + len - 1; /* End of image buffer */ uint8_t *native; /* Pointer to native keys */ unsigned u; /* Local index variable */ H5B_t *ret_value = NULL; /* Return value */ @@ -156,7 +146,8 @@ H5B__cache_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, void *_uda /* Get a pointer to the shared info, for convenience */ shared = (H5B_shared_t *)H5UC_GET_OBJ(bt->rc_shared); - HDassert(shared); + if (NULL == shared) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, NULL, "can't get a pointer to shared data") /* Allocate space for the native keys and child addresses */ if (NULL == (bt->native = H5FL_BLK_MALLOC(native_block, shared->sizeof_keys))) @@ -164,49 +155,61 @@ H5B__cache_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, void *_uda if (NULL == (bt->child = H5FL_SEQ_MALLOC(haddr_t, (size_t)shared->two_k))) HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, NULL, "can't allocate buffer for child addresses") - /* magic number */ + /* Magic number */ + if (H5_IS_BUFFER_OVERFLOW(image, H5_SIZEOF_MAGIC, p_end)) + HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); if (HDmemcmp(image, H5B_MAGIC, (size_t)H5_SIZEOF_MAGIC) != 0) HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "wrong B-tree signature") image += H5_SIZEOF_MAGIC; - /* node type and level */ + /* Node type and level */ + if (H5_IS_BUFFER_OVERFLOW(image, 2, p_end)) + HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); if (*image++ != (uint8_t)udata->type->id) HGOTO_ERROR(H5E_BTREE, H5E_CANTLOAD, NULL, "incorrect B-tree node type") bt->level = *image++; - /* entries used */ + /* Entries used */ + if (H5_IS_BUFFER_OVERFLOW(image, 2, p_end)) + HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); UINT16DECODE(image, bt->nchildren); /* Check if bt->nchildren is greater than two_k */ if (bt->nchildren > shared->two_k) HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "number of children is greater than maximum") - /* sibling pointers */ + /* Sibling pointers */ + if (H5_IS_BUFFER_OVERFLOW(image, H5F_sizeof_addr(udata->f), p_end)) + HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5F_addr_decode(udata->f, (const uint8_t **)&image, &(bt->left)); + + if (H5_IS_BUFFER_OVERFLOW(image, H5F_sizeof_addr(udata->f), p_end)) + HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5F_addr_decode(udata->f, (const uint8_t **)&image, &(bt->right)); - /* the child/key pairs */ + /* Child/key pairs */ native = bt->native; for (u = 0; u < bt->nchildren; u++) { /* Decode native key value */ + if (H5_IS_BUFFER_OVERFLOW(image, shared->sizeof_rkey, p_end)) + HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); if ((udata->type->decode)(shared, image, native) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTDECODE, NULL, "unable to decode key") image += shared->sizeof_rkey; native += udata->type->sizeof_nkey; /* Decode address value */ + if (H5_IS_BUFFER_OVERFLOW(image, H5F_sizeof_addr(udata->f), p_end)) + HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5F_addr_decode(udata->f, (const uint8_t **)&image, bt->child + u); - } /* end for */ + } - /* Decode final key */ + /* Final key */ if (bt->nchildren > 0) { /* Decode native key value */ if ((udata->type->decode)(shared, image, native) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTDECODE, NULL, "unable to decode key") - } /* end if */ - - /* Sanity check */ - HDassert((size_t)((const uint8_t *)image - (const uint8_t *)_image) <= len); + } /* Set return value */ ret_value = bt; @@ -224,11 +227,7 @@ done: * * Purpose: Compute the size of the data structure on disk. * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * May 20, 2010 - * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -256,13 +255,9 @@ H5B__cache_image_len(const void *_thing, size_t *image_len) /*------------------------------------------------------------------------- * Function: H5B__cache_serialize * - * Purpose: Serialize the data structure for writing to disk. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * Mar 24, 2008 + * Purpose: Serialize the data structure for writing to disk * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -341,11 +336,7 @@ done: * * Purpose: Destroy/release an "in core representation" of a data structure * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * Mar 26, 2008 - * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t diff --git a/src/H5Bpkg.h b/src/H5Bpkg.h index ef9f56e..f50bd48 100644 --- a/src/H5Bpkg.h +++ b/src/H5Bpkg.h @@ -44,21 +44,21 @@ /* The B-tree node as stored in memory... */ typedef struct H5B_t { H5AC_info_t cache_info; /* Information for H5AC cache functions */ - /* _must_ be first field in structure */ - H5UC_t *rc_shared; /*ref-counted shared info */ - unsigned level; /*node level */ - unsigned nchildren; /*number of child pointers */ - haddr_t left; /*address of left sibling */ - haddr_t right; /*address of right sibling */ - uint8_t *native; /*array of keys in native format */ - haddr_t *child; /*2k child pointers */ + /* MUST be first field in structure */ + H5UC_t *rc_shared; /* Ref-counted shared info */ + unsigned level; /* Node level */ + unsigned nchildren; /* Number of child pointers */ + haddr_t left; /* Address of left sibling */ + haddr_t right; /* Address of right sibling */ + uint8_t *native; /* Array of keys in native format */ + haddr_t *child; /* 2k child pointers */ } H5B_t; /* Callback info for loading a B-tree node into the cache */ typedef struct H5B_cache_ud_t { - H5F_t *f; /* File that B-tree node is within */ - const struct H5B_class_t *type; /* Type of tree */ - H5UC_t *rc_shared; /* Ref-counted shared info */ + H5F_t *f; /* File that B-tree node is within */ + const struct H5B_class_t *type; /* Type of tree */ + H5UC_t *rc_shared; /* Ref-counted shared info */ } H5B_cache_ud_t; /*****************************/ diff --git a/src/H5Bprivate.h b/src/H5Bprivate.h index 49e400c..0017c43 100644 --- a/src/H5Bprivate.h +++ b/src/H5Bprivate.h @@ -82,16 +82,16 @@ typedef int (*H5B_operator_t)(H5F_t *f, const void *_lt_key, haddr_t addr, const * the instances of nodes in that B-tree. */ typedef struct H5B_shared_t { - const struct H5B_class_t *type; /* Type of tree */ - unsigned two_k; /* 2*"K" value for tree's nodes */ - size_t sizeof_rkey; /* Size of raw (disk) key */ - size_t sizeof_rnode; /* Size of raw (disk) node */ - size_t sizeof_keys; /* Size of native (memory) key node */ - size_t sizeof_addr; /* Size of file address (in bytes) */ - size_t sizeof_len; /* Size of file lengths (in bytes) */ + const struct H5B_class_t *type; /* Type of tree */ + unsigned two_k; /* 2*"K" value for tree's nodes */ + size_t sizeof_rkey; /* Size of raw (disk) key */ + size_t sizeof_rnode; /* Size of raw (disk) node */ + size_t sizeof_keys; /* Size of native (memory) key node */ + size_t sizeof_addr; /* Size of file address (in bytes) */ + size_t sizeof_len; /* Size of file lengths (in bytes) */ uint8_t *page; /* Disk page */ size_t *nkey; /* Offsets of each native key in native key buffer */ - void *udata; /* 'Local' info for a B-tree */ + void *udata; /* 'Local' info for a B-tree */ } H5B_shared_t; /* @@ -101,7 +101,6 @@ typedef struct H5B_shared_t { * has an array of K values indexed by the `id' class field below. The * array is initialized with the HDF5_BTREE_K_DEFAULT macro. */ - typedef struct H5B_class_t { H5B_subid_t id; /*id as found in file*/ size_t sizeof_nkey; /*size of native (memory) key*/ diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 9ec6f70..4315c7b 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -461,6 +461,20 @@ H5_DLL hid_t H5Dget_space_async(hid_t dset_id, hid_t es_id); * \details H5Dget_space_status() determines whether space has been allocated * for the dataset \p dset_id. * + * \note \Bold{BUG:} Prior to the HDF5 1.14.0, 1.12.2 and 1.10.9 releases, + * H5Dget_space_status() may return incorrect space allocation status + * values for datasets with filters applied to them. + * H5Dget_space_status() calculated the space allocation status by + * comparing the sum of the sizes of all the allocated chunks in the + * dataset against the total data size of the dataset, as calculated by + * the number of elements in the dataset's dataspace multiplied by the + * dataset's datatype size. If the dataset had any compression filters + * applied to it and the dataset chunks were successfully compressed, + * the sum of the sizes of the allocated dataset chunks would generally + * always be less than the total data size of the dataset, and + * H5Dget_space_status() wouldn't ever return + * `H5D_SPACE_STATUS_ALLOCATED`. + * * \since 1.6.0 * */ diff --git a/src/H5FDsubfiling/H5FDioc.c b/src/H5FDsubfiling/H5FDioc.c index 2fd8b64..7d20021 100644 --- a/src/H5FDsubfiling/H5FDioc.c +++ b/src/H5FDsubfiling/H5FDioc.c @@ -887,16 +887,20 @@ done: static herr_t H5FD__ioc_close_int(H5FD_ioc_t *file_ptr) { + int mpi_finalized; + int mpi_code; herr_t ret_value = SUCCEED; HDassert(file_ptr); + if (MPI_SUCCESS != (mpi_code = MPI_Finalized(&mpi_finalized))) + H5_SUBFILING_MPI_GOTO_ERROR(FAIL, "MPI_Finalized failed", mpi_code); + if (file_ptr->context_id >= 0) { subfiling_context_t *sf_context = H5_get_subfiling_object(file_ptr->context_id); - int mpi_code; /* Don't allow IOC threads to be finalized until everyone gets here */ - if (file_ptr->mpi_size > 1) + if (!mpi_finalized && (file_ptr->mpi_size > 1)) if (MPI_SUCCESS != (mpi_code = MPI_Barrier(file_ptr->comm))) H5_SUBFILING_MPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code); @@ -911,10 +915,12 @@ H5FD__ioc_close_int(H5FD_ioc_t *file_ptr) file_ptr->context_id = -1; } - if (H5_mpi_comm_free(&file_ptr->comm) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free MPI Communicator"); - if (H5_mpi_info_free(&file_ptr->info) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free MPI Info object"); + if (!mpi_finalized) { + if (H5_mpi_comm_free(&file_ptr->comm) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free MPI Communicator"); + if (H5_mpi_info_free(&file_ptr->info) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free MPI Info object"); + } done: HDfree(file_ptr->file_path); diff --git a/src/H5FDsubfiling/H5FDsubfiling.c b/src/H5FDsubfiling/H5FDsubfiling.c index e086190..64c92ed 100644 --- a/src/H5FDsubfiling/H5FDsubfiling.c +++ b/src/H5FDsubfiling/H5FDsubfiling.c @@ -374,12 +374,29 @@ H5FD__subfiling_term(void) herr_t ret_value = SUCCEED; if (H5FD_SUBFILING_g >= 0) { + int mpi_finalized; int mpi_code; + /* + * Retrieve status of whether MPI has already been terminated. + * This can happen if an HDF5 ID is left unclosed and HDF5 + * shuts down after MPI_Finalize() is called in an application. + */ + if (MPI_SUCCESS != (mpi_code = MPI_Finalized(&mpi_finalized))) + H5_SUBFILING_MPI_GOTO_ERROR(FAIL, "MPI_Finalized failed", mpi_code); + /* Free RPC message MPI Datatype */ - if (H5_subfiling_rpc_msg_type != MPI_DATATYPE_NULL) - if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&H5_subfiling_rpc_msg_type))) - H5_SUBFILING_MPI_GOTO_ERROR(FAIL, "MPI_Type_free failed", mpi_code); + if (H5_subfiling_rpc_msg_type != MPI_DATATYPE_NULL) { + if (!mpi_finalized) { + if (MPI_SUCCESS != (mpi_code = MPI_Type_free(&H5_subfiling_rpc_msg_type))) + H5_SUBFILING_MPI_GOTO_ERROR(FAIL, "MPI_Type_free failed", mpi_code); + } +#ifdef H5FD_SUBFILING_DEBUG + else + HDprintf("** WARNING **: HDF5 is terminating the Subfiling VFD after MPI_Finalize() was " + "called - an HDF5 ID was probably left unclosed\n"); +#endif + } /* Clean up resources */ if (H5_subfiling_terminate() < 0) @@ -1297,10 +1314,15 @@ done: static herr_t H5FD__subfiling_close_int(H5FD_subfiling_t *file_ptr) { + int mpi_finalized; + int mpi_code; herr_t ret_value = SUCCEED; HDassert(file_ptr); + if (MPI_SUCCESS != (mpi_code = MPI_Finalized(&mpi_finalized))) + H5_SUBFILING_MPI_GOTO_ERROR(FAIL, "MPI_Finalized failed", mpi_code); + if (file_ptr->sf_file && H5FD_close(file_ptr->sf_file) < 0) H5_SUBFILING_GOTO_ERROR(H5E_IO, H5E_CANTCLOSEFILE, FAIL, "unable to close subfile"); if (file_ptr->stub_file && H5FD_close(file_ptr->stub_file) < 0) @@ -1311,13 +1333,15 @@ H5FD__subfiling_close_int(H5FD_subfiling_t *file_ptr) H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_ARGS, FAIL, "can't close IOC FAPL"); file_ptr->fa.ioc_fapl_id = H5I_INVALID_HID; - if (H5_mpi_comm_free(&file_ptr->comm) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free MPI Communicator"); - if (H5_mpi_info_free(&file_ptr->info) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free MPI Info object"); + if (!mpi_finalized) { + if (H5_mpi_comm_free(&file_ptr->comm) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free MPI Communicator"); + if (H5_mpi_info_free(&file_ptr->info) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free MPI Info object"); - if (H5_mpi_comm_free(&file_ptr->ext_comm) < 0) - H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "can't free MPI communicator"); + if (H5_mpi_comm_free(&file_ptr->ext_comm) < 0) + H5_SUBFILING_GOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "can't free MPI communicator"); + } file_ptr->fail_to_encode = FALSE; diff --git a/src/H5FDsubfiling/H5subfiling_common.c b/src/H5FDsubfiling/H5subfiling_common.c index 58f3643..8fea794 100644 --- a/src/H5FDsubfiling/H5subfiling_common.c +++ b/src/H5FDsubfiling/H5subfiling_common.c @@ -338,8 +338,18 @@ done: static herr_t H5_free_subfiling_object_int(subfiling_context_t *sf_context) { + int mpi_finalized; + int mpi_code; + herr_t ret_value = SUCCEED; + HDassert(sf_context); + if (MPI_SUCCESS != (mpi_code = MPI_Finalized(&mpi_finalized))) { + /* Assume MPI is finalized or worse, and try to clean up what we can */ + H5_SUBFILING_MPI_DONE_ERROR(FAIL, "MPI_Finalized failed", mpi_code); + mpi_finalized = 1; + } + sf_context->sf_context_id = -1; sf_context->h5_file_id = UINT64_MAX; sf_context->sf_num_fids = 0; @@ -352,28 +362,38 @@ H5_free_subfiling_object_int(subfiling_context_t *sf_context) sf_context->sf_base_addr = -1; if (sf_context->sf_msg_comm != MPI_COMM_NULL) { - if (H5_mpi_comm_free(&sf_context->sf_msg_comm) < 0) - return FAIL; + if (!mpi_finalized) { + if (H5_mpi_comm_free(&sf_context->sf_msg_comm) < 0) + return FAIL; + } sf_context->sf_msg_comm = MPI_COMM_NULL; } if (sf_context->sf_data_comm != MPI_COMM_NULL) { - if (H5_mpi_comm_free(&sf_context->sf_data_comm) < 0) - return FAIL; + if (!mpi_finalized) { + if (H5_mpi_comm_free(&sf_context->sf_data_comm) < 0) + return FAIL; + } sf_context->sf_data_comm = MPI_COMM_NULL; } if (sf_context->sf_eof_comm != MPI_COMM_NULL) { - if (H5_mpi_comm_free(&sf_context->sf_eof_comm) < 0) - return FAIL; + if (!mpi_finalized) { + if (H5_mpi_comm_free(&sf_context->sf_eof_comm) < 0) + return FAIL; + } sf_context->sf_eof_comm = MPI_COMM_NULL; } if (sf_context->sf_node_comm != MPI_COMM_NULL) { - if (H5_mpi_comm_free(&sf_context->sf_node_comm) < 0) - return FAIL; + if (!mpi_finalized) { + if (H5_mpi_comm_free(&sf_context->sf_node_comm) < 0) + return FAIL; + } sf_context->sf_node_comm = MPI_COMM_NULL; } if (sf_context->sf_group_comm != MPI_COMM_NULL) { - if (H5_mpi_comm_free(&sf_context->sf_group_comm) < 0) - return FAIL; + if (!mpi_finalized) { + if (H5_mpi_comm_free(&sf_context->sf_group_comm) < 0) + return FAIL; + } sf_context->sf_group_comm = MPI_COMM_NULL; } @@ -402,16 +422,24 @@ H5_free_subfiling_object_int(subfiling_context_t *sf_context) HDfree(sf_context); - return SUCCEED; + H5_SUBFILING_FUNC_LEAVE; } static herr_t H5_free_subfiling_topology(sf_topology_t *topology) { + int mpi_finalized; + int mpi_code; herr_t ret_value = SUCCEED; HDassert(topology); + if (MPI_SUCCESS != (mpi_code = MPI_Finalized(&mpi_finalized))) { + /* Assume MPI is finalized or worse, but clean up what we can */ + H5_SUBFILING_MPI_DONE_ERROR(FAIL, "MPI_Finalized failed", mpi_code); + mpi_finalized = 1; + } + #ifndef NDEBUG { hbool_t topology_cached = FALSE; @@ -442,8 +470,9 @@ H5_free_subfiling_topology(sf_topology_t *topology) HDfree(topology->io_concentrators); topology->io_concentrators = NULL; - if (H5_mpi_comm_free(&topology->app_comm) < 0) - H5_SUBFILING_DONE_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "can't free MPI communicator"); + if (!mpi_finalized) + if (H5_mpi_comm_free(&topology->app_comm) < 0) + H5_SUBFILING_DONE_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "can't free MPI communicator"); HDfree(topology); diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c index 467e287..7dbaf22 100644 --- a/src/H5Fsuper_cache.c +++ b/src/H5Fsuper_cache.c @@ -13,10 +13,8 @@ /*------------------------------------------------------------------------- * * Created: H5Fsuper_cache.c - * Aug 15 2009 - * Quincey Koziol * - * Purpose: Implement file superblock & driver info metadata cache methods. + * Purpose: Implement file superblock & driver info metadata cache methods * *------------------------------------------------------------------------- */ @@ -76,10 +74,10 @@ static herr_t H5F__cache_drvrinfo_serialize(const H5F_t *f, void *image, size_t static herr_t H5F__cache_drvrinfo_free_icr(void *thing); /* Local encode/decode routines */ -static herr_t H5F__superblock_prefix_decode(H5F_super_t *sblock, const uint8_t **image_ref, +static herr_t H5F__superblock_prefix_decode(H5F_super_t *sblock, const uint8_t **image_ref, size_t len, const H5F_superblock_cache_ud_t *udata, hbool_t extend_eoa); static herr_t H5F__drvrinfo_prefix_decode(H5O_drvinfo_t *drvinfo, char *drv_name, const uint8_t **image_ref, - H5F_drvrinfo_cache_ud_t *udata, hbool_t extend_eoa); + size_t len, H5F_drvrinfo_cache_ud_t *udata, hbool_t extend_eoa); /*********************/ /* Package Variables */ @@ -135,25 +133,21 @@ H5FL_EXTERN(H5F_super_t); /*------------------------------------------------------------------------- * Function: H5F__superblock_prefix_decode * - * Purpose: Decode a superblock prefix - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * December 15, 2016 + * Purpose: Decode a superblock prefix * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t -H5F__superblock_prefix_decode(H5F_super_t *sblock, const uint8_t **image_ref, +H5F__superblock_prefix_decode(H5F_super_t *sblock, const uint8_t **image_ref, size_t len, const H5F_superblock_cache_ud_t *udata, hbool_t extend_eoa) { const uint8_t *image = (const uint8_t *)*image_ref; /* Pointer into raw data buffer */ - htri_t ret_value = SUCCEED; /* Return value */ + const uint8_t *end = image + len - 1; /* Pointer to end of buffer */ + htri_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(sblock); HDassert(image_ref); HDassert(image); @@ -161,27 +155,37 @@ H5F__superblock_prefix_decode(H5F_super_t *sblock, const uint8_t **image_ref, HDassert(udata->f); /* Skip over signature (already checked when locating the superblock) */ + if (H5_IS_BUFFER_OVERFLOW(image, H5F_SIGNATURE_LEN, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); image += H5F_SIGNATURE_LEN; /* Superblock version */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); sblock->super_vers = *image++; if (sblock->super_vers > HDF5_SUPERBLOCK_VERSION_LATEST) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad superblock version number") - /* Sanity check */ - HDassert(((size_t)(image - (const uint8_t *)*image_ref)) == H5F_SUPERBLOCK_FIXED_SIZE); + /* Size check */ + if (((size_t)(image - (const uint8_t *)*image_ref)) != H5F_SUPERBLOCK_FIXED_SIZE) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad superblock (fixed) size") /* Determine the size of addresses & size of offsets, for computing the * variable-sized portion of the superblock. */ if (sblock->super_vers < HDF5_SUPERBLOCK_VERSION_2) { + if (H5_IS_BUFFER_OVERFLOW(image, 6, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); sblock->sizeof_addr = image[4]; sblock->sizeof_size = image[5]; - } /* end if */ + } else { + if (H5_IS_BUFFER_OVERFLOW(image, 2, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); sblock->sizeof_addr = image[0]; sblock->sizeof_size = image[1]; - } /* end else */ + } + if (sblock->sizeof_addr != 2 && sblock->sizeof_addr != 4 && sblock->sizeof_addr != 8 && sblock->sizeof_addr != 16 && sblock->sizeof_addr != 32) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number in an address") @@ -196,12 +200,13 @@ H5F__superblock_prefix_decode(H5F_super_t *sblock, const uint8_t **image_ref, /* Determine the size of the variable-length part of the superblock */ variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(sblock->super_vers, sblock->sizeof_addr, sblock->sizeof_size); - HDassert(variable_size > 0); + if (variable_size == 0) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "variable size can't be zero") /* Make certain we can read the variable-sized portion of the superblock */ if (H5F__set_eoa(udata->f, H5FD_MEM_SUPER, (haddr_t)(H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") - } /* end if */ + } /* Update the image buffer pointer */ *image_ref = image; @@ -211,28 +216,24 @@ done: } /* end H5F__superblock_prefix_decode() */ /*------------------------------------------------------------------------- - * Function: H5F__drvrinfo_prefix_decode - * - * Purpose: Decode a driver info prefix + * Function: H5F__drvrinfo_prefix_decode * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * December 15, 2016 + * Purpose: Decode a driver info prefix * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t -H5F__drvrinfo_prefix_decode(H5O_drvinfo_t *drvrinfo, char *drv_name, const uint8_t **image_ref, +H5F__drvrinfo_prefix_decode(H5O_drvinfo_t *drvrinfo, char *drv_name, const uint8_t **image_ref, size_t len, H5F_drvrinfo_cache_ud_t *udata, hbool_t extend_eoa) { const uint8_t *image = (const uint8_t *)*image_ref; /* Pointer into raw data buffer */ + const uint8_t *end = image + len - 1; /* Pointer to end of buffer */ unsigned drv_vers; /* Version of driver info block */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Sanity check */ HDassert(drvrinfo); HDassert(image_ref); HDassert(image); @@ -240,21 +241,30 @@ H5F__drvrinfo_prefix_decode(H5O_drvinfo_t *drvrinfo, char *drv_name, const uint8 HDassert(udata->f); /* Version number */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); drv_vers = *image++; if (drv_vers != HDF5_DRIVERINFO_VERSION_0) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad driver information block version number") - image += 3; /* reserved bytes */ + /* Reserved bytes */ + if (H5_IS_BUFFER_OVERFLOW(image, 3, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); + image += 3; /* Driver info size */ + if (H5_IS_BUFFER_OVERFLOW(image, 4, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT32DECODE(image, drvrinfo->len); /* Driver name and/or version */ if (drv_name) { + if (H5_IS_BUFFER_OVERFLOW(image, 8, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); H5MM_memcpy(drv_name, (const char *)image, (size_t)8); drv_name[8] = '\0'; image += 8; /* advance past name/version */ - } /* end if */ + } /* Extend the EOA if required so that we can read the complete driver info block */ if (extend_eoa) { @@ -273,7 +283,7 @@ H5F__drvrinfo_prefix_decode(H5O_drvinfo_t *drvrinfo, char *drv_name, const uint8 if (H5F_addr_gt(min_eoa, eoa)) if (H5FD_set_eoa(udata->f->shared->lf, H5FD_MEM_SUPER, min_eoa) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") - } /* end if */ + } /* Update the image buffer pointer */ *image_ref = image; @@ -285,13 +295,9 @@ done: /*------------------------------------------------------------------------- * Function: H5F__cache_superblock_get_initial_load_size * - * Purpose: Compute the size of the data structure on disk. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * July 17, 2013 + * Purpose: Compute the size of the data structure on disk * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -299,7 +305,6 @@ H5F__cache_superblock_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t { FUNC_ENTER_PACKAGE_NOERR - /* Check arguments */ HDassert(image_len); /* Set the initial image length size */ @@ -312,27 +317,22 @@ H5F__cache_superblock_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t /*------------------------------------------------------------------------- * Function: H5F__cache_superblock_get_final_load_size * - * Purpose: Compute the final size of the data structure on disk. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * November 17, 2016 + * Purpose: Compute the final size of the data structure on disk * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t -H5F__cache_superblock_get_final_load_size(const void *_image, size_t H5_ATTR_NDEBUG_UNUSED image_len, - void *_udata, size_t *actual_len) +H5F__cache_superblock_get_final_load_size(const void *_image, size_t image_len, void *_udata, + size_t *actual_len) { const uint8_t *image = _image; /* Pointer into raw data buffer */ H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */ H5F_super_t sblock; /* Temporary file superblock */ - htri_t ret_value = SUCCEED; /* Return value */ + htri_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(image); HDassert(udata); HDassert(actual_len); @@ -340,7 +340,7 @@ H5F__cache_superblock_get_final_load_size(const void *_image, size_t H5_ATTR_NDE HDassert(image_len >= H5F_SUPERBLOCK_FIXED_SIZE + 6); /* Deserialize the file superblock's prefix */ - if (H5F__superblock_prefix_decode(&sblock, &image, udata, TRUE) < 0) + if (H5F__superblock_prefix_decode(&sblock, &image, image_len, udata, TRUE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, FAIL, "can't decode file superblock prefix") /* Save the version to be used in verify_chksum callback */ @@ -357,14 +357,11 @@ done: /*------------------------------------------------------------------------- * Function: H5F__cache_superblock_verify_chksum * - * Purpose: Verify the computed checksum of the data structure is the - * same as the stored chksum. - * - * Return: Success: TRUE/FALSE - * Failure: Negative - * - * Programmer: Vailin Choi; Aug 2015 + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. * + * Return: Success: TRUE/FALSE + * Failure: Negative *------------------------------------------------------------------------- */ static htri_t @@ -372,13 +369,12 @@ H5F__cache_superblock_verify_chksum(const void *_image, size_t len, void *_udata { const uint8_t *image = _image; /* Pointer into raw data buffer */ H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ - htri_t ret_value = TRUE; /* Return value */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; FUNC_ENTER_PACKAGE_NOERR - /* Check arguments */ HDassert(image); HDassert(udata); @@ -390,36 +386,31 @@ H5F__cache_superblock_verify_chksum(const void *_image, size_t len, void *_udata if (stored_chksum != computed_chksum) ret_value = FALSE; - } /* end if */ + } FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__cache_superblock_verify_chksum() */ /*------------------------------------------------------------------------- - * Function: H5F__cache_superblock_deserialize - * - * Purpose: Loads an object from the disk. + * Function: H5F__cache_superblock_deserialize * - * Return: Success: Pointer to new object - * Failure: NULL - * - * Programmer: Quincey Koziol - * July 18 2013 + * Purpose: Load an object from the disk * + * Return: Success: Pointer to new object + * Failure: NULL *------------------------------------------------------------------------- */ static void * -H5F__cache_superblock_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUSED len, void *_udata, - hbool_t H5_ATTR_UNUSED *dirty) +H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, hbool_t H5_ATTR_UNUSED *dirty) { H5F_super_t *sblock = NULL; /* File's superblock */ H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */ - const uint8_t *image = _image; /* Pointer into raw data buffer */ - H5F_super_t *ret_value = NULL; /* Return value */ + const uint8_t *image = _image; /* Pointer into raw data buffer */ + const uint8_t *end = image + len - 1; /* Pointer to end of buffer */ + H5F_super_t *ret_value = NULL; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(image); HDassert(udata); HDassert(udata->f); @@ -430,11 +421,9 @@ H5F__cache_superblock_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUS HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* Deserialize the file superblock's prefix */ - if (H5F__superblock_prefix_decode(sblock, &image, udata, FALSE) < 0) + if (H5F__superblock_prefix_decode(sblock, &image, len, udata, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode file superblock prefix") - const uint8_t *image_end = image + len - 1; - /* Check for older version of superblock format */ if (sblock->super_vers < HDF5_SUPERBLOCK_VERSION_2) { uint32_t status_flags; /* File status flags */ @@ -442,122 +431,113 @@ H5F__cache_superblock_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUS unsigned snode_btree_k; /* B-tree symbol table internal node 'K' value */ unsigned chunk_btree_k; /* B-tree chunk internal node 'K' value */ - /* Check whether the image pointer is out of bounds */ - if (H5_IS_BUFFER_OVERFLOW(image, 1, image_end)) - HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") - /* Freespace version (hard-wired) */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") if (HDF5_FREESPACE_VERSION != *image++) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad free space version number") - /* Check whether the image pointer is out of bounds */ - if (H5_IS_BUFFER_OVERFLOW(image, 1, image_end)) - HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") - /* Root group version number (hard-wired) */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") if (HDF5_OBJECTDIR_VERSION != *image++) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad object directory version number") /* Skip over reserved byte */ - image++; - - /* Check whether the image pointer is out of bounds */ - if (H5_IS_BUFFER_OVERFLOW(image, 1, image_end)) + if (H5_IS_BUFFER_OVERFLOW(image, 1, end)) HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") + image++; /* Shared header version number (hard-wired) */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") if (HDF5_SHAREDHEADER_VERSION != *image++) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad shared-header format version number") /* Skip over size of file addresses (already decoded) */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") image++; udata->f->shared->sizeof_addr = sblock->sizeof_addr; /* Keep a local copy also */ /* Skip over size of file sizes (already decoded) */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") image++; udata->f->shared->sizeof_size = sblock->sizeof_size; /* Keep a local copy also */ /* Skip over reserved byte */ - image++; - - /* Check whether the image pointer is out of bounds */ - if (H5_IS_BUFFER_OVERFLOW(image, sizeof(uint16_t), image_end)) + if (H5_IS_BUFFER_OVERFLOW(image, 1, end)) HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") + image++; /* Various B-tree sizes */ + if (H5_IS_BUFFER_OVERFLOW(image, 2, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") UINT16DECODE(image, sym_leaf_k); if (sym_leaf_k == 0) HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad symbol table leaf node 1/2 rank") udata->sym_leaf_k = sym_leaf_k; /* Keep a local copy also */ - /* Check whether the image pointer is out of bounds */ - if (H5_IS_BUFFER_OVERFLOW(image, sizeof(uint16_t), image_end)) - HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") - /* Need 'get' call to set other array values */ + if (H5_IS_BUFFER_OVERFLOW(image, 2, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") UINT16DECODE(image, snode_btree_k); if (snode_btree_k == 0) HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad 1/2 rank for btree internal nodes") udata->btree_k[H5B_SNODE_ID] = snode_btree_k; - /* - * Delay setting the value in the property list until we've checked + /* Delay setting the value in the property list until we've checked * for the indexed storage B-tree internal 'K' value later. */ - /* Check whether the image pointer is out of bounds */ - if (H5_IS_BUFFER_OVERFLOW(image, sizeof(uint32_t), image_end)) - HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") - /* File status flags (not really used yet) */ + if (H5_IS_BUFFER_OVERFLOW(image, 4, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") UINT32DECODE(image, status_flags); - HDassert(status_flags <= 255); + if (status_flags > 255) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad superblock status flags") sblock->status_flags = (uint8_t)status_flags; if (sblock->status_flags & ~H5F_SUPER_ALL_FLAGS) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock") - /* - * If the superblock version # is greater than 0, read in the indexed + /* If the superblock version # is greater than 0, read in the indexed * storage B-tree internal 'K' value */ if (sblock->super_vers > HDF5_SUPERBLOCK_VERSION_DEF) { - /* Check whether the image pointer is out of bounds */ - if (H5_IS_BUFFER_OVERFLOW(image, sizeof(uint16_t), image_end)) + if (H5_IS_BUFFER_OVERFLOW(image, 2, end)) HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") - UINT16DECODE(image, chunk_btree_k); /* Reserved bytes are present only in version 1 */ if (sblock->super_vers == HDF5_SUPERBLOCK_VERSION_1) { - image += 2; /* reserved */ - - /* Check whether the image pointer is out of bounds */ - if (H5_IS_BUFFER_OVERFLOW(image, 1, image_end)) + /* Reserved */ + if (H5_IS_BUFFER_OVERFLOW(image, 2, end)) HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") + image += 2; } - } /* end if */ + } else chunk_btree_k = HDF5_BTREE_CHUNK_IK_DEF; udata->btree_k[H5B_CHUNK_ID] = chunk_btree_k; - /* Check whether the image pointer will be out of bounds */ - if (H5_IS_BUFFER_OVERFLOW(image, H5F_SIZEOF_ADDR(udata->f) * 4, image_end)) - HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") - /* Remainder of "variable-sized" portion of superblock */ + if (H5_IS_BUFFER_OVERFLOW(image, H5F_sizeof_addr(udata->f) * 4, end)) + HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr /*out*/); H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr /*out*/); H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof /*out*/); H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->driver_addr /*out*/); /* Allocate space for the root group symbol table entry */ - HDassert(!sblock->root_ent); + if (sblock->root_ent) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "root entry should not exist yet") if (NULL == (sblock->root_ent = (H5G_entry_t *)H5MM_calloc(sizeof(H5G_entry_t)))) HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for root group symbol table entry") - /* decode the root group symbol table entry */ - if (H5G_ent_decode(udata->f, (const uint8_t **)&image, sblock->root_ent, image_end) < 0) + /* Decode the root group symbol table entry */ + if (H5G_ent_decode(udata->f, (const uint8_t **)&image, sblock->root_ent, end) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode root group symbol table entry") /* Set the root group address to the correct value */ @@ -572,11 +552,10 @@ H5F__cache_superblock_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUS /* Eliminate the driver info */ sblock->driver_addr = HADDR_UNDEF; udata->drvrinfo_removed = TRUE; - } /* end if */ + } /* NOTE: Driver info block is decoded separately, later */ - - } /* end if */ + } else { uint32_t read_chksum; /* Checksum read from file */ @@ -588,7 +567,7 @@ H5F__cache_superblock_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUS udata->f->shared->sizeof_size = sblock->sizeof_size; /* Keep a local copy also */ /* Check whether the image pointer is out of bounds */ - if (H5_IS_BUFFER_OVERFLOW(image, 1, image_end)) + if (H5_IS_BUFFER_OVERFLOW(image, 1, end)) HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") /* File status flags (not really used yet) */ @@ -597,7 +576,7 @@ H5F__cache_superblock_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUS HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock") /* Check whether the image pointer will be out of bounds */ - if (H5_IS_BUFFER_OVERFLOW(image, H5F_SIZEOF_ADDR(udata->f) * 4, image_end)) + if (H5_IS_BUFFER_OVERFLOW(image, H5F_SIZEOF_ADDR(udata->f) * 4, end)) HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") /* Base, superblock extension, end of file & root group object header addresses */ @@ -609,7 +588,7 @@ H5F__cache_superblock_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUS /* checksum verification already done in verify_chksum cb */ /* Check whether the image pointer will be out of bounds */ - if (H5_IS_BUFFER_OVERFLOW(image, sizeof(uint32_t), image_end)) + if (H5_IS_BUFFER_OVERFLOW(image, sizeof(uint32_t), end)) HGOTO_ERROR(H5E_FILE, H5E_OVERFLOW, NULL, "image pointer is out of bounds") /* Decode checksum */ @@ -621,12 +600,12 @@ H5F__cache_superblock_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUS * any attempt to load the Driver Information Block. */ sblock->driver_addr = HADDR_UNDEF; - } /* end else */ + } - /* Sanity check */ - HDassert((size_t)(image - (const uint8_t *)_image) <= len); + /* Size check */ + if ((size_t)(image - (const uint8_t *)_image) > len) + HDONE_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad decoded superblock size") - /* Set return value */ ret_value = sblock; done: @@ -641,13 +620,9 @@ done: /*------------------------------------------------------------------------- * Function: H5F__cache_superblock_image_len * - * Purpose: Compute the size of the data structure on disk. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * July 19, 2013 + * Purpose: Compute the size of the data structure on disk * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -657,7 +632,6 @@ H5F__cache_superblock_image_len(const void *_thing, size_t *image_len) FUNC_ENTER_PACKAGE_NOERR - /* Check arguments */ HDassert(sblock); HDassert(sblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(sblock->cache_info.type == H5AC_SUPERBLOCK); @@ -672,13 +646,9 @@ H5F__cache_superblock_image_len(const void *_thing, size_t *image_len) /*------------------------------------------------------------------------- * Function: H5F__cache_superblock_serialize * - * Purpose: Flushes a dirty object to disk. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * July 19 2013 + * Purpose: Flush a dirty object to disk * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -687,11 +657,10 @@ H5F__cache_superblock_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNU H5F_super_t *sblock = (H5F_super_t *)_thing; /* Pointer to the object */ uint8_t *image = _image; /* Pointer into raw data buffer */ haddr_t rel_eof; /* Relative EOF for file */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Sanity check */ HDassert(f); HDassert(image); HDassert(sblock); @@ -796,7 +765,7 @@ H5F__cache_superblock_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNU /* Sanity check */ HDassert((size_t)(image - (uint8_t *)_image) == (size_t)H5F_SUPERBLOCK_SIZE(sblock)); - } /* end else */ + } /* Sanity check */ HDassert((size_t)(image - (uint8_t *)_image) == len); @@ -808,29 +777,24 @@ done: /*------------------------------------------------------------------------- * Function: H5F__cache_superblock_free_icr * - * Purpose: Destroy/release an "in core representation" of a data + * Purpose: Destroy/release an "in core representation" of a data * structure * - * Note: The metadata cache sets the object's cache_info.magic to - * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr - * callback (checked in assert). - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * July 20, 2013 + * Note: The metadata cache sets the object's cache_info.magic to + * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr + * callback (checked in assert). * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t H5F__cache_superblock_free_icr(void *_thing) { H5F_super_t *sblock = (H5F_super_t *)_thing; /* Pointer to the object */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Sanity check */ HDassert(sblock); HDassert(sblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC); HDassert(sblock->cache_info.type == H5AC_SUPERBLOCK); @@ -848,11 +812,7 @@ done: * * Purpose: Compute the initial size of the data structure on disk. * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * July 20, 2013 - * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -860,7 +820,6 @@ H5F__cache_drvrinfo_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *i { FUNC_ENTER_PACKAGE_NOERR - /* Check arguments */ HDassert(image_len); /* Set the initial image length size */ @@ -874,25 +833,20 @@ H5F__cache_drvrinfo_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *i * * Purpose: Compute the final size of the data structure on disk. * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * November 17, 2016 - * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t -H5F__cache_drvrinfo_get_final_load_size(const void *_image, size_t H5_ATTR_NDEBUG_UNUSED image_len, - void *_udata, size_t *actual_len) +H5F__cache_drvrinfo_get_final_load_size(const void *_image, size_t image_len, void *_udata, + size_t *actual_len) { const uint8_t *image = _image; /* Pointer into raw data buffer */ H5F_drvrinfo_cache_ud_t *udata = (H5F_drvrinfo_cache_ud_t *)_udata; /* User data */ H5O_drvinfo_t drvrinfo; /* Driver info */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Check arguments */ HDassert(image); HDassert(udata); HDassert(actual_len); @@ -900,7 +854,7 @@ H5F__cache_drvrinfo_get_final_load_size(const void *_image, size_t H5_ATTR_NDEBU HDassert(image_len == H5F_DRVINFOBLOCK_HDR_SIZE); /* Deserialize the file driver info's prefix */ - if (H5F__drvrinfo_prefix_decode(&drvrinfo, NULL, &image, udata, TRUE) < 0) + if (H5F__drvrinfo_prefix_decode(&drvrinfo, NULL, &image, image_len, udata, TRUE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, FAIL, "can't decode file driver info prefix") /* Set the final size for the cache image */ @@ -913,29 +867,23 @@ done: /*------------------------------------------------------------------------- * Function: H5F__cache_drvrinfo_deserialize * - * Purpose: Loads an object from the disk. - * - * Return: Success: Pointer to a new driver info struct - * Failure: NULL - * - * Programmer: Quincey Koziol - * July 20 2013 + * Purpose: Loads an object from the disk * + * Return: Success: Pointer to a new driver info struct + * Failure: NULL *------------------------------------------------------------------------- */ static void * -H5F__cache_drvrinfo_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUSED len, void *_udata, - hbool_t H5_ATTR_UNUSED *dirty) +H5F__cache_drvrinfo_deserialize(const void *_image, size_t len, void *_udata, hbool_t H5_ATTR_UNUSED *dirty) { H5O_drvinfo_t *drvinfo = NULL; /* Driver info */ H5F_drvrinfo_cache_ud_t *udata = (H5F_drvrinfo_cache_ud_t *)_udata; /* User data */ const uint8_t *image = _image; /* Pointer into raw data buffer */ char drv_name[9]; /* Name of driver */ - H5O_drvinfo_t *ret_value = NULL; /* Return value */ + H5O_drvinfo_t *ret_value = NULL; FUNC_ENTER_PACKAGE - /* Sanity check */ HDassert(image); HDassert(len >= H5F_DRVINFOBLOCK_HDR_SIZE); HDassert(udata); @@ -946,7 +894,7 @@ H5F__cache_drvrinfo_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUSED HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "memory allocation failed for driver info message") /* Deserialize the file driver info's prefix */ - if (H5F__drvrinfo_prefix_decode(drvinfo, drv_name, &image, udata, FALSE) < 0) + if (H5F__drvrinfo_prefix_decode(drvinfo, drv_name, &image, len, udata, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode file driver info prefix") /* Sanity check */ @@ -959,7 +907,6 @@ H5F__cache_drvrinfo_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUSED /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) <= len); - /* Set return value */ ret_value = drvinfo; done: @@ -973,13 +920,9 @@ done: /*------------------------------------------------------------------------- * Function: H5F__cache_drvrinfo_image_len * - * Purpose: Compute the size of the data structure on disk. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * July 20, 2013 + * Purpose: Compute the size of the data structure on disk * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -989,7 +932,6 @@ H5F__cache_drvrinfo_image_len(const void *_thing, size_t *image_len) FUNC_ENTER_PACKAGE_NOERR - /* Check arguments */ HDassert(drvinfo); HDassert(drvinfo->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(drvinfo->cache_info.type == H5AC_DRVRINFO); @@ -1005,13 +947,9 @@ H5F__cache_drvrinfo_image_len(const void *_thing, size_t *image_len) /*------------------------------------------------------------------------- * Function: H5F__cache_drvrinfo_serialize * - * Purpose: Flushes a dirty object to disk. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * July 20 2013 + * Purpose: Flush a dirty object to disk * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -1020,11 +958,10 @@ H5F__cache_drvrinfo_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_NDEBU H5O_drvinfo_t *drvinfo = (H5O_drvinfo_t *)_thing; /* Pointer to the object */ uint8_t *image = _image; /* Pointer into raw data buffer */ uint8_t *dbuf; /* Pointer to beginning of driver info */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* check arguments */ HDassert(f); HDassert(image); HDassert(drvinfo); @@ -1061,18 +998,14 @@ done: /*------------------------------------------------------------------------- * Function: H5F__cache_drvrinfo_free_icr * - * Purpose: Destroy/release an "in core representation" of a data + * Purpose: Destroy/release an "in core representation" of a data * structure * - * Note: The metadata cache sets the object's cache_info.magic to - * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr - * callback (checked in assert). - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * July 20, 2013 + * Note: The metadata cache sets the object's cache_info.magic to + * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr + * callback (checked in assert). * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -1082,7 +1015,6 @@ H5F__cache_drvrinfo_free_icr(void *_thing) FUNC_ENTER_PACKAGE_NOERR - /* Check arguments */ HDassert(drvinfo); HDassert(drvinfo->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC); HDassert(drvinfo->cache_info.type == H5AC_DRVRINFO); diff --git a/src/H5Gcache.c b/src/H5Gcache.c index b6c6a85..e088fd8 100644 --- a/src/H5Gcache.c +++ b/src/H5Gcache.c @@ -13,10 +13,8 @@ /*------------------------------------------------------------------------- * * Created: H5Gcache.c - * Feb 5 2008 - * Quincey Koziol * - * Purpose: Implement group metadata cache methods. + * Purpose: Implement group metadata cache methods * *------------------------------------------------------------------------- */ @@ -101,15 +99,10 @@ H5FL_SEQ_EXTERN(H5G_entry_t); /*------------------------------------------------------------------------- * Function: H5G__cache_node_get_initial_load_size() * - * Purpose: Determine the size of the on-disk image of the node, and - * return this value in *image_len. - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/21/14 + * Purpose: Determine the size of the on-disk image of the node, and + * return this value in *image_len. * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -119,7 +112,6 @@ H5G__cache_node_get_initial_load_size(void *_udata, size_t *image_len) FUNC_ENTER_PACKAGE_NOERR - /* Sanity checks */ HDassert(f); HDassert(image_len); @@ -132,22 +124,18 @@ H5G__cache_node_get_initial_load_size(void *_udata, size_t *image_len) /*------------------------------------------------------------------------- * Function: H5G__cache_node_deserialize * - * Purpose: Given a buffer containing the on disk image of a symbol table - * node, allocate an instance of H5G_node_t, load the contents of the - * image into it, and return a pointer to the instance. + * Purpose: Given a buffer containing the on disk image of a symbol table + * node, allocate an instance of H5G_node_t, load the contents of the + * image into it, and return a pointer to the instance. * - * Note that deserializing the image requires access to the file - * pointer, which is not included in the parameter list for this - * callback. Finesse this issue by passing in the file pointer - * twice to the H5AC_protect() call -- once as the file pointer - * proper, and again as the user data + * Note that deserializing the image requires access to the file + * pointer, which is not included in the parameter list for this + * callback. Finesse this issue by passing in the file pointer + * twice to the H5AC_protect() call -- once as the file pointer + * proper, and again as the user data * * Return: Success: Pointer to in core representation * Failure: NULL - * - * Programmer: John Mainzer - * 6/21/14 - * *------------------------------------------------------------------------- */ static void * @@ -157,11 +145,10 @@ H5G__cache_node_deserialize(const void *_image, size_t len, void *_udata, hbool_ H5G_node_t *sym = NULL; /* Symbol table node created */ const uint8_t *image = (const uint8_t *)_image; /* Pointer to image to deserialize */ const uint8_t *image_end = image + len - 1; /* Pointer to end of image buffer */ - void *ret_value = NULL; /* Return value */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* Sanity checks */ HDassert(image); HDassert(len > 0); HDassert(f); @@ -174,22 +161,30 @@ H5G__cache_node_deserialize(const void *_image, size_t len, void *_udata, hbool_ if (NULL == (sym->entry = H5FL_SEQ_CALLOC(H5G_entry_t, (size_t)(2 * H5F_SYM_LEAF_K(f))))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* magic */ + /* Magic */ + if (H5_IS_BUFFER_OVERFLOW(image, H5_SIZEOF_MAGIC, image_end)) + HGOTO_ERROR(H5E_SYM, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); if (HDmemcmp(image, H5G_NODE_MAGIC, (size_t)H5_SIZEOF_MAGIC) != 0) HGOTO_ERROR(H5E_SYM, H5E_BADVALUE, NULL, "bad symbol table node signature") image += H5_SIZEOF_MAGIC; - /* version */ + /* Version */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, image_end)) + HGOTO_ERROR(H5E_SYM, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); if (H5G_NODE_VERS != *image++) HGOTO_ERROR(H5E_SYM, H5E_VERSION, NULL, "bad symbol table node version") - /* reserved */ + /* Reserved */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, image_end)) + HGOTO_ERROR(H5E_SYM, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); image++; - /* number of symbols */ + /* Number of symbols */ + if (H5_IS_BUFFER_OVERFLOW(image, 2, image_end)) + HGOTO_ERROR(H5E_SYM, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); UINT16DECODE(image, sym->nsyms); - /* entries */ + /* Entries */ if (H5G__ent_decode_vec(f, &image, image_end, sym->entry, sym->nsyms) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTLOAD, NULL, "unable to decode symbol table entries") @@ -208,14 +203,9 @@ done: * Function: H5G__cache_node_image_len * * Purpose: Compute the size of the data structure on disk and return - * it in *image_len. - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 6/21/14 + * it in *image_len * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -225,7 +215,6 @@ H5G__cache_node_image_len(const void *_thing, size_t *image_len) FUNC_ENTER_PACKAGE_NOERR - /* Sanity checks */ HDassert(sym); HDassert(sym->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(sym->cache_info.type == H5AC_SNODE); @@ -239,17 +228,12 @@ H5G__cache_node_image_len(const void *_thing, size_t *image_len) /*------------------------------------------------------------------------- * Function: H5G__cache_node_serialize * - * Purpose: Given a correctly sized buffer and an instance of H5G_node_t, - * serialize the contents of the instance of H5G_node_t, and write - * this data into the supplied buffer. This buffer will be written - * to disk. - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/21/14 + * Purpose: Given a correctly sized buffer and an instance of H5G_node_t, + * serialize the contents of the instance of H5G_node_t, and write + * this data into the supplied buffer. This buffer will be written + * to disk. * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -257,11 +241,10 @@ H5G__cache_node_serialize(const H5F_t *f, void *_image, size_t len, void *_thing { H5G_node_t *sym = (H5G_node_t *)_thing; /* Pointer to object */ uint8_t *image = (uint8_t *)_image; /* Pointer into raw data buffer */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Sanity checks */ HDassert(f); HDassert(image); HDassert(sym); @@ -269,20 +252,20 @@ H5G__cache_node_serialize(const H5F_t *f, void *_image, size_t len, void *_thing HDassert(sym->cache_info.type == H5AC_SNODE); HDassert(len == sym->node_size); - /* magic number */ + /* Magic number */ H5MM_memcpy(image, H5G_NODE_MAGIC, (size_t)H5_SIZEOF_MAGIC); image += H5_SIZEOF_MAGIC; - /* version number */ + /* Version number */ *image++ = H5G_NODE_VERS; - /* reserved */ + /* Reserved */ *image++ = 0; - /* number of symbols */ + /* Number of symbols */ UINT16ENCODE(image, sym->nsyms); - /* entries */ + /* Entries */ if (H5G__ent_encode_vec(f, &image, sym->entry, sym->nsyms) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTENCODE, FAIL, "can't serialize") @@ -296,29 +279,23 @@ done: /*------------------------------------------------------------------------- * Function: H5G__cache_node_free_icr * - * Purpose: Destroys a symbol table node in memory. - * - * Note: The metadata cache sets the object's cache_info.magic to - * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr - * callback (checked in assert). - * - * Return: Success: SUCCEED - * Failure: FAIL + * Purpose: Destroy a symbol table node in memory * - * Programmer: John Mainzer - * 6/21/14 + * Note: The metadata cache sets the object's cache_info.magic to + * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr + * callback (checked in assert). * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t H5G__cache_node_free_icr(void *_thing) { H5G_node_t *sym = (H5G_node_t *)_thing; /* Pointer to the object */ - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Sanity checks */ HDassert(sym); HDassert(sym->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC); HDassert(sym->cache_info.type == H5AC_SNODE); diff --git a/src/H5HGcache.c b/src/H5HGcache.c index 235a990..bbfae7c 100644 --- a/src/H5HGcache.c +++ b/src/H5HGcache.c @@ -13,10 +13,8 @@ /*------------------------------------------------------------------------- * * Created: H5HGcache.c - * Feb 5 2008 - * Quincey Koziol * - * Purpose: Implement global heap metadata cache methods. + * Purpose: Implement global heap metadata cache methods * *------------------------------------------------------------------------- */ @@ -30,12 +28,12 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fprivate.h" /* File access */ -#include "H5HGpkg.h" /* Global heaps */ -#include "H5MFprivate.h" /* File memory management */ -#include "H5MMprivate.h" /* Memory management */ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5HGpkg.h" /* Global heaps */ +#include "H5MFprivate.h" /* File memory management */ +#include "H5MMprivate.h" /* Memory management */ /****************/ /* Local Macros */ @@ -63,7 +61,7 @@ static herr_t H5HG__cache_heap_serialize(const H5F_t *f, void *image, size_t len static herr_t H5HG__cache_heap_free_icr(void *thing); /* Prefix deserialization */ -static herr_t H5HG__hdr_deserialize(H5HG_heap_t *heap, const uint8_t *image, const H5F_t *f); +static herr_t H5HG__hdr_deserialize(H5HG_heap_t *heap, const uint8_t *image, size_t len, const H5F_t *f); /*********************/ /* Package Variables */ @@ -96,65 +94,64 @@ const H5AC_class_t H5AC_GHEAP[1] = {{ /*******************/ /*------------------------------------------------------------------------- - * Function: H5HG__hdr_deserialize() + * Function: H5HG__hdr_deserialize * - * Purpose: Decode a global heap's header - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: Quincey Koziol - * December 15, 2016 + * Purpose: Decode a global heap's header * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t -H5HG__hdr_deserialize(H5HG_heap_t *heap, const uint8_t *image, const H5F_t *f) +H5HG__hdr_deserialize(H5HG_heap_t *heap, const uint8_t *image, size_t len, const H5F_t *f) { - herr_t ret_value = SUCCEED; /* Return value */ + const uint8_t *p_end = image + len - 1; /* End of image buffer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Sanity check */ HDassert(heap); HDassert(image); HDassert(f); /* Magic number */ + if (H5_IS_BUFFER_OVERFLOW(image, H5_SIZEOF_MAGIC, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); if (HDmemcmp(image, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC) != 0) HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad global heap collection signature") image += H5_SIZEOF_MAGIC; /* Version */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); if (H5HG_VERSION != *image++) HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in global heap") /* Reserved */ + if (H5_IS_BUFFER_OVERFLOW(image, 3, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); image += 3; /* Size */ + if (H5_IS_BUFFER_OVERFLOW(image, H5F_sizeof_size(f), p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); H5F_DECODE_LENGTH(f, image, heap->size); - HDassert(heap->size >= H5HG_MINSIZE); + if (heap->size < H5HG_MINSIZE) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "global heap size is too small"); done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HG__hdr_deserialize() */ /*------------------------------------------------------------------------- - * Function: H5HG__cache_heap_get_initial_load_size() - * - * Purpose: Return the initial speculative read size to the metadata - * cache. This size will be used in the initial attempt to read - * the global heap. If this read is too small, the cache will - * try again with the correct value obtained from - * H5HG__cache_get_final_load_size(). + * Function: H5HG__cache_heap_get_initial_load_size * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/27/14 + * Purpose: Return the initial speculative read size to the metadata + * cache. This size will be used in the initial attempt to read + * the global heap. If this read is too small, the cache will + * try again with the correct value obtained from + * H5HG__cache_get_final_load_size(). * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -162,39 +159,30 @@ H5HG__cache_heap_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *imag { FUNC_ENTER_PACKAGE_NOERR - /* Sanity check */ HDassert(image_len); - /* Set the image length size */ - *image_len = (size_t)H5HG_MINSIZE; + *image_len = H5HG_MINSIZE; FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HG__cache_heap_get_initial_load_size() */ /*------------------------------------------------------------------------- - * Function: H5HG__cache_heap_get_initial_load_size() - * - * Purpose: Return the final read size for a speculatively ready heap to - * the metadata cache. + * Function: H5HG__cache_heap_get_final_load_size * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: Quincey Koziol - * November 18, 2016 + * Purpose: Return the final read size for a speculatively ready heap to + * the metadata cache. * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t -H5HG__cache_heap_get_final_load_size(const void *image, size_t H5_ATTR_NDEBUG_UNUSED image_len, void *udata, - size_t *actual_len) +H5HG__cache_heap_get_final_load_size(const void *image, size_t image_len, void *udata, size_t *actual_len) { - H5HG_heap_t heap; /* Global heap */ - herr_t ret_value = SUCCEED; /* Return value */ + H5HG_heap_t heap; + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Sanity check */ HDassert(image); HDassert(udata); HDassert(actual_len); @@ -202,10 +190,10 @@ H5HG__cache_heap_get_final_load_size(const void *image, size_t H5_ATTR_NDEBUG_UN HDassert(image_len == H5HG_MINSIZE); /* Deserialize the heap's header */ - if (H5HG__hdr_deserialize(&heap, (const uint8_t *)image, (const H5F_t *)udata) < 0) + if (H5HG__hdr_deserialize(&heap, (const uint8_t *)image, image_len, (const H5F_t *)udata) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, FAIL, "can't decode global heap prefix") - /* Set the final size for the cache image */ + /* Set the actual global heap size */ *actual_len = heap.size; done: @@ -215,31 +203,28 @@ done: /*------------------------------------------------------------------------- * Function: H5HG__cache_heap_deserialize * - * Purpose: Given a buffer containing the on disk image of the global - * heap, deserialize it, load its contents into a newly allocated - * instance of H5HG_heap_t, and return a pointer to the new instance. - * - * Return: Success: Pointer to in core representation - * Failure: NULL - * - * Programmer: John Mainzer - * 7/27/14 + * Purpose: Given a buffer containing the on disk image of the global + * heap, deserialize it, load its contents into a newly allocated + * instance of H5HG_heap_t, and return a pointer to the new + * instance. * + * Return: Success: Pointer to a new global heap + * Failure: NULL *------------------------------------------------------------------------- */ static void * H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata, hbool_t H5_ATTR_UNUSED *dirty) { - H5F_t *f = (H5F_t *)_udata; /* File pointer -- obtained from user data */ - H5HG_heap_t *heap = NULL; /* New global heap */ - uint8_t *image; /* Pointer to image to decode */ - size_t max_idx = 0; /* Maximum heap object index seen */ - size_t nalloc; /* Number of objects allocated */ - void *ret_value = NULL; /* Return value */ + H5F_t *f = (H5F_t *)_udata; /* File pointer */ + H5HG_heap_t *heap = NULL; /* New global heap */ + uint8_t *p = NULL; /* Pointer to objects in (copied) image buffer */ + const uint8_t *p_end = NULL; /* End of (copied) image buffer */ + size_t max_idx = 0; /* Maximum heap object index seen */ + size_t nalloc = 0; /* Number of objects allocated */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* Sanity checks */ HDassert(_image); HDassert(len >= (size_t)H5HG_MINSIZE); HDassert(f); @@ -252,15 +237,28 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata, hbool if (NULL == (heap->chunk = H5FL_BLK_MALLOC(gheap_chunk, len))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Copy the image buffer into the newly allocate chunk */ + /* Copy the image buffer into the newly allocated chunk */ H5MM_memcpy(heap->chunk, _image, len); + /* Set p_end + * + * Note that parsing moves along p / heap->chunk, so p_end + * has to refer to the end of that buffer and NOT _image + */ + p_end = heap->chunk + len - 1; + /* Deserialize the heap's header */ - if (H5HG__hdr_deserialize(heap, (const uint8_t *)heap->chunk, f) < 0) + if (H5_IS_BUFFER_OVERFLOW(heap->chunk, H5HG_SIZEOF_HDR(f), p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + if (H5HG__hdr_deserialize(heap, (const uint8_t *)heap->chunk, len, f) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode global heap header") /* Decode each object */ - image = heap->chunk + H5HG_SIZEOF_HDR(f); + + /* Set the p pointer to the objects in heap->chunk */ + p = heap->chunk + H5HG_SIZEOF_HDR(f); + + /* Set the number of allocated objects */ nalloc = H5HG_NOBJS(f, heap->size); /* Calloc the obj array because the file format spec makes no guarantee @@ -270,32 +268,43 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata, hbool HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") heap->nalloc = nalloc; - while (image < (heap->chunk + heap->size)) { - if ((image + H5HG_SIZEOF_OBJHDR(f)) > (heap->chunk + heap->size)) { - /* - * The last bit of space is too tiny for an object header, so + while (p < (heap->chunk + heap->size)) { + + if ((p + H5HG_SIZEOF_OBJHDR(f)) > (heap->chunk + heap->size)) { + + /* The last bit of space is too tiny for an object header, so * we assume that it's free space. */ - HDassert(NULL == heap->obj[0].begin); - heap->obj[0].size = (size_t)(((const uint8_t *)heap->chunk + heap->size) - image); - heap->obj[0].begin = image; - image += heap->obj[0].size; - } /* end if */ + if (NULL != heap->obj[0].begin) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "object 0 should not be set"); + heap->obj[0].size = (size_t)(((const uint8_t *)heap->chunk + heap->size) - p); + heap->obj[0].begin = p; + + /* No buffer overflow check here since this just moves the pointer + * to the end of the buffer, which was calculated above + */ + p += heap->obj[0].size; + } else { - size_t need = 0; - unsigned idx; - uint8_t *begin = image; + size_t need = 0; /* # bytes needed to store the object */ + unsigned idx; /* Heap object index */ + uint8_t *begin = p; /* Pointer to start of object */ - UINT16DECODE(image, idx); + /* Parse a normal heap entry */ + + if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + UINT16DECODE(p, idx); /* Check if we need more room to store heap objects */ if (idx >= heap->nalloc) { size_t new_alloc; /* New allocation number */ - H5HG_obj_t *new_obj; /* New array of object descriptions */ + H5HG_obj_t *new_obj; /* New array of object descriptions */ /* Determine the new number of objects to index */ new_alloc = MAX(heap->nalloc * 2, (idx + 1)); - HDassert(idx < new_alloc); + if (idx >= new_alloc) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "inappropriate heap index") /* Reallocate array of objects */ if (NULL == (new_obj = H5FL_SEQ_REALLOC(H5HG_obj_t, heap->obj, new_alloc))) @@ -307,16 +316,32 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata, hbool /* Update heap information */ heap->nalloc = new_alloc; heap->obj = new_obj; - HDassert(heap->nalloc > heap->nused); - } /* end if */ - - UINT16DECODE(image, heap->obj[idx].nrefs); - image += 4; /*reserved*/ - H5F_DECODE_LENGTH(f, image, heap->obj[idx].size); + if (heap->nalloc <= heap->nused) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "inappropriate # allocated slots") + } + + /* Number of references */ + if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + UINT16DECODE(p, heap->obj[idx].nrefs); + + /* Reserved bytes */ + if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + p += 4; + + /* Object length */ + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + H5F_DECODE_LENGTH(f, p, heap->obj[idx].size); + + /* Object + * + * Points to beginning of object, INCLUDING the header. + */ heap->obj[idx].begin = begin; - /* - * The total storage size includes the size of the object + /* The total storage size includes the size of the object * header and is zero padded so the next object header is * properly aligned. The entire obj array was calloc'ed, * so no need to zero the space here. The last bit of space @@ -327,26 +352,33 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata, hbool need = H5HG_SIZEOF_OBJHDR(f) + H5HG_ALIGN(heap->obj[idx].size); if (idx > max_idx) max_idx = idx; - } /* end if */ + } else need = heap->obj[idx].size; - image = begin + need; - } /* end else */ - } /* end while */ - - /* Sanity checks */ - HDassert(image == heap->chunk + heap->size); - HDassert(H5HG_ISALIGNED(heap->obj[0].size)); - - /* Set the next index value to use */ + /* Make sure the extra padding doesn't cause us to overrun + * the buffer + */ + if (H5_IS_BUFFER_OVERFLOW(begin, need, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + p = begin + need; + } + } + + /* Post-parse checks */ + if (p != heap->chunk + heap->size) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "partially decoded global heap"); + if (FALSE == H5HG_ISALIGNED(heap->obj[0].size)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "decoded global heap is not aligned"); + + /* Set the next index value to use when creating a new object */ if (max_idx > 0) heap->nused = max_idx + 1; else heap->nused = 1; - /* Sanity check */ - HDassert(max_idx < heap->nused); + if (max_idx >= heap->nused) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad `next unused` heap index value"); /* Add the new heap to the CWFS list for the file */ if (H5F_cwfs_add(f, heap) < 0) @@ -365,15 +397,10 @@ done: /*------------------------------------------------------------------------- * Function: H5HG__cache_heap_image_len * - * Purpose: Return the on disk image size of the global heap to the - * metadata cache via the image_len. - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/27/14 + * Purpose: Return the on disk image size of the global heap to the + * metadata cache via the image_len. * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -383,7 +410,6 @@ H5HG__cache_heap_image_len(const void *_thing, size_t *image_len) FUNC_ENTER_PACKAGE_NOERR - /* Sanity checks */ HDassert(heap); HDassert(heap->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(heap->cache_info.type == H5AC_GHEAP); @@ -398,17 +424,12 @@ H5HG__cache_heap_image_len(const void *_thing, size_t *image_len) /*------------------------------------------------------------------------- * Function: H5HG__cache_heap_serialize * - * Purpose: Given an appropriately sized buffer and an instance of - * H5HG_heap_t, serialize the global heap for writing to file, - * and copy the serialized version into the buffer. - * + * Purpose: Given an appropriately sized buffer and an instance of + * H5HG_heap_t, serialize the global heap for writing to file, + * and copy the serialized version into the buffer. * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/27/14 * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t @@ -426,7 +447,7 @@ H5HG__cache_heap_serialize(const H5F_t H5_ATTR_NDEBUG_UNUSED *f, void *image, si HDassert(heap->size == len); HDassert(heap->chunk); - /* copy the image into the buffer */ + /* Copy the image into the buffer */ H5MM_memcpy(image, heap->chunk, len); FUNC_LEAVE_NOAPI(SUCCEED) @@ -435,29 +456,23 @@ H5HG__cache_heap_serialize(const H5F_t H5_ATTR_NDEBUG_UNUSED *f, void *image, si /*------------------------------------------------------------------------- * Function: H5HG__cache_heap_free_icr * - * Purpose: Free the in memory representation of the supplied global heap. - * - * Note: The metadata cache sets the object's cache_info.magic to - * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr - * callback (checked in assert). - * - * Return: Success: SUCCEED - * Failure: FAIL + * Purpose: Free the in memory representation of the supplied global heap. * - * Programmer: John Mainzer - * 7/27/14 + * Note: The metadata cache sets the object's cache_info.magic to + * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr + * callback (checked in assert). * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t H5HG__cache_heap_free_icr(void *_thing) { H5HG_heap_t *heap = (H5HG_heap_t *)_thing; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Sanity checks */ HDassert(heap); HDassert(heap->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC); HDassert(heap->cache_info.type == H5AC_GHEAP); diff --git a/src/H5HGpkg.h b/src/H5HGpkg.h index ab7cd09..99725d8 100644 --- a/src/H5HGpkg.h +++ b/src/H5HGpkg.h @@ -103,27 +103,32 @@ H5FL_BLK_EXTERN(gheap_chunk); /****************************/ typedef struct H5HG_obj_t { - int nrefs; /* reference count */ - size_t size; /* total size of object */ - uint8_t *begin; /* ptr to object into heap->chunk */ + int nrefs; /* Reference count */ + size_t size; /* Total size of object */ + uint8_t *begin; /* Pointer to object into heap->chunk (INCLUDES header) */ } H5HG_obj_t; /* Forward declarations for fields */ struct H5F_shared_t; struct H5HG_heap_t { - H5AC_info_t cache_info; /* Information for H5AC cache functions, _must_ be */ - /* first field in structure */ - haddr_t addr; /*collection address */ - size_t size; /*total size of collection */ - uint8_t *chunk; /*the collection, incl. header */ - size_t nalloc; /*numb object slots allocated */ - size_t nused; /*number of slots used */ - /* If this value is >65535 then all indices */ - /* have been used at some time and the */ - /* correct new index should be searched for */ - struct H5F_shared_t *shared; /* shared file */ - H5HG_obj_t *obj; /*array of object descriptions */ + H5AC_info_t cache_info; /* Information for H5AC cache functions, MUST be + * the first field in structure + */ + haddr_t addr; /* Collection address */ + size_t size; /* Total size of collection */ + uint8_t *chunk; /* Collection of elements - note that this + * INCLUDES the header, so it's not just + * the objects! + */ + size_t nalloc; /* # object slots allocated */ + size_t nused; /* # of slots used + * If this value is >65535 then all indices + * have been used at some time and the + * correct new index should be searched for + */ + struct H5F_shared_t *shared; /* Shared file */ + H5HG_obj_t *obj; /* Array of object descriptions */ }; /******************************/ diff --git a/src/H5HLcache.c b/src/H5HLcache.c index 72af9b4..c04efb6 100644 --- a/src/H5HLcache.c +++ b/src/H5HLcache.c @@ -13,10 +13,8 @@ /*------------------------------------------------------------------------- * * Created: H5HLcache.c - * Feb 5 2008 - * Quincey Koziol * - * Purpose: Implement local heap metadata cache methods. + * Purpose: Implement local heap metadata cache methods * *------------------------------------------------------------------------- */ @@ -81,7 +79,8 @@ static herr_t H5HL__cache_datablock_notify(H5C_notify_action_t action, void *_th static herr_t H5HL__cache_datablock_free_icr(void *thing); /* Header deserialization */ -static herr_t H5HL__hdr_deserialize(H5HL_t *heap, const uint8_t *image, H5HL_cache_prfx_ud_t *udata); +static herr_t H5HL__hdr_deserialize(H5HL_t *heap, const uint8_t *image, size_t len, + H5HL_cache_prfx_ud_t *udata); /* Free list de/serialization */ static herr_t H5HL__fl_deserialize(H5HL_t *heap); @@ -137,38 +136,39 @@ const H5AC_class_t H5AC_LHEAP_DBLK[1] = {{ /*------------------------------------------------------------------------- * Function: H5HL__hdr_deserialize() * - * Purpose: Decode a local heap's header - * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: Quincey Koziol - * December 15, 2016 + * Purpose: Decode a local heap's header * + * Return: SUCCEED/FAIL *------------------------------------------------------------------------- */ static herr_t -H5HL__hdr_deserialize(H5HL_t *heap, const uint8_t *image, H5HL_cache_prfx_ud_t *udata) +H5HL__hdr_deserialize(H5HL_t *heap, const uint8_t *image, size_t len, H5HL_cache_prfx_ud_t *udata) { - herr_t ret_value = SUCCEED; /* Return value */ + const uint8_t *p_end = image + len - 1; /* End of image buffer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE - /* Sanity checks */ HDassert(heap); HDassert(image); HDassert(udata); - /* Check magic number */ + /* Magic number */ + if (H5_IS_BUFFER_OVERFLOW(image, H5_SIZEOF_MAGIC, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); if (HDmemcmp(image, H5HL_MAGIC, (size_t)H5_SIZEOF_MAGIC) != 0) HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad local heap signature") image += H5_SIZEOF_MAGIC; /* Version */ + if (H5_IS_BUFFER_OVERFLOW(image, 1, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); if (H5HL_VERSION != *image++) HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in local heap") /* Reserved */ + if (H5_IS_BUFFER_OVERFLOW(image, 3, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); image += 3; /* Store the prefix's address & length */ @@ -176,14 +176,20 @@ H5HL__hdr_deserialize(H5HL_t *heap, const uint8_t *image, H5HL_cache_prfx_ud_t * heap->prfx_size = udata->sizeof_prfx; /* Heap data size */ + if (H5_IS_BUFFER_OVERFLOW(image, udata->sizeof_size, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); H5F_DECODE_LENGTH_LEN(image, heap->dblk_size, udata->sizeof_size); /* Free list head */ + if (H5_IS_BUFFER_OVERFLOW(image, udata->sizeof_size, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); H5F_DECODE_LENGTH_LEN(image, heap->free_block, udata->sizeof_size); if (heap->free_block != H5HL_FREE_NULL && heap->free_block >= heap->dblk_size) HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap free list") /* Heap data address */ + if (H5_IS_BUFFER_OVERFLOW(image, udata->sizeof_addr, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); H5F_addr_decode_len(udata->sizeof_addr, &image, &(heap->dblk_addr)); done: @@ -344,8 +350,7 @@ H5HL__cache_prefix_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *im *------------------------------------------------------------------------- */ static herr_t -H5HL__cache_prefix_get_final_load_size(const void *_image, size_t H5_ATTR_NDEBUG_UNUSED image_len, - void *_udata, size_t *actual_len) +H5HL__cache_prefix_get_final_load_size(const void *_image, size_t image_len, void *_udata, size_t *actual_len) { const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ H5HL_cache_prfx_ud_t *udata = (H5HL_cache_prfx_ud_t *)_udata; /* User data for callback */ @@ -363,7 +368,7 @@ H5HL__cache_prefix_get_final_load_size(const void *_image, size_t H5_ATTR_NDEBUG HDmemset(&heap, 0, sizeof(H5HL_t)); /* Deserialize the heap's header */ - if (H5HL__hdr_deserialize(&heap, (const uint8_t *)image, udata) < 0) + if (H5HL__hdr_deserialize(&heap, (const uint8_t *)image, image_len, udata) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, FAIL, "can't decode local heap header") /* Set the final size for the cache image */ @@ -383,25 +388,22 @@ done: /*------------------------------------------------------------------------- * Function: H5HL__cache_prefix_deserialize * - * Purpose: Given a buffer containing the on disk image of the local - * heap prefix, deserialize it, load its contents into a newly allocated - * instance of H5HL_prfx_t, and return a pointer to the new instance. - * - * Return: Success: Pointer to in core representation - * Failure: NULL - * - * Programmer: John Mainzer - * 6/21/14 + * Purpose: Given a buffer containing the on disk image of the local + * heap prefix, deserialize it, load its contents into a newly + * allocated instance of H5HL_prfx_t, and return a pointer to + * the new instance. * + * Return: Success: Pointer to in core representation + * Failure: NULL *------------------------------------------------------------------------- */ static void * -H5HL__cache_prefix_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUSED len, void *_udata, - hbool_t H5_ATTR_UNUSED *dirty) +H5HL__cache_prefix_deserialize(const void *_image, size_t len, void *_udata, hbool_t H5_ATTR_UNUSED *dirty) { H5HL_t *heap = NULL; /* Local heap */ H5HL_prfx_t *prfx = NULL; /* Heap prefix deserialized */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into decoding buffer */ + const uint8_t *p_end = image + len - 1; /* End of image buffer */ H5HL_cache_prfx_ud_t *udata = (H5HL_cache_prfx_ud_t *)_udata; /* User data for callback */ void *ret_value = NULL; /* Return value */ @@ -422,7 +424,7 @@ H5HL__cache_prefix_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUSED HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "can't allocate local heap structure"); /* Deserialize the heap's header */ - if (H5HL__hdr_deserialize(heap, (const uint8_t *)image, udata) < 0) + if (H5HL__hdr_deserialize(heap, (const uint8_t *)image, len, udata) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode local heap header") /* Allocate the heap prefix */ @@ -446,6 +448,8 @@ H5HL__cache_prefix_deserialize(const void *_image, size_t H5_ATTR_NDEBUG_UNUSED image = ((const uint8_t *)_image) + heap->prfx_size; /* Copy the heap data from the speculative read buffer */ + if (H5_IS_BUFFER_OVERFLOW(image, heap->dblk_size, p_end)) + HGOTO_ERROR(H5E_HEAP, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5MM_memcpy(heap->dblk_image, image, heap->dblk_size); /* Build free list */ diff --git a/src/H5Obogus.c b/src/H5Obogus.c index 549c3e9..1b83ed1 100644 --- a/src/H5Obogus.c +++ b/src/H5Obogus.c @@ -13,8 +13,6 @@ /*------------------------------------------------------------------------- * * Created: H5Obogus.c - * Jan 21 2003 - * Quincey Koziol * * Purpose: "bogus" message. This message is guaranteed to never * be found in a valid HDF5 file and is only used to @@ -95,25 +93,20 @@ const H5O_msg_class_t H5O_MSG_BOGUS_INVALID[1] = {{ * Purpose: Decode a "bogus" message and return a pointer to a new * native message struct. * - * Return: Success: Ptr to new message in native struct. - * + * Return: Success: Pointer to new message in native struct * Failure: NULL - * - * Programmer: Quincey Koziol - * Jan 21 2003 - * *------------------------------------------------------------------------- */ static void * -H5O__bogus_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, - unsigned H5_ATTR_UNUSED *ioflags, size_t H5_ATTR_UNUSED p_size, const uint8_t *p) +H5O__bogus_decode(H5F_t *f, H5O_t H5_ATTR_NDEBUG_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, + unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, const uint8_t *p) { - H5O_bogus_t *mesg = NULL; - void *ret_value; /* Return value */ + const uint8_t *p_end = p + p_size - 1; + H5O_bogus_t *mesg = NULL; + void *ret_value; FUNC_ENTER_PACKAGE - /* check args */ HDassert(f); HDassert(p); @@ -121,7 +114,8 @@ H5O__bogus_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUS if (NULL == (mesg = (H5O_bogus_t *)H5MM_calloc(sizeof(H5O_bogus_t)))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* decode */ + if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); UINT32DECODE(p, mesg->u); /* Validate the bogus info */ diff --git a/src/H5Ocache_image.c b/src/H5Ocache_image.c index bd273ec..a06bebc 100644 --- a/src/H5Ocache_image.c +++ b/src/H5Ocache_image.c @@ -13,14 +13,12 @@ /*------------------------------------------------------------------------- * * Created: H5Ocache_image.c - * June 21, 2015 - * John Mainzer * * Purpose: A message indicating that a metadata cache image block - * of the indicated length exists at the specified offset - * in the HDF5 file. + * of the indicated length exists at the specified offset + * in the HDF5 file. * - * The mdci_msg only appears in the superblock extension. + * The mdci_msg only appears in the superblock extension * *------------------------------------------------------------------------- */ @@ -79,30 +77,28 @@ H5FL_DEFINE(H5O_mdci_t); * Function: H5O__mdci_decode * * Purpose: Decode a metadata cache image message and return a - * pointer to a newly allocated H5O_mdci_t struct. - * - * Return: Success: Ptr to new message in native struct. - * Failure: NULL - * - * Programmer: John Mainzer - * 6/22/15 + * pointer to a newly allocated H5O_mdci_t struct. * + * Return: Success: Pointer to new message in native struct + * Failure: NULL *------------------------------------------------------------------------- */ static void * H5O__mdci_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, - unsigned H5_ATTR_UNUSED *ioflags, size_t H5_ATTR_UNUSED p_size, const uint8_t *p) + unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, const uint8_t *p) { - H5O_mdci_t *mesg; /* Native message */ - void *ret_value = NULL; /* Return value */ + H5O_mdci_t *mesg = NULL; /* New cache image message */ + const uint8_t *p_end = p + p_size - 1; /* End of the p buffer */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* Sanity check */ HDassert(f); HDassert(p); /* Version of message */ + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); if (*p++ != H5O_MDCI_VERSION_0) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad version number for message") @@ -111,14 +107,21 @@ H5O__mdci_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSE HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for metadata cache image message") - /* Decode */ + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5F_addr_decode(f, &p, &(mesg->addr)); + + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5F_DECODE_LENGTH(f, p, mesg->size); /* Set return value */ ret_value = (void *)mesg; done: + if (!ret_value && mesg) + H5FL_FREE(H5O_mdci_t, mesg); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__mdci_decode() */ diff --git a/src/H5Ocopy.c b/src/H5Ocopy.c index 926d0da..9852d1f 100644 --- a/src/H5Ocopy.c +++ b/src/H5Ocopy.c @@ -532,10 +532,15 @@ H5O__copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out*/, H5 HDassert((oh_dst->flags & H5O_HDR_CHUNK0_SIZE) == H5O_HDR_CHUNK0_1); /* Determine whether to create gap or NULL message */ - if (delta < H5O_SIZEOF_MSGHDR_OH(oh_dst)) + if ((oh_dst->version > H5O_VERSION_1) && (delta < H5O_SIZEOF_MSGHDR_OH(oh_dst))) dst_oh_gap = delta; - else + else { + /* NULL message must be at least size of message header */ + if (delta < H5O_SIZEOF_MSGHDR_OH(oh_dst)) + delta = H5O_SIZEOF_MSGHDR_OH(oh_dst); + dst_oh_null = delta; + } /* Increase destination object header size */ dst_oh_size += delta; diff --git a/src/H5Odrvinfo.c b/src/H5Odrvinfo.c index 923856f..53de66d 100644 --- a/src/H5Odrvinfo.c +++ b/src/H5Odrvinfo.c @@ -60,34 +60,32 @@ const H5O_msg_class_t H5O_MSG_DRVINFO[1] = {{ #define H5O_DRVINFO_VERSION 0 /*------------------------------------------------------------------------- - * Function: H5O__drvinfo_decode + * Function: H5O__drvinfo_decode * - * Purpose: Decode a shared message table message and return a pointer + * Purpose: Decode a shared message table message and return a pointer * to a newly allocated H5O_drvinfo_t struct. * - * Return: Success: Ptr to new message in native struct. - * Failure: NULL - * - * Programmer: Quincey Koziol - * Mar 1, 2007 - * + * Return: Success: Pointer to new message in native struct + * Failure: NULL *------------------------------------------------------------------------- */ static void * H5O__drvinfo_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, - unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, - size_t H5_ATTR_UNUSED p_size, const uint8_t *p) + unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, + const uint8_t *p) { - H5O_drvinfo_t *mesg; /* Native message */ - void *ret_value = NULL; /* Return value */ + H5O_drvinfo_t *mesg = NULL; /* Native message */ + const uint8_t *p_end = p + p_size - 1; /* End of the p buffer */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* Sanity check */ HDassert(f); HDassert(p); /* Version of message */ + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); if (*p++ != H5O_DRVINFO_VERSION) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad version number for message") @@ -96,27 +94,37 @@ H5O__drvinfo_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for driver info message") /* Retrieve driver name */ + if (H5_IS_BUFFER_OVERFLOW(p, 8, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5MM_memcpy(mesg->name, p, 8); mesg->name[8] = '\0'; p += 8; /* Decode buffer size */ + if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); UINT16DECODE(p, mesg->len); - HDassert(mesg->len); + if (0 == mesg->len) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "message length can't be zero"); /* Allocate space for buffer */ - if (NULL == (mesg->buf = (uint8_t *)H5MM_malloc(mesg->len))) { - mesg = (H5O_drvinfo_t *)H5MM_xfree(mesg); + if (NULL == (mesg->buf = (uint8_t *)H5MM_malloc(mesg->len))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for driver info buffer") - } /* end if */ /* Copy encoded driver info into buffer */ + if (H5_IS_BUFFER_OVERFLOW(p, mesg->len, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5MM_memcpy(mesg->buf, p, mesg->len); /* Set return value */ ret_value = (void *)mesg; done: + if (!ret_value && mesg) { + H5MM_xfree(mesg->buf); + H5MM_xfree(mesg); + } + FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__drvinfo_decode() */ diff --git a/src/H5Odtype.c b/src/H5Odtype.c index e5e8996..b6e1b90 100644 --- a/src/H5Odtype.c +++ b/src/H5Odtype.c @@ -24,6 +24,15 @@ #include "H5Tpkg.h" /* Datatypes */ #include "H5VMprivate.h" /* Vectors and arrays */ +/* Variant boundary-checking macro, used here since H5Tdecode() doesn't take a + * size parameter so we need to ignore the bounds checks. + * + * This is a separate macro since we don't want to inflict that behavior on + * the rest of the library. + */ +#define H5_DTYPE_IS_BUFFER_OVERFLOW(skip, ptr, size, buffer_end) \ + (skip ? FALSE : ((ptr) + (size)-1) > (buffer_end)) + /* PRIVATE PROTOTYPES */ static herr_t H5O__dtype_encode(H5F_t *f, uint8_t *p, const void *mesg); static void *H5O__dtype_decode(H5F_t *f, H5O_t *open_oh, unsigned mesg_flags, unsigned *ioflags, @@ -108,35 +117,46 @@ const H5O_msg_class_t H5O_MSG_DTYPE[1] = {{ }}; /*------------------------------------------------------------------------- - * Function: H5O__dtype_decode_helper + * Function: H5O__dtype_decode_helper * - * Purpose: Decodes a datatype + * Purpose: Decodes a datatype * - * Return: TRUE if we can upgrade the parent type's version even + * Return: TRUE if we can upgrade the parent type's version even * with strict format checks * FALSE if we cannot - * Negative on failure - * - * Programmer: Robb Matzke - * Monday, December 8, 1997 - * + * NEGATIVE on failure *------------------------------------------------------------------------- */ static htri_t -H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t *dt) +H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t *dt, hbool_t skip, + const uint8_t *p_end) { - unsigned flags, version; - unsigned i; - size_t z; - htri_t ret_value = FALSE; /* Return value */ + unsigned flags; + unsigned version; + htri_t ret_value = FALSE; FUNC_ENTER_PACKAGE - /* check args */ HDassert(pp && *pp); HDassert(dt && dt->shared); + /* XXX NOTE! + * + * H5Tencode() does not take a buffer size, so normal bounds checking in + * that case is impossible. + * + * Instead of using our normal H5_IS_BUFFER_OVERFLOW macro, use + * H5_DTYPE_IS_BUFFER_OVERFLOW, which will skip the check when the + * we're decoding a buffer from H5Tconvert(). + * + * Even if this is fixed at some point in the future, as long as we + * support the old, size-less API call, we will need to use the modified + * macros. + */ + /* Version, class & flags */ + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT32DECODE(*pp, flags); version = (flags >> 4) & 0x0f; if (version < H5O_DTYPE_VERSION_1 || version > H5O_DTYPE_VERSION_LATEST) @@ -146,6 +166,8 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t flags >>= 8; /* Size */ + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT32DECODE(*pp, dt->shared->size); /* Check for invalid datatype size */ @@ -161,6 +183,8 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t dt->shared->u.atomic.lsb_pad = (flags & 0x2) ? H5T_PAD_ONE : H5T_PAD_ZERO; dt->shared->u.atomic.msb_pad = (flags & 0x4) ? H5T_PAD_ONE : H5T_PAD_ZERO; dt->shared->u.atomic.u.i.sign = (flags & 0x8) ? H5T_SGN_2 : H5T_SGN_NONE; + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 2 + 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT16DECODE(*pp, dt->shared->u.atomic.offset); UINT16DECODE(*pp, dt->shared->u.atomic.prec); break; @@ -178,7 +202,7 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t /* VAX order if both 1st and 6th bits are turned on*/ if (flags & 0x40) dt->shared->u.atomic.order = H5T_ORDER_VAX; - } /* end if */ + } dt->shared->u.atomic.lsb_pad = (flags & 0x2) ? H5T_PAD_ONE : H5T_PAD_ZERO; dt->shared->u.atomic.msb_pad = (flags & 0x4) ? H5T_PAD_ONE : H5T_PAD_ZERO; dt->shared->u.atomic.u.f.pad = (flags & 0x8) ? H5T_PAD_ONE : H5T_PAD_ZERO; @@ -197,21 +221,40 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t default: HGOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unknown floating-point normalization") - } /* end switch */ + } dt->shared->u.atomic.u.f.sign = (flags >> 8) & 0xff; + + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 2 + 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT16DECODE(*pp, dt->shared->u.atomic.offset); UINT16DECODE(*pp, dt->shared->u.atomic.prec); + + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 1 + 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); dt->shared->u.atomic.u.f.epos = *(*pp)++; dt->shared->u.atomic.u.f.esize = *(*pp)++; - HDassert(dt->shared->u.atomic.u.f.esize > 0); + if (dt->shared->u.atomic.u.f.esize == 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "exponent size can't be zero") + + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 1 + 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); dt->shared->u.atomic.u.f.mpos = *(*pp)++; dt->shared->u.atomic.u.f.msize = *(*pp)++; - HDassert(dt->shared->u.atomic.u.f.msize > 0); + if (dt->shared->u.atomic.u.f.msize == 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "mantissa size can't be zero") + + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT32DECODE(*pp, dt->shared->u.atomic.u.f.ebias); break; - case H5T_TIME: /* Time datatypes */ + case H5T_TIME: + /* + * Time datatypes... + */ dt->shared->u.atomic.order = (flags & 0x1) ? H5T_ORDER_BE : H5T_ORDER_LE; + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT16DECODE(*pp, dt->shared->u.atomic.prec); break; @@ -236,22 +279,35 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t dt->shared->u.atomic.order = (flags & 0x1) ? H5T_ORDER_BE : H5T_ORDER_LE; dt->shared->u.atomic.lsb_pad = (flags & 0x2) ? H5T_PAD_ONE : H5T_PAD_ZERO; dt->shared->u.atomic.msb_pad = (flags & 0x4) ? H5T_PAD_ONE : H5T_PAD_ZERO; + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 2 + 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); UINT16DECODE(*pp, dt->shared->u.atomic.offset); UINT16DECODE(*pp, dt->shared->u.atomic.prec); break; - case H5T_OPAQUE: + case H5T_OPAQUE: { + size_t z; + /* * Opaque types... */ + + /* The opaque tag flag field must be aligned */ z = flags & (H5T_OPAQUE_TAG_MAX - 1); - HDassert(0 == (z & 0x7)); /*must be aligned*/ + if (0 != (z & 0x7)) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "opaque flag field must be aligned") + if (NULL == (dt->shared->u.opaque.tag = (char *)H5MM_malloc(z + 1))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, z, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); H5MM_memcpy(dt->shared->u.opaque.tag, *pp, z); dt->shared->u.opaque.tag[z] = '\0'; + *pp += z; break; + } case H5T_COMPOUND: { unsigned nmembs; /* Number of compound members */ @@ -274,15 +330,26 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t HGOTO_ERROR(H5E_DATATYPE, H5E_CANTALLOC, FAIL, "memory allocation failed") dt->shared->u.compnd.nalloc = nmembs; - HDassert(dt->shared->u.compnd.memb_size == 0); + if (dt->shared->u.compnd.memb_size != 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADVALUE, FAIL, "member size not initialized to zero") for (dt->shared->u.compnd.nmembs = 0; dt->shared->u.compnd.nmembs < nmembs; dt->shared->u.compnd.nmembs++) { - unsigned ndims = 0; /* Number of dimensions of the array field */ - htri_t can_upgrade; /* Whether we can upgrade this type's version */ - hsize_t dim[H5O_LAYOUT_NDIMS]; /* Dimensions of the array */ - H5T_t *array_dt; /* Temporary pointer to the array datatype */ - H5T_t *temp_type; /* Temporary pointer to the field's datatype */ + + size_t actual_name_length; /* Actual length of name */ + size_t max = (size_t)(p_end - *pp + 1); /* Max possible name length */ + unsigned ndims = 0; /* Number of dimensions of the array field */ + htri_t can_upgrade; /* Whether we can upgrade this type's version */ + hsize_t dim[H5O_LAYOUT_NDIMS]; /* Dimensions of the array */ + H5T_t *array_dt; /* Temporary pointer to the array datatype */ + H5T_t *temp_type; /* Temporary pointer to the field's datatype */ + + /* Get the length of the field name */ + actual_name_length = HDstrnlen((const char *)*pp, max); + if (actual_name_length == max) + HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, FAIL, "field name not null terminated") + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, actual_name_length, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); /* Decode the field name */ if (NULL == (dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].name = @@ -291,26 +358,45 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t "can't duplicate compound member name string") /* Version 3 of the datatype message eliminated the padding to multiple of 8 bytes */ - if (version >= H5O_DTYPE_VERSION_3) + if (version >= H5O_DTYPE_VERSION_3) { /* Advance past name, including null terminator */ - *pp += HDstrlen((const char *)*pp) + 1; - else + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, actual_name_length + 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, + "ran off end of input buffer while decoding"); + *pp += actual_name_length + 1; + } + else { /* Advance multiple of 8 w/ null terminator */ - *pp += ((HDstrlen((const char *)*pp) + 8) / 8) * 8; + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, ((actual_name_length + 8) / 8) * 8, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, + "ran off end of input buffer while decoding"); + *pp += ((actual_name_length + 8) / 8) * 8; + } /* Decode the field offset */ /* (starting with version 3 of the datatype message, use the minimum # of bytes required) */ - if (version >= H5O_DTYPE_VERSION_3) + if (version >= H5O_DTYPE_VERSION_3) { + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, offset_nbytes, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, + "ran off end of input buffer while decoding"); UINT32DECODE_VAR(*pp, dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset, offset_nbytes) - else + } + else { + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, + "ran off end of input buffer while decoding"); UINT32DECODE(*pp, dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset) + } /* Older versions of the library allowed a field to have * intrinsic 'arrayness'. Newer versions of the library * use the separate array datatypes. */ if (version == H5O_DTYPE_VERSION_1) { /* Decode the number of dimensions */ + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, + "ran off end of input buffer while decoding"); ndims = *(*pp)++; /* Check that ndims is valid */ @@ -320,18 +406,31 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "invalid number of dimensions for array") } - *pp += 3; /*reserved bytes */ + /* Skip reserved bytes */ + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 3, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, + "ran off end of input buffer while decoding"); + *pp += 3; /* Skip dimension permutation */ + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, + "ran off end of input buffer while decoding"); *pp += 4; /* Skip reserved bytes */ + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, + "ran off end of input buffer while decoding"); *pp += 4; /* Decode array dimension sizes */ - for (i = 0; i < 4; i++) + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, (4 * 4), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, + "ran off end of input buffer while decoding"); + for (int i = 0; i < 4; i++) UINT32DECODE(*pp, dim[i]); - } /* end if */ + } /* Allocate space for the field's datatype */ if (NULL == (temp_type = H5T__alloc())) { @@ -341,14 +440,15 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t } /* Decode the field's datatype information */ - if ((can_upgrade = H5O__dtype_decode_helper(ioflags, pp, temp_type)) < 0) { + if ((can_upgrade = H5O__dtype_decode_helper(ioflags, pp, temp_type, skip, p_end)) < 0) { dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].name = H5MM_xfree(dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].name); if (H5T_close_real(temp_type) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CANTRELEASE, FAIL, "can't release datatype info") HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "unable to decode member type") - } /* end if */ - HDassert(temp_type->shared->size > 0); + } + if (temp_type->shared->size == 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "type size can't be zero") /* Upgrade the version if we can and it is necessary */ if (can_upgrade && temp_type->shared->version > version) { @@ -356,7 +456,7 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t /* Pass "can_upgrade" flag down to parent type */ ret_value = TRUE; - } /* end if */ + } /* Go create the array datatype now, for older versions of the datatype message */ if (version == H5O_DTYPE_VERSION_1) { @@ -371,7 +471,7 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t "can't release datatype info") HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to create array datatype") - } /* end if */ + } /* Close the base type for the array */ if (H5T_close_real(temp_type) < 0) { @@ -394,16 +494,15 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t /* Set the return value to indicate that we should freely * upgrade parent types */ ret_value = TRUE; - } /* end else */ - } /* end if */ - } /* end if */ + } + } + } /* Keep track of the maximum member version found */ if (temp_type->shared->version > max_version) max_version = temp_type->shared->version; - /* - * Set the "force conversion" flag if VL datatype fields exist in this + /* Set the "force conversion" flag if VL datatype fields exist in this * type or any component types */ if (temp_type->shared->force_conv == TRUE) @@ -416,29 +515,30 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t /* Set the field datatype (finally :-) */ dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].type = temp_type; - /* Check if this field overlaps with a prior field */ - /* (probably indicates that the file is corrupt) */ + /* Check if this field overlaps with a prior field + * (probably indicates that the file is corrupt) + */ if (dt->shared->u.compnd.nmembs > 0 && dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset < max_memb_pos) { - for (i = 0; i < dt->shared->u.compnd.nmembs; i++) + for (unsigned u = 0; u < dt->shared->u.compnd.nmembs; u++) if ((dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset >= - dt->shared->u.compnd.memb[i].offset && + dt->shared->u.compnd.memb[u].offset && dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset < - (dt->shared->u.compnd.memb[i].offset + dt->shared->u.compnd.memb[i].size)) || + (dt->shared->u.compnd.memb[u].offset + dt->shared->u.compnd.memb[u].size)) || (dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset < - dt->shared->u.compnd.memb[i].offset && + dt->shared->u.compnd.memb[u].offset && (dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset + dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].size) > - dt->shared->u.compnd.memb[i].offset)) + dt->shared->u.compnd.memb[u].offset)) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "member overlaps with previous member") - } /* end if */ + } /* Update the maximum member position covered */ max_memb_pos = MAX(max_memb_pos, (dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].offset + dt->shared->u.compnd.memb[dt->shared->u.compnd.nmembs].size)); - } /* end for */ + } /* Check if the compound type is packed */ H5T__update_packed(dt); @@ -451,14 +551,17 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t /* We won't mark the message dirty since there were no * errors in the file, simply type versions that we will no * longer encode. */ - } /* end if */ + } /* Check that no member of this compound has a version greater * than the compound itself. */ H5O_DTYPE_CHECK_VERSION(dt, version, max_version, ioflags, "compound", FAIL) } break; - case H5T_REFERENCE: /* Reference datatypes... */ + case H5T_REFERENCE: + /* + * Reference datatypes... + */ dt->shared->u.atomic.order = H5T_ORDER_NONE; dt->shared->u.atomic.prec = 8 * dt->shared->size; dt->shared->u.atomic.offset = 0; @@ -501,7 +604,7 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t nmembs = flags & 0xffff; if (NULL == (dt->shared->parent = H5T__alloc())) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate parent datatype") - if (H5O__dtype_decode_helper(ioflags, pp, dt->shared->parent) < 0) + if (H5O__dtype_decode_helper(ioflags, pp, dt->shared->parent, skip, p_end) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "unable to decode parent datatype") if (dt->shared->parent->shared->size != dt->shared->size) HGOTO_ERROR(H5E_DATATYPE, H5E_BADSIZE, FAIL, "ENUM datatype size does not match parent") @@ -520,37 +623,61 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t /* Names */ for (dt->shared->u.enumer.nmembs = 0; dt->shared->u.enumer.nmembs < nmembs; dt->shared->u.enumer.nmembs++) { + + size_t actual_name_length; /* Actual length of name */ + size_t max = (size_t)(p_end - *pp + 1); /* Max possible name length */ + + actual_name_length = HDstrnlen((const char *)*pp, max); + if (actual_name_length == max) + HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, FAIL, "enum name not null terminated") + + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, actual_name_length, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); if (NULL == (dt->shared->u.enumer.name[dt->shared->u.enumer.nmembs] = H5MM_xstrdup((const char *)*pp))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTCOPY, FAIL, "can't duplicate enum name string") /* Version 3 of the datatype message eliminated the padding to multiple of 8 bytes */ - if (version >= H5O_DTYPE_VERSION_3) + if (version >= H5O_DTYPE_VERSION_3) { /* Advance past name, including null terminator */ - *pp += HDstrlen((const char *)*pp) + 1; - else + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, actual_name_length + 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, + "ran off end of input buffer while decoding"); + *pp += actual_name_length + 1; + } + else { /* Advance multiple of 8 w/ null terminator */ - *pp += ((HDstrlen((const char *)*pp) + 8) / 8) * 8; - } /* end for */ - HDassert(dt->shared->u.enumer.nmembs == nmembs); + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, ((actual_name_length + 8) / 8) * 8, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, + "ran off end of input buffer while decoding"); + *pp += ((actual_name_length + 8) / 8) * 8; + } + } + if (dt->shared->u.enumer.nmembs != nmembs) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "incorrect number of enum members decoded"); /* Values */ + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, nmembs * dt->shared->parent->shared->size, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); H5MM_memcpy(dt->shared->u.enumer.value, *pp, nmembs * dt->shared->parent->shared->size); *pp += nmembs * dt->shared->parent->shared->size; } break; - case H5T_VLEN: /* Variable length datatypes... */ + case H5T_VLEN: + /* + * Variable length datatypes... + */ /* Set the type of VL information, either sequence or string */ dt->shared->u.vlen.type = (H5T_vlen_type_t)(flags & 0x0f); if (dt->shared->u.vlen.type == H5T_VLEN_STRING) { dt->shared->u.vlen.pad = (H5T_str_t)((flags >> 4) & 0x0f); dt->shared->u.vlen.cset = (H5T_cset_t)((flags >> 8) & 0x0f); - } /* end if */ + } /* Decode base type of VL information */ if (NULL == (dt->shared->parent = H5T__alloc())) HGOTO_ERROR(H5E_DATATYPE, H5E_NOSPACE, FAIL, "memory allocation failed") - if (H5O__dtype_decode_helper(ioflags, pp, dt->shared->parent) < 0) + if (H5O__dtype_decode_helper(ioflags, pp, dt->shared->parent, skip, p_end) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "unable to decode VL parent type") /* Check if the parent of this vlen has a version greater than the @@ -565,8 +692,13 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "invalid datatype location") break; - case H5T_ARRAY: /* Array datatypes */ + case H5T_ARRAY: + /* + * Array datatypes... + */ /* Decode the number of dimensions */ + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); dt->shared->u.array.ndims = *(*pp)++; /* Double-check the number of dimensions */ @@ -574,23 +706,32 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t HGOTO_ERROR(H5E_DATATYPE, H5E_CANTLOAD, FAIL, "too many dimensions for array datatype") /* Skip reserved bytes, if version has them */ - if (version < H5O_DTYPE_VERSION_3) + if (version < H5O_DTYPE_VERSION_3) { + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, 3, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); *pp += 3; + } /* Decode array dimension sizes & compute number of elements */ - for (i = 0, dt->shared->u.array.nelem = 1; i < (unsigned)dt->shared->u.array.ndims; i++) { - UINT32DECODE(*pp, dt->shared->u.array.dim[i]); - dt->shared->u.array.nelem *= dt->shared->u.array.dim[i]; - } /* end for */ + dt->shared->u.array.nelem = 1; + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, (dt->shared->u.array.ndims * 4), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); + for (unsigned u = 0; u < dt->shared->u.array.ndims; u++) { + UINT32DECODE(*pp, dt->shared->u.array.dim[u]); + dt->shared->u.array.nelem *= dt->shared->u.array.dim[u]; + } /* Skip array dimension permutations, if version has them */ - if (version < H5O_DTYPE_VERSION_3) + if (version < H5O_DTYPE_VERSION_3) { + if (H5_DTYPE_IS_BUFFER_OVERFLOW(skip, *pp, (dt->shared->u.array.ndims * 4), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, FAIL, "ran off end of input buffer while decoding"); *pp += dt->shared->u.array.ndims * 4; + } /* Decode base type of array */ if (NULL == (dt->shared->parent = H5T__alloc())) HGOTO_ERROR(H5E_DATATYPE, H5E_NOSPACE, FAIL, "memory allocation failed") - if (H5O__dtype_decode_helper(ioflags, pp, dt->shared->parent) < 0) + if (H5O__dtype_decode_helper(ioflags, pp, dt->shared->parent, skip, p_end) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "unable to decode array parent type") /* Check if the parent of this array has a version greater than the @@ -600,8 +741,7 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t /* There should be no array datatypes with version < 2. */ H5O_DTYPE_CHECK_VERSION(dt, version, H5O_DTYPE_VERSION_2, ioflags, "array", FAIL) - /* - * Set the "force conversion" flag if a VL base datatype is used or + /* Set the "force conversion" flag if a VL base datatype is used or * or if any components of the base datatype are VL types. */ if (dt->shared->parent->shared->force_conv == TRUE) @@ -612,7 +752,7 @@ H5O__dtype_decode_helper(unsigned *ioflags /*in,out*/, const uint8_t **pp, H5T_t case H5T_NCLASSES: default: HGOTO_ERROR(H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unknown datatype class found") - } /* end switch */ + } done: /* Cleanup on error */ @@ -1150,27 +1290,36 @@ done: Pointer to the new message in native order on success, NULL on failure DESCRIPTION This function decodes the "raw" disk form of a simple datatype message - into a struct in memory native format. The struct is allocated within this - function using malloc() and is returned to the caller. + into a struct in memory native format. The struct is allocated within this + function using malloc() and is returned to the caller. --------------------------------------------------------------------------*/ static void * H5O__dtype_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, - unsigned *ioflags /*in,out*/, size_t H5_ATTR_UNUSED p_size, const uint8_t *p) + unsigned *ioflags /*in,out*/, size_t p_size, const uint8_t *p) { - H5T_t *dt = NULL; - void *ret_value = NULL; /* Return value */ + hbool_t skip; + H5T_t *dt = NULL; + const uint8_t *p_end = p + p_size - 1; + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* check args */ + HDassert(f); HDassert(p); /* Allocate datatype message */ if (NULL == (dt = H5T__alloc())) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + /* If we are decoding a buffer from H5Tdecode(), we won't have the size + * of the buffer and bounds checking will be impossible. In this case, + * the library will have set p_size to SIZE_MAX and we can use that + * as a signal to skip bounds checking. + */ + skip = (p_size == SIZE_MAX ? TRUE : FALSE); + /* Perform actual decode of message */ - if (H5O__dtype_decode_helper(ioflags, &p, dt) < 0) + if (H5O__dtype_decode_helper(ioflags, &p, dt, skip, p_end) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, NULL, "can't decode type") /* Set return value */ diff --git a/src/H5Ofill.c b/src/H5Ofill.c index 094edaca..99c8950 100644 --- a/src/H5Ofill.c +++ b/src/H5Ofill.c @@ -10,11 +10,9 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Robb Matzke - * Wednesday, September 30, 1998 - * +/* * Purpose: The fill message indicates a bit pattern to use for - * uninitialized data points of a dataset. + * uninitialized data points of a dataset. */ #include "H5Omodule.h" /* This source code file is part of the H5O module */ @@ -178,16 +176,12 @@ H5FL_BLK_EXTERN(type_conv); /*------------------------------------------------------------------------- * Function: H5O__fill_new_decode * - * Purpose: Decode a new fill value message. The new fill value - * message is fill value plus space allocation time and - * fill value writing time and whether fill value is defined. - * - * Return: Success: Ptr to new message in native struct. - * Failure: NULL - * - * Programmer: Raymond Lu - * Feb 26, 2002 + * Purpose: Decode a new fill value message. The new fill value + * message is fill value plus space allocation time and + * fill value writing time and whether fill value is defined. * + * Return: Success: Pointer to new message in native struct + * Failure: NULL *------------------------------------------------------------------------- */ static void * @@ -208,12 +202,21 @@ H5O__fill_new_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for fill value message") /* Version */ + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); fill->version = *p++; if (fill->version < H5O_FILL_VERSION_1 || fill->version > H5O_FILL_VERSION_LATEST) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad version number for fill value message") /* Decode each version */ if (fill->version < H5O_FILL_VERSION_3) { + + /* Versions 1 & 2 */ + + /* Buffer size check for the next three bytes */ + if (H5_IS_BUFFER_OVERFLOW(p, 3, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + /* Space allocation time */ fill->alloc_time = (H5D_alloc_time_t)*p++; @@ -225,26 +228,34 @@ H5O__fill_new_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, /* Only decode fill value information if one is defined */ if (fill->fill_defined) { + + if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); INT32DECODE(p, fill->size); + if (fill->size > 0) { H5_CHECK_OVERFLOW(fill->size, ssize_t, size_t); - /* Ensure that fill size doesn't exceed buffer size, due to possible data corruption */ - if (p + fill->size - 1 > p_end) - HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "fill size exceeds buffer size") + if (H5_IS_BUFFER_OVERFLOW(p, fill->size, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); if (NULL == (fill->buf = H5MM_malloc((size_t)fill->size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for fill value") H5MM_memcpy(fill->buf, p, (size_t)fill->size); - } /* end if */ - } /* end if */ + } + } else - fill->size = (-1); - } /* end if */ + fill->size = -1; + } else { + + /* Version 3 */ + unsigned flags; /* Status flags */ /* Flags */ + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); flags = *p++; /* Check for unknown flags */ @@ -260,39 +271,45 @@ H5O__fill_new_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, /* Check for undefined fill value */ if (flags & H5O_FILL_FLAG_UNDEFINED_VALUE) { - /* Sanity check */ - HDassert(!(flags & H5O_FILL_FLAG_HAVE_VALUE)); + + if (flags & (unsigned)~H5O_FILL_FLAG_HAVE_VALUE) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "have value and undefined value flags both set") /* Set value for "undefined" fill value */ - fill->size = (-1); - } /* end if */ + fill->size = -1; + } else if (flags & H5O_FILL_FLAG_HAVE_VALUE) { /* Fill value size */ + if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); UINT32DECODE(p, fill->size); /* Fill value */ H5_CHECK_OVERFLOW(fill->size, ssize_t, size_t); + + if (H5_IS_BUFFER_OVERFLOW(p, fill->size, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + if (NULL == (fill->buf = H5MM_malloc((size_t)fill->size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for fill value") H5MM_memcpy(fill->buf, p, (size_t)fill->size); /* Set the "defined" flag */ fill->fill_defined = TRUE; - } /* end else */ + } else /* Set the "defined" flag */ fill->fill_defined = TRUE; - } /* end else */ + } /* Set return value */ ret_value = (void *)fill; done: if (!ret_value && fill) { - if (fill->buf) - H5MM_xfree(fill->buf); + H5MM_xfree(fill->buf); fill = H5FL_FREE(H5O_fill_t, fill); - } /* end if */ + } FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__fill_new_decode() */ @@ -300,14 +317,10 @@ done: /*------------------------------------------------------------------------- * Function: H5O__fill_old_decode * - * Purpose: Decode an old fill value message. - * - * Return: Success: Ptr to new message in native struct. - * Failure: NULL - * - * Programmer: Robb Matzke - * Wednesday, September 30, 1998 + * Purpose: Decode an old fill value message * + * Return: Success: Pointer to new message in native struct + * Failure: NULL *------------------------------------------------------------------------- */ static void * @@ -334,6 +347,8 @@ H5O__fill_old_decode(H5F_t *f, H5O_t *open_oh, unsigned H5_ATTR_UNUSED mesg_flag fill->fill_time = H5D_FILL_TIME_IFSET; /* Fill value size */ + if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); UINT32DECODE(p, fill->size); /* Only decode the fill value itself if there is one */ @@ -341,8 +356,8 @@ H5O__fill_old_decode(H5F_t *f, H5O_t *open_oh, unsigned H5_ATTR_UNUSED mesg_flag H5_CHECK_OVERFLOW(fill->size, ssize_t, size_t); /* Ensure that fill size doesn't exceed buffer size, due to possible data corruption */ - if (p + fill->size - 1 > p_end) - HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "fill size exceeds buffer size") + if (H5_IS_BUFFER_OVERFLOW(p, fill->size, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); /* Get the datatype message */ if ((exists = H5O_msg_exists_oh(open_oh, H5O_DTYPE_ID)) < 0) @@ -353,15 +368,15 @@ H5O__fill_old_decode(H5F_t *f, H5O_t *open_oh, unsigned H5_ATTR_UNUSED mesg_flag /* Verify size */ if (fill->size != (ssize_t)H5T_GET_SIZE(dt)) HGOTO_ERROR(H5E_SYM, H5E_CANTGET, NULL, "inconsistent fill value size") - } /* end if */ + } if (NULL == (fill->buf = H5MM_malloc((size_t)fill->size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for fill value") H5MM_memcpy(fill->buf, p, (size_t)fill->size); fill->fill_defined = TRUE; - } /* end if */ + } else - fill->size = (-1); + fill->size = -1; /* Set return value */ ret_value = (void *)fill; @@ -371,10 +386,9 @@ done: H5O_msg_free(H5O_DTYPE_ID, dt); if (!ret_value && fill) { - if (fill->buf) - H5MM_xfree(fill->buf); - fill = H5FL_FREE(H5O_fill_t, fill); - } /* end if */ + H5MM_xfree(fill->buf); + H5FL_FREE(H5O_fill_t, fill); + } FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__fill_old_decode() */ diff --git a/src/H5Ofsinfo.c b/src/H5Ofsinfo.c index 4d5934d..01cf3c3 100644 --- a/src/H5Ofsinfo.c +++ b/src/H5Ofsinfo.c @@ -13,10 +13,8 @@ /*------------------------------------------------------------------------- * * Created: H5Ofsinfo.c - * Feb 2009 - * Vailin Choi * - * Purpose: File space info message. + * Purpose: File space info message * *------------------------------------------------------------------------- */ @@ -81,27 +79,22 @@ H5FL_DEFINE_STATIC(H5O_fsinfo_t); * * Purpose: Decode a message and return a pointer to a newly allocated one. * - * Return: Success: Ptr to new message in native form. - * Failure: NULL - * - * Programmer: Vailin Choi; Feb 2009 - * + * Return: Success: Pointer to new message in native form + * Failure: NULL *------------------------------------------------------------------------- */ - static void * H5O__fsinfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, const uint8_t *p) { - H5O_fsinfo_t *fsinfo = NULL; /* File space info message */ - H5F_mem_page_t ptype; /* Memory type for iteration */ - unsigned vers; /* message version */ - const uint8_t *p_end = p + p_size; - void *ret_value = NULL; /* Return value */ + H5O_fsinfo_t *fsinfo = NULL; /* File space info message */ + H5F_mem_page_t ptype; /* Memory type for iteration */ + unsigned vers; /* Message version */ + const uint8_t *p_end = p + p_size - 1; /* End of the p buffer */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* check args */ HDassert(f); HDassert(p); @@ -113,8 +106,8 @@ H5O__fsinfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU fsinfo->fs_addr[ptype - 1] = HADDR_UNDEF; /* Version of message */ - if (p + 1 - 1 > p_end) /* one byte for version */ - HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "ran off end of input buffer while decoding") + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); vers = *p++; if (vers == H5O_FSINFO_VERSION_0) { @@ -128,8 +121,8 @@ H5O__fsinfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU fsinfo->pgend_meta_thres = H5F_FILE_SPACE_PGEND_META_THRES; fsinfo->eoa_pre_fsm_fsalloc = HADDR_UNDEF; - if (p + 1 + H5F_SIZEOF_SIZE(f) - 1 > p_end) /* one byte for strategy + sizeof(f) */ - HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "ran off end of input buffer while decoding") + if (H5_IS_BUFFER_OVERFLOW(p, 1 + H5F_sizeof_size(f), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); strategy = (H5F_file_space_type_t)*p++; /* File space strategy */ H5F_DECODE_LENGTH(f, p, threshold); /* Free-space section threshold */ @@ -142,9 +135,9 @@ H5O__fsinfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU if (HADDR_UNDEF == (fsinfo->eoa_pre_fsm_fsalloc = H5F_get_eoa(f, H5FD_MEM_DEFAULT))) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file size") for (type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; type++) { - if (p + H5_SIZEOF_HADDR_T > p_end) - HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, - "ran off end of input buffer while decoding") + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, + "ran off end of input buffer while decoding"); H5F_addr_decode(f, &p, &(fsinfo->fs_addr[type - 1])); } break; @@ -166,32 +159,43 @@ H5O__fsinfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU case H5F_FILE_SPACE_DEFAULT: default: HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file space strategy") - } /* end switch */ + } fsinfo->version = H5O_FSINFO_VERSION_1; fsinfo->mapped = TRUE; } else { - HDassert(vers >= H5O_FSINFO_VERSION_1); + if (vers < H5O_FSINFO_VERSION_1) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "bad version number") fsinfo->version = vers; - /* strategy (1) + persist (1) + sizeof(f) + sizeof(f) + pgend_meta_thres (2) + sizeofaddr(f) */ - if (p + 1 + 1 + 2 * H5F_SIZEOF_SIZE(f) + 2 + H5F_SIZEOF_ADDR(f) - 1 > p_end) - HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "ran off end of input buffer while decoding") + if (H5_IS_BUFFER_OVERFLOW(p, 1 + 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); fsinfo->strategy = (H5F_fspace_strategy_t)*p++; /* File space strategy */ fsinfo->persist = *p++; /* Free-space persist or not */ - H5F_DECODE_LENGTH(f, p, fsinfo->threshold); /* Free-space section threshold */ + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + H5F_DECODE_LENGTH(f, p, fsinfo->threshold); /* Free-space section threshold */ + + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5F_DECODE_LENGTH(f, p, fsinfo->page_size); /* File space page size */ - UINT16DECODE(p, fsinfo->pgend_meta_thres); /* Page end metadata threshold */ + + if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); + UINT16DECODE(p, fsinfo->pgend_meta_thres); /* Page end metadata threshold */ + + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5F_addr_decode(f, &p, &(fsinfo->eoa_pre_fsm_fsalloc)); /* EOA before free-space header and section info */ /* Decode addresses of free space managers, if persisting */ if (fsinfo->persist) for (ptype = H5F_MEM_PAGE_SUPER; ptype < H5F_MEM_PAGE_NTYPES; ptype++) { - if (p + H5F_SIZEOF_SIZE(f) - 1 > p_end) /* one byte for sizeof(f) */ - HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "ran off end of input buffer while decoding") + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5F_addr_decode(f, &p, &(fsinfo->fs_addr[ptype - 1])); } fsinfo->mapped = FALSE; @@ -201,8 +205,8 @@ H5O__fsinfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU ret_value = fsinfo; done: - if (ret_value == NULL && fsinfo != NULL) - fsinfo = H5FL_FREE(H5O_fsinfo_t, fsinfo); + if (!ret_value && fsinfo) + H5FL_FREE(H5O_fsinfo_t, fsinfo); FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__fsinfo_decode() */ diff --git a/src/H5Oginfo.c b/src/H5Oginfo.c index 54d8b8b..df45e53 100644 --- a/src/H5Oginfo.c +++ b/src/H5Oginfo.c @@ -13,10 +13,8 @@ /*------------------------------------------------------------------------- * * Created: H5Oginfo.c - * Aug 23 2005 - * Quincey Koziol * - * Purpose: Group Information messages. + * Purpose: Group Information messages * *------------------------------------------------------------------------- */ @@ -78,34 +76,24 @@ H5FL_DEFINE_STATIC(H5O_ginfo_t); * Purpose: Decode a message and return a pointer to * a newly allocated one. * - * Return: Success: Ptr to new message in native order. - * - * Failure: NULL - * - * Programmer: Quincey Koziol - * Aug 30 2005 - * + * Return: Success: Pointer to new message in native order + * Failure: NULL *------------------------------------------------------------------------- */ static void * H5O__ginfo_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, const uint8_t *p) { - H5O_ginfo_t *ginfo = NULL; /* Pointer to group information message */ - unsigned char flags; /* Flags for encoding group info */ - void *ret_value = NULL; /* Return value */ + H5O_ginfo_t *ginfo = NULL; /* Pointer to group information message */ + unsigned char flags; /* Flags for encoding group info */ + const uint8_t *p_end = p + p_size - 1; /* End of the p buffer */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* check args */ + HDassert(f); HDassert(p); - if (p_size == 0) - HGOTO_ERROR(H5E_OHDR, H5E_ARGS, NULL, "size of given ginfo was zero") - - /* Points at last valid byte in buffer */ - const uint8_t *p_end = p + p_size - 1; - /* Version of message */ if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") @@ -132,11 +120,11 @@ H5O__ginfo_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsign HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT16DECODE(p, ginfo->max_compact) UINT16DECODE(p, ginfo->min_dense) - } /* end if */ + } else { ginfo->max_compact = H5G_CRT_GINFO_MAX_COMPACT; ginfo->min_dense = H5G_CRT_GINFO_MIN_DENSE; - } /* end else */ + } /* Get the estimated # of entries & name lengths */ if (ginfo->store_est_entry_info) { @@ -144,19 +132,18 @@ H5O__ginfo_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsign HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT16DECODE(p, ginfo->est_num_entries) UINT16DECODE(p, ginfo->est_name_len) - } /* end if */ + } else { ginfo->est_num_entries = H5G_CRT_GINFO_EST_NUM_ENTRIES; ginfo->est_name_len = H5G_CRT_GINFO_EST_NAME_LEN; - } /* end if */ + } /* Set return value */ ret_value = ginfo; done: - if (ret_value == NULL) - if (ginfo != NULL) - ginfo = H5FL_FREE(H5O_ginfo_t, ginfo); + if (!ret_value && ginfo) + H5FL_FREE(H5O_ginfo_t, ginfo); FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__ginfo_decode() */ diff --git a/src/H5Olayout.c b/src/H5Olayout.c index a58fc0c..f784f24 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -10,10 +10,8 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Robb Matzke - * Wednesday, October 8, 1997 - * - * Purpose: Messages related to data layout. +/* + * Purpose: Messages related to data layout */ #define H5D_FRIEND /*suppress error about including H5Dpkg */ @@ -78,13 +76,8 @@ H5FL_DEFINE(H5O_layout_t); * Purpose: Decode an data layout message and return a pointer to a * new one created with malloc(). * - * Return: Success: Ptr to new message in native order. - * + * Return: Success: Pointer to new message in native order * Failure: NULL - * - * Programmer: Robb Matzke - * Wednesday, October 8, 1997 - * *------------------------------------------------------------------------- */ static void * @@ -94,16 +87,13 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU const uint8_t *p_end = p + p_size - 1; /* End of the p buffer */ H5O_layout_t *mesg = NULL; uint8_t *heap_block = NULL; - unsigned u; - void *ret_value = NULL; /* Return value */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* check args */ HDassert(f); HDassert(p); - /* decode */ if (NULL == (mesg = H5FL_CALLOC(H5O_layout_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "memory allocation failed") mesg->storage.type = H5D_LAYOUT_ERROR; @@ -144,33 +134,33 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU /* Address */ if (mesg->type == H5D_CONTIGUOUS) { - if (H5_IS_BUFFER_OVERFLOW(p, H5F_SIZEOF_ADDR(f), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") H5F_addr_decode(f, &p, &(mesg->storage.u.contig.addr)); /* Set the layout operations */ mesg->ops = H5D_LOPS_CONTIG; - } /* end if */ + } else if (mesg->type == H5D_CHUNKED) { - if (H5_IS_BUFFER_OVERFLOW(p, H5F_SIZEOF_ADDR(f), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") H5F_addr_decode(f, &p, &(mesg->storage.u.chunk.idx_addr)); /* Set the layout operations */ mesg->ops = H5D_LOPS_CHUNK; - /* Set the chunk operations */ - /* (Only "btree" indexing type currently supported in this version) */ + /* Set the chunk operations + * (Only "btree" indexing type currently supported in this version) + */ mesg->storage.u.chunk.idx_type = H5D_CHUNK_IDX_BTREE; mesg->storage.u.chunk.ops = H5D_COPS_BTREE; - } /* end if */ - else { - /* Sanity check */ - HDassert(mesg->type == H5D_COMPACT); - + } + else if (mesg->type == H5D_COMPACT) { /* Set the layout operations */ mesg->ops = H5D_LOPS_COMPACT; - } /* end else */ + } + else + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "invalid layout type") /* Read the size */ if (mesg->type != H5D_CHUNKED) { @@ -178,24 +168,24 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU * truncation of the dimension sizes when they were stored in this * version of the layout message. Compute the contiguous storage * size in the dataset code, where we've got the dataspace - * information available also. - QAK 5/26/04 + * information available also. */ - if (H5_IS_BUFFER_OVERFLOW(p, (ndims * sizeof(uint32_t)), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, (ndims * 4), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") p += ndims * sizeof(uint32_t); /* Skip over dimension sizes */ - } /* end if */ + } else { if (ndims < 2) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad dimensions for chunked storage") mesg->u.chunk.ndims = ndims; - if (H5_IS_BUFFER_OVERFLOW(p, (ndims * sizeof(uint32_t)), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, (ndims * 4), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") + for (unsigned u = 0; u < ndims; u++) { - for (u = 0; u < ndims; u++) { UINT32DECODE(p, mesg->u.chunk.dim[u]); - /* Just in case that something goes very wrong, such as file corruption. */ + /* Just in case that something goes very wrong, such as file corruption */ if (mesg->u.chunk.dim[u] == 0) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad chunk dimension value when parsing layout message - chunk dimension " @@ -204,12 +194,13 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU } /* Compute chunk size */ - for (u = 1, mesg->u.chunk.size = mesg->u.chunk.dim[0]; u < ndims; u++) + mesg->u.chunk.size = mesg->u.chunk.dim[0]; + for (unsigned u = 1; u < ndims; u++) mesg->u.chunk.size *= mesg->u.chunk.dim[u]; - } /* end if */ + } if (mesg->type == H5D_COMPACT) { - if (H5_IS_BUFFER_OVERFLOW(p, sizeof(uint32_t), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT32DECODE(p, mesg->storage.u.compact.size); @@ -223,9 +214,9 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU "memory allocation failed for compact data buffer") H5MM_memcpy(mesg->storage.u.compact.buf, p, mesg->storage.u.compact.size); p += mesg->storage.u.compact.size; - } /* end if */ - } /* end if */ - } /* end if */ + } + } + } else { /* Layout & storage class */ if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) @@ -236,7 +227,7 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU switch (mesg->type) { case H5D_COMPACT: /* Compact data size */ - if (H5_IS_BUFFER_OVERFLOW(p, sizeof(uint16_t), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT16DECODE(p, mesg->storage.u.compact.size); @@ -254,7 +245,7 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU /* Compact data */ H5MM_memcpy(mesg->storage.u.compact.buf, p, mesg->storage.u.compact.size); p += mesg->storage.u.compact.size; - } /* end if */ + } /* Set the layout operations */ mesg->ops = H5D_LOPS_COMPACT; @@ -262,12 +253,12 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU case H5D_CONTIGUOUS: /* Contiguous storage address */ - if (H5_IS_BUFFER_OVERFLOW(p, H5F_SIZEOF_ADDR(f), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") H5F_addr_decode(f, &p, &(mesg->storage.u.contig.addr)); /* Contiguous storage size */ - if (H5_IS_BUFFER_OVERFLOW(p, H5F_SIZEOF_SIZE(f), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") H5F_DECODE_LENGTH(f, p, mesg->storage.u.contig.size); @@ -292,17 +283,18 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad dimensions for chunked storage") /* B-tree address */ - if (H5_IS_BUFFER_OVERFLOW(p, H5F_SIZEOF_ADDR(f), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") H5F_addr_decode(f, &p, &(mesg->storage.u.chunk.idx_addr)); - if (H5_IS_BUFFER_OVERFLOW(p, (mesg->u.chunk.ndims * sizeof(uint32_t)), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, (mesg->u.chunk.ndims * 4), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") /* Chunk dimensions */ - for (u = 0; u < mesg->u.chunk.ndims; u++) { + for (unsigned u = 0; u < mesg->u.chunk.ndims; u++) { + UINT32DECODE(p, mesg->u.chunk.dim[u]); /* Just in case that something goes very wrong, such as file corruption. */ @@ -311,17 +303,19 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU "bad chunk dimension value when parsing layout message - chunk " "dimension must be positive: mesg->u.chunk.dim[%u] = %u", u, mesg->u.chunk.dim[u]) - } /* end for */ + } /* Compute chunk size */ - for (u = 1, mesg->u.chunk.size = mesg->u.chunk.dim[0]; u < mesg->u.chunk.ndims; u++) + mesg->u.chunk.size = mesg->u.chunk.dim[0]; + for (unsigned u = 1; u < mesg->u.chunk.ndims; u++) mesg->u.chunk.size *= mesg->u.chunk.dim[u]; - /* Set the chunk operations */ - /* (Only "btree" indexing type supported with v3 of message format) */ + /* Set the chunk operations + * (Only "btree" indexing type supported with v3 of message format) + */ mesg->storage.u.chunk.idx_type = H5D_CHUNK_IDX_BTREE; mesg->storage.u.chunk.ops = H5D_COPS_BTREE; - } /* end if */ + } else { /* Get the chunked layout flags */ if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) @@ -360,7 +354,7 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU "ran off end of input buffer while decoding") /* Chunk dimensions */ - for (u = 0; u < mesg->u.chunk.ndims; u++) { + for (unsigned u = 0; u < mesg->u.chunk.ndims; u++) { UINT64DECODE_VAR(p, mesg->u.chunk.dim[u], mesg->u.chunk.enc_bytes_per_dim); /* Just in case that something goes very wrong, such as file corruption. */ @@ -372,7 +366,8 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU } /* Compute chunk size */ - for (u = 1, mesg->u.chunk.size = mesg->u.chunk.dim[0]; u < mesg->u.chunk.ndims; u++) + mesg->u.chunk.size = mesg->u.chunk.dim[0]; + for (unsigned u = 1; u < mesg->u.chunk.ndims; u++) mesg->u.chunk.size *= mesg->u.chunk.dim[u]; /* Chunk index type */ @@ -397,12 +392,12 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU case H5D_CHUNK_IDX_SINGLE: /* Single Chunk Index */ if (mesg->u.chunk.flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER) { - if (H5_IS_BUFFER_OVERFLOW(p, H5F_SIZEOF_SIZE(f) + sizeof(uint32_t), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_size(f) + 4, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") H5F_DECODE_LENGTH(f, p, mesg->storage.u.chunk.u.single.nbytes); UINT32DECODE(p, mesg->storage.u.chunk.u.single.filter_mask); - } /* end if */ + } /* Set the chunk operations */ mesg->storage.u.chunk.ops = H5D_COPS_SINGLE; @@ -475,7 +470,7 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU break; case H5D_CHUNK_IDX_BT2: /* v2 B-tree index */ - if (H5_IS_BUFFER_OVERFLOW(p, sizeof(uint32_t), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT32DECODE(p, mesg->u.chunk.u.btree2.cparam.node_size); @@ -511,14 +506,14 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU case H5D_CHUNK_IDX_NTYPES: default: HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "Invalid chunk index type") - } /* end switch */ + } /* Chunk index address */ - if (H5_IS_BUFFER_OVERFLOW(p, H5F_SIZEOF_ADDR(f), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") H5F_addr_decode(f, &p, &(mesg->storage.u.chunk.idx_addr)); - } /* end else */ + } /* Set the layout operations */ mesg->ops = H5D_LOPS_CHUNK; @@ -530,12 +525,12 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "invalid layout version with virtual layout") /* Heap information */ - if (H5_IS_BUFFER_OVERFLOW(p, H5F_SIZEOF_ADDR(f), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") H5F_addr_decode(f, &p, &(mesg->storage.u.virt.serial_list_hobjid.addr)); /* NOTE: virtual mapping global heap entry address could be undefined */ - if (H5_IS_BUFFER_OVERFLOW(p, sizeof(uint32_t), p_end)) + if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT32DECODE(p, mesg->storage.u.virt.serial_list_hobjid.idx); @@ -580,7 +575,7 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU (unsigned)H5O_LAYOUT_VDS_GH_ENC_VERS, (unsigned)heap_vers) /* Number of entries */ - if (H5_IS_BUFFER_OVERFLOW(heap_block_p, H5F_SIZEOF_SIZE(f), heap_block_p_end)) + if (H5_IS_BUFFER_OVERFLOW(heap_block_p, H5F_sizeof_size(f), heap_block_p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") H5F_DECODE_LENGTH(f, heap_block_p, tmp_hsize) @@ -679,9 +674,9 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU else mesg->storage.u.virt.list[i].source_dset.dset_name = mesg->storage.u.virt.list[i].source_dset_name; - } /* end if */ + } - /* unlim_dim fields */ + /* Unlim_dim fields */ mesg->storage.u.virt.list[i].unlim_dim_source = H5S_get_select_unlim_dim(mesg->storage.u.virt.list[i].source_select); mesg->storage.u.virt.list[i].unlim_dim_virtual = @@ -697,7 +692,7 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU mesg->storage.u.virt.list[i].source_select; mesg->storage.u.virt.list[i].source_dset.clipped_virtual_select = mesg->storage.u.virt.list[i].source_dset.virtual_select; - } /* end if */ + } /* Check mapping for validity (do both pre and post * checks here, since we had to allocate the entry list @@ -713,10 +708,10 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU if (H5D_virtual_update_min_dims(mesg, i) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "unable to update virtual dataset minimum dimensions") - } /* end for */ + } /* Read stored checksum */ - if (H5_IS_BUFFER_OVERFLOW(heap_block_p, sizeof(uint32_t), heap_block_p_end)) + if (H5_IS_BUFFER_OVERFLOW(heap_block_p, 4, heap_block_p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT32DECODE(heap_block_p, stored_chksum) @@ -743,8 +738,8 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU case H5D_NLAYOUTS: default: HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "Invalid layout class") - } /* end switch */ - } /* end else */ + } + } /* Set return value */ ret_value = mesg; @@ -755,8 +750,8 @@ done: if (mesg->type == H5D_VIRTUAL) if (H5D__virtual_reset_layout(mesg) < 0) HDONE_ERROR(H5E_OHDR, H5E_CANTFREE, NULL, "unable to reset virtual layout") - mesg = H5FL_FREE(H5O_layout_t, mesg); - } /* end if */ + H5FL_FREE(H5O_layout_t, mesg); + } heap_block = (uint8_t *)H5MM_xfree(heap_block); diff --git a/src/H5Olinfo.c b/src/H5Olinfo.c index 11138df..a82be72 100644 --- a/src/H5Olinfo.c +++ b/src/H5Olinfo.c @@ -13,16 +13,14 @@ /*------------------------------------------------------------------------- * * Created: H5Olinfo.c - * Aug 23 2005 - * Quincey Koziol * - * Purpose: Link Information messages. + * Purpose: Link information messages * *------------------------------------------------------------------------- */ -#define H5G_FRIEND /*suppress error about including H5Gpkg */ -#define H5L_FRIEND /*suppress error about including H5Lpkg */ +#define H5G_FRIEND /* Suppress error about including H5Gpkg */ +#define H5L_FRIEND /* Suppress error about including H5Lpkg */ #include "H5Omodule.h" /* This source code file is part of the H5O module */ #include "H5private.h" /* Generic Functions */ @@ -95,12 +93,8 @@ H5FL_DEFINE_STATIC(H5O_linfo_t); * * Purpose: Decode a message and return a pointer to a newly allocated one. * - * Return: Success: Ptr to new message in native form. + * Return: Success: Pointer to new message in native form * Failure: NULL - * - * Programmer: Quincey Koziol - * Aug 23 2005 - * *------------------------------------------------------------------------- */ static void * @@ -115,7 +109,6 @@ H5O__linfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUS FUNC_ENTER_PACKAGE - /* check args */ HDassert(f); HDassert(p); diff --git a/src/H5Olink.c b/src/H5Olink.c index dabf87e..160b1d0 100644 --- a/src/H5Olink.c +++ b/src/H5Olink.c @@ -13,10 +13,8 @@ /*------------------------------------------------------------------------- * * Created: H5Olink.c - * Aug 29 2005 - * Quincey Koziol * - * Purpose: Link messages. + * Purpose: Link messages * *------------------------------------------------------------------------- */ @@ -100,32 +98,27 @@ H5FL_DEFINE_STATIC(H5O_link_t); * Purpose: Decode a message and return a pointer to * a newly allocated one. * - * Return: Success: Ptr to new message in native order. - * - * Failure: NULL - * - * Programmer: Quincey Koziol - * Aug 29 2005 - * + * Return: Success: Pointer to new message in native order + * Failure: NULL *------------------------------------------------------------------------- */ static void * H5O__link_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, const uint8_t *p) { - H5O_link_t *lnk = NULL; /* Pointer to link message */ - size_t len = 0; /* Length of a string in the message */ - unsigned char link_flags; /* Flags for encoding link info */ - const uint8_t *p_end = p + p_size; /* End of the p buffer */ - void *ret_value = NULL; /* Return value */ + H5O_link_t *lnk = NULL; /* Pointer to link message */ + size_t len = 0; /* Length of a string in the message */ + unsigned char link_flags; /* Flags for encoding link info */ + const uint8_t *p_end = p + p_size - 1; /* End of the p buffer */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* check args */ HDassert(f); HDassert(p); - /* decode */ + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") if (*p++ != H5O_LINK_VERSION) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad version number for message") @@ -134,6 +127,8 @@ H5O__link_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSE HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* Get the encoding flags for the link */ + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") link_flags = *p++; if (link_flags & ~H5O_LINK_ALL_FLAGS) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad flag value for message") @@ -141,63 +136,74 @@ H5O__link_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSE /* Check for non-default link type */ if (link_flags & H5O_LINK_STORE_LINK_TYPE) { /* Get the type of the link */ + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") lnk->type = (H5L_type_t)*p++; if (lnk->type < H5L_TYPE_HARD || lnk->type > H5L_TYPE_MAX) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad link type") - } /* end if */ + } else lnk->type = H5L_TYPE_HARD; /* Get the link creation time from the file */ if (link_flags & H5O_LINK_STORE_CORDER) { + if (H5_IS_BUFFER_OVERFLOW(p, 8, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") INT64DECODE(p, lnk->corder) lnk->corder_valid = TRUE; - } /* end if */ + } else { lnk->corder = 0; lnk->corder_valid = FALSE; - } /* end else */ + } /* Check for non-default name character set */ if (link_flags & H5O_LINK_STORE_NAME_CSET) { /* Get the link name's character set */ + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") lnk->cset = (H5T_cset_t)*p++; if (lnk->cset < H5T_CSET_ASCII || lnk->cset > H5T_CSET_UTF8) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad cset type") - } /* end if */ + } else lnk->cset = H5T_CSET_ASCII; /* Get the length of the link's name */ switch (link_flags & H5O_LINK_NAME_SIZE) { case 0: /* 1 byte size */ + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") len = *p++; break; case 1: /* 2 byte size */ + if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT16DECODE(p, len); break; case 2: /* 4 byte size */ + if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT32DECODE(p, len); break; case 3: /* 8 byte size */ + if (H5_IS_BUFFER_OVERFLOW(p, 8, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT64DECODE(p, len); break; default: - HDassert(0 && "bad size for name"); - } /* end switch */ + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "no appropriate size for name length") + } if (len == 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "invalid name length") - /* Make sure that length doesn't exceed buffer size, which could occur - when the file is corrupted */ - if (p + len > p_end) - HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "name length causes read past end of buffer") - /* Get the link's name */ + if (H5_IS_BUFFER_OVERFLOW(p, len, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") if (NULL == (lnk->name = (char *)H5MM_malloc(len + 1))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") H5MM_memcpy(lnk->name, p, len); @@ -208,20 +214,21 @@ H5O__link_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSE switch (lnk->type) { case H5L_TYPE_HARD: /* Get the address of the object the link points to */ + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") H5F_addr_decode(f, &p, &(lnk->u.hard.addr)); break; case H5L_TYPE_SOFT: /* Get the link value */ + if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT16DECODE(p, len) if (len == 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "invalid link length") - /* Make sure that length doesn't exceed buffer size, which could occur - when the file is corrupted */ - if (p + len > p_end) - HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "name length causes read past end of buffer") - + if (H5_IS_BUFFER_OVERFLOW(p, len, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") if (NULL == (lnk->u.soft.name = (char *)H5MM_malloc((size_t)len + 1))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") H5MM_memcpy(lnk->u.soft.name, p, len); @@ -238,16 +245,15 @@ H5O__link_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSE HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "unknown link type") /* A UD link. Get the user-supplied data */ + if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT16DECODE(p, len) if (lnk->type == H5L_TYPE_EXTERNAL && len < 3) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "external link information length < 3") lnk->u.ud.size = len; if (len > 0) { - /* Make sure that length doesn't exceed buffer size, which could - occur when the file is corrupted */ - if (p + len > p_end) - HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "name length causes read past end of buffer") - + if (H5_IS_BUFFER_OVERFLOW(p, len, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") if (NULL == (lnk->u.ud.udata = H5MM_malloc((size_t)len))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") H5MM_memcpy(lnk->u.ud.udata, p, len); @@ -255,22 +261,20 @@ H5O__link_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSE } else lnk->u.ud.udata = NULL; - } /* end switch */ + } /* Set return value */ ret_value = lnk; done: - if (ret_value == NULL) - if (lnk != NULL) { - if (lnk->name != NULL) - H5MM_xfree(lnk->name); - if (lnk->type == H5L_TYPE_SOFT && lnk->u.soft.name != NULL) - H5MM_xfree(lnk->u.soft.name); - if (lnk->type >= H5L_TYPE_UD_MIN && lnk->u.ud.size > 0 && lnk->u.ud.udata != NULL) - H5MM_xfree(lnk->u.ud.udata); - lnk = H5FL_FREE(H5O_link_t, lnk); - } /* end if */ + if (!ret_value && lnk) { + H5MM_xfree(lnk->name); + if (lnk->type == H5L_TYPE_SOFT && lnk->u.soft.name != NULL) + H5MM_xfree(lnk->u.soft.name); + if (lnk->type >= H5L_TYPE_UD_MIN && lnk->u.ud.size > 0 && lnk->u.ud.udata != NULL) + H5MM_xfree(lnk->u.ud.udata); + H5FL_FREE(H5O_link_t, lnk); + } FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__link_decode() */ diff --git a/src/H5Opline.c b/src/H5Opline.c index 58c729f..bc66165 100644 --- a/src/H5Opline.c +++ b/src/H5Opline.c @@ -11,10 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke - * Wednesday, April 15, 1998 - * - * Purpose: Data filter pipeline message. + * Purpose: Data filter pipeline message */ #include "H5Omodule.h" /* This source code file is part of the H5O module */ @@ -102,12 +99,8 @@ H5FL_DEFINE(H5O_pline_t); * * Purpose: Decodes a filter pipeline message. * - * Return: Success: Ptr to the native message. + * Return: Success: Pointer to a new pipeline message * Failure: NULL - * - * Programmer: Robb Matzke - * Wednesday, April 15, 1998 - * *------------------------------------------------------------------------- */ @@ -120,11 +113,11 @@ H5O__pline_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsign size_t name_length; /* Length of filter name */ size_t i; /* Local index variable */ const uint8_t *p_end = p + p_size - 1; /* End of the p buffer */ - void *ret_value = NULL; /* Return value */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* check args */ + HDassert(f); HDassert(p); /* Allocate space for I/O pipeline message */ @@ -132,14 +125,15 @@ H5O__pline_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsign HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* Version */ - if (p + 4 - 1 > p_end) /* 4 byte is minimum for all versions */ - HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "ran off the end of the buffer: current p = %p, p_end = %p", - (const void *)(p + 4), (const void *)p_end) + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") pline->version = *p++; if (pline->version < H5O_PLINE_VERSION_1 || pline->version > H5O_PLINE_VERSION_LATEST) HGOTO_ERROR(H5E_PLINE, H5E_CANTLOAD, NULL, "bad version number for filter pipeline message") /* Number of filters */ + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") pline->nused = *p++; if (pline->nused > H5Z_MAX_NFILTERS) { @@ -152,8 +146,11 @@ H5O__pline_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsign } /* Reserved */ - if (pline->version == H5O_PLINE_VERSION_1) + if (pline->version == H5O_PLINE_VERSION_1) { + if (H5_IS_BUFFER_OVERFLOW(p, 6, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") p += 6; + } /* Allocate array for filters */ pline->nalloc = pline->nused; @@ -163,94 +160,95 @@ H5O__pline_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, unsign /* Decode filters */ for (i = 0, filter = &pline->filter[0]; i < pline->nused; i++, filter++) { /* Filter ID */ - if (p + 6 - 1 > p_end) /* 6 bytes minimum */ - HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, - "ran off the end of the buffer: current p = %p, p_end = %p", (const void *)(p + 6), - (const void *)p_end) + if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT16DECODE(p, filter->id); /* Length of filter name */ if (pline->version > H5O_PLINE_VERSION_1 && filter->id < H5Z_FILTER_RESERVED) name_length = 0; else { + if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT16DECODE(p, name_length); if (pline->version == H5O_PLINE_VERSION_1 && name_length % 8) HGOTO_ERROR(H5E_PLINE, H5E_CANTLOAD, NULL, "filter name length is not a multiple of eight") - if (p + 4 - 1 > p_end) /* with name_length 4 bytes to go */ - HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, - "ran off the end of the buffer: current p = %p, p_end = %p", - (const void *)(p + 4), (const void *)p_end) - } /* end if */ + } /* Filter flags */ + if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT16DECODE(p, filter->flags); /* Number of filter parameters ("client data elements") */ + if (H5_IS_BUFFER_OVERFLOW(p, 2, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT16DECODE(p, filter->cd_nelmts); /* Filter name, if there is one */ if (name_length) { - size_t actual_name_length; /* Actual length of name */ - size_t len = (size_t)(p_end - p + 1); + size_t actual_name_length; /* Actual length of name */ + size_t max = (size_t)(p_end - p + 1); /* Max possible name length */ + /* Determine actual name length (without padding, but with null terminator) */ - actual_name_length = HDstrnlen((const char *)p, len); - if (actual_name_length == len) + actual_name_length = HDstrnlen((const char *)p, max); + if (actual_name_length == max) HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "filter name not null terminated") actual_name_length += 1; /* include \0 byte */ - HDassert(actual_name_length <= name_length); /* Allocate space for the filter name, or use the internal buffer */ if (actual_name_length > H5Z_COMMON_NAME_LEN) { filter->name = (char *)H5MM_malloc(actual_name_length); if (NULL == filter->name) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for filter name") - } /* end if */ + } else filter->name = filter->_name; HDstrncpy(filter->name, (const char *)p, actual_name_length); + + if (H5_IS_BUFFER_OVERFLOW(p, name_length, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") p += name_length; - } /* end if */ + } /* Filter parameters */ if (filter->cd_nelmts) { - size_t j; /* Local index variable */ /* Allocate space for the client data elements, or use the internal buffer */ if (filter->cd_nelmts > H5Z_COMMON_CD_VALUES) { filter->cd_values = (unsigned *)H5MM_malloc(filter->cd_nelmts * sizeof(unsigned)); if (NULL == filter->cd_values) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for client data") - } /* end if */ + } else filter->cd_values = filter->_cd_values; - /* - * Read the client data values and the padding - */ - for (j = 0; j < filter->cd_nelmts; j++) { - if (p + 4 - 1 <= p_end) - UINT32DECODE(p, filter->cd_values[j]) - else - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, - "ran off the end of the buffer: current p = %p, p_size = %zu, p_end = %p", - (const void *)p, p_size, (const void *)p_end) + /* Read the client data values and the padding */ + for (size_t j = 0; j < filter->cd_nelmts; j++) { + if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") + UINT32DECODE(p, filter->cd_values[j]) } if (pline->version == H5O_PLINE_VERSION_1) - if (filter->cd_nelmts % 2) - p += 4; /*padding*/ - } /* end if */ - } /* end for */ + if (filter->cd_nelmts % 2) { + if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, + "ran off end of input buffer while decoding") + p += 4; /* padding */ + } + } + } /* Set return value */ ret_value = pline; done: - if (NULL == ret_value && pline) { + if (!ret_value && pline) { H5O__pline_reset(pline); H5O__pline_free(pline); - } /* end if */ + } FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__pline_decode() */ diff --git a/src/H5Orefcount.c b/src/H5Orefcount.c index 51da22c..f4d3b5c 100644 --- a/src/H5Orefcount.c +++ b/src/H5Orefcount.c @@ -13,10 +13,8 @@ /*------------------------------------------------------------------------- * * Created: H5Orefcount.c - * Mar 10 2007 - * Quincey Koziol * - * Purpose: Object ref. count messages. + * Purpose: Object reference count messages * *------------------------------------------------------------------------- */ @@ -72,31 +70,30 @@ H5FL_DEFINE_STATIC(H5O_refcount_t); /*------------------------------------------------------------------------- * Function: H5O__refcount_decode * - * Purpose: Decode a message and return a pointer to a newly allocated one. - * - * Return: Success: Ptr to new message in native form. - * Failure: NULL - * - * Programmer: Quincey Koziol - * Mar 10 2007 + * Purpose: Decode a message and return a pointer to a newly allocated + * one. * + * Return: Success: Pointer to new message in native form + * Failure: NULL *------------------------------------------------------------------------- */ static void * H5O__refcount_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, - unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, - size_t H5_ATTR_UNUSED p_size, const uint8_t *p) + unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, + const uint8_t *p) { - H5O_refcount_t *refcount = NULL; /* Reference count */ - void *ret_value = NULL; /* Return value */ + H5O_refcount_t *refcount = NULL; /* Reference count */ + const uint8_t *p_end = p + p_size - 1; /* End of the p buffer */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* check args */ HDassert(f); HDassert(p); /* Version of message */ + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") if (*p++ != H5O_REFCOUNT_VERSION) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad version number for message") @@ -104,15 +101,17 @@ H5O__refcount_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, if (NULL == (refcount = H5FL_MALLOC(H5O_refcount_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Get ref. count for object */ + /* Get reference count for object */ + if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") UINT32DECODE(p, *refcount) /* Set return value */ ret_value = refcount; done: - if (ret_value == NULL && refcount != NULL) - refcount = H5FL_FREE(H5O_refcount_t, refcount); + if (!ret_value && refcount) + H5FL_FREE(H5O_refcount_t, refcount); FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__refcount_decode() */ diff --git a/src/H5Osdspace.c b/src/H5Osdspace.c index e9a0dc6..9bf5d6f 100644 --- a/src/H5Osdspace.c +++ b/src/H5Osdspace.c @@ -115,11 +115,9 @@ H5O__sdspace_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UN FUNC_ENTER_PACKAGE - /* check args */ HDassert(f); HDassert(p); - /* decode */ if (NULL == (sdim = H5FL_CALLOC(H5S_extent_t))) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, NULL, "dataspace structure allocation failed") sdim->type = H5S_NO_CLASS; @@ -154,9 +152,11 @@ H5O__sdspace_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UN if (sdim->type != H5S_SIMPLE && sdim->rank > 0) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "invalid rank for scalar or NULL dataspace") - } /* end if */ + } else { - /* Set the dataspace type to be simple or scalar as appropriate */ + /* Set the dataspace type to be simple or scalar as appropriate + * (version 1 does not allow H5S_NULL) + */ if (sdim->rank > 0) sdim->type = H5S_SIMPLE; else @@ -166,50 +166,46 @@ H5O__sdspace_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UN if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") p++; - } /* end else */ - HDassert(sdim->type != H5S_NULL || sdim->version >= H5O_SDSPACE_VERSION_2); + } - /* Only Version 1 has these reserved bytes */ + /* Version 1 has 4 reserved bytes */ if (version == H5O_SDSPACE_VERSION_1) { if (H5_IS_BUFFER_OVERFLOW(p, 4, p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") - p += 4; /*reserved*/ + p += 4; } /* Decode dimension sizes */ if (sdim->rank > 0) { - uint8_t sizeof_size = H5F_SIZEOF_SIZE(f); - /* - * Ensure that decoding doesn't cause reading past buffer's end, - * due to possible data corruption - check that we have space to - * decode a "sdim->rank" number of hsize_t values - */ - if (H5_IS_BUFFER_OVERFLOW(p, (sizeof_size * sdim->rank), p_end)) + /* Sizes */ + + /* Check that we have space to decode sdim->rank values */ + if (H5_IS_BUFFER_OVERFLOW(p, (H5F_sizeof_size(f) * sdim->rank), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") if (NULL == (sdim->size = (hsize_t *)H5FL_ARR_MALLOC(hsize_t, (size_t)sdim->rank))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "memory allocation failed") - for (i = 0; i < sdim->rank; i++) H5F_DECODE_LENGTH(f, p, sdim->size[i]); + /* Max sizes */ + if (flags & H5S_VALID_MAX) { if (NULL == (sdim->max = (hsize_t *)H5FL_ARR_MALLOC(hsize_t, (size_t)sdim->rank))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, NULL, "memory allocation failed") - /* - * Ensure that decoding doesn't cause reading past buffer's end, - * due to possible data corruption - check that we have space to - * decode a "sdim->rank" number of hsize_t values - */ - if (H5_IS_BUFFER_OVERFLOW(p, (sizeof_size * sdim->rank), p_end)) + /* Check that we have space to decode sdim->rank values */ + if (H5_IS_BUFFER_OVERFLOW(p, (H5F_sizeof_size(f) * sdim->rank), p_end)) HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding") - for (i = 0; i < sdim->rank; i++) H5F_DECODE_LENGTH(f, p, sdim->max[i]); - } /* end if */ - } /* end if */ + } + + /* NOTE: The version 1 permutation indexes were never implemented so + * there is nothing to decode. + */ + } /* Compute the number of elements in the extent */ if (sdim->type == H5S_NULL) @@ -217,16 +213,16 @@ H5O__sdspace_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UN else { for (i = 0, sdim->nelem = 1; i < sdim->rank; i++) sdim->nelem *= sdim->size[i]; - } /* end else */ + } /* Set return value */ - ret_value = (void *)sdim; /*success*/ + ret_value = (void *)sdim; done: if (!ret_value && sdim) { H5S__extent_release(sdim); - sdim = H5FL_FREE(H5S_extent_t, sdim); - } /* end if */ + H5FL_FREE(H5S_extent_t, sdim); + } FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__sdspace_decode() */ diff --git a/src/H5Oshmesg.c b/src/H5Oshmesg.c index 586e2ce..8510c6e 100644 --- a/src/H5Oshmesg.c +++ b/src/H5Oshmesg.c @@ -56,29 +56,25 @@ const H5O_msg_class_t H5O_MSG_SHMESG[1] = {{ }}; /*------------------------------------------------------------------------- - * Function: H5O__shmesg_decode + * Function: H5O__shmesg_decode * - * Purpose: Decode a shared message table message and return a pointer + * Purpose: Decode a shared message table message and return a pointer * to a newly allocated H5O_shmesg_table_t struct. * - * Return: Success: Ptr to new message in native struct. - * Failure: NULL - * - * Programmer: James Laird - * Jan 29, 2007 - * + * Return: Success: Ptr to new message in native struct. + * Failure: NULL *------------------------------------------------------------------------- */ static void * H5O__shmesg_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, - unsigned H5_ATTR_UNUSED *ioflags, size_t H5_ATTR_UNUSED p_size, const uint8_t *p) + unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, const uint8_t *p) { - H5O_shmesg_table_t *mesg; /* Native message */ - void *ret_value = NULL; /* Return value */ + H5O_shmesg_table_t *mesg; /* New shared message table */ + const uint8_t *p_end = p + p_size - 1; /* End of the p buffer */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* Sanity check */ HDassert(f); HDassert(p); @@ -87,14 +83,25 @@ H5O__shmesg_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU "memory allocation failed for shared message table message") /* Retrieve version, table address, and number of indexes */ + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); mesg->version = *p++; + + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5F_addr_decode(f, &p, &(mesg->addr)); + + if (H5_IS_BUFFER_OVERFLOW(p, 1, p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); mesg->nindexes = *p++; /* Set return value */ ret_value = (void *)mesg; done: + if (!ret_value && mesg) + H5MM_xfree(mesg); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__shmesg_decode() */ diff --git a/src/H5Ostab.c b/src/H5Ostab.c index ae4635e..2428f06 100644 --- a/src/H5Ostab.c +++ b/src/H5Ostab.c @@ -13,10 +13,8 @@ /*------------------------------------------------------------------------- * * Created: H5Ostab.c - * Aug 6 1997 - * Robb Matzke * - * Purpose: Symbol table messages. + * Purpose: Symbol table messages * *------------------------------------------------------------------------- */ @@ -78,41 +76,39 @@ H5FL_DEFINE_STATIC(H5O_stab_t); * Purpose: Decode a symbol table message and return a pointer to * a newly allocated one. * - * Return: Success: Ptr to new message in native order. - * - * Failure: NULL - * - * Programmer: Robb Matzke - * Aug 6 1997 - * + * Return: Success: Pointer to new message in native order + * Failure: NULL *------------------------------------------------------------------------- */ static void * H5O__stab_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, - unsigned H5_ATTR_UNUSED *ioflags, size_t H5_ATTR_UNUSED p_size, const uint8_t *p) + unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, const uint8_t *p) { - H5O_stab_t *stab = NULL; - void *ret_value = NULL; /* Return value */ + H5O_stab_t *stab = NULL; + const uint8_t *p_end = p + p_size - 1; /* End of the p buffer */ + void *ret_value = NULL; FUNC_ENTER_PACKAGE - /* check args */ HDassert(f); HDassert(p); - /* decode */ if (NULL == (stab = H5FL_CALLOC(H5O_stab_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5F_addr_decode(f, &p, &(stab->btree_addr)); + + if (H5_IS_BUFFER_OVERFLOW(p, H5F_sizeof_addr(f), p_end)) + HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding"); H5F_addr_decode(f, &p, &(stab->heap_addr)); - /* Set return value */ ret_value = stab; done: - if (ret_value == NULL) - if (stab != NULL) - stab = H5FL_FREE(H5O_stab_t, stab); + if (!ret_value && stab) + H5FL_FREE(H5O_stab_t, stab); FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__stab_decode() */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 8e9b680..84d2522 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -6147,6 +6147,9 @@ H5Pget_vol_id(hid_t plist_id, hid_t *vol_id /*out*/) FUNC_ENTER_API(FAIL) H5TRACE2("e", "ix", plist_id, vol_id); + if (H5P_DEFAULT == plist_id) + plist_id = H5P_FILE_ACCESS_DEFAULT; + /* Get property list for ID */ if (NULL == (plist = (H5P_genplist_t *)H5I_object_verify(plist_id, H5I_GENPROP_LST))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") @@ -6263,6 +6266,9 @@ H5Pget_vol_cap_flags(hid_t plist_id, uint64_t *cap_flags) /* Get the 'cap_flags' from the connector */ if (cap_flags) { + if (H5P_DEFAULT == plist_id) + plist_id = H5P_FILE_ACCESS_DEFAULT; + if (TRUE == H5P_isa_class(plist_id, H5P_FILE_ACCESS)) { H5P_genplist_t *plist; /* Property list pointer */ H5VL_connector_prop_t connector_prop; /* Property for VOL connector ID & info */ diff --git a/test/dtypes.c b/test/dtypes.c index 37fc8c7..bd55491 100644 --- a/test/dtypes.c +++ b/test/dtypes.c @@ -5965,34 +5965,37 @@ error: * * Purpose: Tests functions of encoding and decoding datatype. * - * Return: Success: 0 - * - * Failure: number of errors - * - * Programmer: Raymond Lu - * July 14, 2004 - * + * Return: Success: 0 + * Failure: number of errors *------------------------------------------------------------------------- */ static int test_encode(void) { - struct s1 { + struct cmpd { int a; float b; long c; double d; }; - hid_t file = -1, tid1 = -1, tid2 = -1, tid3 = -1; - hid_t decoded_tid1 = -1, decoded_tid2 = -1, decoded_tid3 = -1; + hid_t file = H5I_INVALID_HID; + hid_t tid1 = H5I_INVALID_HID; + hid_t tid2 = H5I_INVALID_HID; + hid_t tid3 = H5I_INVALID_HID; + hid_t decoded_tid1 = H5I_INVALID_HID; + hid_t decoded_tid2 = H5I_INVALID_HID; + hid_t decoded_tid3 = H5I_INVALID_HID; char filename[1024]; - char compnd_type[] = "Compound_type", enum_type[] = "Enum_type"; - char vlstr_type[] = "VLstring_type"; + char compnd_type[] = "Compound_type"; + char enum_type[] = "Enum_type"; + char vlstr_type[] = "VLstring_type"; short enum_val; size_t cmpd_buf_size = 0; size_t enum_buf_size = 0; size_t vlstr_buf_size = 0; - unsigned char *cmpd_buf = NULL, *enum_buf = NULL, *vlstr_buf = NULL; + unsigned char *cmpd_buf = NULL; + unsigned char *enum_buf = NULL; + unsigned char *vlstr_buf = NULL; hid_t ret_id; herr_t ret; @@ -6008,75 +6011,75 @@ test_encode(void) *----------------------------------------------------------------------- */ /* Create a compound datatype */ - if ((tid1 = H5Tcreate(H5T_COMPOUND, sizeof(struct s1))) < 0) { + if ((tid1 = H5Tcreate(H5T_COMPOUND, sizeof(struct cmpd))) < 0) { H5_FAILED(); HDprintf("Can't create datatype!\n"); goto error; - } /* end if */ - if (H5Tinsert(tid1, "a", HOFFSET(struct s1, a), H5T_NATIVE_INT) < 0) { + } + if (H5Tinsert(tid1, "a", HOFFSET(struct cmpd, a), H5T_NATIVE_INT) < 0) { H5_FAILED(); HDprintf("Can't insert field 'a'\n"); goto error; - } /* end if */ - if (H5Tinsert(tid1, "b", HOFFSET(struct s1, b), H5T_NATIVE_FLOAT) < 0) { + } + if (H5Tinsert(tid1, "b", HOFFSET(struct cmpd, b), H5T_NATIVE_FLOAT) < 0) { H5_FAILED(); HDprintf("Can't insert field 'b'\n"); goto error; - } /* end if */ - if (H5Tinsert(tid1, "c", HOFFSET(struct s1, c), H5T_NATIVE_LONG) < 0) { + } + if (H5Tinsert(tid1, "c", HOFFSET(struct cmpd, c), H5T_NATIVE_LONG) < 0) { H5_FAILED(); HDprintf("Can't insert field 'c'\n"); goto error; - } /* end if */ - if (H5Tinsert(tid1, "d", HOFFSET(struct s1, d), H5T_NATIVE_DOUBLE) < 0) { + } + if (H5Tinsert(tid1, "d", HOFFSET(struct cmpd, d), H5T_NATIVE_DOUBLE) < 0) { H5_FAILED(); HDprintf("Can't insert field 'd'\n"); goto error; - } /* end if */ + } /* Create a enumerate datatype */ if ((tid2 = H5Tcreate(H5T_ENUM, sizeof(short))) < 0) { H5_FAILED(); HDprintf("Can't create enumerate type\n"); goto error; - } /* end if */ + } if (H5Tenum_insert(tid2, "RED", (enum_val = 0, &enum_val)) < 0) { H5_FAILED(); HDprintf("Can't insert field into enumeration type\n"); goto error; - } /* end if */ + } if (H5Tenum_insert(tid2, "GREEN", (enum_val = 1, &enum_val)) < 0) { H5_FAILED(); HDprintf("Can't insert field into enumeration type\n"); goto error; - } /* end if */ + } if (H5Tenum_insert(tid2, "BLUE", (enum_val = 2, &enum_val)) < 0) { H5_FAILED(); HDprintf("Can't insert field into enumeration type\n"); goto error; - } /* end if */ + } if (H5Tenum_insert(tid2, "ORANGE", (enum_val = 3, &enum_val)) < 0) { H5_FAILED(); HDprintf("Can't insert field into enumeration type\n"); goto error; - } /* end if */ + } if (H5Tenum_insert(tid2, "YELLOW", (enum_val = 4, &enum_val)) < 0) { H5_FAILED(); HDprintf("Can't insert field into enumeration type\n"); goto error; - } /* end if */ + } /* Create a variable-length string type */ if ((tid3 = H5Tcopy(H5T_C_S1)) < 0) { H5_FAILED(); HDprintf("Can't copy a string type\n"); goto error; - } /* end if */ + } if (H5Tset_size(tid3, H5T_VARIABLE) < 0) { H5_FAILED(); HDprintf("Can't the string type to be variable-length\n"); goto error; - } /* end if */ + } /*----------------------------------------------------------------------- * Test encoding and decoding compound, enumerate, and VL string datatypes @@ -6087,12 +6090,12 @@ test_encode(void) H5_FAILED(); HDprintf("Can't encode compound type\n"); goto error; - } /* end if */ + } if (cmpd_buf_size > 0) cmpd_buf = (unsigned char *)HDcalloc((size_t)1, cmpd_buf_size); - /* Try decoding bogus buffer */ + /* Try decoding an incorrect (empty) buffer (should fail) */ H5E_BEGIN_TRY { ret_id = H5Tdecode(cmpd_buf); @@ -6100,7 +6103,7 @@ test_encode(void) H5E_END_TRY; if (ret_id != FAIL) { H5_FAILED(); - HDprintf("Decoded bogus buffer!\n"); + HDprintf("Decoded an empty buffer!\n"); goto error; } @@ -6108,7 +6111,7 @@ test_encode(void) H5_FAILED(); HDprintf("Can't encode compound type\n"); goto error; - } /* end if */ + } /* Decode from the compound buffer and return an object handle */ if ((decoded_tid1 = H5Tdecode(cmpd_buf)) < 0) @@ -6119,26 +6122,26 @@ test_encode(void) H5_FAILED(); HDprintf("Datatype wasn't encoded & decoded identically\n"); goto error; - } /* end if */ + } /* Query member number and member index by name, for compound type. */ if (H5Tget_nmembers(decoded_tid1) != 4) { H5_FAILED(); HDprintf("Can't get member number\n"); goto error; - } /* end if */ + } if (H5Tget_member_index(decoded_tid1, "c") != 2) { H5_FAILED(); HDprintf("Can't get correct index number\n"); goto error; - } /* end if */ + } /* Encode enumerate type in a buffer */ if (H5Tencode(tid2, NULL, &enum_buf_size) < 0) { H5_FAILED(); HDprintf("Can't encode enumerate type\n"); goto error; - } /* end if */ + } if (enum_buf_size > 0) enum_buf = (unsigned char *)HDcalloc((size_t)1, enum_buf_size); @@ -6147,40 +6150,40 @@ test_encode(void) H5_FAILED(); HDprintf("Can't encode enumerate type\n"); goto error; - } /* end if */ + } /* Decode from the enumerate buffer and return an object handle */ if ((decoded_tid2 = H5Tdecode(enum_buf)) < 0) { H5_FAILED(); HDprintf("Can't decode enumerate type\n"); goto error; - } /* end if */ + } /* Verify that the datatype was copied exactly */ if (H5Tequal(decoded_tid2, tid2) <= 0) { H5_FAILED(); HDprintf("Datatype wasn't encoded & decoded identically\n"); goto error; - } /* end if */ + } /* Query member number and member index by name, for enumeration type. */ if (H5Tget_nmembers(decoded_tid2) != 5) { H5_FAILED(); HDprintf("Can't get member number\n"); goto error; - } /* end if */ + } if (H5Tget_member_index(decoded_tid2, "ORANGE") != 3) { H5_FAILED(); HDprintf("Can't get correct index number\n"); goto error; - } /* end if */ + } /* Encode VL string type in a buffer */ if (H5Tencode(tid3, NULL, &vlstr_buf_size) < 0) { H5_FAILED(); HDprintf("Can't encode VL string type\n"); goto error; - } /* end if */ + } if (vlstr_buf_size > 0) vlstr_buf = (unsigned char *)HDcalloc((size_t)1, vlstr_buf_size); @@ -6189,26 +6192,26 @@ test_encode(void) H5_FAILED(); HDprintf("Can't encode VL string type\n"); goto error; - } /* end if */ + } /* Decode from the VL string buffer and return an object handle */ if ((decoded_tid3 = H5Tdecode(vlstr_buf)) < 0) { H5_FAILED(); HDprintf("Can't decode VL string type\n"); goto error; - } /* end if */ + } /* Verify that the datatype was copied exactly */ if (H5Tequal(decoded_tid3, tid3) <= 0) { H5_FAILED(); HDprintf("Datatype wasn't encoded & decoded identically\n"); goto error; - } /* end if */ + } if (!H5Tis_variable_str(decoded_tid3)) { H5_FAILED(); HDprintf("Datatype wasn't encoded & decoded identically\n"); goto error; - } /* end if */ + } /*----------------------------------------------------------------------- * Commit and reopen the compound, enumerate, VL string datatypes @@ -6219,17 +6222,17 @@ test_encode(void) H5_FAILED(); HDprintf("Can't commit compound datatype\n"); goto error; - } /* end if */ + } if (H5Tclose(tid1) < 0) { H5_FAILED(); HDprintf("Can't close datatype\n"); goto error; - } /* end if */ + } if (H5Tclose(decoded_tid1) < 0) { H5_FAILED(); HDprintf("Can't close datatype\n"); goto error; - } /* end if */ + } HDfree(cmpd_buf); cmpd_buf_size = 0; @@ -6238,17 +6241,17 @@ test_encode(void) H5_FAILED(); HDprintf("Can't commit compound datatype\n"); goto error; - } /* end if */ + } if (H5Tclose(tid2) < 0) { H5_FAILED(); HDprintf("Can't close datatype\n"); goto error; - } /* end if */ + } if (H5Tclose(decoded_tid2) < 0) { H5_FAILED(); HDprintf("Can't close datatype\n"); goto error; - } /* end if */ + } HDfree(enum_buf); enum_buf_size = 0; @@ -6257,17 +6260,17 @@ test_encode(void) H5_FAILED(); HDprintf("Can't commit vl string datatype\n"); goto error; - } /* end if */ + } if (H5Tclose(tid3) < 0) { H5_FAILED(); HDprintf("Can't close datatype\n"); goto error; - } /* end if */ + } if (H5Tclose(decoded_tid3) < 0) { H5_FAILED(); HDprintf("Can't close datatype\n"); goto error; - } /* end if */ + } HDfree(vlstr_buf); vlstr_buf_size = 0; @@ -6288,7 +6291,7 @@ test_encode(void) H5_FAILED(); HDprintf("Can't encode compound type\n"); goto error; - } /* end if */ + } if (cmpd_buf_size > 0) cmpd_buf = (unsigned char *)HDcalloc((size_t)1, cmpd_buf_size); @@ -6297,7 +6300,7 @@ test_encode(void) H5_FAILED(); HDprintf("Can't encode compound type\n"); goto error; - } /* end if */ + } /* Decode from the compound buffer and return an object handle */ if ((decoded_tid1 = H5Tdecode(cmpd_buf)) < 0) @@ -6308,26 +6311,26 @@ test_encode(void) H5_FAILED(); HDprintf("Datatype wasn't encoded & decoded identically\n"); goto error; - } /* end if */ + } /* Query member number and member index by name, for compound type. */ if (H5Tget_nmembers(decoded_tid1) != 4) { H5_FAILED(); HDprintf("Can't get member number\n"); goto error; - } /* end if */ + } if (H5Tget_member_index(decoded_tid1, "c") != 2) { H5_FAILED(); HDprintf("Can't get correct index number\n"); goto error; - } /* end if */ + } /* Encode enumerate type in a buffer */ if (H5Tencode(tid2, NULL, &enum_buf_size) < 0) { H5_FAILED(); HDprintf("Can't encode enumerate type\n"); goto error; - } /* end if */ + } if (enum_buf_size > 0) enum_buf = (unsigned char *)HDcalloc((size_t)1, enum_buf_size); @@ -6336,40 +6339,40 @@ test_encode(void) H5_FAILED(); HDprintf("Can't encode enumerate type\n"); goto error; - } /* end if */ + } /* Decode from the enumerate buffer and return an object handle */ if ((decoded_tid2 = H5Tdecode(enum_buf)) < 0) { H5_FAILED(); HDprintf("Can't decode enumerate type\n"); goto error; - } /* end if */ + } /* Verify that the datatype was copied exactly */ if (H5Tequal(decoded_tid2, tid2) <= 0) { H5_FAILED(); HDprintf("Datatype wasn't encoded & decoded identically\n"); goto error; - } /* end if */ + } /* Query member number and member index by name, for enumeration type. */ if (H5Tget_nmembers(decoded_tid2) != 5) { H5_FAILED(); HDprintf("Can't get member number\n"); goto error; - } /* end if */ + } if (H5Tget_member_index(decoded_tid2, "ORANGE") != 3) { H5_FAILED(); HDprintf("Can't get correct index number\n"); goto error; - } /* end if */ + } /* Encode VL string type in a buffer */ if (H5Tencode(tid3, NULL, &vlstr_buf_size) < 0) { H5_FAILED(); HDprintf("Can't encode VL string type\n"); goto error; - } /* end if */ + } if (vlstr_buf_size > 0) vlstr_buf = (unsigned char *)HDcalloc((size_t)1, vlstr_buf_size); @@ -6378,14 +6381,14 @@ test_encode(void) H5_FAILED(); HDprintf("Can't encode VL string type\n"); goto error; - } /* end if */ + } /* Decode from the VL string buffer and return an object handle */ if ((decoded_tid3 = H5Tdecode(vlstr_buf)) < 0) { H5_FAILED(); HDprintf("Can't decode VL string type\n"); goto error; - } /* end if */ + } HDfree(vlstr_buf); /* Verify that the datatype was copied exactly */ @@ -6393,12 +6396,12 @@ test_encode(void) H5_FAILED(); HDprintf("Datatype wasn't encoded & decoded identically\n"); goto error; - } /* end if */ + } if (!H5Tis_variable_str(decoded_tid3)) { H5_FAILED(); HDprintf("Datatype wasn't encoded & decoded identically\n"); goto error; - } /* end if */ + } /*----------------------------------------------------------------------- * Test the reference count of the decoded datatypes @@ -6410,19 +6413,19 @@ test_encode(void) H5_FAILED(); HDprintf("Decoded datatype has incorrect reference count\n"); goto error; - } /* end if */ + } if (H5Iget_ref(decoded_tid2) != 1) { H5_FAILED(); HDprintf("Decoded datatype has incorrect reference count\n"); goto error; - } /* end if */ + } if (H5Iget_ref(decoded_tid3) != 1) { H5_FAILED(); HDprintf("Decoded datatype has incorrect reference count\n"); goto error; - } /* end if */ + } /* Make sure the reference counts for the decoded datatypes can be * decremented and the datatypes are closed. */ @@ -6430,19 +6433,19 @@ test_encode(void) H5_FAILED(); HDprintf("Decoded datatype can't close\n"); goto error; - } /* end if */ + } if (H5Idec_ref(decoded_tid2) != 0) { H5_FAILED(); HDprintf("Decoded datatype can't close\n"); goto error; - } /* end if */ + } if (H5Idec_ref(decoded_tid3) != 0) { H5_FAILED(); HDprintf("Decoded datatype can't close\n"); goto error; - } /* end if */ + } /* Make sure the decoded datatypes are already closed. */ H5E_BEGIN_TRY @@ -6487,23 +6490,23 @@ test_encode(void) H5_FAILED(); HDprintf("Can't close datatype\n"); goto error; - } /* end if */ + } if (H5Tclose(tid2) < 0) { H5_FAILED(); HDprintf("Can't close datatype\n"); goto error; - } /* end if */ + } if (H5Tclose(tid3) < 0) { H5_FAILED(); HDprintf("Can't close datatype\n"); goto error; - } /* end if */ + } if (H5Fclose(file) < 0) { H5_FAILED(); HDprintf("Can't close file\n"); goto error; - } /* end if */ + } HDfree(cmpd_buf); HDfree(enum_buf); diff --git a/test/objcopy.c b/test/objcopy.c index 012c81d..eac99e0 100644 --- a/test/objcopy.c +++ b/test/objcopy.c @@ -16397,6 +16397,68 @@ error: return (H5_ITER_ERROR); } /* end test_copy_iterate_cb */ +/* + * Test for a bug with copying of v1 object headers where the + * new object header would end up with a gap in the header data, + * which v1 object header shouldn't have. + */ +static int +test_copy_cdt_v1_header_bug(hid_t fcpl_src, hid_t src_fapl) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t type_id = H5I_INVALID_HID; + hid_t ocpypl_id = H5I_INVALID_HID; + char src_filename[NAME_BUF_SIZE]; + + TESTING("H5Ocopy(): bug with copying v1 object headers"); + + /* Initialize the filenames */ + h5_fixname(FILENAME[0], src_fapl, src_filename, sizeof src_filename); + + if ((file_id = H5Fcreate(src_filename, H5F_ACC_TRUNC, fcpl_src, src_fapl)) < 0) + TEST_ERROR; + + if ((type_id = H5Tcreate(H5T_STRING, 385)) < 0) + TEST_ERROR; + if (H5Tset_strpad(type_id, H5T_STR_NULLPAD) < 0) + TEST_ERROR; + if (H5Tset_cset(type_id, H5T_CSET_ASCII) < 0) + TEST_ERROR; + + if (H5Tcommit2(file_id, "committed_str_type", type_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) + TEST_ERROR; + + if ((ocpypl_id = H5Pcreate(H5P_OBJECT_COPY)) < 0) + TEST_ERROR; + if (H5Pset_copy_object(ocpypl_id, H5O_COPY_WITHOUT_ATTR_FLAG) < 0) + TEST_ERROR; + + if (H5Ocopy(file_id, "committed_str_type", file_id, "committed_str_type2", ocpypl_id, H5P_DEFAULT) < 0) + TEST_ERROR; + + if (H5Tclose(type_id) < 0) + TEST_ERROR; + if (H5Pclose(ocpypl_id) < 0) + TEST_ERROR; + if (H5Fclose(file_id) < 0) + TEST_ERROR; + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY + { + H5Tclose(type_id); + H5Pclose(ocpypl_id); + H5Fclose(file_id); + } + H5E_END_TRY; + + return 1; +} + static int test_copy_iterate(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, hid_t dst_fapl) { @@ -17577,6 +17639,8 @@ main(void) nerrors += test_copy_null_ref(fcpl_src, fcpl_dst, src_fapl, dst_fapl); nerrors += test_copy_null_ref_open(fcpl_src, fcpl_dst, src_fapl, dst_fapl); + nerrors += test_copy_cdt_v1_header_bug(fcpl_src, src_fapl); + nerrors += test_copy_iterate(fcpl_src, fcpl_dst, src_fapl, dst_fapl); } /* end if */ @@ -2227,9 +2227,10 @@ test_vol_cap_flags(void) hid_t fapl_id = H5I_INVALID_HID; hid_t vol_id = H5I_INVALID_HID; uint64_t vol_cap_flags = H5VL_CAP_FLAG_NONE; + char *vol_env = NULL; H5VL_pass_through_info_t passthru_info; - TESTING("VOL capacity flags"); + TESTING("VOL capability flags"); /* Register a fake VOL */ if ((vol_id = H5VLregister_connector(&fake_vol_g, H5P_DEFAULT)) < 0) @@ -2251,6 +2252,29 @@ test_vol_cap_flags(void) if (vol_cap_flags & H5VL_CAP_FLAG_ATTR_BASIC) TEST_ERROR; + /* If using the native VOL by default, check flags again with H5P_DEFAULT */ + vol_env = HDgetenv(HDF5_VOL_CONNECTOR); + if (!vol_env || (0 == HDstrcmp(vol_env, "native"))) { + H5VL_class_t *cls; + hid_t connector_id; + + if (H5Pget_vol_id(H5P_DEFAULT, &connector_id) < 0) + TEST_ERROR; + if (NULL == (cls = H5I_object(connector_id))) + TEST_ERROR; + + vol_cap_flags = H5VL_CAP_FLAG_NONE; + + if (H5Pget_vol_cap_flags(H5P_DEFAULT, &vol_cap_flags) < 0) + TEST_ERROR; + + if (vol_cap_flags != cls->cap_flags) + TEST_ERROR; + + if (H5VLclose(connector_id) < 0) + TEST_ERROR; + } + /* Stack the [internal] passthrough VOL connector on top of the fake connector */ passthru_info.under_vol_id = vol_id; passthru_info.under_vol_info = NULL; diff --git a/testpar/t_subfiling_vfd.c b/testpar/t_subfiling_vfd.c index 0c2bca7..85df3bd 100644 --- a/testpar/t_subfiling_vfd.c +++ b/testpar/t_subfiling_vfd.c @@ -1858,7 +1858,7 @@ test_subfiling_h5fuse(void) args[0] = HDstrdup("env"); args[1] = HDstrdup("sh"); args[2] = HDstrdup("h5fuse.sh"); - args[3] = HDstrdup("-f"); + args[3] = HDstrdup("-q -f"); args[4] = tmp_filename; args[5] = NULL; diff --git a/utils/subfiling_vfd/h5fuse.sh.in b/utils/subfiling_vfd/h5fuse.sh.in index 48e3e61..2085033 100755 --- a/utils/subfiling_vfd/h5fuse.sh.in +++ b/utils/subfiling_vfd/h5fuse.sh.in @@ -14,6 +14,7 @@ BLD='\033[1m' GRN='\033[0;32m' RED='\033[0;31m' PUR='\033[0;35m' +CYN='\033[0;36m' NC='\033[0m' # No Color ############################################################ @@ -23,15 +24,40 @@ function usage() { echo "" # Display usage echo "Purpose: Combine subfiles into a single HDF5 file. Requires the subfiling - configuration file either as a command-line argument, or the script will + configuration file either as a command-line argument or the script will search for the *.config file in the current directory." echo "" - echo "usage: h5fuse.sh [-h] [-f filename]" - echo "-h Print this help." - echo "-f filename Subfile configuration file." + echo "usage: h5fuse.sh [-f filename] [-h] [-p] [-q] [-r] [-v] " + echo "-f filename Subfile configuration file." + echo "-h Print this help." + echo "-q Quiet all output. [no]" + echo "-p h5fuse.sh is being run in parallel, with more than one rank. [no]" + echo "-r Remove subfiles after being processed. [no]" + echo "-v Verbose output. [no]" echo "" } +function gen_mpi() { + +# Program to determine MPI rank and size if being run in parallel (-p). + +cat > "${c_src}" << EOL +#include <mpi.h> +#include <stdio.h> +int main() { + MPI_Init(NULL, NULL); + int world_size; + MPI_Comm_size(MPI_COMM_WORLD, &world_size); + int world_rank; + MPI_Comm_rank(MPI_COMM_WORLD, &world_rank); + printf("%d %d", world_rank, world_size); + MPI_Barrier(MPI_COMM_WORLD); + MPI_Finalize(); +} +EOL + +} + ############################################################ ############################################################ # Main program # @@ -43,14 +69,25 @@ function usage() { ############################################################ # Get the options file_config="" - -while getopts ":h:f:" option; do +verbose="false" +quiet="false" +rm_subf="false" +parallel="false" +while getopts "hpqrvf:" option; do case $option in + f) # subfiling configuration file + file_config=$OPTARG;; h) # display Help usage exit;; - f) # subfiling configuration file - file_config=$OPTARG;; + p) # HDF5 fused file + parallel="true";; + q) # quiet all output + quiet="true";; + r) # remove completed subfiles + rm_subf="true";; + v) # verbose output + verbose="true";; \?) # Invalid option echo -e "$RED ERROR: Invalid option ${BLD}-${OPTARG}${RED} $NC" usage @@ -61,24 +98,27 @@ while getopts ":h:f:" option; do done FAILED=1 -nfiles=1 ############################################################ # Configure file checks # ############################################################ +# +SUBF_CONFDIR="${H5FD_SUBFILING_CONFIG_FILE_PREFIX:-$PWD}" + +# Try to find the config file if [ -z "$file_config" ]; then - nfiles=$(find . -maxdepth 1 -type f -iname "*.config" -printf '.' | wc -m) + nfiles=$(find "$SUBF_CONFDIR" -maxdepth 1 -type f -iname "*.config" -printf '.' | wc -m) if [[ "$nfiles" != "1" ]]; then if [[ "$nfiles" == "0" ]]; then - echo -e "$RED Failed to find .config file in current directory. $NC" + echo -e "$RED Failed to find .config file in ${SUBF_CONFDIR} $NC" usage exit $FAILED else - echo -e "$RED More than one .config file found in current directory. $NC" + echo -e "$RED More than one .config file found in ${SUBF_CONFDIR} $NC" usage exit $FAILED fi fi - file_config=$(find . -maxdepth 1 -type f -iname "*.config") + file_config=$(find "${SUBF_CONFDIR}" -maxdepth 1 -type f -iname '*.config') fi if [ ! -f "$file_config" ]; then @@ -104,62 +144,126 @@ if test -z "$subfile_dir"; then exit $FAILED fi -subfiles=( $( sed -e '1,/subfile_dir=/d' "$file_config" ) ) -#for i in "${subfiles[@]}"; do -# echo "$i" -#done +# For bash 4.4+ +subfs=$(sed -e '1,/subfile_dir=/d' "$file_config") +mapfile -t subfiles <<< "$subfs" if [ ${#subfiles[@]} -eq 0 ]; then echo -e "$RED failed to find subfiles list in $file_config $NC" exit $FAILED fi +nsubfiles=${#subfiles[@]} + +# Get the number of local subfiles +subfiles_loc=() +subfiles_size=() +for i in "${subfiles[@]}"; do + subfile="${subfile_dir}/${i}" + if [ -f "${subfile}" ]; then + subfiles_loc+=("$subfile") + subfiles_size+=($(wc -c "${subfile}" | awk '{print $1}')) + else + subfiles_size+=(0) + fi +done + +START="$(date +%s%N)" -rm -f "$hdf5_file" +mpi_rank=0 +mpi_size=1 +nstart=1 +nend=$nsubfiles -## COMBINE SUBFILES INTO AN HDF5 FILE ## +if [ "$parallel" == "true" ]; then + hex=$(hexdump -n 16 -v -e '/1 "%02X"' /dev/urandom) + c_exec="h5fuse_"${hex} + c_src=${c_exec}.c + + # Generate and compile an MPI program to get MPI rank and size + if [ ! -f "${c_src}" ]; then + gen_mpi + CC=@CC@ + ${CC} "${c_src}" -o "${c_exec}" + fi + wait + rank_size=$(./"${c_exec}") + read -r mpi_rank mpi_size <<<"$rank_size" + + rm -f "${c_src}" "${c_exec}" + + # Divide the subfiles among the ranks + iwork1=$(( nsubfiles / mpi_size )) + iwork2=$(( nsubfiles % mpi_size )) + min=$(( mpi_rank < iwork2 ? mpi_rank : iwork2 )) + nstart=$(( mpi_rank * iwork1 + 1 + min )) + nend=$(( nstart + iwork1 - 1 )) + if [ $iwork2 -gt "$mpi_rank" ]; then + nend=$(( nend + 1 )) + fi +fi + +############################################################ +# COMBINE SUBFILES INTO AN HDF5 FILE # +############################################################ +icnt=1 skip=0 -status=$nfiles -START="$(date +%s%N)" -while [ "$status" -gt 0 ]; do - icnt=0 - for i in "${subfiles[@]}"; do - subfile="${subfile_dir}/${i}" - # Verify the file exists - if [ ! -f "${subfile}" ]; then - echo -e "$RED ERROR: file \"${subfile}\" does not exist. $NC" - exit $FAILED - fi +seek=0 +seek_cnt=0 +for i in "${subfiles[@]}"; do - # Verify the file is not being accessed by a process - t_max=60 - t_sleep=1 - t_elapsed=0 + subfile="${subfile_dir}/${i}" - while fuser -s "${subfile}"; do - if [[ $((t_elapsed % 5)) -eq 0 ]]; then - echo -e "$GRN waiting for process to finish accessing file \"${subfile}\" ... [${t_elapsed}s/${t_max}s] $NC" - fi - sleep $t_sleep - t_elapsed=$((t_elapsed+t_sleep)) - if [[ $t_elapsed -ge $t_max ]]; then - echo -e "$RED ERROR: file \"${subfile}\" still has process accessing it after ${t_elapsed}s $NC" - exit $FAILED + # bs=BYTES read and write up to BYTES bytes at a time; overrides ibs and obs + # ibs=BYTES read up to BYTES bytes at a time + # obs=BYTES write BYTES bytes at a time + # seek=N skip N obs-sized blocks at start of output + # skip=N skip N ibs-sized blocks at start of input + + status=1 + fsize=${subfiles_size[icnt-1]} + if [ "$fsize" -eq "0" ]; then + seek_cnt=$((seek_cnt+1)) + seek=$seek_cnt + if [ "$rm_subf" == "true" ]; then + if [ -f "${subfile}" ]; then + \rm -f "$subfile" + fi + fi + else + if [ $icnt -ge "$nstart" ] && [ $icnt -le "$nend" ]; then + records_left=$fsize + while [ "$status" -gt 0 ]; do + if [ $((skip*stripe_size)) -le "$fsize" ] && [ "$records_left" -gt 0 ]; then + EXEC="dd count=1 bs=$stripe_size if=$subfile of=$hdf5_file skip=$skip seek=$seek conv=notrunc" + if [ "$verbose" == "true" ]; then + echo -e "$GRN $EXEC $NC" + fi + err=$( $EXEC 2>&1 1>/dev/null ) + if [ $? -ne 0 ]; then + echo -e "$CYN ERR: dd Utility Failed $NC" + echo -e "$CYN MSG: $err $NC" + exit $FAILED + fi + records_left=$((records_left-stripe_size)) + skip=$((skip+1)) + seek=$((seek_cnt+skip*nsubfiles)) + else + status=0 + skip=0 + fi + done; wait + if [ "$rm_subf" == "true" ]; then + \rm -f "$subfile" fi - done - - fsize=$(wc -c "${subfile}" | awk '{print $1}') - if [ $((skip*stripe_size)) -le "$fsize" ]; then - EXEC="dd count=1 bs=$stripe_size if=$subfile of=$hdf5_file skip=$skip oflag=append conv=notrunc" - echo -e "$GRN $EXEC $NC" - err="$( $EXEC 2>&1 > /dev/null &)" - icnt=$((icnt+1)) - else - subfiles=("${subfiles[@]:0:icnt}" "${subfiles[@]:$((icnt+1))}") - status=${#subfiles[@]} - fi - done; wait - skip=$((skip+1)) -done + fi + seek_cnt=$((seek_cnt+1)) + seek=$seek_cnt + fi + icnt=$(( icnt +1 )) +done; wait + END=$(( $(date +%s%N) - START )) DURATION_SEC=$(awk -vp="$END" -vq=0.000000001 'BEGIN{printf "%.4f" ,p * q}') -echo -e "$PUR COMPLETION TIME = $DURATION_SEC s $NC" +if [ "$quiet" == "false" ]; then + echo -e "$PUR COMPLETION TIME = $DURATION_SEC s $NC" +fi
\ No newline at end of file |