From 65b1670861c62228cc68f98d4356edd180cfcb1f Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Mon, 6 Dec 2021 10:39:22 -0600 Subject: Merge branch 'selection_io' into selection_io_with_subfiling_vfd (#1265) --- .github/workflows/main.yml | 8 +- .github/workflows/pr-check.yml | 8 +- CMakeLists.txt | 25 + MANIFEST | 30 +- Makefile.am | 4 +- README.txt | 2 +- c++/src/cpp_doc_config | 2 +- config/cmake/FindCIRCLE.cmake | 49 + config/cmake/FindDTCMP.cmake | 48 + config/cmake/FindMFU.cmake | 100 ++ config/cmake/H5pubconf.h.in | 8 +- config/cmake/HDF5UseFortran.cmake | 8 +- config/cmake/scripts/HDF5config.cmake | 4 +- configure.ac | 155 +- doxygen/CMakeLists.txt | 1 + doxygen/Doxyfile.in | 3 +- doxygen/aliases | 16 +- doxygen/dox/About.dox | 120 +- doxygen/dox/FTS.dox | 8 + doxygen/dox/FileFormatSpec.dox | 23 - doxygen/dox/OtherSpecs.dox | 11 - doxygen/dox/RFC.dox | 11 + doxygen/dox/Specifications.dox | 38 +- doxygen/dox/TechnicalNotes.dox | 28 + doxygen/examples/DebuggingHDF5Applications.html | 392 +++++ doxygen/examples/FileFormat.html | 1275 +++++++++++++++ doxygen/examples/Filters.html | 450 +++++ doxygen/examples/IOFlow.html | 137 ++ doxygen/hdf5doxy_layout.xml | 1 + doxygen/img/IOFlow.gif | Bin 0 -> 57285 bytes doxygen/img/IOFlow2.gif | Bin 0 -> 29805 bytes doxygen/img/IOFlow3.gif | Bin 0 -> 21442 bytes java/src/hdf/hdf5lib/H5.java | 4 +- java/test/TestH5.java | 4 +- release_docs/RELEASE.txt | 1067 ++++++------ src/H5.c | 2 + src/H5CX.c | 12 +- src/H5Dmpio.c | 20 +- src/H5Dpkg.h | 12 +- src/H5FDmpio.c | 27 +- src/H5FDsec2.c | 1 - src/H5FDsplitter.c | 1 - src/H5Pocpl.c | 4 + src/H5Ppublic.h | 8 +- src/H5VLint.c | 1 - src/H5public.h | 6 +- test/filter_plugin.c | 99 ++ test/h5test.h | 59 +- test/vfd_plugin.c | 65 +- tools/lib/h5tools.c | 14 +- tools/test/h5dump/CMakeTests.cmake | 2 +- tools/test/h5dump/h5dumpgentest.c | 731 +++++---- tools/test/h5repack/h5repacktst.c | 132 +- .../h5repack_layout.h5-plugin_version_test.ddl | 14 +- tools/test/misc/h5clear_gentest.c | 26 +- tools/test/perform/chunk_cache.c | 18 +- utils/CMakeLists.txt | 5 + utils/Makefile.am | 8 +- utils/tools/CMakeLists.txt | 12 + utils/tools/Makefile.am | 38 + utils/tools/h5dwalk/CMakeLists.txt | 66 + utils/tools/h5dwalk/Makefile.am | 37 + utils/tools/h5dwalk/h5dwalk.1 | 42 + utils/tools/h5dwalk/h5dwalk.c | 1712 ++++++++++++++++++++ utils/tools/test/CMakeLists.txt | 8 + utils/tools/test/Makefile.am | 32 + utils/tools/test/h5dwalk/CMakeLists.txt | 15 + utils/tools/test/h5dwalk/CMakeTests.cmake | 56 + utils/tools/test/h5dwalk/Makefile.am | 43 + utils/tools/test/h5dwalk/copy_demo_files.sh.in | 86 + utils/tools/test/h5dwalk/help.h5dwalk | 13 + utils/tools/test/h5dwalk/testh5dwalk.sh.in | 249 +++ 72 files changed, 6614 insertions(+), 1102 deletions(-) create mode 100644 config/cmake/FindCIRCLE.cmake create mode 100644 config/cmake/FindDTCMP.cmake create mode 100644 config/cmake/FindMFU.cmake create mode 100644 doxygen/dox/FTS.dox delete mode 100644 doxygen/dox/FileFormatSpec.dox delete mode 100644 doxygen/dox/OtherSpecs.dox create mode 100644 doxygen/examples/DebuggingHDF5Applications.html create mode 100644 doxygen/examples/FileFormat.html create mode 100644 doxygen/examples/Filters.html create mode 100644 doxygen/examples/IOFlow.html create mode 100644 doxygen/img/IOFlow.gif create mode 100644 doxygen/img/IOFlow2.gif create mode 100644 doxygen/img/IOFlow3.gif create mode 100644 utils/tools/CMakeLists.txt create mode 100644 utils/tools/Makefile.am create mode 100644 utils/tools/h5dwalk/CMakeLists.txt create mode 100644 utils/tools/h5dwalk/Makefile.am create mode 100644 utils/tools/h5dwalk/h5dwalk.1 create mode 100644 utils/tools/h5dwalk/h5dwalk.c create mode 100644 utils/tools/test/CMakeLists.txt create mode 100644 utils/tools/test/Makefile.am create mode 100644 utils/tools/test/h5dwalk/CMakeLists.txt create mode 100644 utils/tools/test/h5dwalk/CMakeTests.cmake create mode 100644 utils/tools/test/h5dwalk/Makefile.am create mode 100644 utils/tools/test/h5dwalk/copy_demo_files.sh.in create mode 100644 utils/tools/test/h5dwalk/help.h5dwalk create mode 100644 utils/tools/test/h5dwalk/testh5dwalk.sh.in diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ae429a2..7b1d0c4 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,7 @@ jobs: include: - name: "Windows Latest MSVC" artifact: "Windows-MSVC.tar.xz" - os: windows-latest + os: windows-2022 build_type: "Release" toolchain: "" cpp: ON @@ -30,7 +30,7 @@ jobs: ts: OFF hl: ON parallel: OFF - generator: "-G \"Visual Studio 16 2019\" -A x64" + generator: "-G \"Visual Studio 17 2022\" -A x64" - name: "Ubuntu Latest GCC" artifact: "Linux.tar.xz" os: ubuntu-latest @@ -82,7 +82,7 @@ jobs: # Threadsafe runs - name: "Windows TS MSVC" artifact: "Windows-MSVCTS.tar.xz" - os: windows-2016 + os: windows-2019 build_type: "Release" toolchain: "" cpp: OFF @@ -91,7 +91,7 @@ jobs: ts: ON hl: OFF parallel: OFF - generator: "-G \"Visual Studio 15 2017 Win64\"" + generator: "-G \"Visual Studio 16 2019\" -A x64" - name: "Ubuntu TS GCC" artifact: "LinuxTS.tar.xz" os: ubuntu-latest diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 0acaf2d..79d5c83 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -16,7 +16,7 @@ jobs: include: - name: "Windows Latest MSVC" artifact: "Windows-MSVC.tar.xz" - os: windows-latest + os: windows-2022 build_type: "Release" toolchain: "" cpp: ON @@ -25,7 +25,7 @@ jobs: ts: OFF hl: ON parallel: OFF - generator: "-G \"Visual Studio 16 2019\" -A x64" + generator: "-G \"Visual Studio 17 2022\" -A x64" - name: "Ubuntu Latest GCC" artifact: "Linux.tar.xz" os: ubuntu-latest @@ -77,7 +77,7 @@ jobs: # Threadsafe runs - name: "Windows TS MSVC" artifact: "Windows-MSVCTS.tar.xz" - os: windows-2016 + os: windows-2019 build_type: "Release" toolchain: "" cpp: OFF @@ -86,7 +86,7 @@ jobs: ts: ON hl: OFF parallel: OFF - generator: "-G \"Visual Studio 15 2017 Win64\"" + generator: "-G \"Visual Studio 16 2019\" -A x64" - name: "Ubuntu TS GCC" artifact: "LinuxTS.tar.xz" os: ubuntu-latest diff --git a/CMakeLists.txt b/CMakeLists.txt index 250898c..19ea9f1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1017,6 +1017,31 @@ endif () # Option to build HDF5 Utilities #----------------------------------------------------------------------------- if (EXISTS "${HDF5_SOURCE_DIR}/utils" AND IS_DIRECTORY "${HDF5_SOURCE_DIR}/utils") + option (HDF5_BUILD_PARALLEL_TOOLS "Build Parallel HDF5 Tools" OFF) + if (HDF5_BUILD_PARALLEL_TOOLS AND HDF5_ENABLE_PARALLEL) + set (CMAKE_PREFIX_PATH "$HDF_RESOURCES_DIR") + find_package(MFU REQUIRED) + if (MFU_FOUND) + message(STATUS "LL_PATH=${LL_PATH}") + set (H5_HAVE_LIBMFU 1) + set (H5_HAVE_MFU_H 1) + set (CMAKE_REQUIRED_INCLUDES "${MFU_INCLUDE_DIR}") + set (MFU_LIBRARY_DEBUG "$MFU_LIBRARY") + set (MFU_LIBRARY_RELEASE "$MFU_LIBRARY") + endif () + find_package(CIRCLE REQUIRED) + if (CIRCLE_FOUND) + set (H5_HAVE_LIBCIRCLE 1) + set (H5_HAVE_CIRCLE_H 1) + set (CMAKE_REQUIRED_INCLUDES "${CIRCLE_INCLUDE_DIR}") + endif () + find_package(DTCMP REQUIRED) + if (DTCMP_FOUND) + set (H5_HAVE_LIBDTCMP 1) + set (H5_HAVE_DTCMP_H 1) + set (CMAKE_REQUIRED_INCLUDES "${DTCMP_INCLUDE_DIR}") + endif () + endif () add_subdirectory (utils) endif () diff --git a/MANIFEST b/MANIFEST index 03da027..34d937f 100644 --- a/MANIFEST +++ b/MANIFEST @@ -202,6 +202,10 @@ ./config/intel-warnings/win-general ./config/intel-warnings/ifort-general +./config/cmake/FindMFU.cmake +./config/cmake/FindDTCMP.cmake +./config/cmake/FindCIRCLE.cmake + ./config/site-specific/BlankForm ./doc/branches-explained.md @@ -216,11 +220,10 @@ ./doxygen/dox/Cookbook.dox ./doxygen/dox/DDLBNF110.dox ./doxygen/dox/DDLBNF112.dox -./doxygen/dox/FileFormatSpec.dox +./doxygen/dox/FTS.dox ./doxygen/dox/GettingStarted.dox ./doxygen/dox/H5AC_cache_config_t.dox ./doxygen/dox/MetadataCachingInHDF5.dox -./doxygen/dox/OtherSpecs.dox ./doxygen/dox/Overview.dox ./doxygen/dox/ReferenceManual.dox ./doxygen/dox/RFC.dox @@ -236,9 +239,12 @@ ./doxygen/dox/cookbook/Files.c ./doxygen/dox/cookbook/Files.dox ./doxygen/dox/cookbook/Performance.dox +./doxygen/examples/DebuggingHDF5Applications.html ./doxygen/examples/FF-IH_FileGroup.gif ./doxygen/examples/FF-IH_FileObject.gif +./doxygen/examples/FileFormat.html ./doxygen/examples/FileFormatSpecChunkDiagram.jpg +./doxygen/examples/Filters.html ./doxygen/examples/H5Pset_metadata_read_attempts.c ./doxygen/examples/H5Pset_object_flush_cb.c ./doxygen/examples/H5.format.1.0.html @@ -267,6 +273,7 @@ ./doxygen/examples/H5Z_examples.c ./doxygen/examples/H5_examples.c ./doxygen/examples/ImageSpec.html +./doxygen/examples/IOFlow.html ./doxygen/examples/PaletteExample1.gif ./doxygen/examples/Palettes.fm.anc.gif ./doxygen/examples/TableSpec.html @@ -282,6 +289,9 @@ ./doxygen/img/FF-IH_FileObject.gif ./doxygen/img/FileFormatSpecChunkDiagram.jpg ./doxygen/img/HDFG-logo.png +./doxygen/img/IOFlow.gif +./doxygen/img/IOFlow2.gif +./doxygen/img/IOFlow3.gif ./doxygen/img/PaletteExample1.gif ./doxygen/img/Palettes.fm.anc.gif ./doxygen/img/ftv2node.png @@ -3005,6 +3015,22 @@ ./utils/test/Makefile.am ./utils/test/swmr_check_compat_vfd.c +# parallel tools (h5dwalk) and tests +./utils/tools/CMakeLists.txt +./utils/tools/Makefile.am +./utils/tools/h5dwalk/CMakeLists.txt +./utils/tools/h5dwalk/Makefile.am +./utils/tools/h5dwalk/h5dwalk.1 +./utils/tools/h5dwalk/h5dwalk.c +./utils/tools/test/CMakeLists.txt +./utils/tools/test/Makefile.am +./utils/tools/test/h5dwalk/CMakeLists.txt +./utils/tools/test/h5dwalk/CMakeTests.cmake +./utils/tools/test/h5dwalk/Makefile.am +./utils/tools/test/h5dwalk/copy_demo_files.sh.in +./utils/tools/test/h5dwalk/help.h5dwalk +./utils/tools/test/h5dwalk/testh5dwalk.sh.in + # high level libraries ./hl/Makefile.am ./hl/examples/Makefile.am diff --git a/Makefile.am b/Makefile.am index 8518114..2a544f4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -88,9 +88,9 @@ else TOOLS_DIR= endif -SUBDIRS = src $(TESTSERIAL_DIR) $(TESTPARALLEL_DIR) bin utils $(TOOLS_DIR) . \ +SUBDIRS = src $(TESTSERIAL_DIR) $(TESTPARALLEL_DIR) bin $(TOOLS_DIR) utils . \ $(CXX_DIR) $(FORTRAN_DIR) $(JAVA_DIR) $(HDF5_HL_DIR) -DIST_SUBDIRS = src test testpar utils tools . c++ fortran hl examples java +DIST_SUBDIRS = src test testpar tools utils . c++ fortran hl examples java # Some files generated during configure that should be cleaned DISTCLEANFILES=config/stamp1 config/stamp2 diff --git a/README.txt b/README.txt index b9494db..b9cfe8a 100644 --- a/README.txt +++ b/README.txt @@ -1,4 +1,4 @@ -HDF5 version 1.13.0-7 currently under development +HDF5 version 1.13.1-1 currently under development ------------------------------------------------------------------------------ Please refer to the release_docs/INSTALL file for installation instructions. diff --git a/c++/src/cpp_doc_config b/c++/src/cpp_doc_config index 002db5d..3eb7645 100644 --- a/c++/src/cpp_doc_config +++ b/c++/src/cpp_doc_config @@ -38,7 +38,7 @@ PROJECT_NAME = # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "1.13.0-7, currently under development" +PROJECT_NUMBER = "1.13.1-1, currently under development" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/config/cmake/FindCIRCLE.cmake b/config/cmake/FindCIRCLE.cmake new file mode 100644 index 0000000..b36d76c --- /dev/null +++ b/config/cmake/FindCIRCLE.cmake @@ -0,0 +1,49 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#[=======================================================================[.rst: +FindCIRCLE +-------- + +Find the native CIRCLE includes and library + +This module defines + +:: + + CIRCLE_INCLUDE_DIR, where to find CIRCLE.h, etc. + CIRCLE_LIBRARIES, the libraries required to use CIRCLE. + CIRCLE_FOUND, If false, do not try to use CIRCLE. + +also defined, but not for general use are + +:: + + CIRCLE_LIBRARY, where to find the CIRCLE library. +#]=======================================================================] + +if(DEFINED ENV{MFU_ROOT}) + set(ENV{MFU_INCLUDE} "$ENV{MFU_ROOT}/include") + set(ENV{MFU_LIB} "$ENV{MFU_ROOT}/lib") + set(ENV{MFU_LIB64} "$ENV{MFU_ROOT}/lib64") +else() + message("CIRCLE_LIBRARY: If you have problems building this library,\nconsider setting the MFU_ROOT environment variable to indicate\nwhere to find the support libraries and header files!") +endif() + +find_path(CIRCLE_INCLUDE_DIR + NAMES libcircle.h + HINTS ENV MFU_INCLUDE) + +find_library(CIRCLE_LIBRARY + NAMES circle + HINTS ENV MFU_LIB ENV MFU_LIB64 + ) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(CIRCLE REQUIRED_VARS CIRCLE_LIBRARY CIRCLE_INCLUDE_DIR) + +if(CIRCLE_FOUND) + set(CIRCLE_LIBRARIES ${CIRCLE_LIBRARY} ) +endif() + +mark_as_advanced(CIRCLE_INCLUDE_DIR CIRCLE_LIBRARY) diff --git a/config/cmake/FindDTCMP.cmake b/config/cmake/FindDTCMP.cmake new file mode 100644 index 0000000..b95ef20 --- /dev/null +++ b/config/cmake/FindDTCMP.cmake @@ -0,0 +1,48 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#[=======================================================================[.rst: +FindDTCMP +-------- + +Find the native DTCMP includes and library + +This module defines + +:: + + DTCMP_INCLUDE_DIR, where to find DTCMP.h, etc. + DTCMP_LIBRARIES, the libraries required to use DTCMP. + DTCMP_FOUND, If false, do not try to use DTCMP. + +also defined, but not for general use are + +:: + + DTCMP_LIBRARY, where to find the DTCMP library. +#]=======================================================================] + +if(DEFINED ENV{MFU_ROOT}) + set(ENV{MFU_INCLUDE} "$ENV{MFU_ROOT}/include") + set(ENV{MFU_LIB} "$ENV{MFU_ROOT}/lib") + set(ENV{MFU_LIB64} "$ENV{MFU_ROOT}/lib64") +else() + message("DTCMP_LIBRARY: If you have problems building this library,\nconsider setting the MFU_ROOT environment variable to indicate\nwhere to find the support libraries and header files!") +endif() + +find_path(DTCMP_INCLUDE_DIR + NAMES dtcmp.h + HINTS ENV MFU_INCLUDE) + +find_library(DTCMP_LIBRARY + NAMES dtcmp + HINTS ENV MFU_LIB ENV MFU_LIB64) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(DTCMP REQUIRED_VARS DTCMP_LIBRARY DTCMP_INCLUDE_DIR) + +if(DTCMP_FOUND) + set(DTCMP_LIBRARIES ${DTCMP_LIBRARY} ) +endif() + +mark_as_advanced(DTCMP_INCLUDE_DIR DTCMP_LIBRARY) diff --git a/config/cmake/FindMFU.cmake b/config/cmake/FindMFU.cmake new file mode 100644 index 0000000..37699e2 --- /dev/null +++ b/config/cmake/FindMFU.cmake @@ -0,0 +1,100 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +######################################################################### + +# - Derived from the FindTiff.cmake and FindJPEG.cmake that is included with cmake +# FindMFU + +# Find the native MFU includes and library + +# Imported targets +################## + +# This module defines the following :prop_tgt:`IMPORTED` targets: +# +# MFU::MFU +# The MFU library, if found. +# +# Result variables +################### + +# This module will set the following variables in your project: + +# MFU_FOUND, true if the MFU headers and libraries were found. +# MFU_INCLUDE_DIR, the directory containing the MFU headers. +# MFU_INCLUDE_DIRS, the directory containing the MFU headers. +# MFU_LIBRARIES, libraries to link against to use MFU. + +# Cache variables +################# + +# The following variables may also be set: + +# MFU_LIBRARY, where to find the MFU library. +# message (STATUS "Finding MFU library and headers..." ) +######################################################################### + + + +FIND_PATH(MFU_INCLUDE_DIR + NAMES mfu.h + HINTS "$ENV{MFU_ROOT}/include" +) +FIND_LIBRARY(MFU_LIBRARY + NAMES mfu + HINTS "$ENV{MFU_ROOT}/lib64" +) + +if(NOT MFU_LIBRARY) + set(mfu_names ${MFU_NAMES} mfu libmfu) + find_library(MFU_LIBRARY NAMES ${mfu_names}) + include(SelectLibraryConfigurations) + select_library_configurations(MFU) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(MFU + REQUIRED_VARS MFU_LIBRARY MFU_INCLUDE_DIR) + +if(MFU_FOUND) + set(MFU_LIBRARIES "${MFU_LIBRARY}") + set(MFU_INCLUDE_DIRS "${MFU_INCLUDE_DIR}") + set(LL_PATH "$ENV{MFU_ROOT}/lib64:$ENV{MFU_ROOT}/lib") + if(NOT TARGET MFU::MFU) + add_library(MFU::MFU UNKNOWN IMPORTED) + if(MFU_INCLUDE_DIRS) + set_target_properties(MFU::MFU PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${MFU_INCLUDE_DIRS}") + endif() + if(EXISTS "${MFU_LIBRARY}") + set_target_properties(MFU::MFU PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${MFU_LIBRARY}") + endif() + endif() +endif() + +# Report the results. +if (NOT MFU_FOUND) + set (MFU_DIR_MESSAGE + "Mfu was not found. Make sure MFU_LIBRARY and MFU_INCLUDE_DIR are set or set the MFU_INSTALL environment variable." + ) + if (NOT MFU_FIND_QUIETLY) + if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") + message (VERBOSE "${MFU_DIR_MESSAGE}") + endif () + else () + if (MFU_FIND_REQUIRED) + message (FATAL_ERROR "Mfu was NOT found and is Required by this project") + endif () + endif () +endif () diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index 1a1c03a..a6b238c 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -71,13 +71,13 @@ /* Define Fortran compiler ID */ #define H5_Fortran_COMPILER_ID @CMAKE_Fortran_COMPILER_ID@ +/* Define number of valid Fortran INTEGER KINDs (must be defined before F_IKIND)*/ +#cmakedefine H5_H5CONFIG_F_NUM_IKIND @H5_H5CONFIG_F_NUM_IKIND@ + /* Define valid Fortran INTEGER KINDs */ #cmakedefine H5_H5CONFIG_F_IKIND @H5_H5CONFIG_F_IKIND@ -/* Define number of valid Fortran INTEGER KINDs */ -#cmakedefine H5_H5CONFIG_F_NUM_IKIND @H5_H5CONFIG_F_NUM_IKIND@ - -/* Define number of valid Fortran REAL KINDs */ +/* Define number of valid Fortran REAL KINDs (must be defined before F_RKIND) */ #cmakedefine H5_H5CONFIG_F_NUM_RKIND @H5_H5CONFIG_F_NUM_RKIND@ /* Define valid Fortran REAL KINDs */ diff --git a/config/cmake/HDF5UseFortran.cmake b/config/cmake/HDF5UseFortran.cmake index 40027ea..0d05e20 100644 --- a/config/cmake/HDF5UseFortran.cmake +++ b/config/cmake/HDF5UseFortran.cmake @@ -141,7 +141,7 @@ FORTRAN_RUN ("REAL and INTEGER KINDs" # dnl -- LINE 5 -- number of valid real kinds # # Convert the string to a list of strings by replacing the carriage return with a semicolon -string (REGEX REPLACE "\n" ";" PROG_OUTPUT "${PROG_OUTPUT}") +string (REGEX REPLACE "[\r\n]+" ";" PROG_OUTPUT "${PROG_OUTPUT}") list (GET PROG_OUTPUT 0 pac_validIntKinds) list (GET PROG_OUTPUT 1 pac_validRealKinds) @@ -195,7 +195,7 @@ foreach (KIND ${VAR}) " ) FORTRAN_RUN("INTEGER KIND SIZEOF" ${PROG_SRC_${KIND}} XX YY VALIDINTKINDS_RESULT_${KIND} PROG_OUTPUT1) - string (REGEX REPLACE "\n" "" PROG_OUTPUT1 "${PROG_OUTPUT1}") + string (REGEX REPLACE "[\r\n]+" "" PROG_OUTPUT1 "${PROG_OUTPUT1}") set (pack_int_sizeof "${pack_int_sizeof} ${PROG_OUTPUT1},") endforeach () @@ -238,7 +238,7 @@ foreach (KIND ${VAR} ) " ) FORTRAN_RUN ("REAL KIND SIZEOF" ${PROG_SRC2_${KIND}} XX YY VALIDREALKINDS_RESULT_${KIND} PROG_OUTPUT2) - string (REGEX REPLACE "\n" "" PROG_OUTPUT2 "${PROG_OUTPUT2}") + string (REGEX REPLACE "[\r\n]+" "" PROG_OUTPUT2 "${PROG_OUTPUT2}") set (pack_real_sizeof "${pack_real_sizeof} ${PROG_OUTPUT2},") endforeach () @@ -296,7 +296,7 @@ FORTRAN_RUN ("SIZEOF NATIVE KINDs" ${PROG_SRC3} XX YY PAC_SIZEOF_NATIVE_KINDS_RE # dnl -- LINE 6 -- kind of DOUBLE PRECISION # # Convert the string to a list of strings by replacing the carriage return with a semicolon -string (REGEX REPLACE "\n" ";" PROG_OUTPUT3 "${PROG_OUTPUT3}") +string (REGEX REPLACE "[\r\n]+" ";" PROG_OUTPUT3 "${PROG_OUTPUT3}") list (GET PROG_OUTPUT3 0 PAC_FORTRAN_NATIVE_INTEGER_SIZEOF) list (GET PROG_OUTPUT3 1 PAC_FORTRAN_NATIVE_INTEGER_KIND) diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake index ee898b0..2f5af77 100644 --- a/config/cmake/scripts/HDF5config.cmake +++ b/config/cmake/scripts/HDF5config.cmake @@ -37,8 +37,8 @@ cmake_minimum_required (VERSION 3.12) # CTEST_SOURCE_NAME - source folder ############################################################################## -set (CTEST_SOURCE_VERSION "1.13.0") -set (CTEST_SOURCE_VERSEXT "-7") +set (CTEST_SOURCE_VERSION "1.13.1") +set (CTEST_SOURCE_VERSEXT "-1") ############################################################################## # handle input parameters to script. diff --git a/configure.ac b/configure.ac index 4882e58..6431710 100644 --- a/configure.ac +++ b/configure.ac @@ -23,7 +23,7 @@ AC_PREREQ([2.69]) ## NOTE: Do not forget to change the version number here when we do a ## release!!! ## -AC_INIT([HDF5], [1.13.0-7], [help@hdfgroup.org]) +AC_INIT([HDF5], [1.13.1-1], [help@hdfgroup.org]) AC_CONFIG_SRCDIR([src/H5.c]) AC_CONFIG_HEADERS([src/H5config.h]) @@ -823,11 +823,14 @@ AC_LANG_POP(C++) AC_SUBST([HDF5_HL]) AC_SUBST([HDF5_HL_TOOLS]) -## The high-level library is enabled unless the build mode is clean. +## The high-level library and high-level tools are enabled unless the build mode +## is clean. if test "X-$BUILD_MODE" = "X-clean" ; then HDF5_HL=no + HDF5_HL_TOOLS=no else HDF5_HL=yes + HDF5_HL_TOOLS=yes fi ## high-level library directories (set when needed, blank until then) @@ -892,7 +895,7 @@ if test "X-$DIMENSION_SCALES_WITH_NEW_REF" = X- ; then DIMENSION_SCALES_WITH_NEW_REF=no fi -case "X-$DIMENSION_SCALES_WITH_NEW_REF" in +case "X-$DIMENSION_SCALES_WITH_NEW_REF" in X-yes) AC_MSG_RESULT([yes]) AC_DEFINE([DIMENSION_SCALES_WITH_NEW_REF], [1], @@ -1193,6 +1196,7 @@ if test "X$HDF5_DOXYGEN" = "Xyes"; then AC_SUBST([DOXYGEN_EXTERNAL_SEARCH]) AC_SUBST([DOXYGEN_SEARCHENGINE_URL]) AC_SUBST([DOXYGEN_STRIP_FROM_PATH]) + AC_SUBST([DOXYGEN_STRIP_FROM_INC_PATH]) AC_SUBST([DOXYGEN_PREDEFINED]) # SRCDIR Environment variables used inside doxygen macro for the source location: @@ -1200,7 +1204,7 @@ if test "X$HDF5_DOXYGEN" = "Xyes"; then DOXYGEN_VERSION_STRING=${PACKAGE_VERSION} DOXYGEN_INCLUDE_ALIASES='$(SRCDIR)/doxygen/aliases' DOXYGEN_PROJECT_LOGO='$(SRCDIR)/doxygen/img/HDFG-logo.png' - DOXYGEN_PROJECT_BRIEF='C-API Reference' + DOXYGEN_PROJECT_BRIEF='' DOXYGEN_INPUT_DIRECTORY='$(SRCDIR) $(SRCDIR)/doxygen/dox' DOXYGEN_OPTIMIZE_OUTPUT_FOR_C=YES DOXYGEN_MACRO_EXPANSION=YES @@ -1216,6 +1220,7 @@ if test "X$HDF5_DOXYGEN" = "Xyes"; then DOXYGEN_EXTERNAL_SEARCH=NO DOXYGEN_SEARCHENGINE_URL= DOXYGEN_STRIP_FROM_PATH='$(SRCDIR)' + DOXYGEN_STRIP_FROM_INC_PATH='$(SRCDIR)' DOXYGEN_PREDEFINED='H5_HAVE_DIRECT H5_HAVE_LIBHDFS H5_HAVE_MAP_API H5_HAVE_PARALLEL H5_HAVE_ROS3_VFD' DX_INIT_DOXYGEN([HDF5], [./doxygen/Doxyfile], [hdf5lib_docs]) @@ -1572,6 +1577,7 @@ case "X-$withval" in ;; esac + ## ---------------------------------------------------------------------- ## Is mercury(an external RPC Framework library) requested? It has a ## header file `mercury.h' and a library `-lmercury_util' and their @@ -1742,6 +1748,7 @@ fi ## command-line switch. The value is an include path and/or a library path. ## If the library path is specified then it must be preceded by a comma. ## +AC_SUBST([LL_PATH]) AC_SUBST([USE_FILTER_SZIP]) USE_FILTER_SZIP="no" AC_ARG_WITH([szlib], [AS_HELP_STRING([--with-szlib=DIR], @@ -1826,7 +1833,7 @@ if test "x$HAVE_SZLIB" = "xyes" -a "x$HAVE_SZLIB_H" = "xyes"; then export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$szlib_lib" fi - AC_SUBST([LL_PATH]) LL_PATH="$LD_LIBRARY_PATH" + LL_PATH="$LD_LIBRARY_PATH" AC_CACHE_VAL([hdf5_cv_szlib_can_encode], [AC_RUN_IFELSE( @@ -3092,6 +3099,138 @@ if test -n "$PARALLEL"; then fi ## ---------------------------------------------------------------------- +## Build parallel tools if parallel tools, parallel, and build tools options +## are all enabled. +## +AC_SUBST([PARALLEL_TOOLS]) + +## Default is no parallel tools +PARALLEL_TOOLS=no + +AC_MSG_CHECKING([parallel tools]) +AC_ARG_ENABLE([parallel-tools], + [AS_HELP_STRING([--enable-parallel-tools], + [Enable building parallel tools. + [default=no]])], + [PARALLEL_TOOLS=$enableval]) + +if test "X${PARALLEL_TOOLS}" = "Xyes"; then + if test "X${HDF5_TOOLS}" != "Xyes"; then + AC_MSG_ERROR([--enable-tools is required for --enable-parallel-tools]) + fi + if test "X${PARALLEL}" != "Xyes"; then + AC_MSG_ERROR([--enable-parallel is required for --enable-parallel-tools]) + fi +fi + +case "X-$PARALLEL_TOOLS" in + X-|X-no) + AC_MSG_RESULT([no]) + ;; + X-yes) + AC_MSG_RESULT([yes]) + ;; + *) + ;; +esac + +if test "X${PARALLEL_TOOLS}" = "Xyes"; then + ## Is the mpiFileUtils library (libmfu) required and available? + ## + AC_SUBST([H5DWALK_LDFLAGS]) + AC_SUBST([H5DWALK_LIBS]) + AC_SUBST([H5DWALK_CPPFLAGS]) + + H5DWALK_LDFLAGS="${H5DWALK_LDFLAGS}" + + ## Default is not present + HAVE_LIBMFU=no + + AC_ARG_WITH([libmfu], + [AS_HELP_STRING([--with-libmfu=DIR], + [Use the libmfu library [default=no]])],, + [withval=no]) + + case "X-$withval" in + X-yes) + HAVE_LIBMFU="yes" + AC_CHECK_HEADERS([mfu.h],, [unset HAVE_LIBMFU]) + if test "x$HAVE_LIBMFU" = "xyes"; then + AC_CHECK_LIB([mfu], [mfu_init], [H5DWALK_LIBS="-lmfu"], [unset HAVE_LIBMFU]) + fi + if test -z "$HAVE_LIBMFU" -a -n "$HDF5_CONFIG_ABORT"; then + AC_MSG_ERROR([couldn't find libmfu library]) + fi + ;; + X-|X-no|X-none) + HAVE_LIBMFU="no" + AC_MSG_CHECKING([for libmfu library]) + AC_MSG_RESULT([suppressed]) + ;; + *) + HAVE_LIBMFU="yes" + case "$withval" in + *,*) + libmfu_inc="`echo $withval |cut -f1 -d,`" + libmfu_lib="`echo $withval |cut -f2 -d, -s`" + ;; + *) + if test -n "$withval"; then + libmfu_inc="$withval/include" + libmfu_lib="$withval/lib64" + libcircle_lib="$withval/lib" + fi + ;; + esac + + saved_CPPFLAGS="$CPPFLAGS" + saved_AM_CPPFLAGS="$AM_CPPFLAGS" + saved_LDFLAGS="$LDFLAGS" + saved_AM_LDFLAGS="$AM_LDFLAGS" + + ## For these checks we need the libmfu locations added to CPPFLAGS, + ## AM_CPPFLAGS, LDFLAGS, and AM_LDFLAGS. The third param should set them + ## back to these saved values. If the checks pass, then normally these four + ## flag variables would be updated, but in this case we put the changes in + ## variables specific to H5DWALK since they aren't used elsewhere. + if test -n "$libmfu_inc"; then + CPPFLAGS="$CPPFLAGS -I$libmfu_inc" + AM_CPPFLAGS="$AM_CPPFLAGS -I$libmfu_inc" + fi + + if test -n "$libmfu_lib"; then + LDFLAGS="$LDFLAGS -L$libmfu_lib -L$libcircle_lib" + AM_LDFLAGS="$AM_LDFLAGS -L$libmfu_lib -L$libcircle_lib" + fi + + if test "x$HAVE_LIBMFU" = "xyes"; then + AC_CHECK_LIB([mfu], [mfu_init],[H5DWALK_LIBS="-lmfu"], [CPPFLAGS="$saved_CPPFLAGS"; AM_CPPFLAGS="$saved_AM_CPPFLAGS"; LDFLAGS="$saved_LDFLAGS"; AM_LDFLAGS="$saved_AM_LDFLAGS"; unset HAVE_LIBMFU]) + if test -n "$HAVE_LIBMFU"; then + AC_CHECK_HEADERS([mfu.h],[H5DWALK_CPPFLAGS="-I$libmfu_inc"],[CPPFLAGS="$saved_CPPFLAGS"; AM_CPPFLAGS="$saved_AM_CPPFLAGS"; LDFLAGS="$saved_LDFLAGS"; AM_LDFLAGS="$saved_AM_LDFLAGS"; unset HAVE_LIBMFU]) + fi + fi + + if test -z "$HAVE_LIBMFU" -a -n "$HDF5_CONFIG_ABORT"; then + AC_MSG_ERROR([couldn't find libmfu library]) + else + H5DWALK_LDFLAGS="-L$libmfu_lib -L$libcircle_lib" + fi + + if test -z "$LD_LIBRARY_PATH"; then + export LD_LIBRARY_PATH="$libmfu_lib:$libcircle_lib" + else + export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$libmfu_lib:$libcircle_lib" + fi + + LL_PATH="$LD_LIBRARY_PATH" + ;; + esac +fi + +## +AM_CONDITIONAL([PARALLEL_TOOLS_CONDITIONAL], [test "X$HAVE_LIBMFU" = "Xyes"]) + +## ---------------------------------------------------------------------- ## Check if the map API is enabled by --enable-map-api ## AC_SUBST([MAP_API]) @@ -4125,6 +4264,12 @@ AC_CONFIG_FILES([src/libhdf5.settings utils/Makefile utils/mirror_vfd/Makefile utils/test/Makefile + utils/tools/Makefile + utils/tools/h5dwalk/Makefile + utils/tools/test/Makefile + utils/tools/test/h5dwalk/Makefile + utils/tools/test/h5dwalk/copy_demo_files.sh + utils/tools/test/h5dwalk/testh5dwalk.sh tools/Makefile tools/lib/Makefile tools/libtest/Makefile diff --git a/doxygen/CMakeLists.txt b/doxygen/CMakeLists.txt index 36ce590..3462d50 100644 --- a/doxygen/CMakeLists.txt +++ b/doxygen/CMakeLists.txt @@ -27,6 +27,7 @@ if (DOXYGEN_FOUND) set (DOXYGEN_EXTERNAL_SEARCH NO) set (DOXYGEN_SEARCHENGINE_URL) set (DOXYGEN_STRIP_FROM_PATH ${HDF5_SOURCE_DIR}) + set (DOXYGEN_STRIP_FROM_INC_PATH ${HDF5_SOURCE_DIR}) set (DOXYGEN_PREDEFINED "H5_HAVE_DIRECT H5_HAVE_LIBHDFS H5_HAVE_MAP_API H5_HAVE_PARALLEL H5_HAVE_ROS3_VFD") # This configure and individual custom targets work together diff --git a/doxygen/Doxyfile.in b/doxygen/Doxyfile.in index 44d9974..8c871de 100644 --- a/doxygen/Doxyfile.in +++ b/doxygen/Doxyfile.in @@ -179,7 +179,7 @@ STRIP_FROM_PATH = @DOXYGEN_STRIP_FROM_PATH@ # specify the list of include paths that are normally passed to the compiler # using the -I flag. -STRIP_FROM_INC_PATH = +STRIP_FROM_INC_PATH = @DOXYGEN_STRIP_FROM_INC_PATH@ # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but # less readable) file names. This can be useful is your file systems doesn't @@ -856,6 +856,7 @@ INPUT_ENCODING = UTF-8 FILE_PATTERNS = H5*public.h \ H5*module.h \ H5FDcore.h \ + H5FDdevelop.h \ H5FDdirect.h \ H5FDfamily.h \ H5FDhdfs.h \ diff --git a/doxygen/aliases b/doxygen/aliases index 06c3445..f83a875 100644 --- a/doxygen/aliases +++ b/doxygen/aliases @@ -24,6 +24,7 @@ ALIASES += htri_t="Returns zero (false), a positive (true) or a negative (failur ALIASES += api_vers_2{3}="\1() is a macro that is mapped to either \2() or \3().\n\see \ref api-compat-macros" ALIASES += api_vers_3{4}="\1() is a macro that is mapped to either \2() or \3() or \4().\n\see \ref api-compat-macros" +ALIASES += api_vers_4{5}="\1() is a macro that is mapped to either \2() or \3() or \4() or \5().\n\see \ref api-compat-macros" ALIASES += deprecation_note{1}="\deprecated Superseded by \1." @@ -252,10 +253,17 @@ ALIASES += ref_vol_doc="VOL documentation" ################################################################################ ALIASES += ref_rfc20210528="Multi-Thread HDF5" +ALIASES += ref_rfc20210219="Selection I/O" +ALIASES += ref_rfc20200213="VFD Sub-filing" +ALIASES += ref_rfc20200210="Onion VFD" ALIASES += ref_rfc20190923="Virtual Object Layer (VOL) API Compatibility" +ALIASES += ref_rfc20190715="Variable-Length Data in HDF5 Sketch Design" ALIASES += ref_rfc20190410="A Plugin Interface for HDF5 Virtual File Drivers" ALIASES += ref_rfc20181231="Dataset Object Header Size" ALIASES += ref_rfc20181220="MS 3.2 – Addressing Scalability: Scalability of open, close, flush CASE STUDY: CGNS Hotspot analysis of CGNS cgp_open" +ALIASES += ref_rfc20180830="Sparse Chunks" +ALIASES += ref_rfc20180829="H5FD_MIRROR Virtual File Driver" +ALIASES += ref_rfc20180815="Splitter_VFD" ALIASES += ref_rfc20180620="Chunk query functionality in HDF5" ALIASES += ref_rfc20180610="VFD SWMR" ALIASES += ref_rfc20180321="API Contexts" @@ -298,7 +306,7 @@ ALIASES += ref_rfc20120305="h5repack: Improved Hyperslab selections for Large Chunked Datasets" ALIASES += ref_rfc20120120="A Maintainer’s Guide for the Datatype Module in HDF5 Library" ALIASES += ref_rfc20120104="Actual I/O Mode" -ALIASES += ref_rfc20111119="New public functions to handle comparison" +ALIASES += ref_rfc20111119="New public functions to handle comparison" ALIASES += ref_rfc20110825="Merging Named Datatypes in H5Ocopy()" ALIASES += ref_rfc20110811="Expanding the HDF5 Hyperslab Selection Interface" ALIASES += ref_rfc20110726="HDF5 File Space Allocation and Aggregation" @@ -318,8 +326,12 @@ ALIASES += ref_rfc20091218="HDF5 Tools Library Functions" ALIASES += ref_rfc20090612="Default EPSILON values for comparing floating point data" ALIASES += ref_rfc20081218="Reporting of Non-Comparable Datasets by h5diff" +ALIASES += ref_rfc20081205="External Link Traversal Callback" +ALIASES += ref_rfc20081030="Setting Raw Data Chunk Cache Parameters in HDF5" ALIASES += ref_rfc20080915="Performance Report for Free-space Manager" ALIASES += ref_rfc20080904="Setting File Access Property List for accessing External Link" +ALIASES += ref_rfc20080728="Native Time Types in HDF5" +ALIASES += ref_rfc20080723="Special Values in HDF5" ALIASES += ref_rfc20080301="Dynamic Transformations to HDF5 Data" ALIASES += ref_rfc20080209="Using SVN branching to improve software development process at THG" ALIASES += ref_rfc20080206="Maintaining the HISTORY.txt and RELEASE.txt files in HDF5" @@ -327,7 +339,7 @@ ALIASES += ref_rfc20071111="NaN detection in HDF5" ALIASES += ref_rfc20070801="Metadata Journaling to Improve Crash Survivability" ALIASES += ref_rfc20070413="API Compatibility Strategies for HDF5" -ALIASES += ref_rfc20070115="A 'Private' Heap for HDF5" +ALIASES += ref_rfc20070115="A \"Private\" Heap for HDF5" ALIASES += ref_rfc20060623="Performance Comparison of Collective I/O and Independent I/O with Derived Datatypes" ALIASES += ref_rfc20060604="h5stat tool" ALIASES += ref_rfc20060505="Simple Performance Test on Fletcher32 Filter" diff --git a/doxygen/dox/About.dox b/doxygen/dox/About.dox index 32930a8..0b21fcc 100644 --- a/doxygen/dox/About.dox +++ b/doxygen/dox/About.dox @@ -12,12 +12,118 @@ of documentation done right. \section about_documentation Documentation about Documentation -\li \todo Describe how to add a reference or a new RFC -\li \todo Describe how to add an example -\li \todo Describe how to include plain HTML -\li \todo Describe how to add an API macro -\li \todo Describe the custom commands -\li \todo Describe the S3 bucket layout and update routine -\li \todo Link the RM template +In this section, we describe common documentation maintenance tasks. + +\subsection plain_html Including Plain HTML Pages + +The most common use case for this is the inclusion of older documentation. +New documentation should, whenever possible, be created using Doxygen markdown! + +Use Doxygen's htmlinclude +special command to include existing plain HTML pages. + +An example from this documentation set can be seen +here. + +\subsection new_rm_entry Creating a New Reference Manual Entry + +Please refer to the \ref RMT for guidance on how to create a new reference manual entry. + +\subsubsection new_example Adding and Referencing API Examples + +For each HDF5 module, such as \Code{H5F}, there is an examples source file called +\Code{H5*_examples.c}. For example, the \Code{H5F} API examples are located in + +H5F_examples.c. Examples are code blocks marked as Doxygen +snippets. +For example, the source code for the H5Fcreate() API sample is located between +the +\verbatim +//! +... +//! +\endverbatim +comments in + +H5F_examples.c. + +Add a new API example by adding a new code block enclosed between matching +snippet tags. The name of the tag is usually the function name stripped of +the module prefix. + +The inclusion of such a block of code can then be triggered via Doxygen's +snippet +special command. For example, the following markup +\verbatim +* \snippet H5F_examples.c create +\endverbatim +yields +\snippet H5F_examples.c create + +\subsubsection api_macro Adding an API Macro + +API macros are handled by the api_vers_2, api_vers_3, api_vers_4 +custom commands. The numbers indicate the number of potential API function +mappings. For example, H5Acreate() has two potential mappings, H5Acreate1() and +H5Acreate2(). To trigger the creation of a reference manual entry for H5Acreate() +use the following markup: +\verbatim +\api_vers_2{H5Acreate,H5Acreate1,H5Acreate2} +\endverbatim +This yields: + +\api_vers_2{H5Acreate,H5Acreate1,H5Acreate2} + +\subsection custom_commands Creating Custom Commands + +See Doxygen's Custom Commands documentation +as a general reference. + +All custom commands for this project are located in the +aliases +file in the doxygen +subdirectory of the main HDF5 repo. + +The custom commands are grouped in sections. Find a suitable section for your command or +ask for help if unsure! + +\subsection new_rfc Adding a New RFC or Referencing an Existing RFC + +For ease of reference, we define custom commands for each RFC in the RFCs section +of the +aliases +file. For example the custom command \Code{ref_rfc20141210} can be used to insert a +reference to "RFC: Virtual Object Layer". In other words, the markup +\verbatim +\ref_rfc20141210 +\endverbatim +yields a clickable link: + +\ref_rfc20141210 + +To add a new RFC, add a custom command for the RFC to the +aliases +file. The naming convention for the custom command is \Code{ref_rfcYYYYMMDD}, +where \Code{YYYYMMDD} is the ID of the RFC. The URL is composed of the prefix +\verbatim +https://docs.hdfgroup.org/hdf5/rfc/ +\endverbatim +and the name of your RFC file, typically, a PDF file, i.e., the full URL would +be +\verbatim +https://docs.hdfgroup.org/hdf5/rfc/my_great_rfc_name.pdf +\endverbatim + +\subsection hosting How Do Updates and Changes Get Published? + +Currently, the files underlying this documentation website are stored in an +bucket on AWS S3. The top-level bucket is 
s3://docs.hdfgroup.org/hdf5/
+There are folders for the development branch and all supported release +version. + +Talk to your friendly IT-team if you need write access, or you need someone to +push an updated version for you! + +\todo Make the publication a GitHub action! */ \ No newline at end of file diff --git a/doxygen/dox/FTS.dox b/doxygen/dox/FTS.dox new file mode 100644 index 0000000..9dae7c1 --- /dev/null +++ b/doxygen/dox/FTS.dox @@ -0,0 +1,8 @@ +/** \page FTS Full-Text Search + +\htmlonly + +
+\endhtmlonly + +*/ \ No newline at end of file diff --git a/doxygen/dox/FileFormatSpec.dox b/doxygen/dox/FileFormatSpec.dox deleted file mode 100644 index fc10574..0000000 --- a/doxygen/dox/FileFormatSpec.dox +++ /dev/null @@ -1,23 +0,0 @@ -/** \page FMT3 HDF5 File Format Specification Version 3.0 - -\htmlinclude H5.format.html - -*/ - -/** \page FMT2 HDF5 File Format Specification Version 2.0 - -\htmlinclude H5.format.2.0.html - -*/ - -/** \page FMT11 HDF5 File Format Specification Version 1.1 - -\htmlinclude H5.format.1.1.html - -*/ - -/** \page FMT1 HDF5 File Format Specification Version 1.0 - -\htmlinclude H5.format.1.0.html - -*/ \ No newline at end of file diff --git a/doxygen/dox/OtherSpecs.dox b/doxygen/dox/OtherSpecs.dox deleted file mode 100644 index e53f26e..0000000 --- a/doxygen/dox/OtherSpecs.dox +++ /dev/null @@ -1,11 +0,0 @@ -/** \page IMG HDF5 Image and Palette Specification Version 1.2 - -\htmlinclude ImageSpec.html - -*/ - -/** \page TBL HDF5 Table Specification Version 1.0 - -\htmlinclude TableSpec.html - -*/ diff --git a/doxygen/dox/RFC.dox b/doxygen/dox/RFC.dox index c16dcea..c2562b0 100644 --- a/doxygen/dox/RFC.dox +++ b/doxygen/dox/RFC.dox @@ -3,10 +3,17 @@ + + + + + + + @@ -69,8 +76,12 @@ + + + + diff --git a/doxygen/dox/Specifications.dox b/doxygen/dox/Specifications.dox index 4ae48d0..5a36d61 100644 --- a/doxygen/dox/Specifications.dox +++ b/doxygen/dox/Specifications.dox @@ -19,4 +19,40 @@ \li HDF5 Dimension Scale Specification -*/ \ No newline at end of file +*/ + +/** \page FMT3 HDF5 File Format Specification Version 3.0 + +\htmlinclude H5.format.html + +*/ + +/** \page FMT2 HDF5 File Format Specification Version 2.0 + +\htmlinclude H5.format.2.0.html + +*/ + +/** \page FMT11 HDF5 File Format Specification Version 1.1 + +\htmlinclude H5.format.1.1.html + +*/ + +/** \page FMT1 HDF5 File Format Specification Version 1.0 + +\htmlinclude H5.format.1.0.html + +*/ + +/** \page IMG HDF5 Image and Palette Specification Version 1.2 + +\htmlinclude ImageSpec.html + +*/ + +/** \page TBL HDF5 Table Specification Version 1.0 + +\htmlinclude TableSpec.html + +*/ diff --git a/doxygen/dox/TechnicalNotes.dox b/doxygen/dox/TechnicalNotes.dox index 2bda175..9bd2802 100644 --- a/doxygen/dox/TechnicalNotes.dox +++ b/doxygen/dox/TechnicalNotes.dox @@ -1,6 +1,10 @@ /** \page TN Technical Notes \li \link api-compat-macros API Compatibility Macros \endlink +\li \ref APPDBG "Debugging HDF5 Applications" +\li \ref FMTDISC "File Format Walkthrough" +\li \ref FILTER "Filters" +\li \ref IOFLOW "HDF5 Raw I/O Flow Notes" \li \ref TNMDC "Metadata Caching in HDF5" \li \ref MT "Thread Safe library" \li \ref VFL "Virtual File Layer" @@ -13,8 +17,32 @@ */ +/** \page IOFLOW HDF5 Raw I/O Flow Notes + +\htmlinclude IOFlow.html + +*/ + /** \page VFL HDF5 Virtual File Layer \htmlinclude VFL.html */ + +/** \page FMTDISC HDF5 File Format Discussion + +\htmlinclude FileFormat.html + +*/ + +/** \page FILTER HDF5 Filters + +\htmlinclude Filters.html + +*/ + +/** \page APPDBG Debugging HDF5 Applications + +\htmlinclude DebuggingHDF5Applications.html + +*/ \ No newline at end of file diff --git a/doxygen/examples/DebuggingHDF5Applications.html b/doxygen/examples/DebuggingHDF5Applications.html new file mode 100644 index 0000000..c6aaf74 --- /dev/null +++ b/doxygen/examples/DebuggingHDF5Applications.html @@ -0,0 +1,392 @@ + + + Debugging HDF5 Applications + +

Introduction

+ +

The HDF5 library contains a number of debugging features to + make programmers' lives easier including the ability to print + detailed error messages, check invariant conditions, display + timings and other statistics, and trace API function calls and + return values. + +

+
Error Messages +
Error messages are normally displayed automatically on the + standard error stream and include a stack trace of the library + including file names, line numbers, and function names. The + application has complete control over how error messages are + displayed and can disable the display on a permanent or + temporary basis. Refer to the documentation for the H5E error + handling package. + +

+
Invariant Conditions +
Unless NDEBUG is defined during compiling, the + library will include code to verify that invariant conditions + have the expected values. When a problem is detected the + library will display the file and line number within the + library and the invariant condition that failed. A core dump + may be generated for post mortem debugging. The code to + perform these checks can be included on a per-package bases. + +

+
Timings and Statistics +
The library can be configured to accumulate certain + statistics about things like cache performance, datatype + conversion, data space conversion, and data filters. The code + is included on a per-package basis and enabled at runtime by + an environment variable. + +

+
API Tracing +
All API calls made by an application can be displayed and + include formal argument names and actual values and the + function return value. This code is also conditionally + included at compile time and enabled at runtime. +
+ +

The statistics and tracing can be displayed on any output + stream (including streams opened by the shell) with output from + different packages even going to different streams. + +

Error Messages

+ +

By default any API function that fails will print an error + stack to the standard error stream. + +

+

+
RFC IDTitleComments
2021-05-28 \ref_rfc20210528
2021-02-19 \ref_rfc20210219
2020-02-13 \ref_rfc20200213
2020-02-10 \ref_rfc20200210
2019-09-23 \ref_rfc20190923
2019-07-15 \ref_rfc20190715
2019-04-10 \ref_rfc20190410
2018-12-31 \ref_rfc20181231
2018-12-20 \ref_rfc20181220
2018-08-30 \ref_rfc20180830
2018-08-29 \ref_rfc20180829
2018-08-15 \ref_rfc20180815
2018-06-20 \ref_rfc20180620
2018-06-10 \ref_rfc20180610
2018-03-21 \ref_rfc20180321
2009-09-07 \ref_rfc20090907
2009-06-12 \ref_rfc20090612
2008-12-18 \ref_rfc20081218
2008-12-05 \ref_rfc20081205
2008-10-30 \ref_rfc20081030
2008-09-15 \ref_rfc20080915
2008-09-04 \ref_rfc20080904
2008-07-28 \ref_rfc20080728
2008-07-23 \ref_rfc20080723
2008-03-01 \ref_rfc20080301
2008-02-09 \ref_rfc20080209
2008-02-06 \ref_rfc20080206
+ + + +
+ 


+HDF5-DIAG: Error detected in thread 0.  Back trace follows.
+  #000: H5F.c line 1245 in H5Fopen(): unable to open file
+    major(04): File interface
+    minor(10): Unable to open file
+  #001: H5F.c line 846 in H5F_open(): file does not exist
+    major(04): File interface
+    minor(10): Unable to open file
+
+
+ + +

The error handling package (H5E) is described + elsewhere. + +

Invariant Conditions

+ +

To include checks for invariant conditions the library should + be configured with --disable-production, the + default for versions before 1.2. The library designers have made + every attempt to handle error conditions gracefully but an + invariant condition assertion may fail in certain cases. The + output from a failure usually looks something like this: + +

+

+ + + + +
+ 


+Assertion failed: H5.c:123: i<NELMTS(H5_debug_g)
+IOT Trap, core dumped.
+
+
+
+ +

Timings and Statistics

+ +

Code to accumulate statistics is included at compile time by + using the --enable-debug configure switch. The + switch can be followed by an equal sign and a comma-separated + list of package names or else a default list is used. + +

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
NameDefaultDescription
aNoAttributes
acYesMeta data cache
bYesB-Trees
dYesDatasets
eYesError handling
fYesFiles
gYesGroups
hgYesGlobal heap
hlNoLocal heaps
iYesInterface abstraction
mfNoFile memory management
mmYesLibrary memory managment
oNoObject headers and messages
pYesProperty lists
sYesData spaces
tYesDatatypes
vYesVectors
zYesRaw data filters
+
+ +

In addition to including the code at compile time the + application must enable each package at runtime. This is done + by listing the package names in the HDF5_DEBUG + environment variable. That variable may also contain file + descriptor numbers (the default is `2') which control the output + for all following packages up to the next file number. The + word all refers to all packages. Any word my be + preceded by a minus sign to turn debugging off for the package. + +

+

+ + + + + + + + + + + + + + +
Sample debug specifications
allThis causes debugging output from all packages to be + sent to the standard error stream.
all -t -sDebugging output for all packages except datatypes + and data spaces will appear on the standard error + stream.
-all ac 255 t,sThis disables all debugging even if the default was to + debug something, then output from the meta data cache is + send to the standard error stream and output from data + types and spaces is sent to file descriptor 255 which + should be redirected by the shell.
+
+ +

The components of the HDF5_DEBUG value may be + separated by any non-lowercase letter. + +

API Tracing

+ +

The HDF5 library can trace API calls by printing the + function name, the argument names and their values, and the + return value. Some people like to see lots of output during + program execution instead of using a good symbolic debugger, and + this feature is intended for their consumption. For example, + the output from h5ls foo after turning on tracing, + includes: + +

+

+ + + + +
+ 
+H5Tcopy(type=184549388) = 184549419 (type);
+H5Tcopy(type=184549392) = 184549424 (type);
+H5Tlock(type=184549424) = SUCCEED;
+H5Tcopy(type=184549393) = 184549425 (type);
+H5Tlock(type=184549425) = SUCCEED;
+H5Fopen(filename="foo", flags=0, access=H5P_DEFAULT) = FAIL;
+HDF5-DIAG: Error detected in thread 0.  Back trace follows.
+  #000: H5F.c line 1245 in H5Fopen(): unable to open file
+    major(04): File interface
+    minor(10): Unable to open file
+  #001: H5F.c line 846 in H5F_open(): file does not exist
+    major(04): File interface
+    minor(10): Unable to open file
+
+
+
+ +

The code that performs the tracing must be included in the + library by specifying the --enable-trace + configuration switch (the default for versions before 1.2). Then + the word trace must appear in the value of the + HDF5_DEBUG variable. The output will appear on the + last file descriptor before the word trace or two + (standard error) by default. + +

+

+ + + + + + + +
To display the trace on the standard error stream: + 
$ env HDF5_DEBUG=trace a.out
+
+
To send the trace to a file: + 
$ env HDF5_DEBUG="55 trace" a.out 55>trace-output
+
+
+
+ +

Performance

+ +

If the library was not configured for tracing then there is no + unnecessary overhead since all tracing code is excluded. + However, if tracing is enabled but not used there is a small + penalty. First, code size is larger because of extra + statically-declared character strings used to store argument + types and names and extra auto variable pointer in each + function. Also, execution is slower because each function sets + and tests a local variable and each API function calls the + H5_trace() function. + +

If tracing is enabled and turned on then the penalties from the + previous paragraph apply plus the time required to format each + line of tracing information. There is also an extra call to + H5_trace() for each API function to print the return value. + +

Safety

+ +

The tracing mechanism is invoked for each API function before + arguments are checked for validity. If bad arguments are passed + to an API function it could result in a segmentation fault. + However, the tracing output is line-buffered so all previous + output will appear. + +

Completeness

+ +

There are two API functions that don't participate in + tracing. They are H5Eprint() and + H5Eprint_cb() because their participation would + mess up output during automatic error reporting. + +

On the other hand, a number of API functions are called during + library initialization and they print tracing information. + +

Implementation

+ +

For those interested in the implementation here is a + description. Each API function should have a call to one of the + H5TRACE() macros immediately after the + FUNC_ENTER() macro. The first argument is the + return type encoded as a string. The second argument is the + types of all the function arguments encoded as a string. The + remaining arguments are the function arguments. This macro was + designed to be as terse and unobtrousive as possible. + +

In order to keep the H5TRACE() calls synchronized + with the source code we've written a perl script which gets + called automatically just before Makefile dependencies are + calculated for the file. However, this only works when one is + using GNU make. To reinstrument the tracing explicitly, invoke + the trace program from the hdf5 bin directory with + the names of the source files that need to be updated. If any + file needs to be modified then a backup is created by appending + a tilde to the file name. + +

+

+ + + + + +
Explicit Instrumentation
+ 
+$ ../bin/trace *.c
+H5E.c: in function `H5Ewalk_cb':
+H5E.c:336: warning: trace info was not inserted
+
+
+
+ +

Note: The warning message is the result of a comment of the + form /*NO TRACE*/ somewhere in the function + body. Tracing information will not be updated or inserted if + such a comment exists. + +

Error messages have the same format as a compiler so that they + can be parsed from program development environments like + Emacs. Any function which generates an error will not be + modified.

+ + diff --git a/doxygen/examples/FileFormat.html b/doxygen/examples/FileFormat.html new file mode 100644 index 0000000..fc35357 --- /dev/null +++ b/doxygen/examples/FileFormat.html @@ -0,0 +1,1275 @@ + + + + HDF5 File Format Discussion + + + + + + + + + + +

HDF5 File Format Discussion

+

Quincey Koziol
+ koziol@ncsa.uiuc.edu
+ May 15, 2003 +

+ +
    + +

  1. Document's Audience:

    + +
      +
    • Current H5 library designers and knowledgable external developers.
      • +
    + +

  2. Background Reading:

    + +
    +
    HDF5 File Format Specification +
    This describes the current HDF5 file format. +
    + +

  3. Introduction:

    + +
    +
    What is this document about?
    +
    This document attempts to explain the HDF5 file format + specification with a few examples and describes some potential + improvements to the format specification. +

    +
    + +

  4. File Format Examples:

    + +

    This section has several small programs and describes the format of a file +created with each of them. +

    + +

    Example program one - Create an empty file: +

     
+#include "hdf5.h"
+#include 
+
+int main()
+{
+    hid_t fid;      /* File ID */
+    herr_t ret;     /* Generic return value */
+
+    /* Create the file */
+    fid=H5Fcreate("example1.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+    assert(fid>=0);
+
+    /* Close the file */
+    ret=H5Fclose(fid);
+    assert(ret>=0);
+
+    return(0);
+}
+
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Super Block +
    bytebytebytebyte
    \211'H''D''F'
    \r\n\032\n
    0000
    0880
    416
    0x00000003
    0

    0xffffffffffffffff

    ?

    0xffffffffffffffff

    + + + + + + + + + + + + + + + + + + + + +
    0

    928

    H5G_CACHED_STAB (1)
    0
    + + + + + + + +

    384


    96

    +
    +
    +
    +
    +
     
+%h5debug example1.h5
+
+Reading signature at address 0 (rel)
+File Super Block...
+File name:                                         example1.h5
+File access flags                                  0x00000000
+File open reference count:                         1
+Address of super block:                            0 (abs)
+Size of user block:                                0 bytes
+Super block version number:                        0
+Free list version number:                          0
+Root group symbol table entry version number:      0
+Shared header version number:                      0
+Size of file offsets (haddr_t type):               8 bytes
+Size of file lengths (hsize_t type):               8 bytes
+Symbol table leaf node 1/2 rank:                   4
+Symbol table internal node 1/2 rank:               16
+File consistency flags:                            0x00000003
+Base address:                                      0 (abs)
+Free list address:                                 UNDEF (rel)
+Address of driver information block:               UNDEF (rel)
+Root group symbol table entry:
+   Name offset into private heap:                  0
+   Object header address:                          928
+   Dirty:                                          Yes
+   Cache info type:                                Symbol Table
+   Cached information:
+      B-tree address:                              384
+      Heap address:                                96
+
    + + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Root Group Object Header +
    bytebytebytebyte
    102
    1
    32
    0x001116
    0x010
    + + + + + + + +

    384


    96

    +
    00
    0x000
    +
    +
    +
     
+%h5debug example1.h5 928
+
+New address: 928
+Reading signature at address 928 (rel)
+Object Header...
+Dirty:                                             0
+Version:                                           1
+Header size (in bytes):                            16
+Number of links:                                   1
+Number of messages (allocated):                    2 (32)
+Number of chunks (allocated):                      1 (8)
+Chunk 0...
+   Dirty:                                          0
+   Address:                                        944
+   Size in bytes:                                  32
+Message 0...
+   Message ID (sequence number):                   0x0011 stab(0)
+   Shared message:                                 No
+   Constant:                                       Yes
+   Raw size in obj header:                         16 bytes
+   Chunk number:                                   0
+   Message Information:
+      B-tree address:                              384
+      Name heap address:                           96
+Message 1...
+   Message ID (sequence number):                   0x0000 null(0)
+   Shared message:                                 No
+   Constant:                                       No
+   Raw size in obj header:                         0 bytes
+   Chunk number:                                   0
+   Message Information:
+      
+
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Root Group Local Heap +
    bytebytebytebyte
    'H''E''A''P'
    0
    256
    8
    128
    +
    +
    + +
     
+%h5debug example1.h5 96
+
+New address: 96
+Reading signature at address 96 (rel)
+Local Heap...
+Dirty:                                             0
+Header size (in bytes):                            32
+Address of heap data:                              128
+Data bytes allocated on disk:                      256
+Data bytes allocated in core:                      256
+Free Blocks (offset, size):
+   Block #0:                                        8,      248
+Percent of heap used:                              3.12%
+Data follows (`__' indicates free region)...
+     0: 00 00 00 00 00 00 00 00  __ __ __ __ __ __ __ __ ........
+    16: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    32: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    48: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    64: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    80: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    96: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+   112: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+   128: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+   144: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+   160: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+   176: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+   192: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+   208: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+   224: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+   240: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+
+
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Root Group B-tree +
    bytebytebytebyte
    'T''R''E''E'
    000

    0xffffffffffffffff


    0xffffffffffffffff

    +
    +
    +
     
+%h5debug example1.h5 384 96
+
+New address: 384
+Reading signature at address 384 (rel)
+Tree type ID:                                      H5B_SNODE_ID
+Size of node:                                      544
+Size of raw (disk) key:                            8
+Dirty flag:                                        False
+Number of initial dirty children:                  0
+Level:                                             0
+Address of left sibling:                           UNDEF
+Address of right sibling:                          UNDEF
+Number of children (max):                          0 (32)
+
+
    + +

    + +

    Example program two - Create a file with a single dataset in it: +

     
+#include "hdf5.h"
+#include 
+
+int main()
+{
+    hid_t fid;      /* File ID */
+    hid_t sid;      /* Dataspace ID */
+    hid_t did;      /* Dataset ID */
+    herr_t ret;     /* Generic return value */
+
+    /* Create the file */
+    fid=H5Fcreate("example2.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+    assert(fid>=0);
+
+    /* Create a scalar dataspace for the dataset */
+    sid=H5Screate(H5S_SCALAR);
+    assert(sid>=0);
+
+    /* Create a trivial dataset */
+    did=H5Dcreate(fid, "Dataset", H5T_NATIVE_INT, sid, H5P_DEFAULT);
+    assert(did>=0);
+
+    /* Close the dataset */
+    ret=H5Dclose(did);
+    assert(ret>=0);
+
+    /* Close the dataspace */
+    ret=H5Sclose(sid);
+    assert(ret>=0);
+
+    /* Close the file */
+    ret=H5Fclose(fid);
+    assert(ret>=0);
+
+    return(0);
+}
+
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Super Block +
    bytebytebytebyte
    \211'H''D''F'
    \r\n\032\n
    0000
    0880
    416
    0x00000003
    0

    0xffffffffffffffff

    ?

    0xffffffffffffffff

    + + + + + + + + + + + + + + + + + + + + +
    0

    928

    H5G_CACHED_STAB (1)
    0
    + + + + + + + +

    384


    96

    +
    +
    +
    +
    +
     
+%h5debug example2.h5
+
+Reading signature at address 0 (rel)
+File Super Block...
+File name:                                         example2.h5
+File access flags                                  0x00000000
+File open reference count:                         1
+Address of super block:                            0 (abs)
+Size of user block:                                0 bytes
+Super block version number:                        0
+Free list version number:                          0
+Root group symbol table entry version number:      0
+Shared header version number:                      0
+Size of file offsets (haddr_t type):               8 bytes
+Size of file lengths (hsize_t type):               8 bytes
+Symbol table leaf node 1/2 rank:                   4
+Symbol table internal node 1/2 rank:               16
+File consistency flags:                            0x00000003
+Base address:                                      0 (abs)
+Free list address:                                 UNDEF (rel)
+Address of driver information block:               UNDEF (rel)
+Root group symbol table entry:
+   Name offset into private heap:                  0
+   Object header address:                          928
+   Dirty:                                          Yes
+   Cache info type:                                Symbol Table
+   Cached entry information:
+      B-tree address:                              384
+      Heap address:                                96
+
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Root Group Object Header +
    bytebytebytebyte
    102
    1
    32
    0x001116
    0x010
    + + + + + + + +

    384


    96

    +
    00
    0x000
    +
    +
    +
     
+%h5debug example2.h5 928
+
+New address: 928
+Reading signature at address 928 (rel)
+Object Header...
+Dirty:                                             0
+Version:                                           1
+Header size (in bytes):                            16
+Number of links:                                   1
+Number of messages (allocated):                    2 (32)
+Number of chunks (allocated):                      1 (8)
+Chunk 0...
+   Dirty:                                          0
+   Address:                                        944
+   Size in bytes:                                  32
+Message 0...
+   Message ID:                                     0x0011 stab(0)
+   Shared message:                                 No
+   Constant:                                       Yes
+   Raw size in obj header:                         16 bytes
+   Chunk number:                                   0
+   Message Information:
+      B-tree address:                              384
+      Name heap address:                           96
+Message 1...
+   Message ID:                                     0x0000 null(0)
+   Shared message:                                 No
+   Constant:                                       No
+   Raw size in obj header:                         0 bytes
+   Chunk number:                                   0
+   Message Information:
+      
+
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Root Group Local Heap +
    bytebytebytebyte
    'H''E''A''P'
    0
    256
    16
    128
    +
    +
    + +
     
+%h5debug example2.h5 96
+
+New address: 96
+Reading signature at address 96 (rel)
+Local Heap...
+Dirty:                                             0
+Header size (in bytes):                            32
+Address of heap data:                              128
+Data bytes allocated on disk:                      256
+Data bytes allocated in core:                      256
+Free Blocks (offset, size):
+   Block #0:                                       16,      240
+Percent of heap used:                              6.25%
+Data follows (`__' indicates free region)...
+      0: 00 00 00 00 00 00 00 00  44 61 74 61 73 65 74 00 ........Dataset.
+     16: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+     32: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+     48: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+     64: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+     80: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+     96: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    112: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    128: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    144: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    160: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    176: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    192: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    208: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    224: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+    240: __ __ __ __ __ __ __ __  __ __ __ __ __ __ __ __
+
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Root Group B-tree +
    bytebytebytebyte
    'T''R''E''E'
    001

    0xffffffffffffffff


    0xffffffffffffffff


    0


    1248


    8

    +
    +
    +
     
+%h5debug example2.h5 384 96
+
+New address: 384
+Reading signature at address 384 (rel)
+Tree type ID:                                      H5B_SNODE_ID
+Size of node:                                      544
+Size of raw (disk) key:                            8
+Dirty flag:                                        False
+Number of initial dirty children:                  0
+Level:                                             0
+Address of left sibling:                           UNDEF
+Address of right sibling:                          UNDEF
+Number of children (max):                          1 (32)
+Child 0...
+   Address:                                        1248
+   Left Key:
+      Heap offset:                                 0
+      Name :
+   Right Key:
+      Heap offset:                                 8
+      Name :                                       Dataset
+
    + +
    + + + + + + + + + + + + + + + + + + + + + + + +
    + Root Group B-tree Symbol Table Node +
    bytebytebytebyte
    'S''N''O''D'
    101
    + + + + + + + + + + + + + + + + + + + + +
    8

    976

    0
    0


    0


    +
    +
    +
    +
     
+%h5debug example2.h5 1248 96
+
+New address: 1248
+Reading signature at address 1248 (rel)
+Symbol Table Node...
+Dirty:                                             No
+Size of Node (in bytes):                           328
+Number of Symbols:                                 1 of 8
+Symbol 0:
+   Name:                                           `Dataset'
+   Name offset into private heap:                  8
+   Object header address:                          976
+   Dirty:                                          No
+   Cache info type:                                Nothing Cached
+
    + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + '/Dataset' Object Header +
    bytebytebytebyte
    Version: 1Reserved: 0Number of Header Messages: 6
    Object Reference Count: 1
    Total Object Header Size: 256
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Fill Value Header Message +
    Message Type: 0x0005Message Data Size: 8
    Flags: 0x01Reserved: 0
    Version: 1Space Allocation Time: 2 (Late)Fill Value Writing Time: 0 (At allocation)Fill Value Defined: 0 (Undefined)
    Fill Value Datatype Size: 0 (Use dataset's datatype for fill-value datatype)
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Datatype Header Message +
    Message Type: 0x0003Message Data Size: 16
    Flags: 0x01Reserved: 0
    + + + + + +
    Version: 0x1Class: 0x0 (Fixed-Point)
    +     		Fixed-Point Bit-Field: 0x08 (Little-endian, No padding, Signed)
    Size: 4
    Bit Offset: 0Bit Precision: 32
    Message Alignment Filler: -
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Dataspace Header Message +
    Message Type: 0x0001Message Data Size: 8
    Flags: 0x00Reserved: 0
    Version: 1Rank: 0 (Scalar)Flags: 0x00 (No maximum dimensions, no permutation information)Reserved: 0
    Reserved: 0
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Layout Header Message +
    Message Type: 0x0008Message Data Size: 24
    Flags: 0x00Reserved: 0
    Version: 1Rank: 1 (Dataspace rank+1)Class: 1 (Contiguous)Reserved: 0
    Reserved: 0

    Address: 0xffffffffffffffff (Undefined)

    Dimension 0 Size: 4 (Datatype size)
    Message Alignment Filler: -
    +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    + Modification Date & Time Header Message +
    Message Type: 0x0012Message Data Size: 8
    Flags: 0x00Reserved: 0
    Version: 1Reserved: 0
    Seconds Since Epoch: 1052401700 (2003-05-08 08:48:20 CDT)
    +
    + + + + + + + + + + + + + + + + + +
    + Null Header Message +
    Message Type: 0x0000Message Data Size: 144
    Flags: 0x00Reserved: 0
    +
    +
    +
    +
     
+%h5debug example2.h5 976
+
+New address: 976
+Reading signature at address 976 (rel)
+Object Header...
+Dirty:                                             0
+Version:                                           1
+Header size (in bytes):                            16
+Number of links:                                   1
+Number of messages (allocated):                    6 (32)
+Number of chunks (allocated):                      1 (8)
+Chunk 0...
+   Dirty:                                          0
+   Address:                                        992
+   Size in bytes:                                  256
+Message 0...
+   Message ID (sequence number):                   0x0005 `fill_new' (0)
+   Shared:                                         No
+   Constant:                                       Yes
+   Raw size in obj header:                         8 bytes
+   Chunk number:                                   0
+   Message Information:
+      Version:                                     1
+      Space Allocation Time:                       Late
+      Fill Time:                                   On Allocation
+      Fill Value Defined:                          Undefined
+      Size:                                        0
+      Data type:                                   
+Message 1...
+   Message ID (sequence number):                   0x0003 data_type(0)
+   Shared message:                                 No
+   Constant:                                       Yes
+   Raw size in obj header:                         16 bytes
+   Chunk number:                                   0
+   Message Information:
+      Type class:                                  integer
+      Size:                                        4 bytes
+      Byte order:                                  little endian
+      Precision:                                   32 bits
+      Offset:                                      0 bits
+      Low pad type:                                zero
+      High pad type:                               zero
+      Sign scheme:                                 2's comp
+Message 2...
+   Message ID (sequence number):                   0x0001 simple_dspace(0)
+   Shared message:                                 No
+   Constant:                                       No
+   Raw size in obj header:                         8 bytes
+   Chunk number:                                   0
+   Message Information:
+      Rank:                                        0
+Message 3...
+   Message ID (sequence number):                   0x0008 layout(0)
+   Shared message:                                 No
+   Constant:                                       No
+   Raw size in obj header:                         24 bytes
+   Chunk number:                                   0
+   Message Information:
+      Data address:                                UNDEF
+      Number of dimensions:                        1
+      Size:                                        {4}
+Message 4...
+   Message ID (sequence number):                   0x0012 mtime_new(0)
+   Shared message:                                 No
+   Constant:                                       No
+   Raw size in obj header:                         8 bytes
+   Chunk number:                                   0
+   Message Information:
+      Time:                                        2003-03-05 14:52:00 CST
+Message 5...
+   Message ID (sequence number):                   0x0000 null(0)
+   Shared message:                                 No
+   Constant:                                       No
+   Raw size in obj header:                         144 bytes
+   Chunk number:                                   0
+   Message Information:
+      
+
    + +

    + +
+ + + + diff --git a/doxygen/examples/Filters.html b/doxygen/examples/Filters.html new file mode 100644 index 0000000..2d5bc5e --- /dev/null +++ b/doxygen/examples/Filters.html @@ -0,0 +1,450 @@ + + + Filters +

Filters in HDF5

+ + Note: Transient pipelines described in this document have not + been implemented. + +

Introduction

+ +

HDF5 allows chunked data to pass through user-defined filters + on the way to or from disk. The filters operate on chunks of an + H5D_CHUNKED dataset can be arranged in a pipeline + so output of one filter becomes the input of the next filter. + +

Each filter has a two-byte identification number (type + H5Z_filter_t) allocated by The HDF Group and can also be + passed application-defined integer resources to control its + behavior. Each filter also has an optional ASCII comment + string. + +

+ + + + + + + + + + + + + + + + + + + +
Values for H5Z_filter_tDescription
0-255These values are reserved for filters predefined and + registered by the HDF5 library and of use to the general + public. They are described in a separate section + below.
256-511Filter numbers in this range are used for testing only + and can be used temporarily by any organization. No + attempt is made to resolve numbering conflicts since all + definitions are by nature temporary.
512-65535Reserved for future assignment. Please contact the + HDF5 development team + to reserve a value or range of values for + use by your filters.
+ +

Defining and Querying the Filter Pipeline

+ +

Two types of filters can be applied to raw data I/O: permanent + filters and transient filters. The permanent filter pipeline is + defned when the dataset is created while the transient pipeline + is defined for each I/O operation. During an + H5Dwrite() the transient filters are applied first + in the order defined and then the permanent filters are applied + in the order defined. For an H5Dread() the + opposite order is used: permanent filters in reverse order, then + transient filters in reverse order. An H5Dread() + must result in the same amount of data for a chunk as the + original H5Dwrite(). + +

The permanent filter pipeline is defined by calling + H5Pset_filter() for a dataset creation property + list while the transient filter pipeline is defined by calling + that function for a dataset transfer property list. + +

+
herr_t H5Pset_filter (hid_t plist, + H5Z_filter_t filter, unsigned int flags, + size_t cd_nelmts, const unsigned int + cd_values[]) +
This function adds the specified filter and + corresponding properties to the end of the transient or + permanent output filter pipeline (depending on whether + plist is a dataset creation or dataset transfer + property list). The flags argument specifies certain + general properties of the filter and is documented below. The + cd_values is an array of cd_nelmts integers + which are auxiliary data for the filter. The integer values + will be stored in the dataset object header as part of the + filter information. +
int H5Pget_nfilters (hid_t plist) +
This function returns the number of filters defined in the + permanent or transient filter pipeline depending on whether + plist is a dataset creation or dataset transfer + property list. In each pipeline the filters are numbered from + 0 through N-1 where N is the value returned + by this function. During output to the file the filters of a + pipeline are applied in increasing order (the inverse is true + for input). Zero is returned if there are no filters in the + pipeline and a negative value is returned for errors. +
H5Z_filter_t H5Pget_filter (hid_t plist, + int filter_number, unsigned int *flags, + size_t *cd_nelmts, unsigned int + *cd_values, size_t namelen, char name[]) +
This is the query counterpart of + H5Pset_filter() and returns information about a + particular filter number in a permanent or transient pipeline + depending on whether plist is a dataset creation or + dataset transfer property list. On input, cd_nelmts + indicates the number of entries in the cd_values + array allocated by the caller while on exit it contains the + number of values defined by the filter. The + filter_number should be a value between zero and + N-1 as described for H5Pget_nfilters() + and the function will return failure (a negative value) if the + filter number is out of range. If name is a pointer + to an array of at least namelen bytes then the filter + name will be copied into that array. The name will be null + terminated if the namelen is large enough. The + filter name returned will be the name appearing in the file or + else the name registered for the filter or else an empty string. +
+ +

The flags argument to the functions above is a bit vector of + the following fields: + +

+ + + + + + + + + + +
Values for flagsDescription
H5Z_FLAG_OPTIONALIf this bit is set then the filter is optional. If + the filter fails (see below) during an + H5Dwrite() operation then the filter is + just excluded from the pipeline for the chunk for which + it failed; the filter will not participate in the + pipeline during an H5Dread() of the chunk. + This is commonly used for compression filters: if the + compression result would be larger than the input then + the compression filter returns failure and the + uncompressed data is stored in the file. If this bit is + clear and a filter fails then the + H5Dwrite() or H5Dread() also + fails.
+ +

Defining Filters

+ +

Each filter is bidirectional, handling both input and output to + the file, and a flag is passed to the filter to indicate the + direction. In either case the filter reads a chunk of data from + a buffer, usually performs some sort of transformation on the + data, places the result in the same or new buffer, and returns + the buffer pointer and size to the caller. If something goes + wrong the filter should return zero to indicate a failure. + +

During output, a filter that fails or isn't defined and is + marked as optional is silently excluded from the pipeline and + will not be used when reading that chunk of data. A required + filter that fails or isn't defined causes the entire output + operation to fail. During input, any filter that has not been + excluded from the pipeline during output and fails or is not + defined will cause the entire input operation to fail. + +

Filters are defined in two phases. The first phase is to + define a function to act as the filter and link the function + into the application. The second phase is to register the + function, associating the function with an + H5Z_filter_t identification number and a comment. + +

+
typedef size_t (*H5Z_func_t)(unsigned int + flags, size_t cd_nelmts, const unsigned int + cd_values[], size_t nbytes, size_t + *buf_size, void **buf) +
The flags, cd_nelmts, and + cd_values are the same as for the + H5Pset_filter() function with the additional flag + H5Z_FLAG_REVERSE which is set when the filter is + called as part of the input pipeline. The input buffer is + pointed to by *buf and has a total size of + *buf_size bytes but only nbytes are valid + data. The filter should perform the transformation in place if + possible and return the number of valid bytes or zero for + failure. If the transformation cannot be done in place then + the filter should allocate a new buffer with + malloc() and assign it to *buf, + assigning the allocated size of that buffer to + *buf_size. The old buffer should be freed + by calling free(). + +

+
herr_t H5Zregister (H5Z_filter_t filter_id, + const char *comment, H5Z_func_t + filter) +
The filter function is associated with a filter + number and a short ASCII comment which will be stored in the + hdf5 file if the filter is used as part of a permanent + pipeline during dataset creation. +
+ +

Predefined Filters

+ +

If zlib version 1.1.2 or later was found + during configuration then the library will define a filter whose + H5Z_filter_t number is + H5Z_FILTER_DEFLATE. Since this compression method + has the potential for generating compressed data which is larger + than the original, the H5Z_FLAG_OPTIONAL flag + should be turned on so such cases can be handled gracefully by + storing the original data instead of the compressed data. The + cd_nvalues should be one with cd_value[0] + being a compression agression level between zero and nine, + inclusive (zero is the fastest compression while nine results in + the best compression ratio). + +

A convenience function for adding the + H5Z_FILTER_DEFLATE filter to a pipeline is: + +

+
herr_t H5Pset_deflate (hid_t plist, unsigned + aggression) +
The deflate compression method is added to the end of the + permanent or transient filter pipeline depending on whether + plist is a dataset creation or dataset transfer + property list. The aggression is a number between + zero and nine (inclusive) to indicate the tradeoff between + speed and compression ratio (zero is fastest, nine is best + ratio). +
+ +

Even if the zlib isn't detected during + configuration the application can define + H5Z_FILTER_DEFLATE as a permanent filter. If the + filter is marked as optional (as with + H5Pset_deflate()) then it will always fail and be + automatically removed from the pipeline. Applications that read + data will fail only if the data is actually compressed; they + won't fail if H5Z_FILTER_DEFLATE was part of the + permanent output pipeline but was automatically excluded because + it didn't exist when the data was written. + +

zlib can be acquired from + + http://www.cdrom.com/pub/infozip/zlib/. + +

Example

+ +

This example shows how to define and register a simple filter + that adds a checksum capability to the data stream. + +

The function that acts as the filter always returns zero + (failure) if the md5() function was not detected at + configuration time (left as an excercise for the reader). + Otherwise the function is broken down to an input and output + half. The output half calculates a checksum, increases the size + of the output buffer if necessary, and appends the checksum to + the end of the buffer. The input half calculates the checksum + on the first part of the buffer and compares it to the checksum + already stored at the end of the buffer. If the two differ then + zero (failure) is returned, otherwise the buffer size is reduced + to exclude the checksum. + +

+ + + + +
+ 


+                  size_t
+                  md5_filter(unsigned int flags, size_t cd_nelmts,
+                  const unsigned int cd_values[], size_t nbytes,
+                  size_t *buf_size, void **buf)
+                  {
+                  #ifdef HAVE_MD5
+                  unsigned char       cksum[16];
+
+                  if (flags & H5Z_REVERSE) {
+                  /* Input */
+                  assert(nbytes>=16);
+                  md5(nbytes-16, *buf, cksum);
+
+                  /* Compare */
+                  if (memcmp(cksum, (char*)(*buf)+nbytes-16, 16)) {
+                  return 0; /*fail*/
+                  }
+
+                  /* Strip off checksum */
+                  return nbytes-16;
+
+                  } else {
+                  /* Output */
+                  md5(nbytes, *buf, cksum);
+
+                  /* Increase buffer size if necessary */
+                  if (nbytes+16>*buf_size) {
+                  *buf_size = nbytes + 16;
+                  *buf = realloc(*buf, *buf_size);
+                  }
+
+                  /* Append checksum */
+                  memcpy((char*)(*buf)+nbytes, cksum, 16);
+                  return nbytes+16;
+                  }
+                  #else
+                  return 0; /*fail*/
+                  #endif
+                  }
+
+
+ +

Once the filter function is defined it must be registered so + the HDF5 library knows about it. Since we're testing this + filter we choose one of the H5Z_filter_t numbers + from the reserved range. We'll randomly choose 305. + +

+

+ + + + +
+ 


+                  #define FILTER_MD5 305
+                  herr_t status = H5Zregister(FILTER_MD5, "md5 checksum", md5_filter);
+
+
+ +

Now we can use the filter in a pipeline. We could have added + the filter to the pipeline before defining or registering the + filter as long as the filter was defined and registered by time + we tried to use it (if the filter is marked as optional then we + could have used it without defining it and the library would + have automatically removed it from the pipeline for each chunk + written before the filter was defined and registered). + +

+

+ + + + +
+ 


+                  hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE);
+                  hsize_t chunk_size[3] = {10,10,10};
+                  H5Pset_chunk(dcpl, 3, chunk_size);
+                  H5Pset_filter(dcpl, FILTER_MD5, 0, 0, NULL);
+                  hid_t dset = H5Dcreate(file, "dset", H5T_NATIVE_DOUBLE, space, dcpl);
+
+
+ +

6. Filter Diagnostics

+ +

If the library is compiled with debugging turned on for the H5Z + layer (usually as a result of configure + --enable-debug=z) then filter statistics are printed when + the application exits normally or the library is closed. The + statistics are written to the standard error stream and include + two lines for each filter that was used: one for input and one + for output. The following fields are displayed: + +

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Field NameDescription
MethodThis is the name of the method as defined with + H5Zregister() with the charaters + "< or ">" prepended to indicate + input or output.
TotalThe total number of bytes processed by the filter + including errors. This is the maximum of the + nbytes argument or the return value. +
ErrorsThis field shows the number of bytes of the Total + column which can be attributed to errors.
User, System, ElapsedThese are the amount of user time, system time, and + elapsed time in seconds spent in the filter function. + Elapsed time is sensitive to system load. These times + may be zero on operating systems that don't support the + required operations.
BandwidthThis is the filter bandwidth which is the total + number of bytes processed divided by elapsed time. + Since elapsed time is subject to system load the + bandwidth numbers cannot always be trusted. + Furthermore, the bandwidth includes bytes attributed to + errors which may significanly taint the value if the + function is able to detect errors without much + expense.
+ +

+

+ + + + + +
+ Example: Filter Statistics +
+ 

H5Z: filter statistics accumulated ov=
+                  er life of library:
+                  Method     Total  Errors  User  System  Elapsed Bandwidth
+                  ------     -----  ------  ----  ------  ------- ---------
+                  >deflate  160000   40000  0.62    0.74     1.33 117.5 kBs
+                  <deflate  120000       0  0.11    0.00     0.12 1.000 MBs
+
+
+ +
+ + +

Footnote 1: Dataset chunks can be compressed + through the use of filters. Developers should be aware that + reading and rewriting compressed chunked data can result in holes + in an HDF5 file. In time, enough such holes can increase the + file size enough to impair application or library performance + when working with that file. See + + Freespace Management + in the chapter + + Performance Analysis and Issues.

+ diff --git a/doxygen/examples/IOFlow.html b/doxygen/examples/IOFlow.html new file mode 100644 index 0000000..6b2c27e --- /dev/null +++ b/doxygen/examples/IOFlow.html @@ -0,0 +1,137 @@ + + + + HDF5 Raw I/O Flow Notes + + + + + + + + +

HDF5 Raw I/O Flow Notes

+

Quincey Koziol
+ koziol@ncsa.uiuc.edu
+ August 20, 2003 +

+ +
    + +

  1. Document's Audience:

    + +
      +
    • Current H5 library designers and knowledgable external developers.
      • +
    + +

  2. Background Reading:

    + +

  3. Introduction:

    + +
    +
    What is this document about?
    +
    This document attempts to supplement the flow charts describing + the flow of control for raw data I/O in the library. +

    +
    + +

  4. Figures:

    +

    The following figures provide the main information:

    + + + + +
    High-Level View of Writing Raw Data
    Perform Serial or Parallel I/O
    Gather/Convert/Scatter
    + +

  5. Notes From Accompanying Figures:

    + +

    This section provides notes to augment the information in the accompanying + figures. +

    + +
      +
    1. Validate Parameters - Resolve any H5S_ALL parameters + for dataspace selections to actual dataspaces, allocate + conversion buffers, etc. +
      2. + +
    2. Space Allocated in File? - Space may not have been allocated + in the file to store the dataset data, if "late allocation" was chosen + for the allocation time when the dataset was created. +
      3. + +
    3. Allocate & Fill Space - These operations allocate both contiguous + and chunked dataset's space in the file. The chunked dataset space + allocation iterates through all the chunks in the file and allocates + both the B-tree information and the raw data in the file. Because of + the way filters work, fill-values are written out for chunked datasets + as they are allocated, instead of as a separate step. + In parallel + I/O, the chunked dataset allocation can potentially be time-consuming, + since all the raw data in the dataset is allocated from one process. +
      4. + +
    4. Datatype Conversion Needed? - This currently is the deciding + factor between doing "direct I/O" (in serial or parallel) and needing + to perform gather/convert/scatter operations. I believe that MPI + is capable of performing a limited range of type conversions and if so, + we should add support to detect when they can be used. This will + allow more I/O operations to be performed collectively. +
      5. + +
    5. Collective I/O Requested/Allowed? - A user has to both request + that collective I/O occur and also their I/O operation must meet the + requirements that the library sets for supporting collective parallel + I/O: +
        +
      • The dataspace must be scalar or simple (which is a no-op really, + since we don't support "complex" dataspaces in the library + currently). +
        • +
      • The selection must be regular. "all" selections + and hyperslab selections that were + made with only one call to H5Sselect_hyperslab() (i.e. not a + hyperslab selection that has been aggregated over multiple + selection calls) are regular. Supporting point and + irregular hyperslab selections are on the "to do" list. +
        • +
      • The dataset must be stored contiguously on disk (as shown in the + figure also). Supporting chunked dataset storage is also + on the "to do" list. +
        • +
      +
      6. + +
    6. Build "chunk map" - This step still has some scalability issues + as it creates a data structure that is proportional to the number of + chunks which will be written to, which could potentially be very large. + Building the "chunk map" information incrementally is on the "to do" + list also. +
      7. + +
    7. Perform Chunked I/O - As the figure shows, there is no support + for collective parallel I/O on chunked datasets currently. As noted + earlier, this is on the "to do" list. +
      8. + +
    8. Perform "Direct" Serial I/O - "Direct" serial I/O writes data + from the application's buffer, without any intervening buffer or memory + copies. For maximum efficiency and performance, the elements in the + selections should be adjoining. +
      9. + +
    9. Perform Collective Parallel I/O - This step also writes data + directly from an application buffer, but additionally uses collective + MPI I/O operations to combine the data from each process in the parallel + application in an efficient manner. +
      10. +
    + +
+ + + + diff --git a/doxygen/hdf5doxy_layout.xml b/doxygen/hdf5doxy_layout.xml index fc20aa1..6efa690 100644 --- a/doxygen/hdf5doxy_layout.xml +++ b/doxygen/hdf5doxy_layout.xml @@ -12,6 +12,7 @@ + diff --git a/doxygen/img/IOFlow.gif b/doxygen/img/IOFlow.gif new file mode 100644 index 0000000..3e79030 Binary files /dev/null and b/doxygen/img/IOFlow.gif differ diff --git a/doxygen/img/IOFlow2.gif b/doxygen/img/IOFlow2.gif new file mode 100644 index 0000000..c75ca79 Binary files /dev/null and b/doxygen/img/IOFlow2.gif differ diff --git a/doxygen/img/IOFlow3.gif b/doxygen/img/IOFlow3.gif new file mode 100644 index 0000000..316cd1e Binary files /dev/null and b/doxygen/img/IOFlow3.gif differ diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 3a485da..4129f50 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -216,7 +216,7 @@ import hdf.hdf5lib.structs.H5O_token_t; * exception handlers to print out the HDF-5 error stack. *
* - * @version HDF5 1.13.0
+ * @version HDF5 1.13.1
* See also: hdf.hdf5lib.HDFArray
* hdf.hdf5lib.HDF5Constants
* hdf.hdf5lib.HDF5CDataTypes
@@ -239,7 +239,7 @@ public class H5 implements java.io.Serializable { * * Make sure to update the versions number when a different library is used. */ - public final static int LIB_VERSION[] = { 1, 13, 0 }; + public final static int LIB_VERSION[] = { 1, 13, 1 }; /** * add system property to load library by path diff --git a/java/test/TestH5.java b/java/test/TestH5.java index 1f298f3..1f81f09 100644 --- a/java/test/TestH5.java +++ b/java/test/TestH5.java @@ -287,7 +287,7 @@ public class TestH5 { */ @Test public void testH5get_libversion() { - int libversion[] = { 1, 13, 0 }; + int libversion[] = { 1, 13, 1 }; try { H5.H5get_libversion(libversion); @@ -326,7 +326,7 @@ public class TestH5 { */ @Test public void testH5check_version() { - int majnum = 1, minnum = 13, relnum = 0; + int majnum = 1, minnum = 13, relnum = 1; try { H5.H5check_version(majnum, minnum, relnum); diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index fbdbf3f..017d750 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -1,4 +1,4 @@ -HDF5 version 1.13.0-7 currently under development +HDF5 version 1.13.1-1 currently under development ================================================================================ @@ -47,6 +47,12 @@ New Features Configuration: ------------- + - Added new configure option to support building parallel tools. + See Tools below (autotools - CMake): + --enable-parallel-tools HDF5_BUILD_PARALLEL_TOOLS + + (RAW - 2021/10/25) + - Added new configure options to enable dimension scales APIs (H5DS*) to use new object references with the native VOL connector (aka native HDF5 library). New references are always used for non-native terminal VOL @@ -193,19 +199,19 @@ New Features - CMake option to build the HDF filter plugins project as an external project - The HDF filter plugins project is a collection of registered compression - filters that can be dynamically loaded when needed to access data stored - in a hdf5 file. This CMake-only option allows the plugins to be built and - distributed with the hdf5 library and tools. Like the options for szip and - zlib, either a tgz file or a git repository can be specified for the source. + The HDF filter plugins project is a collection of registered compression + filters that can be dynamically loaded when needed to access data stored + in a hdf5 file. This CMake-only option allows the plugins to be built and + distributed with the hdf5 library and tools. Like the options for szip and + zlib, either a tgz file or a git repository can be specified for the source. - The option was refactored to use the CMake FetchContent process. This allows - more control over the filter targets, but required external project command - options to be moved to a CMake include file, HDF5PluginCache.cmake. Also - enabled the filter examples to be used as tests for operation of the - filter plugins. + The option was refactored to use the CMake FetchContent process. This allows + more control over the filter targets, but required external project command + options to be moved to a CMake include file, HDF5PluginCache.cmake. Also + enabled the filter examples to be used as tests for operation of the + filter plugins. - (ADB - 2020/12/10, OESS-98) + (ADB - 2020/12/10, OESS-98) - FreeBSD Autotools configuration now defaults to 'cc' and 'c++' compilers @@ -258,244 +264,244 @@ New Features - Autotools and CMake target added to produce doxygen generated documentation - The default is OFF or disabled. - Autoconf option is '--enable-doxygen' - autotools make target is 'doxygen' and will build all doxygen targets - CMake configure option is 'HDF5_BUILD_DOC'. - CMake target is 'doxygen' for all available doxygen targets - CMake target is 'hdf5lib_doc' for the src subdirectory + The default is OFF or disabled. + Autoconf option is '--enable-doxygen' + autotools make target is 'doxygen' and will build all doxygen targets + CMake configure option is 'HDF5_BUILD_DOC'. + CMake target is 'doxygen' for all available doxygen targets + CMake target is 'hdf5lib_doc' for the src subdirectory - (ADB - 2020/11/03) + (ADB - 2020/11/03) - CMake option to use MSVC naming conventions with MinGW - HDF5_MSVC_NAMING_CONVENTION option enable to use MSVC naming conventions - when using a MinGW toolchain + HDF5_MSVC_NAMING_CONVENTION option enable to use MSVC naming conventions + when using a MinGW toolchain - (xan - 2020/10/30) + (xan - 2020/10/30) - CMake option to statically link gcc libs with MinGW - HDF5_MINGW_STATIC_GCC_LIBS allows to statically link libg/libstdc++ - with the MinGW toolchain + HDF5_MINGW_STATIC_GCC_LIBS allows to statically link libg/libstdc++ + with the MinGW toolchain - (xan - 2020/10/30) + (xan - 2020/10/30) - CMake option to build the HDF filter plugins project as an external project - The HDF filter plugins project is a collection of registered compression - filters that can be dynamically loaded when needed to access data stored - in a hdf5 file. This CMake-only option allows the plugins to be built and - distributed with the hdf5 library and tools. Like the options for szip and - zlib, either a tgz file or a git repository can be specified for the source. + The HDF filter plugins project is a collection of registered compression + filters that can be dynamically loaded when needed to access data stored + in a hdf5 file. This CMake-only option allows the plugins to be built and + distributed with the hdf5 library and tools. Like the options for szip and + zlib, either a tgz file or a git repository can be specified for the source. - The necessary options are (see the INSTALL_CMake.txt file): - HDF5_ENABLE_PLUGIN_SUPPORT - PLUGIN_TGZ_NAME or PLUGIN_GIT_URL - There are more options necessary for various filters and the plugin project - documents should be referenced. + The necessary options are (see the INSTALL_CMake.txt file): + HDF5_ENABLE_PLUGIN_SUPPORT + PLUGIN_TGZ_NAME or PLUGIN_GIT_URL + There are more options necessary for various filters and the plugin project + documents should be referenced. - (ADB - 2020/09/27, OESS-98) + (ADB - 2020/09/27, OESS-98) - Added CMake option to format source files - HDF5_ENABLE_FORMATTERS option will enable creation of targets using the - pattern - HDF5_*_SRC_FORMAT - where * corresponds to the source folder - or tool folder. All sources can be formatted by executing the format target; - make format + HDF5_ENABLE_FORMATTERS option will enable creation of targets using the + pattern - HDF5_*_SRC_FORMAT - where * corresponds to the source folder + or tool folder. All sources can be formatted by executing the format target; + make format - (ADB - 2020/08/24) + (ADB - 2020/08/24) - Add file locking configure and CMake options - HDF5 1.10.0 introduced a file locking scheme, primarily to help - enforce SWMR setup. Formerly, the only user-level control of the scheme - was via the HDF5_USE_FILE_LOCKING environment variable. + HDF5 1.10.0 introduced a file locking scheme, primarily to help + enforce SWMR setup. Formerly, the only user-level control of the scheme + was via the HDF5_USE_FILE_LOCKING environment variable. - This change introduces configure-time options that control whether - or not file locking will be used and whether or not the library - ignores errors when locking has been disabled on the file system - (useful on some HPC Lustre installations). + This change introduces configure-time options that control whether + or not file locking will be used and whether or not the library + ignores errors when locking has been disabled on the file system + (useful on some HPC Lustre installations). - In both the Autotools and CMake, the settings have the effect of changing - the default property list settings (see the H5Pset/get_file_locking() - entry, below). + In both the Autotools and CMake, the settings have the effect of changing + the default property list settings (see the H5Pset/get_file_locking() + entry, below). - The yes/no/best-effort file locking configure setting has also been - added to the libhdf5.settings file. + The yes/no/best-effort file locking configure setting has also been + added to the libhdf5.settings file. - Autotools: + Autotools: - An --enable-file-locking=(yes|no|best-effort) option has been added. + An --enable-file-locking=(yes|no|best-effort) option has been added. - yes: Use file locking. - no: Do not use file locking. - best-effort: Use file locking and ignore "disabled" errors. + yes: Use file locking. + no: Do not use file locking. + best-effort: Use file locking and ignore "disabled" errors. - CMake: + CMake: - Two self-explanatory options have been added: + Two self-explanatory options have been added: - HDF5_USE_FILE_LOCKING - HDF5_IGNORE_DISABLED_FILE_LOCKS + HDF5_USE_FILE_LOCKING + HDF5_IGNORE_DISABLED_FILE_LOCKS - Setting both of these to ON is the equivalent to the Autotools' - best-effort setting. + Setting both of these to ON is the equivalent to the Autotools' + best-effort setting. - NOTE: - The precedence order of the various file locking control mechanisms is: + NOTE: + The precedence order of the various file locking control mechanisms is: - 1) HDF5_USE_FILE_LOCKING environment variable (highest) + 1) HDF5_USE_FILE_LOCKING environment variable (highest) - 2) H5Pset_file_locking() + 2) H5Pset_file_locking() - 3) configure/CMake options (which set the property list defaults) + 3) configure/CMake options (which set the property list defaults) - 4) library defaults (currently best-effort) + 4) library defaults (currently best-effort) - (DER - 2020/07/30, HDFFV-11092) + (DER - 2020/07/30, HDFFV-11092) - CMake option to link the generated Fortran MOD files into the include directory. - The Fortran generation of MOD files by a Fortran compile can produce - different binary files between SHARED and STATIC compiles with different - compilers and/or different platforms. Note that it has been found that - different versions of Fortran compilers will produce incompatible MOD - files. Currently, CMake will locate these MOD files in subfolders of - the include directory and add that path to the Fortran library target - in the CMake config file, which can be used by the CMake find library - process. For other build systems using the binary from a CMake install, - a new CMake configuration can be used to copy the pre-chosen version - of the Fortran MOD files into the install include directory. - - The default will depend on the configuration of - BUILD_STATIC_LIBS and BUILD_SHARED_LIBS: - YES YES Default to SHARED - YES NO Default to STATIC - NO YES Default to SHARED - NO NO Default to SHARED - The defaults can be overridden by setting the config option - HDF5_INSTALL_MOD_FORTRAN to one of NO, SHARED, or STATIC - - (ADB - 2020/07/09, HDFFV-11116) + The Fortran generation of MOD files by a Fortran compile can produce + different binary files between SHARED and STATIC compiles with different + compilers and/or different platforms. Note that it has been found that + different versions of Fortran compilers will produce incompatible MOD + files. Currently, CMake will locate these MOD files in subfolders of + the include directory and add that path to the Fortran library target + in the CMake config file, which can be used by the CMake find library + process. For other build systems using the binary from a CMake install, + a new CMake configuration can be used to copy the pre-chosen version + of the Fortran MOD files into the install include directory. + + The default will depend on the configuration of + BUILD_STATIC_LIBS and BUILD_SHARED_LIBS: + YES YES Default to SHARED + YES NO Default to STATIC + NO YES Default to SHARED + NO NO Default to SHARED + The defaults can be overridden by setting the config option + HDF5_INSTALL_MOD_FORTRAN to one of NO, SHARED, or STATIC + + (ADB - 2020/07/09, HDFFV-11116) - CMake option to use AEC (open source SZip) library instead of SZip - The open source AEC library is a replacement library for SZip. In - order to use it for hdf5 the libaec CMake source was changed to add - "-fPIC" and exclude test files. Autotools does not build the - compression libraries within hdf5 builds. New option USE_LIBAEC is - required to compensate for the different files produced by AEC build. + The open source AEC library is a replacement library for SZip. In + order to use it for hdf5 the libaec CMake source was changed to add + "-fPIC" and exclude test files. Autotools does not build the + compression libraries within hdf5 builds. New option USE_LIBAEC is + required to compensate for the different files produced by AEC build. - (ADB - 2020/04/22, OESS-65) + (ADB - 2020/04/22, OESS-65) - CMake ConfigureChecks.cmake file now uses CHECK_STRUCT_HAS_MEMBER - Some handcrafted tests in HDFTests.c has been removed and the CMake - CHECK_STRUCT_HAS_MEMBER module has been used. + Some handcrafted tests in HDFTests.c has been removed and the CMake + CHECK_STRUCT_HAS_MEMBER module has been used. - (ADB - 2020/03/24, TRILAB-24) + (ADB - 2020/03/24, TRILAB-24) - Both build systems use same set of warnings flags - GNU C, C++ and gfortran warnings flags were moved to files in a config - sub-folder named gnu-warnings. Flags that only are available for a specific - version of the compiler are in files named with that version. - Clang C warnings flags were moved to files in a config sub-folder - named clang-warnings. - Intel C, Fortran warnings flags were moved to files in a config sub-folder - named intel-warnings. + GNU C, C++ and gfortran warnings flags were moved to files in a config + sub-folder named gnu-warnings. Flags that only are available for a specific + version of the compiler are in files named with that version. + Clang C warnings flags were moved to files in a config sub-folder + named clang-warnings. + Intel C, Fortran warnings flags were moved to files in a config sub-folder + named intel-warnings. - There are flags in named "error-xxx" files with warnings that may - be promoted to errors. Some source files may still need fixes. + There are flags in named "error-xxx" files with warnings that may + be promoted to errors. Some source files may still need fixes. - There are also pairs of files named "developer-xxx" and "no-developer-xxx" - that are chosen by the CMake option:HDF5_ENABLE_DEV_WARNINGS or the - configure option:--enable-developer-warnings. + There are also pairs of files named "developer-xxx" and "no-developer-xxx" + that are chosen by the CMake option:HDF5_ENABLE_DEV_WARNINGS or the + configure option:--enable-developer-warnings. - In addition, CMake no longer applies these warnings for examples. + In addition, CMake no longer applies these warnings for examples. - (ADB - 2020/03/24, TRILAB-192) + (ADB - 2020/03/24, TRILAB-192) - Added test script for file size compare - If CMake minimum version is at least 3.14, the fileCompareTest.cmake - script will compare file sizes. + If CMake minimum version is at least 3.14, the fileCompareTest.cmake + script will compare file sizes. - (ADB - 2020/02/24, HDFFV-11036) + (ADB - 2020/02/24, HDFFV-11036) - Update CMake minimum version to 3.12 - Updated CMake minimum version to 3.12 and added version checks - for Windows features. + Updated CMake minimum version to 3.12 and added version checks + for Windows features. - (ADB - 2020/02/05, TRILABS-142) + (ADB - 2020/02/05, TRILABS-142) - Fixed CMake include properties for Fortran libraries - Corrected the library properties for Fortran to use the - correct path for the Fortran module files. + Corrected the library properties for Fortran to use the + correct path for the Fortran module files. - (ADB - 2020/02/04, HDFFV-11012) + (ADB - 2020/02/04, HDFFV-11012) - Added common warnings files for gnu and intel - Added warnings files to use one common set of flags - during configure for both autotools and CMake build - systems. The initial implementation only affects a - general set of flags for gnu and intel compilers. + Added warnings files to use one common set of flags + during configure for both autotools and CMake build + systems. The initial implementation only affects a + general set of flags for gnu and intel compilers. - (ADB - 2020/01/17) + (ADB - 2020/01/17) - Added new options to CMake for control of testing - Added CMake options (default ON); - HDF5_TEST_SERIAL AND/OR HDF5_TEST_PARALLEL - combined with: - HDF5_TEST_TOOLS - HDF5_TEST_EXAMPLES - HDF5_TEST_SWMR - HDF5_TEST_FORTRAN - HDF5_TEST_CPP - HDF5_TEST_JAVA + Added CMake options (default ON); + HDF5_TEST_SERIAL AND/OR HDF5_TEST_PARALLEL + combined with: + HDF5_TEST_TOOLS + HDF5_TEST_EXAMPLES + HDF5_TEST_SWMR + HDF5_TEST_FORTRAN + HDF5_TEST_CPP + HDF5_TEST_JAVA - (ADB - 2020/01/15, HDFFV-11001) + (ADB - 2020/01/15, HDFFV-11001) - Added Clang sanitizers to CMake for analyzer support if compiler is clang. - Added CMake code and files to execute the Clang sanitizers if - HDF5_ENABLE_SANITIZERS is enabled and the USE_SANITIZER option - is set to one of the following: - Address - Memory - MemoryWithOrigins - Undefined - Thread - Leak - 'Address;Undefined' + Added CMake code and files to execute the Clang sanitizers if + HDF5_ENABLE_SANITIZERS is enabled and the USE_SANITIZER option + is set to one of the following: + Address + Memory + MemoryWithOrigins + Undefined + Thread + Leak + 'Address;Undefined' - (ADB - 2019/12/12, TRILAB-135) + (ADB - 2019/12/12, TRILAB-135) - Update CMake for VS2019 support - CMake added support for VS2019 in version 3.15. Changes to the CMake - generator setting required changes to scripts. Also updated version - references in CMake files as necessary. + CMake added support for VS2019 in version 3.15. Changes to the CMake + generator setting required changes to scripts. Also updated version + references in CMake files as necessary. - (ADB - 2019/11/18, HDFFV-10962) + (ADB - 2019/11/18, HDFFV-10962) - Update CMake options to match new autotools options - Add configure options (autotools - CMake): - enable-asserts HDF5_ENABLE_ASSERTS - enable-symbols HDF5_ENABLE_SYMBOLS - enable-profiling HDF5_ENABLE_PROFILING - enable-optimization HDF5_ENABLE_OPTIMIZATION - In addition NDEBUG is no longer forced defined and relies on the CMake - process. + Add configure options (autotools - CMake): + enable-asserts HDF5_ENABLE_ASSERTS + enable-symbols HDF5_ENABLE_SYMBOLS + enable-profiling HDF5_ENABLE_PROFILING + enable-optimization HDF5_ENABLE_OPTIMIZATION + In addition NDEBUG is no longer forced defined and relies on the CMake + process. - (ADB - 2019/10/07, HDFFV-100901, HDFFV-10637, TRILAB-97) + (ADB - 2019/10/07, HDFFV-100901, HDFFV-10637, TRILAB-97) Library: @@ -526,11 +532,11 @@ New Features defined as: struct H5FD_ctl_memcpy_args_t { - void * dstbuf; /**< Destination buffer */ - hsize_t dst_off; /**< Offset within destination buffer */ - const void *srcbuf; /**< Source buffer */ - hsize_t src_off; /**< Offset within source buffer */ - size_t len; /**< Length of data to copy from source buffer */ + void * dstbuf; /**< Destination buffer */ + hsize_t dst_off; /**< Offset within destination buffer */ + const void *srcbuf; /**< Source buffer */ + hsize_t src_off; /**< Offset within source buffer */ + size_t len; /**< Length of data to copy from source buffer */ } H5FD_ctl_memcpy_args_t; Further, HDF5's compact dataset I/O routines were @@ -540,7 +546,7 @@ New Features "ctl" callback operation in order to ask the underlying file driver to correctly handle memory copies. - (JTH - 2021/09/28) + (JTH - 2021/09/28) - Adds new "ctl" callback to VFD H5FD_class_t structure with the following prototype: @@ -565,7 +571,7 @@ New Features `output` [out] - A pointer for the receiving VFD to use for output from the operation - (JRM - 2021/08/16) + (JRM - 2021/08/16) - Change how the release part of version, in major.minor.release is checked for compatibility @@ -584,7 +590,7 @@ New Features which case the release part of version, in major.minor.release, must be exact. An environment variable still controls the logic. - (ADB - 2021/07/27) + (ADB - 2021/07/27) - gcc warning suppression macros were moved out of H5public.h @@ -597,7 +603,7 @@ New Features VFD refactoring, the macros have been duplicated in H5FDmulti.c to suppress the format string warnings there. - (DER - 2021/06/03) + (DER - 2021/06/03) - H5Gcreate1() now rejects size_hint parameters larger than UINT32_MAX @@ -614,7 +620,7 @@ New Features The Doxygen documentation has been updated and passing values larger than UINT32_MAX for size_hint will now produce a normal HDF5 error. - (DER - 2021/04/29, HDFFV-11241) + (DER - 2021/04/29, HDFFV-11241) - H5Pset_fapl_log() no longer crashes when passed an invalid fapl ID @@ -628,7 +634,7 @@ New Features The pointer is now correctly initialized and the API call now produces a normal HDF5 error when fed an invalid fapl ID. - (DER - 2021/04/28, HDFFV-11240) + (DER - 2021/04/28, HDFFV-11240) - Fixes a segfault when H5Pset_mdc_log_options() is called multiple times @@ -640,7 +646,7 @@ New Features The string is now handled properly and the segfault no longer occurs. - (DER - 2021/04/27, HDFFV-11239) + (DER - 2021/04/27, HDFFV-11239) - HSYS_GOTO_ERROR now emits the results of GetLastError() on Windows @@ -659,15 +665,15 @@ New Features The format string on Windows has been changed from: - "%s, errno = %d, error message = '%s'" + "%s, errno = %d, error message = '%s'" to: - "%s, errno = %d, error message = '%s', Win32 GetLastError() = %"PRIu32"" + "%s, errno = %d, error message = '%s', Win32 GetLastError() = %"PRIu32"" for those inclined to parse it for error values. - (DER - 2021/03/21) + (DER - 2021/03/21) - File locking now works on Windows @@ -683,26 +689,26 @@ New Features same scheme as POSIX systems. We lock the entire file when we set up the locks (by passing DWORDMAX as both size parameters to LockFileEx()). - (DER - 2021/03/19, HDFFV-10191) + (DER - 2021/03/19, HDFFV-10191) - H5Epush_ret() now requires a trailing semicolon - H5Epush_ret() is a function-like macro that has been changed to - contain a `do {} while(0)` loop. Consequently, a trailing semicolon - is now required to end the `while` statement. Previously, a trailing - semi would work, but was not mandatory. This change was made to allow - clang-format to correctly format the source code. + H5Epush_ret() is a function-like macro that has been changed to + contain a `do {} while(0)` loop. Consequently, a trailing semicolon + is now required to end the `while` statement. Previously, a trailing + semi would work, but was not mandatory. This change was made to allow + clang-format to correctly format the source code. - (SAM - 2021/03/03) + (SAM - 2021/03/03) - Improved performance of H5Sget_select_elem_pointlist - Modified library to cache the point after the last block of points - retrieved by H5Sget_select_elem_pointlist, so a subsequent call to the - same function to retrieve the next block of points from the list can - proceed immediately without needing to iterate over the point list. + Modified library to cache the point after the last block of points + retrieved by H5Sget_select_elem_pointlist, so a subsequent call to the + same function to retrieve the next block of points from the list can + proceed immediately without needing to iterate over the point list. - (NAF - 2021/01/19) + (NAF - 2021/01/19) - Replaced H5E_ATOM with H5E_ID in H5Epubgen.h @@ -716,130 +722,130 @@ New Features - Add a new public function H5Ssel_iter_reset - This function resets a dataspace selection iterator back to an - initial state so that it may be used for iteration once more. - This can be useful when needing to iterate over a selection - multiple times without having to repeatedly create/destroy - a selection iterator for that dataspace selection. + This function resets a dataspace selection iterator back to an + initial state so that it may be used for iteration once more. + This can be useful when needing to iterate over a selection + multiple times without having to repeatedly create/destroy + a selection iterator for that dataspace selection. - (JTH - 2020/09/18) + (JTH - 2020/09/18) - Remove HDFS VFD stubs - The original implementation of the HDFS VFD included non-functional - versions of the following public API calls when the HDFS VFD is - not built as a part of the HDF5 library: + The original implementation of the HDFS VFD included non-functional + versions of the following public API calls when the HDFS VFD is + not built as a part of the HDF5 library: - * H5FD_hdfs_init() - * H5Pget_fapl_hdfs() - * H5Pset_fapl_hdfs() + * H5FD_hdfs_init() + * H5Pget_fapl_hdfs() + * H5Pset_fapl_hdfs() - They will remain present in HDF5 1.10 and HDF5 1.12 releases - for binary compatibility purposes but have been removed as of 1.14.0. + They will remain present in HDF5 1.10 and HDF5 1.12 releases + for binary compatibility purposes but have been removed as of 1.14.0. - Note that this has nothing to do with the real HDFS VFD API calls - that are fully functional when the HDFS VFD is configured and built. + Note that this has nothing to do with the real HDFS VFD API calls + that are fully functional when the HDFS VFD is configured and built. - We simply changed: + We simply changed: - #ifdef LIBHDFS - - #else - - #endif + #ifdef LIBHDFS + + #else + + #endif - to: + to: - #ifdef LIBHDFS - - #endif + #ifdef LIBHDFS + + #endif - Which is how the other optional VFDs are handled. + Which is how the other optional VFDs are handled. - (DER - 2020/08/27) + (DER - 2020/08/27) - Add Mirror VFD - Use TCP/IP sockets to perform write-only (W/O) file I/O on a remote - machine. Must be used in conjunction with the Splitter VFD. + Use TCP/IP sockets to perform write-only (W/O) file I/O on a remote + machine. Must be used in conjunction with the Splitter VFD. - (JOS - 2020/03/13, TBD) + (JOS - 2020/03/13, TBD) - Add Splitter VFD - Maintain separate R/W and W/O channels for "concurrent" file writes - to two files using a single HDF5 file handle. + Maintain separate R/W and W/O channels for "concurrent" file writes + to two files using a single HDF5 file handle. - (JOS - 2020/03/13, TBD) + (JOS - 2020/03/13, TBD) - Refactored public exposure of haddr_t type in favor of "object tokens" - To better accommodate HDF5 VOL connectors where "object addresses in a file" - may not make much sense, the following changes were made to the library: - - * Introduced new H5O_token_t "object token" type, which represents a - unique and permanent identifier for referencing an HDF5 object within - a container; these "object tokens" are meant to replace object addresses. - Along with the new type, a new H5Oopen_by_token API call was introduced - to open an object by a token, similar to how object addresses were - previously used with H5Oopen_by_addr. - - * Introduced new H5Lget_info2, H5Lget_info_by_idx2, H5Literate2, H5Literate_by_name2, - H5Lvisit2 and H5Lvisit_by_name2 API calls, along with their associated H5L_info2_t - struct and H5L_iterate2_t callback function, which work with the newly-introduced - object tokens, instead of object addresses. The original functions have been - renamed to version 1 functions and are deprecated in favor of the new version 2 - functions. The H5L_info_t and H5L_iterate_t types have been renamed to version 1 - types and are now deprecated in favor of their version 2 counterparts. For each of - the functions and types, compatibility macros take place of the original symbols. - - * Introduced new H5Oget_info3, H5Oget_info_by_name3, H5Oget_info_by_idx3, - H5Ovisit3 and H5Ovisit_by_name3 API calls, along with their associated H5O_info2_t - struct and H5O_iterate2_t callback function, which work with the newly-introduced - object tokens, instead of object addresses. The version 2 functions are now - deprecated in favor of the version 3 functions. The H5O_info_t and H5O_iterate_t - types have been renamed to version 1 types and are now deprecated in favor of their - version 2 counterparts. For each, compatibility macros take place of the original - symbols. - - * Introduced new H5Oget_native_info, H5Oget_native_info_by_name and - H5Oget_native_info_by_idx API calls, along with their associated H5O_native_info_t - struct, which are used to retrieve the native HDF5 file format-specific information - about an object. This information (such as object header info and B-tree/heap info) - has been removed from the new H5O_info2_t struct so that the more generic - H5Oget_info(_by_name/_by_idx)3 routines will not try to retrieve it for non-native - VOL connectors. - - * Added new H5Otoken_cmp, H5Otoken_to_str and H5Otoken_from_str routines to compare - two object tokens, convert an object token into a nicely-readable string format and - to convert an object token string back into a real object token, respectively. - - (DER, QAK, JTH - 2020/01/16) + To better accommodate HDF5 VOL connectors where "object addresses in a file" + may not make much sense, the following changes were made to the library: + + * Introduced new H5O_token_t "object token" type, which represents a + unique and permanent identifier for referencing an HDF5 object within + a container; these "object tokens" are meant to replace object addresses. + Along with the new type, a new H5Oopen_by_token API call was introduced + to open an object by a token, similar to how object addresses were + previously used with H5Oopen_by_addr. + + * Introduced new H5Lget_info2, H5Lget_info_by_idx2, H5Literate2, H5Literate_by_name2, + H5Lvisit2 and H5Lvisit_by_name2 API calls, along with their associated H5L_info2_t + struct and H5L_iterate2_t callback function, which work with the newly-introduced + object tokens, instead of object addresses. The original functions have been + renamed to version 1 functions and are deprecated in favor of the new version 2 + functions. The H5L_info_t and H5L_iterate_t types have been renamed to version 1 + types and are now deprecated in favor of their version 2 counterparts. For each of + the functions and types, compatibility macros take place of the original symbols. + + * Introduced new H5Oget_info3, H5Oget_info_by_name3, H5Oget_info_by_idx3, + H5Ovisit3 and H5Ovisit_by_name3 API calls, along with their associated H5O_info2_t + struct and H5O_iterate2_t callback function, which work with the newly-introduced + object tokens, instead of object addresses. The version 2 functions are now + deprecated in favor of the version 3 functions. The H5O_info_t and H5O_iterate_t + types have been renamed to version 1 types and are now deprecated in favor of their + version 2 counterparts. For each, compatibility macros take place of the original + symbols. + + * Introduced new H5Oget_native_info, H5Oget_native_info_by_name and + H5Oget_native_info_by_idx API calls, along with their associated H5O_native_info_t + struct, which are used to retrieve the native HDF5 file format-specific information + about an object. This information (such as object header info and B-tree/heap info) + has been removed from the new H5O_info2_t struct so that the more generic + H5Oget_info(_by_name/_by_idx)3 routines will not try to retrieve it for non-native + VOL connectors. + + * Added new H5Otoken_cmp, H5Otoken_to_str and H5Otoken_from_str routines to compare + two object tokens, convert an object token into a nicely-readable string format and + to convert an object token string back into a real object token, respectively. + + (DER, QAK, JTH - 2020/01/16) - Add new public function H5Sselect_adjust. - This function shifts a dataspace selection by a specified logical offset - within the dataspace extent. This can be useful for VOL developers to - implement chunked datasets. + This function shifts a dataspace selection by a specified logical offset + within the dataspace extent. This can be useful for VOL developers to + implement chunked datasets. - (NAF - 2019/11/18) + (NAF - 2019/11/18) - Add new public function H5Sselect_project_intersection. - This function computes the intersection between two dataspace selections - and projects that intersection into a third selection. This can be useful - for VOL developers to implement chunked or virtual datasets. + This function computes the intersection between two dataspace selections + and projects that intersection into a third selection. This can be useful + for VOL developers to implement chunked or virtual datasets. - (NAF - 2019/11/13, ID-148) + (NAF - 2019/11/13, ID-148) - Add new public function H5VLget_file_type. - This function returns a datatype equivalent to the supplied datatype but - with the location set to be in the file. This datatype can then be used - with H5Tconvert to convert data between file and in-memory representation. - This function is intended for use only by VOL connector developers. + This function returns a datatype equivalent to the supplied datatype but + with the location set to be in the file. This datatype can then be used + with H5Tconvert to convert data between file and in-memory representation. + This function is intended for use only by VOL connector developers. - (NAF - 2019/11/08, ID-127) + (NAF - 2019/11/08, ID-127) Parallel Library: @@ -859,10 +865,10 @@ New Features h5pget_file_locking_f() h5pset_file_locking_f() - See the configure option discussion for HDFFV-11092 (above) for more - information on the file locking feature and how it's controlled. + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. - (DER - 2020/07/30, HDFFV-11092) + (DER - 2020/07/30, HDFFV-11092) C++ Library: ------------ @@ -871,10 +877,10 @@ New Features FileAccPropList::setFileLocking() FileAccPropList::getFileLocking() - See the configure option discussion for HDFFV-11092 (above) for more - information on the file locking feature and how it's controlled. + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. - (DER - 2020/07/30, HDFFV-11092) + (DER - 2020/07/30, HDFFV-11092) Java Library: @@ -891,12 +897,12 @@ New Features - Added new H5S functions. - H5Sselect_copy, H5Sselect_shape_same, H5Sselect_adjust, - H5Sselect_intersect_block, H5Sselect_project_intersection, - H5Scombine_hyperslab, H5Smodify_select, H5Scombine_select - wrapper functions added. + H5Sselect_copy, H5Sselect_shape_same, H5Sselect_adjust, + H5Sselect_intersect_block, H5Sselect_project_intersection, + H5Scombine_hyperslab, H5Smodify_select, H5Scombine_select + wrapper functions added. - (ADB - 2020/10/27, HDFFV-10868) + (ADB - 2020/10/27, HDFFV-10868) - Add wrappers for H5Pset/get_file_locking() API calls @@ -904,120 +910,133 @@ New Features H5Pget_use_file_locking() H5Pget_ignore_disabled_file_locking() - Unlike the C++ and Fortran wrappers, there are separate getters for the - two file locking settings, each of which returns a boolean value. + Unlike the C++ and Fortran wrappers, there are separate getters for the + two file locking settings, each of which returns a boolean value. - See the configure option discussion for HDFFV-11092 (above) for more - information on the file locking feature and how it's controlled. + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. - (DER - 2020/07/30, HDFFV-11092) + (DER - 2020/07/30, HDFFV-11092) - Added ability to test java library with VOLs. - Created a new CMake script that combines the java and vol test scripts. + Created a new CMake script that combines the java and vol test scripts. - (ADB - 2020/02/03, HDFFV-10996) + (ADB - 2020/02/03, HDFFV-10996) - Tests fail for non-English locales. - In the JUnit tests with a non-English locale, only the part before - the decimal comma is replaced by XXXX and this leads to a comparison - error. Changed the regex for the Time substitution. + In the JUnit tests with a non-English locale, only the part before + the decimal comma is replaced by XXXX and this leads to a comparison + error. Changed the regex for the Time substitution. - (ADB - 2020/01/09, HDFFV-10995) + (ADB - 2020/01/09, HDFFV-10995) Tools: ------ + - Added a new (unix ONLY) parallel meta tool 'h5dwalk', which utilizes the + mpifileutils (https://hpc.github.io/mpifileutils) open source utility + library to enable parallel execution of other HDF5 tools. + This approach can greatly enhance the serial hdf5 tool performance over large + collections of files by utilizing MPI parallelism to distribute an application + load over many independent MPI ranks and files. + + An introduction to the mpifileutils library and initial 'User Guide' for + the new 'h5dwalk" tool can be found at: + https://github.com/HDFGroup/hdf5doc/tree/master/RFCs/HDF5/tools/parallel_tools + + (RAW - 2021/10/25) + - Refactored the perform tools and removed depends on test library. - Moved the perf and h5perf tools from tools/test/perform to - tools/src/h5perf so that they can be installed. This required - that the test library dependency be removed by copying the - needed functions from h5test.c. - The standalone scripts and other perform tools remain in the - tools/test/perform folder. + Moved the perf and h5perf tools from tools/test/perform to + tools/src/h5perf so that they can be installed. This required + that the test library dependency be removed by copying the + needed functions from h5test.c. + The standalone scripts and other perform tools remain in the + tools/test/perform folder. - (ADB - 2021/08/10) + (ADB - 2021/08/10) - Removed partial long exceptions - Some of the tools accepted shortened versions of the long options - (ex: --datas instead of --dataset). These were implemented inconsistently, - are difficult to maintian, and occasionally block useful long option - names. These partial long options have been removed from all the tools. + Some of the tools accepted shortened versions of the long options + (ex: --datas instead of --dataset). These were implemented inconsistently, + are difficult to maintian, and occasionally block useful long option + names. These partial long options have been removed from all the tools. - (DER - 2021/08/03) + (DER - 2021/08/03) - h5repack added help text for user-defined filters. - Added help text line that states the valid values of the filter flag - for user-defined filters; - filter_flag: 1 is OPTIONAL or 0 is MANDATORY + Added help text line that states the valid values of the filter flag + for user-defined filters; + filter_flag: 1 is OPTIONAL or 0 is MANDATORY - (ADB - 2021/01/14, HDFFV-11099) + (ADB - 2021/01/14, HDFFV-11099) - Added h5delete tool - Deleting HDF5 storage when using the VOL can be tricky when the VOL - does not create files. The h5delete tool is a simple wrapper around - the H5Fdelete() API call that uses the VOL specified in the - HDF5_VOL_CONNECTOR environment variable to delete a "file". If - the call to H5Fdelete() fails, the tool will attempt to use - the POSIX remove(3) call to remove the file. + Deleting HDF5 storage when using the VOL can be tricky when the VOL + does not create files. The h5delete tool is a simple wrapper around + the H5Fdelete() API call that uses the VOL specified in the + HDF5_VOL_CONNECTOR environment variable to delete a "file". If + the call to H5Fdelete() fails, the tool will attempt to use + the POSIX remove(3) call to remove the file. - Note that the HDF5 library does currently have support for - H5Fdelete() in the native VOL connector. + Note that the HDF5 library does currently have support for + H5Fdelete() in the native VOL connector. - (DER - 2020/12/16) + (DER - 2020/12/16) - h5repack added options to control how external links are handled. - Currently h5repack preserves external links and cannot copy and merge - data from the external files. Two options, merge and prune, were added to - control how to merge data from an external link into the resulting file. - --merge Follow external soft link recursively and merge data. - --prune Do not follow external soft links and remove link. - --merge --prune Follow external link, merge data and remove dangling link. + Currently h5repack preserves external links and cannot copy and merge + data from the external files. Two options, merge and prune, were added to + control how to merge data from an external link into the resulting file. + --merge Follow external soft link recursively and merge data. + --prune Do not follow external soft links and remove link. + --merge --prune Follow external link, merge data and remove dangling link. - (ADB - 2020/08/05, HDFFV-9984) + (ADB - 2020/08/05, HDFFV-9984) - h5repack was fixed to repack the reference attributes properly. - The code line that checks if the update of reference inside a compound - datatype is misplaced outside the code block loop that carries out the - check. In consequence, the next attribute that is not the reference - type was repacked again as the reference type and caused the failure of - repacking. The fix is to move the corresponding code line to the correct - code block. + The code line that checks if the update of reference inside a compound + datatype is misplaced outside the code block loop that carries out the + check. In consequence, the next attribute that is not the reference + type was repacked again as the reference type and caused the failure of + repacking. The fix is to move the corresponding code line to the correct + code block. - (KY -2020/02/07, HDFFV-11014) + (KY -2020/02/07, HDFFV-11014) - h5diff was updated to use the new reference APIs. - h5diff uses the new reference APIs to compare references. - Attribute references can also be compared. + h5diff uses the new reference APIs to compare references. + Attribute references can also be compared. - (ADB - 2019/12/19, HDFFV-10980) + (ADB - 2019/12/19, HDFFV-10980) - h5dump and h5ls were updated to use the new reference APIs. - The tools library now use the new reference APIs to inspect a - file. Also the DDL spec was updated to reflect the format - changes produced with the new APIs. The export API and support - functions in the JNI were updated to match. + The tools library now use the new reference APIs to inspect a + file. Also the DDL spec was updated to reflect the format + changes produced with the new APIs. The export API and support + functions in the JNI were updated to match. - (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) + (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) High-Level APIs: ---------------- - added set/get for unsigned long long attributes - the attribute writing high-level API has been expanded to include - public set/get functions for ULL attributes, analogously to the - existing set/get for other types. + The attribute writing high-level API has been expanded to include + public set/get functions for ULL attributes, analogously to the + existing set/get for other types. - (AF - 2021/09/08) + (AF - 2021/09/08) C Packet Table API: ------------------- @@ -1039,6 +1058,14 @@ Bug Fixes since HDF5-1.12.0 release =================================== Library ------- + - Fixed an H5Pget_filter_by_id1/2() assert w/ out of range filter IDs + + Both H5Pget_filter_by_id1 and 2 did not range check the filter ID, which + could trip as assert in debug versions of the library. The library now + returns a normal HDF5 error when the filter ID is out of range. + + (DER - 2021/11/23, HDFFV-11286) + - Fixed an issue with collective metadata reads being permanently disabled after a dataset chunk lookup operation. This would usually cause a mismatched MPI_Bcast and MPI_ERR_TRUNCATE issue in the library for @@ -1049,79 +1076,79 @@ Bug Fixes since HDF5-1.12.0 release - Fixed cross platform incompatibility of references within variable length types - Reference types within variable length types previously could not be - read on a platform with different endianness from where they were - written. Fixed so cross platform portability is restored. + Reference types within variable length types previously could not be + read on a platform with different endianness from where they were + written. Fixed so cross platform portability is restored. - (NAF - 2021/09/30) + (NAF - 2021/09/30) - Detection of simple data transform function "x" - In the case of the simple data transform function "x" the (parallel) - library recognizes this is the same as not applying this data transform - function. This improves the I/O performance. In the case of the parallel - library, it also avoids breaking to independent I/O, which makes it - possible to apply a filter when writing or reading data to or from - teh HDF5 file. + In the case of the simple data transform function "x" the (parallel) + library recognizes this is the same as not applying this data transform + function. This improves the I/O performance. In the case of the parallel + library, it also avoids breaking to independent I/O, which makes it + possible to apply a filter when writing or reading data to or from + the HDF5 file. - (JWSB - 2021/09/13) + (JWSB - 2021/09/13) - Fixed an invalid read and memory leak when parsing corrupt file space info messages - When the corrupt file from CVE-2020-10810 was parsed by the library, - the code that imports the version 0 file space info object header - message to the version 1 struct could read past the buffer read from - the disk, causing an invalid memory read. Not catching this error would - cause downstream errors that eventually resulted in a previously - allocated buffer to be unfreed when the library shut down. In builds - where the free lists are in use, this could result in an infinite loop - and SIGABRT when the library shuts down. + When the corrupt file from CVE-2020-10810 was parsed by the library, + the code that imports the version 0 file space info object header + message to the version 1 struct could read past the buffer read from + the disk, causing an invalid memory read. Not catching this error would + cause downstream errors that eventually resulted in a previously + allocated buffer to be unfreed when the library shut down. In builds + where the free lists are in use, this could result in an infinite loop + and SIGABRT when the library shuts down. - We now track the buffer size and raise an error on attempts to read - past the end of it. + We now track the buffer size and raise an error on attempts to read + past the end of it. - (DER - 2021/08/12, HDFFV-11053) + (DER - 2021/08/12, HDFFV-11053) - Fixed CVE-2018-14460 - The tool h5repack produced a segfault when the rank in dataspace - message was corrupted, causing invalid read while decoding the - dimension sizes. + The tool h5repack produced a segfault when the rank in dataspace + message was corrupted, causing invalid read while decoding the + dimension sizes. - The problem was fixed by ensuring that decoding the dimension sizes - and max values will not go beyong the end of the buffer. + The problem was fixed by ensuring that decoding the dimension sizes + and max values will not go beyong the end of the buffer. - (BMR - 2021/05/12, HDFFV-11223) + (BMR - 2021/05/12, HDFFV-11223) - Fixed CVE-2018-11206 - The tool h5dump produced a segfault when the size of a fill value - message was corrupted and caused a buffer overflow. + The tool h5dump produced a segfault when the size of a fill value + message was corrupted and caused a buffer overflow. - The problem was fixed by verifying the fill value's size - against the buffer size before attempting to access the buffer. + The problem was fixed by verifying the fill value's size + against the buffer size before attempting to access the buffer. - (BMR - 2021/03/15, HDFFV-10480) + (BMR - 2021/03/15, HDFFV-10480) - Fixed CVE-2018-14033 (same issue as CVE-2020-10811) - The tool h5dump produced a segfault when the storage size message - was corrupted and caused a buffer overflow. + The tool h5dump produced a segfault when the storage size message + was corrupted and caused a buffer overflow. - The problem was fixed by verifying the storage size against the - buffer size before attempting to access the buffer. + The problem was fixed by verifying the storage size against the + buffer size before attempting to access the buffer. - (BMR - 2021/03/15, HDFFV-11159/HDFFV-11049) + (BMR - 2021/03/15, HDFFV-11159/HDFFV-11049) - Remove underscores on header file guards - Header file guards used a variety of underscores at the beginning of the define. + Header file guards used a variety of underscores at the beginning of the define. - Removed all leading (some trailing) underscores from header file guards. + Removed all leading (some trailing) underscores from header file guards. - (ADB - 2021/03/03, #361) + (ADB - 2021/03/03, #361) - Fixed a segmentation fault @@ -1134,146 +1161,146 @@ Bug Fixes since HDF5-1.12.0 release - Fixed issue with MPI communicator and info object not being copied into new FAPL retrieved from H5F_get_access_plist - Added logic to copy the MPI communicator and info object into - the output FAPL. MPI communicator is retrieved from the VFD, while - the MPI info object is retrieved from the file's original FAPL. + Added logic to copy the MPI communicator and info object into + the output FAPL. MPI communicator is retrieved from the VFD, while + the MPI info object is retrieved from the file's original FAPL. - (JTH - 2021/02/15, HDFFV-11109) + (JTH - 2021/02/15, HDFFV-11109) - Fixed problems with vlens and refs inside compound using H5VLget_file_type() - Modified library to properly ref count H5VL_object_t structs and only - consider file vlen and reference types to be equal if their files are - the same. + Modified library to properly ref count H5VL_object_t structs and only + consider file vlen and reference types to be equal if their files are + the same. - (NAF - 2021/01/22) + (NAF - 2021/01/22) - Fixed CVE-2018-17432 - The tool h5repack produced a segfault on a corrupted file which had - invalid rank for scalar or NULL datatype. + The tool h5repack produced a segfault on a corrupted file which had + invalid rank for scalar or NULL datatype. - The problem was fixed by modifying the dataspace encode and decode - functions to detect and report invalid rank. h5repack now fails - with an error message for the corrupted file. + The problem was fixed by modifying the dataspace encode and decode + functions to detect and report invalid rank. h5repack now fails + with an error message for the corrupted file. - (BMR - 2020/10/26, HDFFV-10590) + (BMR - 2020/10/26, HDFFV-10590) - Creation of dataset with optional filter - When the combination of type, space, etc doesn't work for filter - and the filter is optional, it was supposed to be skipped but it was - not skipped and the creation failed. + When the combination of type, space, etc doesn't work for filter + and the filter is optional, it was supposed to be skipped but it was + not skipped and the creation failed. - Allowed the creation of the dataset in such a situation. + Allowed the creation of the dataset in such a situation. - (BMR - 2020/08/13, HDFFV-10933) + (BMR - 2020/08/13, HDFFV-10933) - Explicitly declared dlopen to use RTLD_LOCAL - dlopen documentation states that if neither RTLD_GLOBAL nor - RTLD_LOCAL are specified, then the default behavior is unspecified. - The default on linux is usually RTLD_LOCAL while macos will default - to RTLD_GLOBAL. + dlopen documentation states that if neither RTLD_GLOBAL nor + RTLD_LOCAL are specified, then the default behavior is unspecified. + The default on linux is usually RTLD_LOCAL while macos will default + to RTLD_GLOBAL. - (ADB - 2020/08/12, HDFFV-11127) + (ADB - 2020/08/12, HDFFV-11127) - H5Sset_extent_none() sets the dataspace class to H5S_NO_CLASS which causes asserts/errors when passed to other dataspace API calls. - H5S_NO_CLASS is an internal class value that should not have been - exposed via a public API call. + H5S_NO_CLASS is an internal class value that should not have been + exposed via a public API call. - In debug builds of the library, this can cause assert() function to - trip. In non-debug builds, it will produce normal library errors. + In debug builds of the library, this can cause assert() function to + trip. In non-debug builds, it will produce normal library errors. - The new library behavior is for H5Sset_extent_none() to convert - the dataspace into one of type H5S_NULL, which is better handled - by the library and easier for developers to reason about. + The new library behavior is for H5Sset_extent_none() to convert + the dataspace into one of type H5S_NULL, which is better handled + by the library and easier for developers to reason about. - (DER - 2020/07/27, HDFFV-11027) + (DER - 2020/07/27, HDFFV-11027) - Fixed issues CVE-2018-13870 and CVE-2018-13869 - When a buffer overflow occurred because a name length was corrupted - and became very large, h5dump crashed on memory access violation. + When a buffer overflow occurred because a name length was corrupted + and became very large, h5dump crashed on memory access violation. - A check for reading pass the end of the buffer was added to multiple - locations to prevent the crashes and h5dump now simply fails with an - error message when this error condition occurs. + A check for reading pass the end of the buffer was added to multiple + locations to prevent the crashes and h5dump now simply fails with an + error message when this error condition occurs. - (BMR - 2020/07/22, HDFFV-11120 and HDFFV-11121) + (BMR - 2020/07/22, HDFFV-11120 and HDFFV-11121) - Fixed the segmentation fault when reading attributes with multiple threads - It was reported that the reading of attributes with variable length string - datatype will crash with segmentation fault particularly when the number of - threads is high (>16 threads). The problem was due to the file pointer that - was set in the variable length string datatype for the attribute. That file - pointer was already closed when the attribute was accessed. + It was reported that the reading of attributes with variable length string + datatype will crash with segmentation fault particularly when the number of + threads is high (>16 threads). The problem was due to the file pointer that + was set in the variable length string datatype for the attribute. That file + pointer was already closed when the attribute was accessed. - The problem was fixed by setting the file pointer to the current opened file pointer - when the attribute was accessed. Similar patch up was done before when reading - dataset with variable length string datatype. + The problem was fixed by setting the file pointer to the current opened file pointer + when the attribute was accessed. Similar patch up was done before when reading + dataset with variable length string datatype. - (VC - 2020/07/13, HDFFV-11080) + (VC - 2020/07/13, HDFFV-11080) - Fixed CVE-2020-10810 - The tool h5clear produced a segfault during an error recovery in - the superblock decoding. An internal pointer was reset to prevent - further accessing when it is not assigned with a value. + The tool h5clear produced a segfault during an error recovery in + the superblock decoding. An internal pointer was reset to prevent + further accessing when it is not assigned with a value. - (BMR - 2020/06/29, HDFFV-11053) + (BMR - 2020/06/29, HDFFV-11053) - Fixed CVE-2018-17435 - The tool h52gif produced a segfault when the size of an attribute - message was corrupted and caused a buffer overflow. + The tool h52gif produced a segfault when the size of an attribute + message was corrupted and caused a buffer overflow. - The problem was fixed by verifying the attribute message's size - against the buffer size before accessing the buffer. h52gif was - also fixed to display the failure instead of silently exiting - after the segfault was eliminated. + The problem was fixed by verifying the attribute message's size + against the buffer size before accessing the buffer. h52gif was + also fixed to display the failure instead of silently exiting + after the segfault was eliminated. - (BMR - 2020/06/19, HDFFV-10591) + (BMR - 2020/06/19, HDFFV-10591) Java Library ------------ - JNI utility function does not handle new references. - The JNI utility function for converting reference data to string did - not use the new APIs. In addition to fixing that function, added new - java tests for using the new APIs. + The JNI utility function for converting reference data to string did + not use the new APIs. In addition to fixing that function, added new + java tests for using the new APIs. - (ADB - 2021/02/16, HDFFV-11212) + (ADB - 2021/02/16, HDFFV-11212) - The H5FArray.java class, in which virtually the entire execution time - is spent using the HDFNativeData method that converts from an array - of bytes to an array of the destination Java type. + is spent using the HDFNativeData method that converts from an array + of bytes to an array of the destination Java type. 1. Convert the entire byte array into a 1-d array of the desired type, rather than performing 1 conversion per row; 2. Use the Java Arrays method copyOfRange to grab the section of the array from (1) that is desired to be inserted into the destination array. - (PGT,ADB - 2020/12/13, HDFFV-10865) + (PGT,ADB - 2020/12/13, HDFFV-10865) - Added ability to test java library with VOLs. - Created a new CMake script that combines the java and vol test scripts. + Created a new CMake script that combines the java and vol test scripts. - (ADB - 2020/02/03, HDFFV-10996) + (ADB - 2020/02/03, HDFFV-10996) - Tests fail for non-English locales. - In the JUnit tests with a non-English locale, only the part before - the decimal comma is replaced by XXXX and this leads to a comparison - error. Changed the regex for the Time substitution. + In the JUnit tests with a non-English locale, only the part before + the decimal comma is replaced by XXXX and this leads to a comparison + error. Changed the regex for the Time substitution. - (ADB - 2020/01/09, HDFFV-10995) + (ADB - 2020/01/09, HDFFV-10995) Configuration @@ -1391,88 +1418,88 @@ Bug Fixes since HDF5-1.12.0 release ----- - Changed how h5dump and h5ls identify long double. - Long double support is not consistent across platforms. Tools will always - identify long double as 128-bit [little/big]-endian float nn-bit precision. - New test file created for datasets with attributes for float, double and - long double. In addition any unknown integer or float datatype will now - also show the number of bits for precision. - These files are also used in the java tests. + Long double support is not consistent across platforms. Tools will always + identify long double as 128-bit [little/big]-endian float nn-bit precision. + New test file created for datasets with attributes for float, double and + long double. In addition any unknown integer or float datatype will now + also show the number of bits for precision. + These files are also used in the java tests. - (ADB - 2021/03/24, HDFFV-11229,HDFFV-11113) + (ADB - 2021/03/24, HDFFV-11229,HDFFV-11113) - Fixed tools argument parsing. - Tools parsing used the length of the option from the long array to match - the option from the command line. This incorrectly matched a shorter long - name option that happened to be a subset of another long option. - Changed to match whole names. + Tools parsing used the length of the option from the long array to match + the option from the command line. This incorrectly matched a shorter long + name option that happened to be a subset of another long option. + Changed to match whole names. - (ADB - 2021/01/19, HDFFV-11106) + (ADB - 2021/01/19, HDFFV-11106) - The tools library was updated by standardizing the error stack process. - General sequence is: - h5tools_setprogname(PROGRAMNAME); - h5tools_setstatus(EXIT_SUCCESS); - h5tools_init(); - ... process the command-line (check for error-stack enable) ... - h5tools_error_report(); - ... (do work) ... - h5diff_exit(ret); + General sequence is: + h5tools_setprogname(PROGRAMNAME); + h5tools_setstatus(EXIT_SUCCESS); + h5tools_init(); + ... process the command-line (check for error-stack enable) ... + h5tools_error_report(); + ... (do work) ... + h5diff_exit(ret); - (ADB - 2020/07/20, HDFFV-11066) + (ADB - 2020/07/20, HDFFV-11066) - h5diff fixed a command line parsing error. - h5diff would ignore the argument to -d (delta) if it is smaller than DBL_EPSILON. - The macro H5_DBL_ABS_EQUAL was removed and a direct value comparision was used. + h5diff would ignore the argument to -d (delta) if it is smaller than DBL_EPSILON. + The macro H5_DBL_ABS_EQUAL was removed and a direct value comparision was used. - (ADB - 2020/07/20, HDFFV-10897) + (ADB - 2020/07/20, HDFFV-10897) - h5diff added a command line option to ignore attributes. - h5diff would ignore all objects with a supplied path if the exclude-path argument is used. - Adding the exclude-attribute argument will only exclude attributes, with the supplied path, - from comparison. + h5diff would ignore all objects with a supplied path if the exclude-path argument is used. + Adding the exclude-attribute argument will only exclude attributes, with the supplied path, + from comparison. - (ADB - 2020/07/20, HDFFV-5935) + (ADB - 2020/07/20, HDFFV-5935) - h5diff added another level to the verbose argument to print filenames. - Added verbose level 3 that is level 2 plus the filenames. The levels are: - 0 : Identical to '-v' or '--verbose' - 1 : All level 0 information plus one-line attribute status summary - 2 : All level 1 information plus extended attribute status report - 3 : All level 2 information plus file names + Added verbose level 3 that is level 2 plus the filenames. The levels are: + 0 : Identical to '-v' or '--verbose' + 1 : All level 0 information plus one-line attribute status summary + 2 : All level 1 information plus extended attribute status report + 3 : All level 2 information plus file names - (ADB - 2020/07/20, HDFFV-1005) + (ADB - 2020/07/20, HDFFV-1005) - h5repack was fixed to repack the reference attributes properly. - The code line that checks if the update of reference inside a compound - datatype is misplaced outside the code block loop that carries out the - check. In consequence, the next attribute that is not the reference - type was repacked again as the reference type and caused the failure of - repacking. The fix is to move the corresponding code line to the correct - code block. + The code line that checks if the update of reference inside a compound + datatype is misplaced outside the code block loop that carries out the + check. In consequence, the next attribute that is not the reference + type was repacked again as the reference type and caused the failure of + repacking. The fix is to move the corresponding code line to the correct + code block. - (KY -2020/02/10, HDFFV-11014) + (KY -2020/02/10, HDFFV-11014) - h5diff was updated to use the new reference APIs. - h5diff uses the new reference APIs to compare references. - Attribute references can also be compared. + h5diff uses the new reference APIs to compare references. + Attribute references can also be compared. - (ADB - 2019/12/19, HDFFV-10980) + (ADB - 2019/12/19, HDFFV-10980) - h5dump and h5ls were updated to use the new reference APIs. - The tools library now use the new reference APIs to inspect a - file. Also the DDL spec was updated to reflect the format - changes produced with the new APIs. The export API and support - functions in the JNI were updated to match. + The tools library now use the new reference APIs to inspect a + file. Also the DDL spec was updated to reflect the format + changes produced with the new APIs. The export API and support + functions in the JNI were updated to match. - (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) + (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) Performance @@ -1484,7 +1511,7 @@ Bug Fixes since HDF5-1.12.0 release ----------- - Corrected INTERFACE INTENT(IN) to INTENT(OUT) for buf_size in h5fget_file_image_f. - (MSB - 2020/02/18, HDFFV-11029) + (MSB - 2020/02/18, HDFFV-11029) High-Level Library @@ -1493,7 +1520,7 @@ Bug Fixes since HDF5-1.12.0 release Incorrect length assignment. - (ADB - 2021/10/14) + (ADB - 2021/10/14) Fortran High-Level APIs @@ -1515,21 +1542,21 @@ Bug Fixes since HDF5-1.12.0 release -------- - Added DataSet::operator= - Some compilers complain if the copy constructor is given explicitly - but the assignment operator is implicitly set to default. + Some compilers complain if the copy constructor is given explicitly + but the assignment operator is implicitly set to default. - (2021/05/19) + (2021/05/19) Testing ------- - Stopped java/test/junit.sh.in installing libs for testing under ${prefix} - Lib files needed are now copied to a subdirectory in the java/test - directory, and on Macs the loader path for libhdf5.xxxs.so is changed - in the temporary copy of libhdf5_java.dylib. + Lib files needed are now copied to a subdirectory in the java/test + directory, and on Macs the loader path for libhdf5.xxxs.so is changed + in the temporary copy of libhdf5_java.dylib. - (LRK, 2020/07/02, HDFFV-11063) + (LRK, 2020/07/02, HDFFV-11063) Platforms Tested diff --git a/src/H5.c b/src/H5.c index d0fb0f7..dbd53be 100644 --- a/src/H5.c +++ b/src/H5.c @@ -29,6 +29,7 @@ #include "H5Lprivate.h" /* Links */ #include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ +#include "H5PLprivate.h" /* Plugins */ #include "H5SLprivate.h" /* Skip lists */ #include "H5Tprivate.h" /* Datatypes */ @@ -276,6 +277,7 @@ H5_init_library(void) , {H5AC_init, "metadata caching"} , {H5L_init, "link"} , {H5S_init, "dataspace"} + , {H5PL_init, "plugins"} /* Finish initializing interfaces that depend on the interfaces above */ , {H5P_init_phase2, "property list"} , {H5VL_init_phase2, "VOL"} diff --git a/src/H5CX.c b/src/H5CX.c index 1329b9d..7c99055 100644 --- a/src/H5CX.c +++ b/src/H5CX.c @@ -1789,7 +1789,7 @@ H5CX_get_mpi_coll_datatypes(MPI_Datatype *btype, MPI_Datatype *ftype) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ HDassert(btype); @@ -1801,7 +1801,6 @@ H5CX_get_mpi_coll_datatypes(MPI_Datatype *btype, MPI_Datatype *ftype) *btype = (*head)->ctx.btype; *ftype = (*head)->ctx.ftype; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_get_mpi_coll_datatypes() */ @@ -2963,7 +2962,7 @@ H5CX_set_mpi_coll_datatypes(MPI_Datatype btype, MPI_Datatype ftype) herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -2973,7 +2972,6 @@ H5CX_set_mpi_coll_datatypes(MPI_Datatype btype, MPI_Datatype ftype) (*head)->ctx.btype = btype; (*head)->ctx.ftype = ftype; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_set_mpi_coll_datatypes() */ @@ -2995,7 +2993,7 @@ H5CX_set_io_xfer_mode(H5FD_mpio_xfer_t io_xfer_mode) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -3007,7 +3005,6 @@ H5CX_set_io_xfer_mode(H5FD_mpio_xfer_t io_xfer_mode) /* Mark the value as valid */ (*head)->ctx.io_xfer_mode_valid = TRUE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_set_io_xfer_mode() */ @@ -3029,7 +3026,7 @@ H5CX_set_mpio_coll_opt(H5FD_mpio_collective_opt_t mpio_coll_opt) H5CX_node_t **head = NULL; /* Pointer to head of API context list */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI_NOERR /* Sanity check */ head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ @@ -3041,7 +3038,6 @@ H5CX_set_mpio_coll_opt(H5FD_mpio_collective_opt_t mpio_coll_opt) /* Mark the value as valid */ (*head)->ctx.mpio_coll_opt_valid = TRUE; -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_set_mpio_coll_opt() */ diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 448e92d..ddf5e06 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -449,8 +449,8 @@ done: */ herr_t H5D__mpio_select_read(const H5D_io_info_t *io_info, const H5D_type_info_t H5_ATTR_UNUSED *type_info, - hsize_t mpi_buf_count, const H5S_t H5_ATTR_UNUSED *file_space, - const H5S_t H5_ATTR_UNUSED *mem_space) + hsize_t mpi_buf_count, H5S_t H5_ATTR_UNUSED *file_space, + H5S_t H5_ATTR_UNUSED *mem_space) { const H5D_contig_storage_t *store_contig = &(io_info->store->contig); /* Contiguous storage info for this I/O operation */ @@ -480,8 +480,8 @@ done: */ herr_t H5D__mpio_select_write(const H5D_io_info_t *io_info, const H5D_type_info_t H5_ATTR_UNUSED *type_info, - hsize_t mpi_buf_count, const H5S_t H5_ATTR_UNUSED *file_space, - const H5S_t H5_ATTR_UNUSED *mem_space) + hsize_t mpi_buf_count, H5S_t H5_ATTR_UNUSED *file_space, + H5S_t H5_ATTR_UNUSED *mem_space) { const H5D_contig_storage_t *store_contig = &(io_info->store->contig); /* Contiguous storage info for this I/O operation */ @@ -690,7 +690,7 @@ done: */ herr_t H5D__contig_collective_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t H5_ATTR_UNUSED nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t H5_ATTR_UNUSED nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm) { H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_CONTIGUOUS_COLLECTIVE; @@ -729,7 +729,7 @@ done: */ herr_t H5D__contig_collective_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t H5_ATTR_UNUSED nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t H5_ATTR_UNUSED nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t H5_ATTR_UNUSED *fm) { H5D_mpio_actual_io_mode_t actual_io_mode = H5D_MPIO_CONTIGUOUS_COLLECTIVE; @@ -929,8 +929,8 @@ done: */ herr_t H5D__chunk_collective_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t H5_ATTR_UNUSED nelmts, const H5S_t H5_ATTR_UNUSED *file_space, - const H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t *fm) + hsize_t H5_ATTR_UNUSED nelmts, H5S_t H5_ATTR_UNUSED *file_space, + H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t *fm) { herr_t ret_value = SUCCEED; /* Return value */ @@ -959,8 +959,8 @@ done: */ herr_t H5D__chunk_collective_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t H5_ATTR_UNUSED nelmts, const H5S_t H5_ATTR_UNUSED *file_space, - const H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t *fm) + hsize_t H5_ATTR_UNUSED nelmts, H5S_t H5_ATTR_UNUSED *file_space, + H5S_t H5_ATTR_UNUSED *mem_space, H5D_chunk_map_t *fm) { herr_t ret_value = SUCCEED; /* Return value */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 0b25f9d..17d24a2 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -702,26 +702,26 @@ H5_DLL herr_t H5D__fill_term(H5D_fill_buf_info_t *fb_info); #endif /*H5S_DEBUG*/ /* MPI-IO function to read, it will select either regular or irregular read */ H5_DLL herr_t H5D__mpio_select_read(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space); + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space); /* MPI-IO function to write, it will select either regular or irregular read */ H5_DLL herr_t H5D__mpio_select_write(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space); + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space); /* MPI-IO functions to handle contiguous collective IO */ H5_DLL herr_t H5D__contig_collective_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); H5_DLL herr_t H5D__contig_collective_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); /* MPI-IO functions to handle chunked collective IO */ H5_DLL herr_t H5D__chunk_collective_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); H5_DLL herr_t H5D__chunk_collective_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t nelmts, H5S_t *file_space, H5S_t *mem_space, H5D_chunk_map_t *fm); /* MPI-IO function to check if a direct I/O transfer is possible between diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 145f2be..e492f4e 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -223,21 +223,22 @@ H5FD_mpio_init(void) FUNC_ENTER_NOAPI(H5I_INVALID_HID) /* Register the MPI-IO VFD, if it isn't already */ - if (H5I_VFL != H5I_get_type(H5FD_MPIO_g)) + if (H5I_VFL != H5I_get_type(H5FD_MPIO_g)) { H5FD_MPIO_g = H5FD_register((const H5FD_class_t *)&H5FD_mpio_g, sizeof(H5FD_class_t), FALSE); - /* Check if MPI driver has been loaded dynamically */ - env = HDgetenv(HDF5_DRIVER); - if (env && !HDstrcmp(env, "mpio")) { - int mpi_initialized = 0; - - /* Initialize MPI if not already initialized */ - if (MPI_SUCCESS != MPI_Initialized(&mpi_initialized)) - HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, H5I_INVALID_HID, "can't check if MPI is initialized") - if (!mpi_initialized) { - if (MPI_SUCCESS != MPI_Init(NULL, NULL)) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, H5I_INVALID_HID, "can't initialize MPI") - H5FD_mpi_self_initialized = TRUE; + /* Check if MPI driver has been loaded dynamically */ + env = HDgetenv(HDF5_DRIVER); + if (env && !HDstrcmp(env, "mpio")) { + int mpi_initialized = 0; + + /* Initialize MPI if not already initialized */ + if (MPI_SUCCESS != MPI_Initialized(&mpi_initialized)) + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, H5I_INVALID_HID, "can't check if MPI is initialized") + if (!mpi_initialized) { + if (MPI_SUCCESS != MPI_Init(NULL, NULL)) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, H5I_INVALID_HID, "can't initialize MPI") + H5FD_mpi_self_initialized = TRUE; + } } } diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index 4f0e40a..8462eab 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -1084,7 +1084,6 @@ H5FD__sec2_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void H5_AT /* Sanity checks */ HDassert(file); - HDassert(H5FD_SEC2 == file->pub.driver_id); switch (op_code) { /* Unknown op code */ diff --git a/src/H5FDsplitter.c b/src/H5FDsplitter.c index 8cb5509..1c1205f 100644 --- a/src/H5FDsplitter.c +++ b/src/H5FDsplitter.c @@ -1320,7 +1320,6 @@ H5FD__splitter_ctl(H5FD_t *_file, uint64_t op_code, uint64_t flags, const void * /* Sanity checks */ HDassert(file); - HDassert(H5FD_SPLITTER == file->pub.driver_id); switch (op_code) { /* Unknown op code */ diff --git a/src/H5Pocpl.c b/src/H5Pocpl.c index edb0cca..e442030 100644 --- a/src/H5Pocpl.c +++ b/src/H5Pocpl.c @@ -940,6 +940,8 @@ H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t id, unsigned int *flags /*out* H5TRACE8("e", "iZfx*zxzxx", plist_id, id, flags, cd_nelmts, cd_values, namelen, name, filter_config); /* Check args */ + if (id < 0 || id > H5Z_FILTER_MAX) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "filter ID value out of range") if (cd_nelmts || cd_values) { /* * It's likely that users forget to initialize this on input, so @@ -1838,6 +1840,8 @@ H5Pget_filter_by_id1(hid_t plist_id, H5Z_filter_t id, unsigned int *flags /*out* H5TRACE7("e", "iZfx*zxzx", plist_id, id, flags, cd_nelmts, cd_values, namelen, name); /* Check args */ + if (id < 0 || id > H5Z_FILTER_MAX) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "filter ID value out of range") if (cd_nelmts || cd_values) { /* * It's likely that users forget to initialize this on input, so diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 3fa7118..53fe5ac 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -6471,10 +6471,10 @@ H5_DLL herr_t H5Pset_layout(hid_t plist_id, H5D_layout_t layout); * byte 0 * * - * ???????? - * ????SPPP - * PPPPPPPP - * PPPP???? + * ???????? + * ????SPPP + * PPPPPPPP + * PPPP???? * * * Note: S - sign bit, P - significant bit, ? - padding bit; For diff --git a/src/H5VLint.c b/src/H5VLint.c index b602ac0..7035b83 100644 --- a/src/H5VLint.c +++ b/src/H5VLint.c @@ -191,7 +191,6 @@ H5VL_init_phase2(void) , {H5CX_init, "context"} , {H5ES_init, "event set"} , {H5Z_init, "transform"} - , {H5PL_init, "plugin"} , {H5R_init, "reference"} }; diff --git a/src/H5public.h b/src/H5public.h index 3e59e24..78d2a8f 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -83,15 +83,15 @@ /** * For tweaks, bug-fixes, or development */ -#define H5_VERS_RELEASE 0 +#define H5_VERS_RELEASE 1 /** * For pre-releases like \c snap0. Empty string for official releases. */ -#define H5_VERS_SUBRELEASE "7" +#define H5_VERS_SUBRELEASE "1" /** * Full version string */ -#define H5_VERS_INFO "HDF5 library version: 1.13.0-7" +#define H5_VERS_INFO "HDF5 library version: 1.13.1-1" #define H5check() H5check_version(H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE) diff --git a/test/filter_plugin.c b/test/filter_plugin.c index 276141a..c373b3b 100644 --- a/test/filter_plugin.c +++ b/test/filter_plugin.c @@ -1305,6 +1305,102 @@ error: } /* end test_path_api_calls() */ /*------------------------------------------------------------------------- + * Function: test_filter_numbers + * + * Purpose: Tests the filter numbers are handled correctly + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_filter_numbers(void) +{ + hid_t dcpl_id = H5I_INVALID_HID; + H5Z_filter_t id; + herr_t status = SUCCEED; + size_t nelmts = 0; + unsigned int flags; + unsigned int filter_config; + + HDputs("Testing filter number handling"); + + /* Check that out-of-range filter numbers are handled correctly */ + TESTING(" Filter # out of range"); + + /* Create property list */ + if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + nelmts = 0; + + /* Test id > H5Z_FILTER_MAX and < 0, current version */ + + H5E_BEGIN_TRY + { + id = H5Z_FILTER_MAX + 1; + status = H5Pget_filter_by_id2(dcpl_id, id, &flags, &nelmts, NULL, 0, NULL, &filter_config); + } + H5E_END_TRY; + + /* Should fail */ + if (status != FAIL) + TEST_ERROR; + + H5E_BEGIN_TRY + { + id = -1; + status = H5Pget_filter_by_id2(dcpl_id, id, &flags, &nelmts, NULL, 0, NULL, &filter_config); + } + H5E_END_TRY; + + /* Should fail */ + if (status != FAIL) + TEST_ERROR; + + /* Test id > H5Z_FILTER_MAX and < 0, deprecated version */ + +#ifndef H5_NO_DEPRECATED_SYMBOLS + H5E_BEGIN_TRY + { + id = H5Z_FILTER_MAX + 1; + status = H5Pget_filter_by_id1(dcpl_id, id, &flags, &nelmts, NULL, 0, NULL); + } + H5E_END_TRY; + + /* Should fail */ + if (status != FAIL) + TEST_ERROR; + + H5E_BEGIN_TRY + { + id = -1; + status = H5Pget_filter_by_id1(dcpl_id, id, &flags, &nelmts, NULL, 0, NULL); + } + H5E_END_TRY; + + /* Should fail */ + if (status != FAIL) + TEST_ERROR; +#endif + + if (H5Pclose(dcpl_id) < 0) + TEST_ERROR; + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Pclose(dcpl_id); + } + H5E_END_TRY; + return FAIL; +} /* end test_filter_numbers() */ + +/*------------------------------------------------------------------------- * Function: disable_chunk_cache * * Purpose: Turns the chunk cache off @@ -1523,6 +1619,9 @@ main(void) /* Test the APIs for access to the filter plugin path table */ nerrors += (test_path_api_calls() < 0 ? 1 : 0); + /* Test filter numbers */ + nerrors += (test_filter_numbers() < 0 ? 1 : 0); + if (nerrors) TEST_ERROR; diff --git a/test/h5test.h b/test/h5test.h index 914a534..7b82b68 100644 --- a/test/h5test.h +++ b/test/h5test.h @@ -172,60 +172,19 @@ H5TEST_DLLVAR MPI_Info h5_io_info_g; /* MPI INFO object for IO */ #define H5_EXCLUDE_MULTIPART_DRIVERS 0x01 #define H5_EXCLUDE_NON_MULTIPART_DRIVERS 0x02 -/* Macros to create and fill 2D arrays with a single heap allocation. - * These can be used to replace large stack and global arrays which raise - * warnings. - * - * The macros make a single heap allocation large enough to hold all the - * pointers and the data elements. The first part of the allocation holds - * the pointers, and the second part holds the data as a contiguous block - * in row-major order. - * - * To pass the data block to calls like H5Dread(), pass a pointer to the - * first array element as the data pointer (e.g., array[0] in a 2D array). - * - * The fill macro just fills the array with an increasing count value. - * - * Usage: - * - * int **array; - * - * H5TEST_ALLOCATE_2D_ARRAY(array, int, 5, 10); - * - * H5TEST_FILL_2D_ARRAY(array, int, 5, 10); - * - * (do stuff) - * - * HDfree(array); +/* Fill an array on the heap with an increasing count value. BUF + * is expected to point to a `struct { TYPE arr[...][...]; }`. */ -#define H5TEST_ALLOCATE_2D_ARRAY(ARR, TYPE, DIMS_I, DIMS_J) \ - do { \ - /* Prefix with h5taa to avoid shadow warnings */ \ - size_t h5taa_pointers_size = 0; \ - size_t h5taa_data_size = 0; \ - int h5taa_i; \ - \ - h5taa_pointers_size = (DIMS_I) * sizeof(TYPE *); \ - h5taa_data_size = (DIMS_I) * (DIMS_J) * sizeof(TYPE); \ - \ - ARR = (TYPE **)HDmalloc(h5taa_pointers_size + h5taa_data_size); \ - \ - ARR[0] = (TYPE *)(ARR + (DIMS_I)); \ - \ - for (h5taa_i = 1; h5taa_i < (DIMS_I); h5taa_i++) \ - ARR[h5taa_i] = ARR[h5taa_i - 1] + (DIMS_J); \ - } while (0) - -#define H5TEST_FILL_2D_ARRAY(ARR, TYPE, DIMS_I, DIMS_J) \ +#define H5TEST_FILL_2D_HEAP_ARRAY(BUF, TYPE) \ do { \ /* Prefix with h5tfa to avoid shadow warnings */ \ - int h5tfa_i = 0; \ - int h5tfa_j = 0; \ - TYPE h5tfa_count = 0; \ + size_t h5tfa_i = 0; \ + size_t h5tfa_j = 0; \ + TYPE h5tfa_count = 0; \ \ - for (h5tfa_i = 0; h5tfa_i < (DIMS_I); h5tfa_i++) \ - for (h5tfa_j = 0; h5tfa_j < (DIMS_J); h5tfa_j++) { \ - ARR[h5tfa_i][h5tfa_j] = h5tfa_count; \ + for (h5tfa_i = 0; h5tfa_i < NELMTS((BUF)->arr); h5tfa_i++) \ + for (h5tfa_j = 0; h5tfa_j < NELMTS((BUF)->arr[0]); h5tfa_j++) { \ + (BUF)->arr[h5tfa_i][h5tfa_j] = h5tfa_count; \ h5tfa_count++; \ } \ } while (0) diff --git a/test/vfd_plugin.c b/test/vfd_plugin.c index df211cc..182c048 100644 --- a/test/vfd_plugin.c +++ b/test/vfd_plugin.c @@ -313,8 +313,53 @@ test_get_config_str(void) if (H5Pclose(fapl_id) < 0) TEST_ERROR; + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Pclose(fapl_id); + } + H5E_END_TRY; + + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: test_env_var + * + * Purpose: Tests loading of NULL VFD plugin with HDF5_DRIVER + * environment variable and setting of VFD configuration + * string with HDF5_DRIVER_CONFIG environment variable + * + * Return: EXIT_SUCCESS/EXIT_FAILURE + * + *------------------------------------------------------------------------- + */ +static int +test_env_var(void) +{ + const char *const config_str = "{name: null}"; + ssize_t config_str_len = 0; + htri_t driver_is_registered; + char config_str_buf[128]; + + TESTING("Loading of VFD plugin with HDF5_DRIVER environment variable"); + + /* Try to retrieve length of default configuration string - should be 0 */ + HDmemset(config_str_buf, 0, 128); + + if ((config_str_len = H5Pget_driver_config_str(H5P_FILE_ACCESS_DEFAULT, config_str_buf, 128)) < 0) + TEST_ERROR; + if (0 != config_str_len) + TEST_ERROR; + if (HDstrlen(config_str_buf) > 0) + TEST_ERROR; + /* Set default driver and driver configuration using environment variables */ - if (HDsetenv(HDF5_DRIVER, "sec2", 1) < 0) + if (HDsetenv(HDF5_DRIVER, "null_vfd_plugin", 1) < 0) TEST_ERROR; if (HDsetenv(HDF5_DRIVER_CONFIG, config_str, 1) < 0) TEST_ERROR; @@ -325,7 +370,15 @@ test_get_config_str(void) if (H5open() < 0) TEST_ERROR; - /* Retrieve configuration string from default FAPL */ + /* Check driver */ + if ((driver_is_registered = H5FDis_driver_registered_by_name("null_vfd_plugin")) < 0) + TEST_ERROR; + if (!driver_is_registered) + TEST_ERROR; + if (H5Pget_driver(H5P_FILE_ACCESS_DEFAULT) == H5_DEFAULT_VFD) + TEST_ERROR; + + /* Check driver configuration string */ HDmemset(config_str_buf, 0, 128); if ((config_str_len = H5Pget_driver_config_str(H5P_FILE_ACCESS_DEFAULT, config_str_buf, 128)) < 0) TEST_ERROR; @@ -345,11 +398,8 @@ test_get_config_str(void) return SUCCEED; error: - H5E_BEGIN_TRY - { - H5Pclose(fapl_id); - } - H5E_END_TRY; + HDsetenv(HDF5_DRIVER, "", 1); + HDsetenv(HDF5_DRIVER_CONFIG, "", 1); return FAIL; } @@ -376,6 +426,7 @@ main(void) nerrors += (test_set_by_value() < 0) ? 1 : 0; nerrors += (test_set_multi() < 0) ? 1 : 0; nerrors += (test_get_config_str() < 0) ? 1 : 0; + nerrors += (test_env_var() < 0) ? 1 : 0; if (nerrors) { HDprintf("***** %d VFD plugin TEST%s FAILED! *****\n", nerrors, nerrors > 1 ? "S" : ""); diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index 4de2c5c..db8df16 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -1890,15 +1890,21 @@ render_bin_output(FILE *stream, hid_t container, hid_t tid, void *_mem, hsize_t hid_t region_id = H5I_INVALID_HID; hid_t region_space = H5I_INVALID_HID; H5S_sel_type region_type; + H5R_ref_t tref; + + if (size > sizeof(tref)) + H5TOOLS_THROW((-1), "unexpectedly large ref"); + + HDmemset(&tref, 0, sizeof(tref)); for (block_index = 0; block_index < block_nelmts; block_index++) { mem = ((unsigned char *)_mem) + block_index * size; - if ((region_id = H5Ropen_object((H5R_ref_t *)mem, H5P_DEFAULT, H5P_DEFAULT)) < 0) + HDmemcpy(&tref, mem, size); + if ((region_id = H5Ropen_object(&tref, H5P_DEFAULT, H5P_DEFAULT)) < 0) H5TOOLS_INFO("H5Ropen_object H5T_STD_REF failed"); else { - if ((region_space = H5Ropen_region((H5R_ref_t *)mem, H5P_DEFAULT, H5P_DEFAULT)) >= - 0) { - if (!h5tools_is_zero(mem, H5Tget_size(H5T_STD_REF))) { + if ((region_space = H5Ropen_region(&tref, H5P_DEFAULT, H5P_DEFAULT)) >= 0) { + if (!h5tools_is_zero(&tref, H5Tget_size(H5T_STD_REF))) { region_type = H5Sget_select_type(region_space); if (region_type == H5S_SEL_POINTS) render_bin_output_region_points(region_space, region_id, stream, diff --git a/tools/test/h5dump/CMakeTests.cmake b/tools/test/h5dump/CMakeTests.cmake index a8984de..c0f279d 100644 --- a/tools/test/h5dump/CMakeTests.cmake +++ b/tools/test/h5dump/CMakeTests.cmake @@ -412,7 +412,7 @@ # -------------------------------------------------------------------- HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/tbin1.ddl" "${PROJECT_BINARY_DIR}/testfiles/std/tbin1LE.ddl" "h5dump_std_files") - if (WIN32) + if (WIN32 AND CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION VERSION_LESS 10.0.18362.0) configure_file(${HDF5_TOOLS_DIR}/testfiles/tbinregR.exp ${PROJECT_BINARY_DIR}/testfiles/std/tbinregR.exp NEWLINE_STYLE CRLF) #file (READ ${HDF5_TOOLS_DIR}/testfiles/tbinregR.exp TEST_STREAM) #file (WRITE ${PROJECT_BINARY_DIR}/testfiles/std/tbinregR.exp "${TEST_STREAM}") diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index 7efe53c..b62c3a8 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -7264,15 +7264,33 @@ gent_packedbits(void) hid_t space = H5I_INVALID_HID; hsize_t dims[2]; - uint8_t ** dsetu8 = NULL; - uint16_t **dsetu16 = NULL; - uint32_t **dsetu32 = NULL; - uint64_t **dsetu64 = NULL; - int8_t ** dset8 = NULL; - int16_t ** dset16 = NULL; - int32_t ** dset32 = NULL; - int64_t ** dset64 = NULL; - double ** dsetdbl = NULL; + struct { + uint8_t arr[F66_XDIM][F66_YDIM8]; + } * dsetu8; + struct { + uint16_t arr[F66_XDIM][F66_YDIM16]; + } * dsetu16; + struct { + uint32_t arr[F66_XDIM][F66_YDIM32]; + } * dsetu32; + struct { + uint64_t arr[F66_XDIM][F66_YDIM64]; + } * dsetu64; + struct { + int8_t arr[F66_XDIM][F66_YDIM8]; + } * dset8; + struct { + int16_t arr[F66_XDIM][F66_YDIM16]; + } * dset16; + struct { + int32_t arr[F66_XDIM][F66_YDIM32]; + } * dset32; + struct { + int64_t arr[F66_XDIM][F66_YDIM64]; + } * dset64; + struct { + double arr[F66_XDIM][F66_YDIM8]; + } * dsetdbl; uint8_t valu8bits; uint16_t valu16bits; @@ -7286,15 +7304,15 @@ gent_packedbits(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dsetu8, uint8_t, F66_XDIM, F66_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dsetu16, uint16_t, F66_XDIM, F66_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dsetu32, uint32_t, F66_XDIM, F66_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dsetu64, uint64_t, F66_XDIM, F66_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset8, int8_t, F66_XDIM, F66_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dset16, int16_t, F66_XDIM, F66_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dset32, int32_t, F66_XDIM, F66_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, int64_t, F66_XDIM, F66_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dsetdbl, double, F66_XDIM, F66_YDIM8); + dsetu8 = malloc(sizeof(*dsetu8)); + dsetu16 = malloc(sizeof(*dsetu16)); + dsetu32 = malloc(sizeof(*dsetu32)); + dsetu64 = malloc(sizeof(*dsetu64)); + dset8 = malloc(sizeof(*dset8)); + dset16 = malloc(sizeof(*dset16)); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dsetdbl = malloc(sizeof(*dsetdbl)); fid = H5Fcreate(FILE66, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); @@ -7306,13 +7324,13 @@ gent_packedbits(void) valu8bits = (uint8_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu8[i][0] = valu8bits; + dsetu8->arr[i][0] = valu8bits; for (j = 1; j < dims[1]; j++) - dsetu8[i][j] = (uint8_t)(dsetu8[i][j - 1] << 1); + dsetu8->arr[i][j] = (uint8_t)(dsetu8->arr[i][j - 1] << 1); valu8bits = (uint8_t)(valu8bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_UINT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8); H5Sclose(space); H5Dclose(dataset); @@ -7324,13 +7342,13 @@ gent_packedbits(void) valu16bits = (uint16_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu16[i][0] = valu16bits; + dsetu16->arr[i][0] = valu16bits; for (j = 1; j < dims[1]; j++) - dsetu16[i][j] = (uint16_t)(dsetu16[i][j - 1] << 1); + dsetu16->arr[i][j] = (uint16_t)(dsetu16->arr[i][j - 1] << 1); valu16bits = (uint16_t)(valu16bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_UINT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16); H5Sclose(space); H5Dclose(dataset); @@ -7342,13 +7360,13 @@ gent_packedbits(void) valu32bits = (uint32_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu32[i][0] = valu32bits; + dsetu32->arr[i][0] = valu32bits; for (j = 1; j < dims[1]; j++) - dsetu32[i][j] = dsetu32[i][j - 1] << 1; + dsetu32->arr[i][j] = dsetu32->arr[i][j - 1] << 1; valu32bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_UINT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32); H5Sclose(space); H5Dclose(dataset); @@ -7360,13 +7378,13 @@ gent_packedbits(void) valu64bits = (uint64_t)~0Lu; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu64[i][0] = valu64bits; + dsetu64->arr[i][0] = valu64bits; for (j = 1; j < dims[1]; j++) - dsetu64[i][j] = dsetu64[i][j - 1] << 1; + dsetu64->arr[i][j] = dsetu64->arr[i][j - 1] << 1; valu64bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64); H5Sclose(space); H5Dclose(dataset); @@ -7378,13 +7396,13 @@ gent_packedbits(void) val8bits = (int8_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset8[i][0] = val8bits; + dset8->arr[i][0] = val8bits; for (j = 1; j < dims[1]; j++) - dset8[i][j] = (int8_t)(dset8[i][j - 1] << 1); + dset8->arr[i][j] = (int8_t)(dset8->arr[i][j - 1] << 1); val8bits = (int8_t)(val8bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_INT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8[0]); + H5Dwrite(dataset, H5T_NATIVE_INT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8); H5Sclose(space); H5Dclose(dataset); @@ -7396,13 +7414,13 @@ gent_packedbits(void) val16bits = (int16_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset16[i][0] = val16bits; + dset16->arr[i][0] = val16bits; for (j = 1; j < dims[1]; j++) - dset16[i][j] = (int16_t)(dset16[i][j - 1] << 1); + dset16->arr[i][j] = (int16_t)(dset16->arr[i][j - 1] << 1); val16bits = (int16_t)(val16bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_INT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16[0]); + H5Dwrite(dataset, H5T_NATIVE_INT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16); H5Sclose(space); H5Dclose(dataset); @@ -7414,13 +7432,13 @@ gent_packedbits(void) val32bits = (int32_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; + dset32->arr[i][0] = val32bits; for (j = 1; j < dims[1]; j++) - dset32[i][j] = dset32[i][j - 1] << 1; + dset32->arr[i][j] = dset32->arr[i][j - 1] << 1; val32bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32[0]); + H5Dwrite(dataset, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32); H5Sclose(space); H5Dclose(dataset); @@ -7432,13 +7450,13 @@ gent_packedbits(void) val64bits = (int64_t)~0L; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; + dset64->arr[i][0] = val64bits; for (j = 1; j < dims[1]; j++) - dset64[i][j] = dset64[i][j - 1] << 1; + dset64->arr[i][j] = dset64->arr[i][j - 1] << 1; val64bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_INT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64[0]); + H5Dwrite(dataset, H5T_NATIVE_INT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64); H5Sclose(space); H5Dclose(dataset); @@ -7450,9 +7468,9 @@ gent_packedbits(void) for (i = 0; i < dims[0]; i++) for (j = 0; j < dims[1]; j++) - dsetdbl[i][j] = 0.0001 * (double)j + (double)i; + dsetdbl->arr[i][j] = 0.0001 * (double)j + (double)i; - H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl[0]); + H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl); H5Sclose(space); H5Dclose(dataset); @@ -7489,15 +7507,33 @@ gent_attr_intsize(void) hid_t root = H5I_INVALID_HID; hsize_t dims[2]; - uint8_t ** dsetu8 = NULL; - uint16_t **dsetu16 = NULL; - uint32_t **dsetu32 = NULL; - uint64_t **dsetu64 = NULL; - int8_t ** dset8 = NULL; - int16_t ** dset16 = NULL; - int32_t ** dset32 = NULL; - int64_t ** dset64 = NULL; - double ** dsetdbl = NULL; + struct { + uint8_t arr[F66_XDIM][F66_YDIM8]; + } * dsetu8; + struct { + uint16_t arr[F66_XDIM][F66_YDIM16]; + } * dsetu16; + struct { + uint32_t arr[F66_XDIM][F66_YDIM32]; + } * dsetu32; + struct { + uint64_t arr[F66_XDIM][F66_YDIM64]; + } * dsetu64; + struct { + int8_t arr[F66_XDIM][F66_YDIM8]; + } * dset8; + struct { + int16_t arr[F66_XDIM][F66_YDIM16]; + } * dset16; + struct { + int32_t arr[F66_XDIM][F66_YDIM64]; + } * dset32; + struct { + int64_t arr[F66_XDIM][F66_YDIM64]; + } * dset64; + struct { + double arr[F66_XDIM][F66_YDIM8]; + } * dsetdbl; uint8_t valu8bits; uint16_t valu16bits; @@ -7511,15 +7547,15 @@ gent_attr_intsize(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dsetu8, uint8_t, F66_XDIM, F66_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dsetu16, uint16_t, F66_XDIM, F66_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dsetu32, uint32_t, F66_XDIM, F66_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dsetu64, uint64_t, F66_XDIM, F66_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset8, int8_t, F66_XDIM, F66_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dset16, int16_t, F66_XDIM, F66_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dset32, int32_t, F66_XDIM, F66_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, int64_t, F66_XDIM, F66_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dsetdbl, double, F66_XDIM, F66_YDIM8); + dsetu8 = malloc(sizeof(*dsetu8)); + dsetu16 = malloc(sizeof(*dsetu16)); + dsetu32 = malloc(sizeof(*dsetu32)); + dsetu64 = malloc(sizeof(*dsetu64)); + dset8 = malloc(sizeof(*dset8)); + dset16 = malloc(sizeof(*dset16)); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dsetdbl = malloc(sizeof(*dsetdbl)); fid = H5Fcreate(FILE69, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); root = H5Gopen2(fid, "/", H5P_DEFAULT); @@ -7532,14 +7568,14 @@ gent_attr_intsize(void) valu8bits = (uint8_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu8[i][0] = valu8bits; + dsetu8->arr[i][0] = valu8bits; for (j = 1; j < dims[1]; j++) { - dsetu8[i][j] = (uint8_t)(dsetu8[i][j - 1] << 1); + dsetu8->arr[i][j] = (uint8_t)(dsetu8->arr[i][j - 1] << 1); } valu8bits = (uint8_t)(valu8bits << 1); } - H5Awrite(attr, H5T_NATIVE_UINT8, dsetu8[0]); + H5Awrite(attr, H5T_NATIVE_UINT8, dsetu8); H5Sclose(space); H5Aclose(attr); @@ -7551,14 +7587,14 @@ gent_attr_intsize(void) valu16bits = (uint16_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu16[i][0] = valu16bits; + dsetu16->arr[i][0] = valu16bits; for (j = 1; j < dims[1]; j++) { - dsetu16[i][j] = (uint16_t)(dsetu16[i][j - 1] << 1); + dsetu16->arr[i][j] = (uint16_t)(dsetu16->arr[i][j - 1] << 1); } valu16bits = (uint16_t)(valu16bits << 1); } - H5Awrite(attr, H5T_NATIVE_UINT16, dsetu16[0]); + H5Awrite(attr, H5T_NATIVE_UINT16, dsetu16); H5Sclose(space); H5Aclose(attr); @@ -7570,14 +7606,14 @@ gent_attr_intsize(void) valu32bits = (uint32_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu32[i][0] = valu32bits; + dsetu32->arr[i][0] = valu32bits; for (j = 1; j < dims[1]; j++) { - dsetu32[i][j] = dsetu32[i][j - 1] << 1; + dsetu32->arr[i][j] = dsetu32->arr[i][j - 1] << 1; } valu32bits <<= 1; } - H5Awrite(attr, H5T_NATIVE_UINT32, dsetu32[0]); + H5Awrite(attr, H5T_NATIVE_UINT32, dsetu32); H5Sclose(space); H5Aclose(attr); @@ -7589,14 +7625,14 @@ gent_attr_intsize(void) valu64bits = (uint64_t)~0Lu; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu64[i][0] = valu64bits; + dsetu64->arr[i][0] = valu64bits; for (j = 1; j < dims[1]; j++) { - dsetu64[i][j] = dsetu64[i][j - 1] << 1; + dsetu64->arr[i][j] = dsetu64->arr[i][j - 1] << 1; } valu64bits <<= 1; } - H5Awrite(attr, H5T_NATIVE_UINT64, dsetu64[0]); + H5Awrite(attr, H5T_NATIVE_UINT64, dsetu64); H5Sclose(space); H5Aclose(attr); @@ -7608,14 +7644,14 @@ gent_attr_intsize(void) val8bits = (int8_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset8[i][0] = val8bits; + dset8->arr[i][0] = val8bits; for (j = 1; j < dims[1]; j++) { - dset8[i][j] = (int8_t)(dset8[i][j - 1] << 1); + dset8->arr[i][j] = (int8_t)(dset8->arr[i][j - 1] << 1); } val8bits = (int8_t)(val8bits << 1); } - H5Awrite(attr, H5T_NATIVE_INT8, dset8[0]); + H5Awrite(attr, H5T_NATIVE_INT8, dset8); H5Sclose(space); H5Aclose(attr); @@ -7627,14 +7663,14 @@ gent_attr_intsize(void) val16bits = (int16_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset16[i][0] = val16bits; + dset16->arr[i][0] = val16bits; for (j = 1; j < dims[1]; j++) { - dset16[i][j] = (int16_t)(dset16[i][j - 1] << 1); + dset16->arr[i][j] = (int16_t)(dset16->arr[i][j - 1] << 1); } val16bits = (int16_t)(val16bits << 1); } - H5Awrite(attr, H5T_NATIVE_INT16, dset16[0]); + H5Awrite(attr, H5T_NATIVE_INT16, dset16); H5Sclose(space); H5Aclose(attr); @@ -7646,14 +7682,14 @@ gent_attr_intsize(void) val32bits = (int32_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; + dset32->arr[i][0] = val32bits; for (j = 1; j < dims[1]; j++) { - dset32[i][j] = dset32[i][j - 1] << 1; + dset32->arr[i][j] = dset32->arr[i][j - 1] << 1; } val32bits <<= 1; } - H5Awrite(attr, H5T_NATIVE_INT32, dset32[0]); + H5Awrite(attr, H5T_NATIVE_INT32, dset32); H5Sclose(space); H5Aclose(attr); @@ -7665,14 +7701,14 @@ gent_attr_intsize(void) val64bits = (int64_t)~0L; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; + dset64->arr[i][0] = val64bits; for (j = 1; j < dims[1]; j++) { - dset64[i][j] = dset64[i][j - 1] << 1; + dset64->arr[i][j] = dset64->arr[i][j - 1] << 1; } val64bits <<= 1; } - H5Awrite(attr, H5T_NATIVE_INT64, dset64[0]); + H5Awrite(attr, H5T_NATIVE_INT64, dset64); H5Sclose(space); H5Aclose(attr); @@ -7684,9 +7720,9 @@ gent_attr_intsize(void) for (i = 0; i < dims[0]; i++) for (j = 0; j < dims[1]; j++) - dsetdbl[i][j] = 0.0001 * (double)j + (double)i; + dsetdbl->arr[i][j] = 0.0001 * (double)j + (double)i; - H5Awrite(attr, H5T_NATIVE_DOUBLE, dsetdbl[0]); + H5Awrite(attr, H5T_NATIVE_DOUBLE, dsetdbl); H5Sclose(space); H5Aclose(attr); @@ -8586,15 +8622,33 @@ gent_intscalars(void) hid_t tid = H5I_INVALID_HID; hsize_t dims[2]; - uint8_t ** dsetu8 = NULL; - uint16_t **dsetu16 = NULL; - uint32_t **dsetu32 = NULL; - uint64_t **dsetu64 = NULL; - int8_t ** dset8 = NULL; - int16_t ** dset16 = NULL; - int32_t ** dset32 = NULL; - int64_t ** dset64 = NULL; - double ** dsetdbl = NULL; + struct { + uint8_t arr[F73_XDIM][F73_YDIM8]; + } * dsetu8; + struct { + uint16_t arr[F73_XDIM][F73_YDIM16]; + } * dsetu16; + struct { + uint32_t arr[F73_XDIM][F73_YDIM32]; + } * dsetu32; + struct { + uint64_t arr[F73_XDIM][F73_YDIM64]; + } * dsetu64; + struct { + int8_t arr[F73_XDIM][F73_YDIM8]; + } * dset8; + struct { + int16_t arr[F73_XDIM][F73_YDIM16]; + } * dset16; + struct { + int32_t arr[F73_XDIM][F73_YDIM32]; + } * dset32; + struct { + int64_t arr[F73_XDIM][F73_YDIM64]; + } * dset64; + struct { + double arr[F73_XDIM][F73_YDIM8]; + } * dsetdbl; uint8_t valu8bits; uint16_t valu16bits; @@ -8608,15 +8662,15 @@ gent_intscalars(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dsetu8, uint8_t, F73_XDIM, F73_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dsetu16, uint16_t, F73_XDIM, F73_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dsetu32, uint32_t, F73_XDIM, F73_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dsetu64, uint64_t, F73_XDIM, F73_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset8, int8_t, F73_XDIM, F73_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dset16, int16_t, F73_XDIM, F73_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dset32, int32_t, F73_XDIM, F73_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, int64_t, F73_XDIM, F73_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dsetdbl, double, F73_XDIM, F73_YDIM8); + dsetu8 = malloc(sizeof(*dsetu8)); + dsetu16 = malloc(sizeof(*dsetu16)); + dsetu32 = malloc(sizeof(*dsetu32)); + dsetu64 = malloc(sizeof(*dsetu64)); + dset8 = malloc(sizeof(*dset8)); + dset16 = malloc(sizeof(*dset16)); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dsetdbl = malloc(sizeof(*dsetdbl)); fid = H5Fcreate(FILE73, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); @@ -8629,14 +8683,14 @@ gent_intscalars(void) valu8bits = (uint8_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu8[i][0] = valu8bits; + dsetu8->arr[i][0] = valu8bits; for (j = 1; j < dims[1]; j++) { - dsetu8[i][j] = (uint8_t)(dsetu8[i][j - 1] << 1); + dsetu8->arr[i][j] = (uint8_t)(dsetu8->arr[i][j - 1] << 1); } valu8bits = (uint8_t)(valu8bits << 1); } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8); H5Sclose(space); H5Dclose(dataset); @@ -8649,14 +8703,14 @@ gent_intscalars(void) valu16bits = (uint16_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu16[i][0] = valu16bits; + dsetu16->arr[i][0] = valu16bits; for (j = 1; j < dims[1]; j++) { - dsetu16[i][j] = (uint16_t)(dsetu16[i][j - 1] << 1); + dsetu16->arr[i][j] = (uint16_t)(dsetu16->arr[i][j - 1] << 1); } valu16bits = (uint16_t)(valu16bits << 1); } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16); H5Sclose(space); H5Dclose(dataset); @@ -8669,14 +8723,14 @@ gent_intscalars(void) valu32bits = (uint32_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu32[i][0] = valu32bits; + dsetu32->arr[i][0] = valu32bits; for (j = 1; j < dims[1]; j++) { - dsetu32[i][j] = dsetu32[i][j - 1] << 1; + dsetu32->arr[i][j] = dsetu32->arr[i][j - 1] << 1; } valu32bits <<= 1; } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32); H5Sclose(space); H5Dclose(dataset); @@ -8689,14 +8743,14 @@ gent_intscalars(void) valu64bits = (uint64_t)~0Lu; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu64[i][0] = valu64bits; + dsetu64->arr[i][0] = valu64bits; for (j = 1; j < dims[1]; j++) { - dsetu64[i][j] = dsetu64[i][j - 1] << 1; + dsetu64->arr[i][j] = dsetu64->arr[i][j - 1] << 1; } valu64bits <<= 1; } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64); H5Sclose(space); H5Dclose(dataset); @@ -8709,14 +8763,14 @@ gent_intscalars(void) val8bits = (int8_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset8[i][0] = val8bits; + dset8->arr[i][0] = val8bits; for (j = 1; j < dims[1]; j++) { - dset8[i][j] = (int8_t)(dset8[i][j - 1] << 1); + dset8->arr[i][j] = (int8_t)(dset8->arr[i][j - 1] << 1); } val8bits = (int8_t)(val8bits << 1); } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8); H5Sclose(space); H5Dclose(dataset); @@ -8729,14 +8783,14 @@ gent_intscalars(void) val16bits = (int16_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset16[i][0] = val16bits; + dset16->arr[i][0] = val16bits; for (j = 1; j < dims[1]; j++) { - dset16[i][j] = (int16_t)(dset16[i][j - 1] << 1); + dset16->arr[i][j] = (int16_t)(dset16->arr[i][j - 1] << 1); } val16bits = (int16_t)(val16bits << 1); } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16); H5Sclose(space); H5Dclose(dataset); @@ -8749,14 +8803,14 @@ gent_intscalars(void) val32bits = (int32_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; + dset32->arr[i][0] = val32bits; for (j = 1; j < dims[1]; j++) { - dset32[i][j] = dset32[i][j - 1] << 1; + dset32->arr[i][j] = dset32->arr[i][j - 1] << 1; } val32bits <<= 1; } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32); H5Sclose(space); H5Dclose(dataset); @@ -8769,14 +8823,14 @@ gent_intscalars(void) val64bits = (int64_t)~0L; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; + dset64->arr[i][0] = val64bits; for (j = 1; j < dims[1]; j++) { - dset64[i][j] = dset64[i][j - 1] << 1; + dset64->arr[i][j] = dset64->arr[i][j - 1] << 1; } val64bits <<= 1; } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64); H5Sclose(space); H5Dclose(dataset); @@ -8789,9 +8843,9 @@ gent_intscalars(void) for (i = 0; i < dims[0]; i++) for (j = 0; j < dims[1]; j++) - dsetdbl[i][j] = 0.0001 * (double)j + (double)i; + dsetdbl->arr[i][j] = 0.0001 * (double)j + (double)i; - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl); H5Sclose(space); H5Dclose(dataset); @@ -8829,15 +8883,33 @@ gent_attr_intscalars(void) hid_t tid = H5I_INVALID_HID; hsize_t dims[2]; - uint8_t ** dsetu8 = NULL; - uint16_t **dsetu16 = NULL; - uint32_t **dsetu32 = NULL; - uint64_t **dsetu64 = NULL; - int8_t ** dset8 = NULL; - int16_t ** dset16 = NULL; - int32_t ** dset32 = NULL; - int64_t ** dset64 = NULL; - double ** dsetdbl = NULL; + struct { + uint8_t arr[F73_XDIM][F73_YDIM8]; + } * dsetu8; + struct { + uint16_t arr[F73_XDIM][F73_YDIM16]; + } * dsetu16; + struct { + uint32_t arr[F73_XDIM][F73_YDIM32]; + } * dsetu32; + struct { + uint64_t arr[F73_XDIM][F73_YDIM64]; + } * dsetu64; + struct { + int8_t arr[F73_XDIM][F73_YDIM8]; + } * dset8; + struct { + int16_t arr[F73_XDIM][F73_YDIM16]; + } * dset16; + struct { + int32_t arr[F73_XDIM][F73_YDIM32]; + } * dset32; + struct { + int64_t arr[F73_XDIM][F73_YDIM64]; + } * dset64; + struct { + double arr[F73_XDIM][F73_YDIM8]; + } * dsetdbl; uint8_t valu8bits; uint16_t valu16bits; @@ -8851,15 +8923,15 @@ gent_attr_intscalars(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dsetu8, uint8_t, F73_XDIM, F73_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dsetu16, uint16_t, F73_XDIM, F73_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dsetu32, uint32_t, F73_XDIM, F73_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dsetu64, uint64_t, F73_XDIM, F73_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset8, int8_t, F73_XDIM, F73_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dset16, int16_t, F73_XDIM, F73_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dset32, int32_t, F73_XDIM, F73_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, int64_t, F73_XDIM, F73_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dsetdbl, double, F73_XDIM, F73_YDIM8); + dsetu8 = malloc(sizeof(*dsetu8)); + dsetu16 = malloc(sizeof(*dsetu16)); + dsetu32 = malloc(sizeof(*dsetu32)); + dsetu64 = malloc(sizeof(*dsetu64)); + dset8 = malloc(sizeof(*dset8)); + dset16 = malloc(sizeof(*dset16)); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dsetdbl = malloc(sizeof(*dsetdbl)); fid = H5Fcreate(FILE74, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); root = H5Gopen2(fid, "/", H5P_DEFAULT); @@ -8873,14 +8945,14 @@ gent_attr_intscalars(void) valu8bits = (uint8_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu8[i][0] = valu8bits; + dsetu8->arr[i][0] = valu8bits; for (j = 1; j < dims[1]; j++) { - dsetu8[i][j] = (uint8_t)(dsetu8[i][j - 1] << 1); + dsetu8->arr[i][j] = (uint8_t)(dsetu8->arr[i][j - 1] << 1); } valu8bits = (uint8_t)(valu8bits << 1); } - H5Awrite(attr, tid, dsetu8[0]); + H5Awrite(attr, tid, dsetu8); H5Sclose(space); H5Aclose(attr); @@ -8893,14 +8965,14 @@ gent_attr_intscalars(void) valu16bits = (uint16_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu16[i][0] = valu16bits; + dsetu16->arr[i][0] = valu16bits; for (j = 1; j < dims[1]; j++) { - dsetu16[i][j] = (uint16_t)(dsetu16[i][j - 1] << 1); + dsetu16->arr[i][j] = (uint16_t)(dsetu16->arr[i][j - 1] << 1); } valu16bits = (uint16_t)(valu16bits << 1); } - H5Awrite(attr, tid, dsetu16[0]); + H5Awrite(attr, tid, dsetu16); H5Sclose(space); H5Aclose(attr); @@ -8913,14 +8985,14 @@ gent_attr_intscalars(void) valu32bits = (uint32_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu32[i][0] = valu32bits; + dsetu32->arr[i][0] = valu32bits; for (j = 1; j < dims[1]; j++) { - dsetu32[i][j] = dsetu32[i][j - 1] << 1; + dsetu32->arr[i][j] = dsetu32->arr[i][j - 1] << 1; } valu32bits <<= 1; } - H5Awrite(attr, tid, dsetu32[0]); + H5Awrite(attr, tid, dsetu32); H5Sclose(space); H5Aclose(attr); @@ -8933,14 +9005,14 @@ gent_attr_intscalars(void) valu64bits = (uint64_t)~0Lu; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu64[i][0] = valu64bits; + dsetu64->arr[i][0] = valu64bits; for (j = 1; j < dims[1]; j++) { - dsetu64[i][j] = dsetu64[i][j - 1] << 1; + dsetu64->arr[i][j] = dsetu64->arr[i][j - 1] << 1; } valu64bits <<= 1; } - H5Awrite(attr, tid, dsetu64[0]); + H5Awrite(attr, tid, dsetu64); H5Sclose(space); H5Aclose(attr); @@ -8953,14 +9025,14 @@ gent_attr_intscalars(void) val8bits = (int8_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset8[i][0] = val8bits; + dset8->arr[i][0] = val8bits; for (j = 1; j < dims[1]; j++) { - dset8[i][j] = (int8_t)(dset8[i][j - 1] << 1); + dset8->arr[i][j] = (int8_t)(dset8->arr[i][j - 1] << 1); } val8bits = (int8_t)(val8bits << 1); } - H5Awrite(attr, tid, dset8[0]); + H5Awrite(attr, tid, dset8); H5Sclose(space); H5Aclose(attr); @@ -8973,14 +9045,14 @@ gent_attr_intscalars(void) val16bits = (int16_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset16[i][0] = val16bits; + dset16->arr[i][0] = val16bits; for (j = 1; j < dims[1]; j++) { - dset16[i][j] = (int16_t)(dset16[i][j - 1] << 1); + dset16->arr[i][j] = (int16_t)(dset16->arr[i][j - 1] << 1); } val16bits = (int16_t)(val16bits << 1); } - H5Awrite(attr, tid, dset16[0]); + H5Awrite(attr, tid, dset16); H5Sclose(space); H5Aclose(attr); @@ -8993,14 +9065,14 @@ gent_attr_intscalars(void) val32bits = (int32_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; + dset32->arr[i][0] = val32bits; for (j = 1; j < dims[1]; j++) { - dset32[i][j] = dset32[i][j - 1] << 1; + dset32->arr[i][j] = dset32->arr[i][j - 1] << 1; } val32bits <<= 1; } - H5Awrite(attr, tid, dset32[0]); + H5Awrite(attr, tid, dset32); H5Sclose(space); H5Aclose(attr); @@ -9013,14 +9085,14 @@ gent_attr_intscalars(void) val64bits = (int64_t)~0L; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; + dset64->arr[i][0] = val64bits; for (j = 1; j < dims[1]; j++) { - dset64[i][j] = dset64[i][j - 1] << 1; + dset64->arr[i][j] = dset64->arr[i][j - 1] << 1; } val64bits <<= 1; } - H5Awrite(attr, tid, dset64[0]); + H5Awrite(attr, tid, dset64); H5Sclose(space); H5Aclose(attr); @@ -9033,9 +9105,9 @@ gent_attr_intscalars(void) for (i = 0; i < dims[0]; i++) for (j = 0; j < dims[1]; j++) - dsetdbl[i][j] = 0.0001 * (double)j + (double)i; + dsetdbl->arr[i][j] = 0.0001 * (double)j + (double)i; - H5Awrite(attr, tid, dsetdbl[0]); + H5Awrite(attr, tid, dsetdbl); H5Sclose(space); H5Aclose(attr); @@ -9618,15 +9690,33 @@ gent_intattrscalars(void) hid_t tid = H5I_INVALID_HID; hsize_t dims[2]; - uint8_t ** dsetu8 = NULL; - uint16_t **dsetu16 = NULL; - uint32_t **dsetu32 = NULL; - uint64_t **dsetu64 = NULL; - int8_t ** dset8 = NULL; - int16_t ** dset16 = NULL; - int32_t ** dset32 = NULL; - int64_t ** dset64 = NULL; - double ** dsetdbl = NULL; + struct { + uint8_t arr[F73_XDIM][F73_YDIM8]; + } *dsetu8 = NULL; + struct { + uint16_t arr[F73_XDIM][F73_YDIM16]; + } *dsetu16 = NULL; + struct { + uint32_t arr[F73_XDIM][F73_YDIM32]; + } *dsetu32 = NULL; + struct { + uint64_t arr[F73_XDIM][F73_YDIM64]; + } *dsetu64 = NULL; + struct { + int8_t arr[F73_XDIM][F73_YDIM8]; + } *dset8 = NULL; + struct { + int16_t arr[F73_XDIM][F73_YDIM16]; + } *dset16 = NULL; + struct { + int32_t arr[F73_XDIM][F73_YDIM32]; + } *dset32 = NULL; + struct { + int64_t arr[F73_XDIM][F73_YDIM64]; + } *dset64 = NULL; + struct { + double arr[F73_XDIM][F73_YDIM8]; + } *dsetdbl = NULL; uint8_t valu8bits; uint16_t valu16bits; @@ -9640,15 +9730,15 @@ gent_intattrscalars(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dsetu8, uint8_t, F73_XDIM, F73_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dsetu16, uint16_t, F73_XDIM, F73_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dsetu32, uint32_t, F73_XDIM, F73_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dsetu64, uint64_t, F73_XDIM, F73_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset8, int8_t, F73_XDIM, F73_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dset16, int16_t, F73_XDIM, F73_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dset32, int32_t, F73_XDIM, F73_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, int64_t, F73_XDIM, F73_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dsetdbl, double, F73_XDIM, F73_YDIM8); + dsetu8 = malloc(sizeof(*dsetu8)); + dsetu16 = malloc(sizeof(*dsetu16)); + dsetu32 = malloc(sizeof(*dsetu32)); + dsetu64 = malloc(sizeof(*dsetu64)); + dset8 = malloc(sizeof(*dset8)); + dset16 = malloc(sizeof(*dset16)); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dsetdbl = malloc(sizeof(*dsetdbl)); fid = H5Fcreate(FILE78, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); @@ -9661,9 +9751,9 @@ gent_intattrscalars(void) valu8bits = (uint8_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu8[i][0] = valu8bits; + dsetu8->arr[i][0] = valu8bits; for (j = 1; j < dims[1]; j++) { - dsetu8[i][j] = (uint8_t)(dsetu8[i][j - 1] << 1); + dsetu8->arr[i][j] = (uint8_t)(dsetu8->arr[i][j - 1] << 1); } valu8bits = (uint8_t)(valu8bits << 1); } @@ -9671,7 +9761,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8); /* Attribute of 8 bits unsigned int */ attr = H5Acreate2(dataset, F73_DATASETU08, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dsetu8[0]); + H5Awrite(attr, tid, dsetu8); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9685,9 +9775,9 @@ gent_intattrscalars(void) valu16bits = (uint16_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu16[i][0] = valu16bits; + dsetu16->arr[i][0] = valu16bits; for (j = 1; j < dims[1]; j++) { - dsetu16[i][j] = (uint16_t)(dsetu16[i][j - 1] << 1); + dsetu16->arr[i][j] = (uint16_t)(dsetu16->arr[i][j - 1] << 1); } valu16bits = (uint16_t)(valu16bits << 1); } @@ -9695,7 +9785,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16); /* Attribute of 16 bits unsigned int */ attr = H5Acreate2(dataset, F73_DATASETU16, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dsetu16[0]); + H5Awrite(attr, tid, dsetu16); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9709,9 +9799,9 @@ gent_intattrscalars(void) valu32bits = (uint32_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu32[i][0] = valu32bits; + dsetu32->arr[i][0] = valu32bits; for (j = 1; j < dims[1]; j++) { - dsetu32[i][j] = dsetu32[i][j - 1] << 1; + dsetu32->arr[i][j] = dsetu32->arr[i][j - 1] << 1; } valu32bits <<= 1; } @@ -9719,7 +9809,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32); /* Attribute of 32 bits unsigned int */ attr = H5Acreate2(dataset, F73_DATASETU32, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dsetu32[0]); + H5Awrite(attr, tid, dsetu32); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9733,9 +9823,9 @@ gent_intattrscalars(void) valu64bits = (uint64_t)~0Lu; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu64[i][0] = valu64bits; + dsetu64->arr[i][0] = valu64bits; for (j = 1; j < dims[1]; j++) { - dsetu64[i][j] = dsetu64[i][j - 1] << 1; + dsetu64->arr[i][j] = dsetu64->arr[i][j - 1] << 1; } valu64bits <<= 1; } @@ -9743,7 +9833,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64); /* Attribute of 64 bits unsigned int */ attr = H5Acreate2(dataset, F73_DATASETU64, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dsetu64[0]); + H5Awrite(attr, tid, dsetu64); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9757,9 +9847,9 @@ gent_intattrscalars(void) val8bits = (int8_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset8[i][0] = val8bits; + dset8->arr[i][0] = val8bits; for (j = 1; j < dims[1]; j++) { - dset8[i][j] = (int8_t)(dset8[i][j - 1] << 1); + dset8->arr[i][j] = (int8_t)(dset8->arr[i][j - 1] << 1); } val8bits = (int8_t)(val8bits << 1); } @@ -9767,7 +9857,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8); /* Attribute of 8 bits signed int */ attr = H5Acreate2(dataset, F73_DATASETS08, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dset8[0]); + H5Awrite(attr, tid, dset8); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9781,9 +9871,9 @@ gent_intattrscalars(void) val16bits = (int16_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset16[i][0] = val16bits; + dset16->arr[i][0] = val16bits; for (j = 1; j < dims[1]; j++) { - dset16[i][j] = (int16_t)(dset16[i][j - 1] << 1); + dset16->arr[i][j] = (int16_t)(dset16->arr[i][j - 1] << 1); } val16bits = (int16_t)(val16bits << 1); } @@ -9791,7 +9881,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16); /* Attribute of 16 bits signed int */ attr = H5Acreate2(dataset, F73_DATASETS16, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dset16[0]); + H5Awrite(attr, tid, dset16); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9805,9 +9895,9 @@ gent_intattrscalars(void) val32bits = (int32_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; + dset32->arr[i][0] = val32bits; for (j = 1; j < dims[1]; j++) { - dset32[i][j] = dset32[i][j - 1] << 1; + dset32->arr[i][j] = dset32->arr[i][j - 1] << 1; } val32bits <<= 1; } @@ -9815,7 +9905,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32); /* Attribute of 32 bits signed int */ attr = H5Acreate2(dataset, F73_DATASETS32, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dset32[0]); + H5Awrite(attr, tid, dset32); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9829,9 +9919,9 @@ gent_intattrscalars(void) val64bits = (int64_t)~0L; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; + dset64->arr[i][0] = val64bits; for (j = 1; j < dims[1]; j++) { - dset64[i][j] = dset64[i][j - 1] << 1; + dset64->arr[i][j] = dset64->arr[i][j - 1] << 1; } val64bits <<= 1; } @@ -9839,7 +9929,7 @@ gent_intattrscalars(void) H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64); /* Attribute of 64 bits signed int */ attr = H5Acreate2(dataset, F73_DATASETS64, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dset64[0]); + H5Awrite(attr, tid, dset64); H5Aclose(attr); H5Sclose(space); H5Dclose(dataset); @@ -9853,12 +9943,12 @@ gent_intattrscalars(void) for (i = 0; i < dims[0]; i++) for (j = 0; j < dims[1]; j++) - dsetdbl[i][j] = 0.0001 * (double)j + (double)i; + dsetdbl->arr[i][j] = 0.0001 * (double)j + (double)i; H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl); /* Attribute of double */ attr = H5Acreate2(dataset, F73_DUMMYDBL, tid, space, H5P_DEFAULT, H5P_DEFAULT); - H5Awrite(attr, tid, dsetdbl[0]); + H5Awrite(attr, tid, dsetdbl); H5Aclose(attr); H5Sclose(space); @@ -9896,15 +9986,33 @@ gent_intsattrs(void) hid_t aspace = H5I_INVALID_HID; hsize_t dims[2], adims[1]; - uint8_t ** dsetu8 = NULL; - uint16_t **dsetu16 = NULL; - uint32_t **dsetu32 = NULL; - uint64_t **dsetu64 = NULL; - int8_t ** dset8 = NULL; - int16_t ** dset16 = NULL; - int32_t ** dset32 = NULL; - int64_t ** dset64 = NULL; - double ** dsetdbl = NULL; + struct { + uint8_t arr[F66_XDIM][F66_YDIM8]; + } * dsetu8; + struct { + uint16_t arr[F66_XDIM][F66_YDIM16]; + } * dsetu16; + struct { + uint32_t arr[F66_XDIM][F66_YDIM32]; + } * dsetu32; + struct { + uint64_t arr[F66_XDIM][F66_YDIM64]; + } * dsetu64; + struct { + int8_t arr[F66_XDIM][F66_YDIM8]; + } * dset8; + struct { + int16_t arr[F66_XDIM][F66_YDIM16]; + } * dset16; + struct { + int32_t arr[F66_XDIM][F66_YDIM32]; + } * dset32; + struct { + int64_t arr[F66_XDIM][F66_YDIM64]; + } * dset64; + struct { + double arr[F66_XDIM][F66_YDIM8]; + } * dsetdbl; uint8_t * asetu8 = NULL; uint16_t *asetu16 = NULL; @@ -9928,15 +10036,15 @@ gent_intsattrs(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dsetu8, uint8_t, F66_XDIM, F66_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dsetu16, uint16_t, F66_XDIM, F66_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dsetu32, uint32_t, F66_XDIM, F66_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dsetu64, uint64_t, F66_XDIM, F66_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset8, int8_t, F66_XDIM, F66_YDIM8); - H5TEST_ALLOCATE_2D_ARRAY(dset16, int16_t, F66_XDIM, F66_YDIM16); - H5TEST_ALLOCATE_2D_ARRAY(dset32, int32_t, F66_XDIM, F66_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, int64_t, F66_XDIM, F66_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dsetdbl, double, F66_XDIM, F66_YDIM8); + dsetu8 = malloc(sizeof(*dsetu8)); + dsetu16 = malloc(sizeof(*dsetu16)); + dsetu32 = malloc(sizeof(*dsetu32)); + dsetu64 = malloc(sizeof(*dsetu64)); + dset8 = malloc(sizeof(*dset8)); + dset16 = malloc(sizeof(*dset16)); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dsetdbl = malloc(sizeof(*dsetdbl)); asetu8 = HDcalloc(F66_XDIM * F66_YDIM8, sizeof(uint8_t)); asetu16 = HDcalloc(F66_XDIM * F66_YDIM16, sizeof(uint16_t)); @@ -9958,16 +10066,16 @@ gent_intsattrs(void) valu8bits = (uint8_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu8[i][0] = valu8bits; - asetu8[i * dims[1]] = dsetu8[i][0]; + dsetu8->arr[i][0] = valu8bits; + asetu8[i * dims[1]] = dsetu8->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dsetu8[i][j] = (uint8_t)(dsetu8[i][j - 1] << 1); - asetu8[i * dims[1] + j] = dsetu8[i][j]; + dsetu8->arr[i][j] = (uint8_t)(dsetu8->arr[i][j - 1] << 1); + asetu8[i * dims[1] + j] = dsetu8->arr[i][j]; } valu8bits = (uint8_t)(valu8bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_UINT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu8); /* Attribute of 8 bits unsigned int */ adims[0] = F66_XDIM * F66_YDIM8; aspace = H5Screate_simple(1, adims, NULL); @@ -9986,16 +10094,16 @@ gent_intsattrs(void) valu16bits = (uint16_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu16[i][0] = valu16bits; - asetu16[i * dims[1]] = dsetu16[i][0]; + dsetu16->arr[i][0] = valu16bits; + asetu16[i * dims[1]] = dsetu16->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dsetu16[i][j] = (uint16_t)(dsetu16[i][j - 1] << 1); - asetu16[i * dims[1] + j] = dsetu16[i][j]; + dsetu16->arr[i][j] = (uint16_t)(dsetu16->arr[i][j - 1] << 1); + asetu16[i * dims[1] + j] = dsetu16->arr[i][j]; } valu16bits = (uint16_t)(valu16bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_UINT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu16); /* Attribute of 16 bits unsigned int */ adims[0] = F66_XDIM * F66_YDIM16; aspace = H5Screate_simple(1, adims, NULL); @@ -10014,16 +10122,16 @@ gent_intsattrs(void) valu32bits = (uint32_t)~0u; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu32[i][0] = valu32bits; - asetu32[i * dims[1]] = dsetu32[i][0]; + dsetu32->arr[i][0] = valu32bits; + asetu32[i * dims[1]] = dsetu32->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dsetu32[i][j] = dsetu32[i][j - 1] << 1; - asetu32[i * dims[1] + j] = dsetu32[i][j]; + dsetu32->arr[i][j] = dsetu32->arr[i][j - 1] << 1; + asetu32[i * dims[1] + j] = dsetu32->arr[i][j]; } valu32bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_UINT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu32); /* Attribute of 32 bits unsigned int */ adims[0] = F66_XDIM * F66_YDIM32; aspace = H5Screate_simple(1, adims, NULL); @@ -10042,16 +10150,16 @@ gent_intsattrs(void) valu64bits = (uint64_t)~0Lu; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dsetu64[i][0] = valu64bits; - asetu64[i * dims[1]] = dsetu64[i][0]; + dsetu64->arr[i][0] = valu64bits; + asetu64[i * dims[1]] = dsetu64->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dsetu64[i][j] = dsetu64[i][j - 1] << 1; - asetu64[i * dims[1] + j] = dsetu64[i][j]; + dsetu64->arr[i][j] = dsetu64->arr[i][j - 1] << 1; + asetu64[i * dims[1] + j] = dsetu64->arr[i][j]; } valu64bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64[0]); + H5Dwrite(dataset, H5T_NATIVE_UINT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetu64); /* Attribute of 64 bits unsigned int */ adims[0] = F66_XDIM * F66_YDIM64; aspace = H5Screate_simple(1, adims, NULL); @@ -10070,16 +10178,16 @@ gent_intsattrs(void) val8bits = (int8_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset8[i][0] = val8bits; - aset8[i * dims[1]] = dset8[i][0]; + dset8->arr[i][0] = val8bits; + aset8[i * dims[1]] = dset8->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset8[i][j] = (int8_t)(dset8[i][j - 1] << 1); - aset8[i * dims[1] + j] = dset8[i][j]; + dset8->arr[i][j] = (int8_t)(dset8->arr[i][j - 1] << 1); + aset8[i * dims[1] + j] = dset8->arr[i][j]; } val8bits = (int8_t)(val8bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_INT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8[0]); + H5Dwrite(dataset, H5T_NATIVE_INT8, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset8); /* Attribute of 8 bits signed int */ adims[0] = F66_XDIM * F66_YDIM8; aspace = H5Screate_simple(1, adims, NULL); @@ -10098,16 +10206,16 @@ gent_intsattrs(void) val16bits = (int16_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset16[i][0] = val16bits; - aset16[i * dims[1]] = dset16[i][0]; + dset16->arr[i][0] = val16bits; + aset16[i * dims[1]] = dset16->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset16[i][j] = (int16_t)(dset16[i][j - 1] << 1); - aset16[i * dims[1] + j] = dset16[i][j]; + dset16->arr[i][j] = (int16_t)(dset16->arr[i][j - 1] << 1); + aset16[i * dims[1] + j] = dset16->arr[i][j]; } val16bits = (int16_t)(val16bits << 1); } - H5Dwrite(dataset, H5T_NATIVE_INT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16[0]); + H5Dwrite(dataset, H5T_NATIVE_INT16, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset16); /* Attribute of 16 bits signed int */ adims[0] = F66_XDIM * F66_YDIM16; aspace = H5Screate_simple(1, adims, NULL); @@ -10126,16 +10234,16 @@ gent_intsattrs(void) val32bits = (int32_t)~0; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; - aset32[i * dims[1]] = dset32[i][0]; + dset32->arr[i][0] = val32bits; + aset32[i * dims[1]] = dset32->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset32[i][j] = dset32[i][j - 1] << 1; - aset32[i * dims[1] + j] = dset32[i][j]; + dset32->arr[i][j] = dset32->arr[i][j - 1] << 1; + aset32[i * dims[1] + j] = dset32->arr[i][j]; } val32bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32[0]); + H5Dwrite(dataset, H5T_NATIVE_INT32, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32); /* Attribute of 32 bits signed int */ adims[0] = F66_XDIM * F66_YDIM32; aspace = H5Screate_simple(1, adims, NULL); @@ -10154,16 +10262,16 @@ gent_intsattrs(void) val64bits = (int64_t)~0L; /* all 1s */ for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; - aset64[i * dims[1]] = dset64[i][0]; + dset64->arr[i][0] = val64bits; + aset64[i * dims[1]] = dset64->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset64[i][j] = dset64[i][j - 1] << 1; - aset64[i * dims[1] + j] = dset64[i][j]; + dset64->arr[i][j] = dset64->arr[i][j - 1] << 1; + aset64[i * dims[1] + j] = dset64->arr[i][j]; } val64bits <<= 1; } - H5Dwrite(dataset, H5T_NATIVE_INT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64[0]); + H5Dwrite(dataset, H5T_NATIVE_INT64, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64); /* Attribute of 64 bits signed int */ adims[0] = F66_XDIM * F66_YDIM64; aspace = H5Screate_simple(1, adims, NULL); @@ -10182,11 +10290,11 @@ gent_intsattrs(void) for (i = 0; i < dims[0]; i++) for (j = 0; j < dims[1]; j++) { - dsetdbl[i][j] = 0.0001 * (double)j + (double)i; - asetdbl[i * dims[1] + j] = dsetdbl[i][j]; + dsetdbl->arr[i][j] = 0.0001 * (double)j + (double)i; + asetdbl[i * dims[1] + j] = dsetdbl->arr[i][j]; } - H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl[0]); + H5Dwrite(dataset, H5T_NATIVE_DOUBLE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dsetdbl); /* Attribute of double */ adims[0] = F66_XDIM * F66_YDIM8; aspace = H5Screate_simple(1, adims, NULL); @@ -10238,9 +10346,15 @@ gent_floatsattrs(void) hid_t aspace = H5I_INVALID_HID; hsize_t dims[2], adims[1]; - float ** dset32 = NULL; - double ** dset64 = NULL; - long double **dset128 = NULL; + struct { + float arr[F89_XDIM][F89_YDIM32]; + } * dset32; + struct { + double arr[F89_XDIM][F89_YDIM64]; + } * dset64; + struct { + long double arr[F89_XDIM][F89_YDIM128]; + } * dset128; float * aset32 = NULL; double * aset64 = NULL; @@ -10253,9 +10367,9 @@ gent_floatsattrs(void) unsigned int i, j; /* Create arrays */ - H5TEST_ALLOCATE_2D_ARRAY(dset32, float, F89_XDIM, F89_YDIM32); - H5TEST_ALLOCATE_2D_ARRAY(dset64, double, F89_XDIM, F89_YDIM64); - H5TEST_ALLOCATE_2D_ARRAY(dset128, long double, F89_XDIM, F89_YDIM128); + dset32 = malloc(sizeof(*dset32)); + dset64 = malloc(sizeof(*dset64)); + dset128 = malloc(sizeof(*dset128)); aset32 = HDcalloc(F89_XDIM * F89_YDIM32, sizeof(float)); aset64 = HDcalloc(F89_XDIM * F89_YDIM64, sizeof(double)); @@ -10277,16 +10391,16 @@ gent_floatsattrs(void) val32bits = (float)F89_YDIM32; for (i = 0; i < dims[0]; i++) { - dset32[i][0] = val32bits; - aset32[i * dims[1]] = dset32[i][0]; + dset32->arr[i][0] = val32bits; + aset32[i * dims[1]] = dset32->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset32[i][j] = (float)(j * dims[0] + i) / (float)F89_YDIM32; - aset32[i * dims[1] + j] = dset32[i][j]; + dset32->arr[i][j] = (float)(j * dims[0] + i) / (float)F89_YDIM32; + aset32[i * dims[1] + j] = dset32->arr[i][j]; } val32bits -= (float)1; } - H5Dwrite(dataset, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32[0]); + H5Dwrite(dataset, H5T_IEEE_F32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset32); /* Attribute of 32 bits float */ adims[0] = F89_XDIM * F89_YDIM32; aspace = H5Screate_simple(1, adims, NULL); @@ -10305,16 +10419,16 @@ gent_floatsattrs(void) val64bits = (double)F89_YDIM64; for (i = 0; i < dims[0]; i++) { - dset64[i][0] = val64bits; - aset64[i * dims[1]] = dset64[i][0]; + dset64->arr[i][0] = val64bits; + aset64[i * dims[1]] = dset64->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset64[i][j] = (double)(j * dims[0] + i) / (double)F89_YDIM64; - aset64[i * dims[1] + j] = dset64[i][j]; + dset64->arr[i][j] = (double)(j * dims[0] + i) / (double)F89_YDIM64; + aset64[i * dims[1] + j] = dset64->arr[i][j]; } val64bits -= (double)1; } - H5Dwrite(dataset, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64[0]); + H5Dwrite(dataset, H5T_IEEE_F64LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset64); /* Attribute of 64 bits double */ adims[0] = F89_XDIM * F89_YDIM64; aspace = H5Screate_simple(1, adims, NULL); @@ -10333,16 +10447,16 @@ gent_floatsattrs(void) val128bits = (long double)F89_YDIM128; for (i = 0; i < dims[0]; i++) { - dset128[i][0] = val128bits; - aset128[i * dims[1]] = dset128[i][0]; + dset128->arr[i][0] = val128bits; + aset128[i * dims[1]] = dset128->arr[i][0]; for (j = 1; j < dims[1]; j++) { - dset128[i][j] = (long double)(j * dims[0] + i) / (long double)F89_YDIM128; - aset128[i * dims[1] + j] = dset128[i][j]; + dset128->arr[i][j] = (long double)(j * dims[0] + i) / (long double)F89_YDIM128; + aset128[i * dims[1] + j] = dset128->arr[i][j]; } val128bits -= (long double)1; } - H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset128[0]); + H5Dwrite(dataset, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset128); /* Attribute of 128 bits long double */ adims[0] = F89_XDIM * F89_YDIM128; aspace = H5Screate_simple(1, adims, NULL); @@ -10534,13 +10648,17 @@ gent_bitnopaquefields(void) static void gent_intsfourdims(void) { - hid_t fid, dataset, space; - hsize_t dims[F81_RANK]; - uint32_t dset1[F81_ZDIM][F81_YDIM][F81_XDIM][F81_WDIM]; + hid_t fid, dataset, space; + hsize_t dims[F81_RANK]; + struct { + uint32_t arr[F81_ZDIM][F81_YDIM][F81_XDIM][F81_WDIM]; + } * dset1; unsigned int i, j, k, l; fid = H5Fcreate(FILE81, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + dset1 = malloc(sizeof(*dset1)); + /* Dataset of 32 bits unsigned int */ dims[0] = F81_ZDIM; dims[1] = F81_YDIM; @@ -10553,7 +10671,7 @@ gent_intsfourdims(void) for (j = 0; j < F81_YDIM; j++) for (k = 0; k < F81_XDIM; k++) for (l = 0; l < F81_WDIM; l++) - dset1[i][j][k][l] = + dset1->arr[i][j][k][l] = i * F81_YDIM * F81_XDIM * F81_WDIM + j * F81_XDIM * F81_WDIM + k * F81_WDIM + l; H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset1); @@ -10561,6 +10679,7 @@ gent_intsfourdims(void) H5Dclose(dataset); H5Fclose(fid); + free(dset1); } /*------------------------------------------------------------------------- diff --git a/tools/test/h5repack/h5repacktst.c b/tools/test/h5repack/h5repacktst.c index ed3fedd..6ca33e2 100644 --- a/tools/test/h5repack/h5repacktst.c +++ b/tools/test/h5repack/h5repacktst.c @@ -2455,14 +2455,15 @@ make_szip(hid_t loc_id) unsigned szip_pixels_per_block = 8; hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; int szip_can_encode = 0; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2486,7 +2487,7 @@ make_szip(hid_t loc_id) /* set szip data */ if (H5Pset_szip(dcpl, szip_options_mask, szip_pixels_per_block) < 0) goto error; - if (make_dset(loc_id, "dset_szip", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_szip", sid, dcpl, buf) < 0) goto error; } else @@ -2529,15 +2530,16 @@ make_deflate(hid_t loc_id) hid_t sid = H5I_INVALID_HID; /* dataspace ID */ hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; hobj_ref_t bufref[1]; /* reference */ hsize_t dims1r[1] = {1}; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2557,7 +2559,7 @@ make_deflate(hid_t loc_id) /* set deflate data */ if (H5Pset_deflate(dcpl, 9) < 0) goto error; - if (make_dset(loc_id, "dset_deflate", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_deflate", sid, dcpl, buf) < 0) goto error; /* create a reference to the dataset, test second seeep of file for references */ @@ -2608,13 +2610,14 @@ make_shuffle(hid_t loc_id) hid_t sid = H5I_INVALID_HID; /* dataspace ID */ hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2634,7 +2637,7 @@ make_shuffle(hid_t loc_id) /* set the shuffle filter */ if (H5Pset_shuffle(dcpl) < 0) goto error; - if (make_dset(loc_id, "dset_shuffle", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_shuffle", sid, dcpl, buf) < 0) goto error; /*------------------------------------------------------------------------- @@ -2677,13 +2680,14 @@ make_fletcher32(hid_t loc_id) hid_t sid = H5I_INVALID_HID; /* dataspace ID */ hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2706,7 +2710,7 @@ make_fletcher32(hid_t loc_id) /* set the checksum filter */ if (H5Pset_fletcher32(dcpl) < 0) goto error; - if (make_dset(loc_id, "dset_fletcher32", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_fletcher32", sid, dcpl, buf) < 0) goto error; /*------------------------------------------------------------------------- @@ -2752,13 +2756,14 @@ make_nbit(hid_t loc_id) hid_t dxpl = H5P_DEFAULT; hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2791,7 +2796,7 @@ make_nbit(hid_t loc_id) goto error; if ((dsid = H5Dcreate2(loc_id, "dset_nbit", dtid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; - if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf[0]) < 0) + if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf) < 0) goto error; H5Dclose(dsid); @@ -2799,7 +2804,7 @@ make_nbit(hid_t loc_id) goto error; if ((dsid = H5Dcreate2(loc_id, "dset_int31", dtid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; - if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf[0]) < 0) + if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf) < 0) goto error; H5Dclose(dsid); @@ -2853,13 +2858,14 @@ make_scaleoffset(hid_t loc_id) hid_t dxpl = H5P_DEFAULT; hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -2890,12 +2896,12 @@ make_scaleoffset(hid_t loc_id) goto error; if ((dsid = H5Dcreate2(loc_id, "dset_scaleoffset", dtid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; - if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf[0]) < 0) + if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf) < 0) goto error; H5Dclose(dsid); if ((dsid = H5Dcreate2(loc_id, "dset_none", dtid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; - if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf[0]) < 0) + if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf) < 0) goto error; H5Tclose(dtid); H5Dclose(dsid); @@ -2952,16 +2958,17 @@ make_all_filters(hid_t loc_id) #endif /* H5_HAVE_FILTER_SZIP */ hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; #if defined(H5_HAVE_FILTER_SZIP) int szip_can_encode = 0; #endif /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /* create a space */ if ((sid = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -3011,7 +3018,7 @@ make_all_filters(hid_t loc_id) goto error; #endif - if (make_dset(loc_id, "dset_all", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_all", sid, dcpl, buf) < 0) goto error; /* remove the filters from the dcpl */ @@ -3020,7 +3027,7 @@ make_all_filters(hid_t loc_id) /* set the checksum filter */ if (H5Pset_fletcher32(dcpl) < 0) goto error; - if (make_dset(loc_id, "dset_fletcher32", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_fletcher32", sid, dcpl, buf) < 0) goto error; /* Make sure encoding is enabled */ @@ -3032,7 +3039,7 @@ make_all_filters(hid_t loc_id) /* set szip data */ if (H5Pset_szip(dcpl, szip_options_mask, szip_pixels_per_block) < 0) goto error; - if (make_dset(loc_id, "dset_szip", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_szip", sid, dcpl, buf) < 0) goto error; } else { @@ -3046,7 +3053,7 @@ make_all_filters(hid_t loc_id) /* set the shuffle filter */ if (H5Pset_shuffle(dcpl) < 0) goto error; - if (make_dset(loc_id, "dset_shuffle", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_shuffle", sid, dcpl, buf) < 0) goto error; #if defined(H5_HAVE_FILTER_DEFLATE) @@ -3056,7 +3063,7 @@ make_all_filters(hid_t loc_id) /* set deflate data */ if (H5Pset_deflate(dcpl, 1) < 0) goto error; - if (make_dset(loc_id, "dset_deflate", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_deflate", sid, dcpl, buf) < 0) goto error; #endif @@ -3072,7 +3079,7 @@ make_all_filters(hid_t loc_id) goto error; if ((dsid = H5Dcreate2(loc_id, "dset_nbit", dtid, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; - if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf[0]) < 0) + if (H5Dwrite(dsid, dtid, H5S_ALL, H5S_ALL, dxpl, buf) < 0) goto error; /* close */ @@ -3217,15 +3224,16 @@ make_layout(hid_t loc_id) hid_t sid = H5I_INVALID_HID; /* dataspace ID */ hsize_t dims[RANK] = {DIM1, DIM2}; hsize_t chunk_dims[RANK] = {CDIM1, CDIM2}; - int ** buf = NULL; int i; char name[16]; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, DIM1, DIM2); + struct { + int arr[DIM1][DIM2]; + } *buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; - H5TEST_FILL_2D_ARRAY(buf, int, DIM1, DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(buf, int); /*------------------------------------------------------------------------- * make several dataset with no filters @@ -3233,7 +3241,7 @@ make_layout(hid_t loc_id) */ for (i = 0; i < 4; i++) { HDsprintf(name, "dset%d", i + 1); - if (write_dset(loc_id, RANK, dims, name, H5T_NATIVE_INT, buf[0]) < 0) + if (write_dset(loc_id, RANK, dims, name, H5T_NATIVE_INT, buf) < 0) goto error; } @@ -3254,7 +3262,7 @@ make_layout(hid_t loc_id) */ if (H5Pset_layout(dcpl, H5D_COMPACT) < 0) goto error; - if (make_dset(loc_id, "dset_compact", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_compact", sid, dcpl, buf) < 0) goto error; /*------------------------------------------------------------------------- @@ -3263,7 +3271,7 @@ make_layout(hid_t loc_id) */ if (H5Pset_layout(dcpl, H5D_CONTIGUOUS) < 0) goto error; - if (make_dset(loc_id, "dset_contiguous", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_contiguous", sid, dcpl, buf) < 0) goto error; /*------------------------------------------------------------------------- @@ -3272,7 +3280,7 @@ make_layout(hid_t loc_id) */ if (H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) goto error; - if (make_dset(loc_id, "dset_chunk", sid, dcpl, buf[0]) < 0) + if (make_dset(loc_id, "dset_chunk", sid, dcpl, buf) < 0) goto error; /*------------------------------------------------------------------------- @@ -3329,13 +3337,13 @@ make_layout2(hid_t loc_id) hsize_t s_dims[RANK] = {S_DIM1, S_DIM2}; /* Dataspace (< 1 k) */ hsize_t chunk_dims[RANK] = {S_DIM1 / 2, S_DIM2 / 2}; /* Dimension sizes for chunks */ - int **s_buf = NULL; /* Temporary buffer */ - /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(s_buf, int, S_DIM1, S_DIM2); + struct { + int arr[S_DIM1][S_DIM2]; + } *s_buf = malloc(sizeof(*s_buf)); if (NULL == s_buf) goto error; - H5TEST_FILL_2D_ARRAY(s_buf, int, S_DIM1, S_DIM2); + H5TEST_FILL_2D_HEAP_ARRAY(s_buf, int); /* Create dataspaces */ if ((s_sid = H5Screate_simple(RANK, s_dims, NULL)) < 0) @@ -3346,7 +3354,7 @@ make_layout2(hid_t loc_id) goto error; if (H5Pset_layout(contig_dcpl, H5D_CONTIGUOUS) < 0) goto error; - if (make_dset(loc_id, CONTIG_S, s_sid, contig_dcpl, s_buf[0]) < 0) + if (make_dset(loc_id, CONTIG_S, s_sid, contig_dcpl, s_buf) < 0) goto error; /* Create chunked datasets */ @@ -3354,7 +3362,7 @@ make_layout2(hid_t loc_id) goto error; if (H5Pset_chunk(chunked_dcpl, RANK, chunk_dims) < 0) goto error; - if (make_dset(loc_id, CHUNKED_S_FIX, s_sid, chunked_dcpl, s_buf[0]) < 0) + if (make_dset(loc_id, CHUNKED_S_FIX, s_sid, chunked_dcpl, s_buf) < 0) goto error; ret_value = 0; @@ -3408,19 +3416,21 @@ make_layout3(hid_t loc_id) hsize_t chunk_dims1[RANK] = {DIM1_L3 * 2, 5}; hsize_t chunk_dims2[RANK] = {SDIM1_L3 + 2, SDIM2_L3 / 2}; hsize_t chunk_dims3[RANK] = {SDIM1_L3 - 2, SDIM2_L3 / 2}; - int ** buf1 = NULL; - int ** buf2 = NULL; - /* Create and fill arrays */ - H5TEST_ALLOCATE_2D_ARRAY(buf1, int, DIM1_L3, DIM2_L3); - if (NULL == buf1) - goto error; - H5TEST_FILL_2D_ARRAY(buf1, int, DIM1_L3, DIM2_L3); + /* Create arrays */ + struct { + int arr[DIM1_L3][DIM2_L3]; + } *buf1 = malloc(sizeof(*buf1)); + struct { + int arr[SDIM1_L3][SDIM2_L3]; + } *buf2 = malloc(sizeof(*buf2)); - H5TEST_ALLOCATE_2D_ARRAY(buf2, int, SDIM1_L3, SDIM2_L3); - if (NULL == buf2) + if (NULL == buf1 || NULL == buf2) goto error; - H5TEST_FILL_2D_ARRAY(buf2, int, SDIM1_L3, SDIM2_L3); + + /* Fill arrays */ + H5TEST_FILL_2D_HEAP_ARRAY(buf1, int); + H5TEST_FILL_2D_HEAP_ARRAY(buf2, int); /*------------------------------------------------------------------------- * make chunked dataset with @@ -3438,7 +3448,7 @@ make_layout3(hid_t loc_id) if (H5Pset_chunk(dcpl1, RANK, chunk_dims1) < 0) goto error; - if (make_dset(loc_id, "chunk_unlimit1", sid1, dcpl1, buf1[0]) < 0) + if (make_dset(loc_id, "chunk_unlimit1", sid1, dcpl1, buf1) < 0) goto error; /*------------------------------------------------------------------------- @@ -3459,7 +3469,7 @@ make_layout3(hid_t loc_id) if (H5Pset_chunk(dcpl2, RANK, chunk_dims2) < 0) goto error; - if (make_dset(loc_id, "chunk_unlimit2", sid2, dcpl2, buf2[0]) < 0) + if (make_dset(loc_id, "chunk_unlimit2", sid2, dcpl2, buf2) < 0) goto error; /*------------------------------------------------------------------------- @@ -3476,7 +3486,7 @@ make_layout3(hid_t loc_id) if (H5Pset_chunk(dcpl3, RANK, chunk_dims3) < 0) goto error; - if (make_dset(loc_id, "chunk_unlimit3", sid2, dcpl3, buf2[0]) < 0) + if (make_dset(loc_id, "chunk_unlimit3", sid2, dcpl3, buf2) < 0) goto error; /*------------------------------------------------------------------------- diff --git a/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl b/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl index eeb0f2d..15ae813 100644 --- a/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl +++ b/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl @@ -11,7 +11,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { @@ -33,7 +33,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { @@ -55,7 +55,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { @@ -77,7 +77,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { @@ -99,7 +99,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { @@ -121,7 +121,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { @@ -143,7 +143,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { diff --git a/tools/test/misc/h5clear_gentest.c b/tools/test/misc/h5clear_gentest.c index 53fbc42..923c3f2 100644 --- a/tools/test/misc/h5clear_gentest.c +++ b/tools/test/misc/h5clear_gentest.c @@ -56,24 +56,26 @@ const char *FILENAME_ENHANCE[] = { static int gen_cache_image_file(const char *fname) { - hid_t fid = H5I_INVALID_HID; /* File ID */ - hid_t did = -1, sid = H5I_INVALID_HID; /* Dataset ID, dataspace ID */ - hid_t fapl = H5I_INVALID_HID; /* File access property list */ - hid_t dcpl = H5I_INVALID_HID; /* Dataset creation property list */ - hsize_t dims[2]; /* Dimension sizes */ - hsize_t chunks[2]; /* Chunked dimension sizes */ - int i, j; /* Local index variables */ - int ** buf = NULL; /* Buffer for data to write */ - H5AC_cache_image_config_t cache_image_config = /* Cache image input configuration */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t did = -1, sid = H5I_INVALID_HID; /* Dataset ID, dataspace ID */ + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + hid_t dcpl = H5I_INVALID_HID; /* Dataset creation property list */ + hsize_t dims[2]; /* Dimension sizes */ + hsize_t chunks[2]; /* Chunked dimension sizes */ + int i, j; /* Local index variables */ + struct { + int arr[50][100]; + } * buf; /* Buffer for data to write */ + H5AC_cache_image_config_t cache_image_config = /* Cache image input configuration */ {H5AC__CURR_CACHE_IMAGE_CONFIG_VERSION, TRUE, FALSE, H5AC__CACHE_IMAGE__ENTRY_AGEOUT__NONE}; /* Create and fill array */ - H5TEST_ALLOCATE_2D_ARRAY(buf, int, 50, 100); + buf = malloc(sizeof(*buf)); if (NULL == buf) goto error; for (i = 0; i < 50; i++) for (j = 0; j < 100; j++) - buf[i][j] = i * j; + buf->arr[i][j] = i * j; /* Create a copy of file access property list */ if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) @@ -108,7 +110,7 @@ gen_cache_image_file(const char *fname) goto error; /* Write to the dataset */ - if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf[0]) < 0) + if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) goto error; /* Closing */ diff --git a/tools/test/perform/chunk_cache.c b/tools/test/perform/chunk_cache.c index a99334f..832447e 100644 --- a/tools/test/perform/chunk_cache.c +++ b/tools/test/perform/chunk_cache.c @@ -98,7 +98,9 @@ create_dset1(hid_t file) hid_t dcpl = H5I_INVALID_HID; hsize_t dims[RANK] = {DSET1_DIM1, DSET1_DIM2}; hsize_t chunk_dims[RANK] = {CHUNK1_DIM1, CHUNK1_DIM2}; - int ** data = NULL; /* data for writing */ + struct { + int arr[DSET1_DIM1][DSET1_DIM2]; + } *data = malloc(sizeof(*data)); /* Create the data space. */ if ((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -122,9 +124,8 @@ create_dset1(hid_t file) 0) goto error; - /* Create & fill array */ - H5TEST_ALLOCATE_2D_ARRAY(data, int, DSET1_DIM1, DSET1_DIM2); - H5TEST_FILL_2D_ARRAY(data, int, DSET1_DIM1, DSET1_DIM2); + /* Fill array */ + H5TEST_FILL_2D_HEAP_ARRAY(data, int); /* Write data to dataset */ if (H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) @@ -162,7 +163,9 @@ create_dset2(hid_t file) hid_t dcpl = H5I_INVALID_HID; hsize_t dims[RANK] = {DSET2_DIM1, DSET2_DIM2}; hsize_t chunk_dims[RANK] = {CHUNK2_DIM1, CHUNK2_DIM2}; - int ** data = NULL; /* data for writing */ + struct { + int arr[DSET2_DIM1][DSET2_DIM2]; + } *data = malloc(sizeof(*data)); /* Create the data space. */ if ((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) @@ -185,9 +188,8 @@ create_dset2(hid_t file) 0) goto error; - /* Create & fill array */ - H5TEST_ALLOCATE_2D_ARRAY(data, int, DSET2_DIM1, DSET2_DIM2); - H5TEST_FILL_2D_ARRAY(data, int, DSET2_DIM1, DSET2_DIM2); + /* Fill array */ + H5TEST_FILL_2D_HEAP_ARRAY(data, int); /* Write data to dataset */ if (H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 7c263d1..a248ce1 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -9,3 +9,8 @@ option (HDF5_BUILD_UTILS "Build HDF5 Utils" ON) if (HDF5_BUILD_UTILS) add_subdirectory (mirror_vfd) endif () + +#-- Add the h5dwalk and test executables +if (HDF5_BUILD_PARALLEL_TOOLS AND HDF5_ENABLE_PARALLEL) + add_subdirectory(tools) +endif() diff --git a/utils/Makefile.am b/utils/Makefile.am index 876dfb1..cd63db4 100644 --- a/utils/Makefile.am +++ b/utils/Makefile.am @@ -32,7 +32,13 @@ else MIRROR_VFD_DIR= endif +if BUILD_TOOLS_CONDITIONAL + TOOLS_DIR =tools +else + TOOLS_DIR= +endif + # All subdirectories -SUBDIRS=$(MIRROR_VFD_DIR) $(TESTUTIL_DIR) +SUBDIRS=$(MIRROR_VFD_DIR) $(TESTUTIL_DIR) $(TOOLS_DIR) include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/CMakeLists.txt b/utils/tools/CMakeLists.txt new file mode 100644 index 0000000..49562d7 --- /dev/null +++ b/utils/tools/CMakeLists.txt @@ -0,0 +1,12 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_UTILS_TOOLS C) + + +if (HDF5_BUILD_PARALLEL_TOOLS) + add_subdirectory (h5dwalk) +endif() + +#-- Add the tests +if (BUILD_TESTING) + add_subdirectory (test) +endif() diff --git a/utils/tools/Makefile.am b/utils/tools/Makefile.am new file mode 100644 index 0000000..0c89aff --- /dev/null +++ b/utils/tools/Makefile.am @@ -0,0 +1,38 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +## +# +# Tools HDF5 Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +if PARALLEL_TOOLS_CONDITIONAL + H5DWALK=h5dwalk +else + H5DWALK= +endif + +if BUILD_TESTS_CONDITIONAL + TESTSERIAL_DIR =test +else + TESTSERIAL_DIR= +endif + +CONFIG=ordered + +# All subdirectories +SUBDIRS=$(H5DWALK) $(TESTSERIAL_DIR) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/h5dwalk/CMakeLists.txt b/utils/tools/h5dwalk/CMakeLists.txt new file mode 100644 index 0000000..244cc26 --- /dev/null +++ b/utils/tools/h5dwalk/CMakeLists.txt @@ -0,0 +1,66 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_UTILS_TOOLS_H5DWALK C) + +# -------------------------------------------------------------------- +# Add the h5dwalk and test executables +# -------------------------------------------------------------------- +if (NOT ONLY_SHARED_LIBS) + add_executable (h5dwalk ${HDF5_UTILS_TOOLS_H5DWALK_SOURCE_DIR}/h5dwalk.c) +# add_custom_target(generate_demo ALL +# DEPENDS "${HDF5_TOOLS_DIR}/test/demo_destfiles.test" +# ) + target_include_directories (h5dwalk PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${CIRCLE_INCLUDE_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_compile_options(h5dwalk PRIVATE "${HDF5_CMAKE_C_FLAGS}") + TARGET_C_PROPERTIES (h5dwalk STATIC) + target_link_libraries (h5dwalk PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} ${MFU_LIBRARY} "$<$:${MPI_C_LIBRARIES}>") + set_target_properties (h5dwalk PROPERTIES FOLDER tools) + set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5dwalk") + + set (H5_DEP_EXECUTABLES h5dwalk) +endif () + +if (BUILD_SHARED_LIBS) + add_executable (h5dwalk-shared ${HDF5_UTILS_TOOLS_H5DWALK_SOURCE_DIR}/h5dwalk.c) + target_include_directories (h5dwalk-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${CIRCLE_INCLUDE_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + target_compile_options(h5dwalk-shared PRIVATE "${HDF5_CMAKE_C_FLAGS}") + TARGET_C_PROPERTIES (h5dwalk-shared SHARED) + target_link_libraries (h5dwalk-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} ${MFU_LIBRARY} "$<$:${MPI_C_LIBRARIES}>") + set_target_properties (h5dwalk-shared PROPERTIES FOLDER tools) + set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5dwalk-shared") + + set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} h5dwalk-shared) +endif () + +#----------------------------------------------------------------------------- +# Add Target to clang-format +#----------------------------------------------------------------------------- +if (HDF5_ENABLE_FORMATTERS) + if (NOT ONLY_SHARED_LIBS) + clang_format (HDF5_H5DWALK_SRC_FORMAT h5dwalk) + else () + clang_format (HDF5_H5DWALK_SRC_FORMAT h5dwalk-shared) + endif () +endif () + +############################################################################## +############################################################################## +### I N S T A L L A T I O N ### +############################################################################## +############################################################################## + +#----------------------------------------------------------------------------- +# Rules for Installation of tools using make Install target +#----------------------------------------------------------------------------- +if (HDF5_EXPORTED_TARGETS) + foreach (exec ${H5_DEP_EXECUTABLES}) + INSTALL_PROGRAM_PDB (${exec} ${HDF5_INSTALL_BIN_DIR} toolsapplications) + endforeach () + + install ( + TARGETS + ${H5_DEP_EXECUTABLES} + EXPORT + ${HDF5_EXPORTED_TARGETS} + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications + ) +endif () diff --git a/utils/tools/h5dwalk/Makefile.am b/utils/tools/h5dwalk/Makefile.am new file mode 100644 index 0000000..34cdb32 --- /dev/null +++ b/utils/tools/h5dwalk/Makefile.am @@ -0,0 +1,37 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +# Include src directory +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib $(H5DWALK_CPPFLAGS) + +# These are our main targets, the tools +# h5dwalk_SOURCES=h5dwalk.c $(TOOLSOURCES) +bin_PROGRAMS=h5dwalk +#bin_SCRIPTS=install-examples + +# Add h5stat specific linker flags here +h5dwalk_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) $(H5DWALK_LDFLAGS) + +# Tell automake to clean h5redeploy script +CLEANFILES= + +# All programs rely on hdf5 library and h5tools library +h5dwalk_LDADD=$(LIBH5TOOLS) $(LIBHDF5) $(H5DWALK_LIBS) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/h5dwalk/h5dwalk.1 b/utils/tools/h5dwalk/h5dwalk.1 new file mode 100644 index 0000000..60e1080 --- /dev/null +++ b/utils/tools/h5dwalk/h5dwalk.1 @@ -0,0 +1,42 @@ +.TH "h5dwalk" 1 +.SH NAME +h5dwalk \- Provides a means of extending HDF5 tools by using parallelism on groups of files. +.SH SYNOPSIS +h5dwalk [OPTIONS] -T h5tool [H5TOOL_options...] +.SH DESCRIPTION +h5dwalk utilizes the mpiFileUtils library to invoke a selected HDF5 tool on a collection of files. The mpiFileUtils library provides the facilities to walk directory trees and provide a selection of files contained therein. This selection can be filtered in various ways. At present, h5dwalk filters the original file selection to include only HDF5 formatted files. The resulting collection or collections can be utilized as the file inputs to the selected h5tool. +.SH OPTIONS +.TP +.B \-h +or +.B \-\-help +Print a usage message and exit. +.TP +.B \-i +or +.B \-\-input filename +Read command input from a file. Not yet implemented. +.TP +.B \-o +or +.B \-\-output filename +Captures the hdf5 tool output into a named file. +.TP +.B \-l +or +.B \-\-log [file] +Captures hdf5 tool output into a individual log files. If an optional file (directory) is specified, then output from all tool instances will be written in the given file directory. Without the optional filename, each tool instance ouput will be captured in a new log file whose name is associated with the hdf5 tool that was run and is written in the current working directory. +.TP +.B \-E +or +.B \-\-error [file] +Show all HDF5 error reporting. Behavior is similar to --log, i.e. errors can either be logged in a single named file or in individual tool specific files. Not yet implemented. +.TP +.B \-T +or +.B \-\-tool hdf5_tool +Specifies the hdf5 tool that should be invoked for each file in a collection of files. The collection consists of individual HDF5 files found by walking a specified directory tree which is used in place of the normal tool filename argument. The '-T' option should appear on the command line just prior to the HDF5 tool argument options. +.TP +.SH "SEE ALSO" +\&\fIh5dump\fR\|(1), \fIh5diff\fR\|(1), \fIh5repart\fR\|(1), \fIh5diff\fR\|(1), +\&\fIh5import\fR\|(1), \fIgif2h5\fR\|(1), \fIh52gif\fR\|(1), \fIh5perf\fR\|(1) diff --git a/utils/tools/h5dwalk/h5dwalk.c b/utils/tools/h5dwalk/h5dwalk.c new file mode 100644 index 0000000..1f42aed --- /dev/null +++ b/utils/tools/h5dwalk/h5dwalk.c @@ -0,0 +1,1712 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#include "H5private.h" +#include "h5tools.h" +#include "h5tools_utils.h" +#include "hdf5.h" + +#include "libcircle.h" +#include "dtcmp.h" +#include "mfu.h" +#include "mfu_flist.h" +#include "mfu_errors.h" +#include "mfu_flist_internal.h" + +/* Name of tool */ +#define PROGRAMNAME "h5dwalk" + +#ifdef DAOS_SUPPORT +#include "mfu_daos.h" +#endif + +static char *user_cmd = NULL; +static char mpierrstr[MPI_MAX_ERROR_STRING]; +static int mpierrlen; +static int sg_mpi_rank = 0; +static int current_input_index = 0; +static int processing_inputfile = 0; + +static void dh5tool_flist_write_text(const char *name, mfu_flist bflist); +static void run_command(int argc, char **argv, char *cmdline, const char *fname); +static void add_executable(int argc, char **argv, char *cmdstring, int *f_index, int f_count); +static int process_input_file(char *inputname, int myrank, int size); +static void usage(void); + +H5_ATTR_NORETURN void h5dwalk_exit(int status); + +/* keep stats during walk */ +uint64_t total_dirs = 0; +uint64_t total_files = 0; +uint64_t total_links = 0; +uint64_t total_unknown = 0; +uint64_t total_bytes = 0; +/* global flags which indicate whether we need + * to capture tool outputs into a file... + * Related to this is whether the stderr should + * be logged seperately. + */ +#define BUFT_SIZE 131072 +/* FIXME: 'buft_max' should probably be configurable.. */ +size_t buft_max = 64; +size_t buft_count = 0; +buf_t **buf_cache = NULL; + +int log_output_in_single_file = 0; +char *output_log_file = NULL; + +int log_stdout_in_file = 0; +char *txtlog = NULL; + +int log_errors_in_file = 0; +char *errlog = NULL; + +int use_config_file = 0; +int config_index[4] = { + 0, +}; + +#ifndef PATH_MAX +#define PATH_MAX 4096 +#endif + +#define MAX_DISTRIBUTE_SEPARATORS 128 +struct distribute_option { + int separator_number; + uint64_t separators[MAX_DISTRIBUTE_SEPARATORS]; +}; + +static const char * s_opts = "hl*E*i:o:T:"; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, + {"log_text", optional_arg, 'l'}, + {"error", optional_arg, 'E'}, + {"input", require_arg, 'i'}, + {"output", require_arg, 'o'}, + {"tool", require_arg, 'T'}, + {NULL, 0, '\0'}}; +static void +save_command(const char *argv0) +{ + assert(argv0); + user_cmd = HDstrdup(argv0); +} + +static void +create_default_separators(struct distribute_option *option, mfu_flist *flist, uint64_t *size, + size_t *separators, uint64_t *global_max_file_size) +{ + /* get local max file size for Allreduce */ + uint64_t local_max_file_size = 0; + for (uint64_t i = 0; i < *size; i++) { + uint64_t file_size = mfu_flist_file_get_size(*flist, i); + if (file_size > local_max_file_size) { + local_max_file_size = file_size; + } + } + + /* get the max file size across all ranks */ + MPI_Allreduce(&local_max_file_size, global_max_file_size, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD); + + /* print and convert max file size to appropriate units */ + double max_size_tmp; + const char *max_size_units; + mfu_format_bytes(*global_max_file_size, &max_size_tmp, &max_size_units); + HDprintf("Max File Size: %.3lf %s\n", max_size_tmp, max_size_units); + + /* round next_pow_2 to next multiple of 10 */ + uint64_t max_magnitude_bin = (uint64_t)((ceil(log2((double)(*global_max_file_size)) / 10)) * 10); + + /* get bin ranges based on max file size */ + option->separators[0] = 1; + + /* plus one is for zero count bin */ + *separators = (size_t)(max_magnitude_bin / 10); + uint64_t power = 10; + for (int i = 1; power <= max_magnitude_bin; i++) { + double raised_2 = pow(2, (double)(power)); + option->separators[i] = (uint64_t)raised_2; + power += 10; + } +} + +static int +h5dwalk_map_fn(mfu_flist flist __attribute__((unused)), uint64_t idx, int ranks, + void *args __attribute__((unused))) +{ + int rank = (int)((int)idx % ranks); + return rank; +} + +static int +print_flist_distribution(int file_histogram, struct distribute_option *option, mfu_flist *pflist, int rank) +{ + /* file list to use */ + mfu_flist flist = *pflist; + + /* get local size for each rank, and max file sizes */ + uint64_t size = mfu_flist_size(flist); + uint64_t global_max_file_size; + + size_t separators = 0; + if (file_histogram) { + /* create default separators */ + create_default_separators(option, &flist, &size, &separators, &global_max_file_size); + } + else { + separators = (size_t)option->separator_number; + } + + /* allocate a count for each bin, initialize the bin counts to 0 + * it is separator + 1 because the last bin is the last separator + * to the DISTRIBUTE_MAX */ + uint64_t *dist = (uint64_t *)MFU_MALLOC((separators + 1) * sizeof(uint64_t)); + + /* initialize the bin counts to 0 */ + for (size_t i = 0; i <= separators; i++) { + dist[i] = 0; + } + + /* for each file, identify appropriate bin and increment its count */ + for (size_t i = 0; i < size; i++) { + /* get the size of the file */ + uint64_t file_size = mfu_flist_file_get_size(flist, i); + + /* loop through the bins and find the one the file belongs to, + * set last bin to -1, if a bin is not found while looping through the + * list of file size separators, then it belongs in the last bin + * so (last file size - MAX bin) */ + int64_t max_bin_flag = -1; + for (size_t j = 0; j < separators; j++) { + if (file_size <= option->separators[j]) { + /* found the bin set bin index & increment its count */ + dist[j]++; + + /* a file for this bin was found so can't belong to + * last bin (so set the flag) & exit the loop */ + max_bin_flag = 1; + break; + } + } + + /* if max_bin_flag is still -1 then the file belongs to the last bin */ + if (max_bin_flag < 0) { + dist[separators]++; + } + } + + /* get the total sum across all of the bins */ + uint64_t *disttotal = (uint64_t *)MFU_MALLOC((separators + 1) * sizeof(uint64_t)); + MPI_Allreduce(dist, disttotal, (int)(separators + 1), MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD); + + /* Print the file distribution */ + if (rank == 0) { + /* number of files in a bin */ + uint64_t number; + double size_tmp; + const char *size_units; + HDprintf("%-27s %s\n", "Range", "Number"); + for (size_t i = 0; i <= separators; i++) { + HDprintf("%s", "[ "); + if (i == 0) { + HDprintf("%7.3lf %3s", 0.000, "B"); + } + else { + mfu_format_bytes((uint64_t)option->separators[i - 1], &size_tmp, &size_units); + HDprintf("%7.3lf %3s", size_tmp, size_units); + } + + printf("%s", " - "); + + if (file_histogram) { + mfu_format_bytes((uint64_t)option->separators[i], &size_tmp, &size_units); + number = disttotal[i]; + mfu_format_bytes((uint64_t)option->separators[i], &size_tmp, &size_units); + HDprintf("%7.3lf %3s ) %" PRIu64 "\n", size_tmp, size_units, number); + } + else { + if (i == separators) { + number = disttotal[i]; + HDprintf("%10s ) %" PRIu64 "\n", "MAX", number); + } + else { + number = disttotal[i]; + mfu_format_bytes((uint64_t)option->separators[i], &size_tmp, &size_units); + HDprintf("%7.3lf %3s ) %" PRIu64 "\n", size_tmp, size_units, number); + } + } + } + } + + /* free the memory used to hold bin counts */ + mfu_free(&disttotal); + mfu_free(&dist); + + return 0; +} + +/* * Search the right position to insert the separator * If the separator exists already, return failure * + * Otherwise, locate the right position, and move the array forward to save the separator. + */ +static int +distribute_separator_add(struct distribute_option *option, uint64_t separator) +{ + int low = 0; + int high; + int middle; + int pos; + int count; + + count = option->separator_number; + option->separator_number++; + if (option->separator_number > MAX_DISTRIBUTE_SEPARATORS) { + HDprintf("Too many separators"); + return -1; + } + + if (count == 0) { + option->separators[0] = separator; + return 0; + } + + high = count - 1; + while (low < high) { + middle = (high - low) / 2 + low; + if (option->separators[middle] == separator) + return -1; + /* In the left half */ + else if (option->separators[middle] < separator) + low = middle + 1; + /* In the right half */ + else + high = middle; + } + assert(low == high); + if (option->separators[low] == separator) + return -1; + + if (option->separators[low] < separator) + pos = low + 1; + else + pos = low; + + if (pos < count) + HDmemmove(&option->separators[low + 1], &option->separators[low], + sizeof(*option->separators) * (uint64_t)(count - pos)); + + option->separators[pos] = separator; + return 0; +} + +static int +distribution_parse(struct distribute_option *option, const char *string) +{ + char * ptr; + char * next; + unsigned long long separator; + char * str; + int status = 0; + + if (strncmp(string, "size", strlen("size")) != 0) { + return -1; + } + + option->separator_number = 0; + if (strlen(string) == strlen("size")) { + return 0; + } + + if (string[strlen("size")] != ':') { + return -1; + } + + str = HDstrdup(string); + /* Parse separators */ + ptr = str + strlen("size:"); + next = ptr; + while (ptr && ptr < str + strlen(string)) { + next = strchr(ptr, ','); + if (next != NULL) { + *next = '\0'; + next++; + } + + if (mfu_abtoull(ptr, &separator) != MFU_SUCCESS) { + HDprintf("Invalid separator \"%s\"\n", ptr); + status = -1; + goto out; + } + + if (distribute_separator_add(option, separator)) { + HDprintf("Duplicated separator \"%llu\"\n", separator); + status = -1; + goto out; + } + + ptr = next; + } + +out: + mfu_free(&str); + return status; +} + +static void +usage(void) +{ + if (sg_mpi_rank) + return; + + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "Usage: h5dwalk [options] ...\n"); +#ifdef DAOS_SUPPORT + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "DAOS paths can be specified as:\n"); + PRINTVALSTREAM(rawoutstream, " daos:///[/] | \n"); +#endif + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "Options:\n"); + PRINTVALSTREAM(rawoutstream, " -i, --input - read list from file\n"); + PRINTVALSTREAM(rawoutstream, " -o, --output - write output summary to the named file.\n"); + PRINTVALSTREAM(rawoutstream, + " -E, --error - write processed errors to file in text format\n"); + PRINTVALSTREAM( + rawoutstream, + " -l, --log_text - write individual tool outputs to a file. Logs can be written to an " + "optional named directory.\n"); + PRINTVALSTREAM(rawoutstream, " -T, --tool - name of the HDF5 tool to invoke\n"); + PRINTVALSTREAM(rawoutstream, " -h, --help - print usage\n"); + PRINTVALSTREAM(rawoutstream, "\n"); + PRINTVALSTREAM(rawoutstream, "For more information see https://mpifileutils.readthedocs.io. \n"); + PRINTVALSTREAM(rawoutstream, "\n"); +} + +/* given an index, return pointer to that file element, + * NULL if index is not in range */ +static elem_t * +list_get_elem(flist_t *flist, uint64_t idx) +{ + /* return pointer to element if index is within range */ + uint64_t max = flist->list_count; + if (idx < max) { + elem_t *elem = flist->list_index[idx]; + return elem; + } + return NULL; +} + +#ifdef VERBOSE +/* print information about a file given the index and rank (used in print_files) */ +static void +print_file(mfu_flist flist, uint64_t idx) +{ + /* store types as strings for print_file */ + char type_str_unknown[] = "UNK"; + char type_str_dir[] = "DIR"; + char type_str_file[] = "REG"; + char type_str_link[] = "LNK"; + + /* get filename */ + const char *file = mfu_flist_file_get_name(flist, idx); + + if (mfu_flist_have_detail(flist)) { + /* get mode */ + mode_t mode = (mode_t)mfu_flist_file_get_mode(flist, idx); + uint64_t acc = mfu_flist_file_get_atime(flist, idx); + uint64_t mod = mfu_flist_file_get_mtime(flist, idx); + uint64_t cre = mfu_flist_file_get_ctime(flist, idx); + uint64_t size = mfu_flist_file_get_size(flist, idx); + const char *username = mfu_flist_file_get_username(flist, idx); + const char *groupname = mfu_flist_file_get_groupname(flist, idx); + + char access_s[30]; + char modify_s[30]; + char create_s[30]; + time_t access_t = (time_t)acc; + time_t modify_t = (time_t)mod; + time_t create_t = (time_t)cre; + size_t access_rc = strftime(access_s, sizeof(access_s) - 1, "%FT%T", localtime(&access_t)); + size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%b %e %Y %H:%M", localtime(&modify_t)); + size_t create_rc = strftime(create_s, sizeof(create_s) - 1, "%FT%T", localtime(&create_t)); + if (access_rc == 0 || modify_rc == 0 || create_rc == 0) { + /* error */ + access_s[0] = '\0'; + modify_s[0] = '\0'; + create_s[0] = '\0'; + } + + char mode_format[11]; + mfu_format_mode(mode, mode_format); + + double size_tmp; + const char *size_units; + mfu_format_bytes(size, &size_tmp, &size_units); + + HDprintf("%s %s %s %7.3f %3s %s %s\n", mode_format, username, groupname, size_tmp, size_units, + modify_s, file); + } + else { + /* get type */ + mfu_filetype type = mfu_flist_file_get_type(flist, idx); + char * type_str = type_str_unknown; + if (type == MFU_TYPE_DIR) { + type_str = type_str_dir; + } + else if (type == MFU_TYPE_FILE) { + type_str = type_str_file; + } + else if (type == MFU_TYPE_LINK) { + type_str = type_str_link; + } + + HDprintf("Type=%s File=%s\n", type_str, file); + } +} + +/* TODO: move this somewhere or modify existing print_file */ +/* print information about a file given the index and rank (used in print_files) */ +static size_t +print_file_text(mfu_flist flist, uint64_t idx, char *buffer, size_t bufsize) +{ + size_t numbytes = 0; + + /* store types as strings for print_file */ + char type_str_unknown[] = "UNK"; + char type_str_dir[] = "DIR"; + char type_str_file[] = "REG"; + char type_str_link[] = "LNK"; + + /* get filename */ + const char *file = mfu_flist_file_get_name(flist, idx); + + if (mfu_flist_have_detail(flist)) { + /* get mode */ + mode_t mode = (mode_t)mfu_flist_file_get_mode(flist, idx); + + uint64_t acc = mfu_flist_file_get_atime(flist, idx); + uint64_t mod = mfu_flist_file_get_mtime(flist, idx); + uint64_t cre = mfu_flist_file_get_ctime(flist, idx); + uint64_t size = mfu_flist_file_get_size(flist, idx); + const char *username = mfu_flist_file_get_username(flist, idx); + const char *groupname = mfu_flist_file_get_groupname(flist, idx); + + char access_s[30]; + char modify_s[30]; + char create_s[30]; + time_t access_t = (time_t)acc; + time_t modify_t = (time_t)mod; + time_t create_t = (time_t)cre; + size_t access_rc = strftime(access_s, sizeof(access_s) - 1, "%FT%T", localtime(&access_t)); + size_t modify_rc = strftime(modify_s, sizeof(modify_s) - 1, "%b %e %Y %H:%M", localtime(&modify_t)); + size_t create_rc = strftime(create_s, sizeof(create_s) - 1, "%FT%T", localtime(&create_t)); + if (access_rc == 0 || modify_rc == 0 || create_rc == 0) { + /* error */ + access_s[0] = '\0'; + modify_s[0] = '\0'; + create_s[0] = '\0'; + } + + char mode_format[11]; + mfu_format_mode(mode, mode_format); + + double size_tmp; + const char *size_units; + mfu_format_bytes(size, &size_tmp, &size_units); + + numbytes = (size_t)snHDprintf(buffer, bufsize, "%s %s %s %7.3f %3s %s %s\n", mode_format, username, + groupname, size_tmp, size_units, modify_s, file); + } + else { + /* get type */ + mfu_filetype type = mfu_flist_file_get_type(flist, idx); + char * type_str = type_str_unknown; + if (type == MFU_TYPE_DIR) { + type_str = type_str_dir; + } + else if (type == MFU_TYPE_FILE) { + type_str = type_str_file; + } + else if (type == MFU_TYPE_LINK) { + type_str = type_str_link; + } + + numbytes = (size_t)snHDprintf(buffer, bufsize, "Type=%s File=%s\n", type_str, file); + } + + return numbytes; +} +#endif + +static size_t +get_local_bufsize(uint64_t *bufsize) +{ + size_t total = 0; + if (buft_count > 0) { + buf_t *lastbuf = buf_cache[buft_count - 1]; + size_t remaining = lastbuf->count; + total = (lastbuf->bufsize * buft_count) - remaining; + *bufsize = (uint64_t)(lastbuf->bufsize); + } + return total; +} + +static void +dh5tool_flist_write_text(const char *name, mfu_flist bflist) +{ + /* convert handle to flist_t */ + flist_t *flist = (flist_t *)bflist; + + /* get our rank and size of the communicator */ + int rank, ranks; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &ranks); + + /* start timer */ + double start_write = MPI_Wtime(); + + /* total list items */ + uint64_t all_count = mfu_flist_global_size(flist); + + /* report the filename we're writing to */ + if (mfu_rank == 0) { + MFU_LOG(MFU_LOG_INFO, "Writing to output file: %s", name); + } + + uint64_t idx = 0; + char * ptr = NULL; + + /* if we block things up into 128MB chunks, how many iterations + * to write everything? */ + // uint64_t maxwrite = 128 * 1024 * 1024; + uint64_t maxwrite = 0; + size_t local_total = get_local_bufsize(&maxwrite); + uint64_t iters = 0; + if (local_total > 0) + iters = (uint64_t)local_total / maxwrite; + + if (iters * maxwrite < (uint64_t)local_total) { + iters++; + } + + /* get max iterations across all procs */ + uint64_t all_iters; + MPI_Allreduce(&iters, &all_iters, 1, MPI_UINT64_T, MPI_MAX, MPI_COMM_WORLD); + + /* use mpi io hints to stripe across OSTs */ + MPI_Info info; + MPI_Info_create(&info); + + /* change number of ranks to string to pass to MPI_Info */ + char str_buf[12]; + HDprintf(str_buf, "%d", ranks); + + /* no. of I/O devices for lustre striping is number of ranks */ + MPI_Info_set(info, "striping_factor", str_buf); + + /* open file */ + MPI_Status status; + MPI_File fh; + const char *datarep = "native"; + int amode = MPI_MODE_WRONLY | MPI_MODE_CREATE; + + int mpirc = MPI_File_open(MPI_COMM_WORLD, (const char *)name, amode, info, &fh); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to open file for writing: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* truncate file to 0 bytes */ + mpirc = MPI_File_set_size(fh, 0); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to truncate file: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* set file view to be sequence of datatypes past header */ + mpirc = MPI_File_set_view(fh, 0, MPI_BYTE, MPI_BYTE, datarep, MPI_INFO_NULL); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to set view on file: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* compute byte offset to write our element */ + uint64_t offset = 0; + uint64_t bytes = (uint64_t)local_total; + MPI_Exscan(&bytes, &offset, 1, MPI_UINT64_T, MPI_SUM, MPI_COMM_WORLD); + MPI_Offset write_offset = (MPI_Offset)offset; + + uint64_t written = 0; + while (all_iters > 0) { + /* compute number of bytes left to write */ + uint64_t remaining = (uint64_t)local_total - written; + + /* maybe Incr pointer to our next buffer */ + if (remaining == 0) { + idx++; + if (buf_cache[idx]->buf == NULL) { + } + } + + /* compute count we'll write in this iteration */ + int write_count = (int)maxwrite; + if (remaining < maxwrite) { + write_count = (int)remaining; + } + /* Get the buffer to output to the selected file */ + ptr = buf_cache[idx]->buf; + + /* collective write of file data */ + mpirc = MPI_File_write_at_all(fh, write_offset, ptr, write_count, MPI_BYTE, &status); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to write to file: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* update our offset into the file */ + write_offset += (MPI_Offset)write_count; + + /* update number of bytes written so far */ + written += (uint64_t)write_count; + + /* update pointer into our buffer */ + ptr += write_count; + + /* decrement our collective write loop counter */ + all_iters--; + } + + /* free buffer */ + // mfu_free(&buf); + + /* close file */ + mpirc = MPI_File_close(&fh); + if (mpirc != MPI_SUCCESS) { + MPI_Error_string(mpirc, mpierrstr, &mpierrlen); + MFU_ABORT(1, "Failed to close file: `%s' rc=%d %s", name, mpirc, mpierrstr); + } + + /* free mpi info */ + MPI_Info_free(&info); + + /* end timer */ + double end_write = MPI_Wtime(); + + /* report write count, time, and rate */ + if (mfu_rank == 0) { + double secs = end_write - start_write; + double rate = 0.0; + if (secs > 0.0) { + rate = ((double)all_count) / secs; + } + MFU_LOG(MFU_LOG_INFO, "Wrote %lu files in %.3lf seconds (%.3lf files/sec)", all_count, secs, rate); + } + + return; +} + +static void +filter_hdf_files(mfu_flist *pflist, char *regex_exp, int exclude, int name) +{ + mfu_flist flist = *pflist; + mfu_flist eligible = mfu_flist_subset(flist); + uint64_t idx = 0; + uint64_t files = mfu_flist_size(flist); + while (idx < files) { + mfu_filetype type = mfu_flist_file_get_type(flist, idx); + if (type == MFU_TYPE_FILE || type == MFU_TYPE_LINK || type == MFU_TYPE_UNKNOWN) { + const char *file = mfu_flist_file_get_name(flist, idx); + int accessible = H5Fis_accessible(file, H5P_DEFAULT); + if (accessible) + mfu_flist_file_copy(flist, idx, eligible); + } + idx++; + } + + mfu_flist_summarize(eligible); + + /* assume we'll use the full list */ + // mfu_flist srclist = flist; + mfu_flist srclist = eligible; + + /* filter the list if needed */ + mfu_flist filtered_flist = MFU_FLIST_NULL; + if (regex_exp != NULL) { + /* filter the list based on regex */ + filtered_flist = mfu_flist_filter_regex(eligible, regex_exp, exclude, name); + + /* update our source list to use the filtered list instead of the original */ + srclist = filtered_flist; + } + + mfu_flist_free(&flist); + *pflist = srclist; + return; +} + +static int +fill_file_list(mfu_flist new_flist, const char *config_filename, int myrank, int size) +{ + int index = 0; + char linebuf[PATH_MAX] = { + '\0', + }; + FILE *config = HDfopen(config_filename, "r"); + if (config == NULL) + return -1; + while (HDfgets(linebuf, sizeof(linebuf), config) != NULL) { + struct stat statbuf; + char * eol = HDstrchr(linebuf, '\n'); + if (eol) + *eol = '\0'; + if (HDstat(linebuf, &statbuf) == 0) { + if (myrank == (index % size)) { + mfu_flist_insert_stat((flist_t *)new_flist, linebuf, O_RDONLY, &statbuf); + } + index++; + } + linebuf[0] = 0; + } + HDfclose(config); + return index; +} + +static int +count_dirpaths(int argc, int startcnt, const char *argv[], int **index_out) +{ + int k; + int path_cnt = 0; + int idx_count = (argc - startcnt); + int * index = NULL; + struct stat pathcheck; + + if (idx_count > 0) { + index = (int *)malloc((size_t)(argc - startcnt) * sizeof(int)); + assert(index); + } + else + return 0; + + for (k = startcnt; k < argc; k++) { + char *slash = NULL; + int c = *argv[k]; + if ((c == '.') || (c == '/')) { + index[path_cnt++] = k; + } + else if ((c == '@')) { + const char *configFile = argv[k] + 1; + if (stat(configFile, &pathcheck) == 0) { + if (S_ISREG(pathcheck.st_mode)) { + config_index[use_config_file++] = k; + } + } + } + else if ((slash = strchr(argv[k], '/')) != NULL) { + if (stat(argv[k], &pathcheck) == 0) { + if (S_ISDIR(pathcheck.st_mode)) + index[path_cnt++] = k; + } + } + } + if ((path_cnt == 0) && (index != NULL)) { + free(index); + return 0; + } + *index_out = index; + return path_cnt; +} + +static char ** +copy_args(int argc, const char *argv[], int *mfu_argc, int *copy_len) +{ + int i, bytes_copied = 0; + int check_mfu_args = 1; + char **argv_copy = (char **)MFU_MALLOC((size_t)(argc + 2) * sizeof(char **)); + assert(argv_copy); + assert(mfu_argc); + assert(copy_len); + save_command(argv[0]); + + for (i = 0; i < argc; i++) { + argv_copy[i] = HDstrdup(argv[i]); + bytes_copied += (int)(strlen(argv[i]) + 1); + argv_copy[i] = HDstrdup(argv[i]); + if (check_mfu_args && (HDstrncmp(argv[i], "-T", 2) == 0)) { + check_mfu_args = 0; + *mfu_argc = i + 1; + } + } + argv_copy[i] = 0; + *copy_len = bytes_copied; + return argv_copy; +} + +typedef struct hash_entry { + int hash; + char * name; + struct hash_entry *next; /* table Collision */ + int nextCount; +} hash_entry_t; + +#ifndef NAME_ENTRIES +#define NAME_ENTRIES 4096 +#endif + +static hash_entry_t filename_cache[NAME_ENTRIES]; + +static int +get_copy_count(char *fname, char *appname) +{ + int filehash = 0, apphash = 0; + size_t k, applen = strlen(appname); + size_t filelen = strlen(fname); + int hash_index; + + for (k = 0; k < filelen; k++) { + filehash += fname[k]; + } + for (k = 0; k < applen; k++) { + apphash += appname[k]; + } + hash_index = filehash % NAME_ENTRIES; + if (filename_cache[hash_index].name == NULL) { + filename_cache[hash_index].hash = apphash; + filename_cache[hash_index].name = HDstrdup(fname); + filename_cache[hash_index].next = NULL; + filename_cache[hash_index].nextCount = 1; + return 0; + } + else if ((apphash == filename_cache[hash_index].hash) && + (strcmp(filename_cache[hash_index].name, fname) == 0)) { + int retval = filename_cache[hash_index].nextCount++; + return retval; + } + else { /* Collision */ + hash_entry_t *nextEntry = &filename_cache[hash_index]; + hash_entry_t *lastEntry = nextEntry; + while (nextEntry) { + if ((apphash == nextEntry->hash) && (strcmp(nextEntry->name, fname) == 0)) { + /* Match (increment nextCount and return) */ + int retval = nextEntry->nextCount++; + return retval; + } + else { + /* No Match (continue search) */ + lastEntry = nextEntry; + nextEntry = lastEntry->next; + } + } + nextEntry = (hash_entry_t *)malloc(sizeof(hash_entry_t)); + if (nextEntry) { + lastEntry->next = nextEntry; + nextEntry->name = HDstrdup(fname); + nextEntry->hash = apphash; + nextEntry->next = NULL; + nextEntry->nextCount = 1; + } + } + return 0; +} + +static void +run_command(int argc __attribute__((unused)), char **argv, char *cmdline, const char *fname) +{ + char filepath[1024]; + char *toolname = argv[0]; + char *buf = NULL; + int use_stdout = 0; + +#ifdef H5_HAVE_WINDOWS + HDprintf("ERROR: %s %s: Unable to support fork/exec on WINDOWS\n", PROGRAMNAME, __func__); + h5dwalk_exit(EXIT_FAILURE); +#else + + /* create a copy of the 1st file passed to the application */ + HDstrcpy(filepath, fname); + + if (log_output_in_single_file || use_stdout) { + pid_t pid; + int pipefd[2]; + buf_t * thisbuft = NULL; + buf_t **bufs = buf_cache; + + if (bufs == NULL) { + bufs = (buf_t **)MFU_CALLOC(buft_max, sizeof(buf_t *)); + assert((bufs != NULL)); + buf_cache = bufs; +#ifdef VERBOSE + if (buft_count == 0) { + HDprintf("[%d] Initial buf_cache allocation: buft_count=%d\n", sg_mpi_rank, buft_count); + } +#endif + bufs[buft_count++] = thisbuft = (buf_t *)MFU_CALLOC(1, sizeof(buf_t)); + assert((thisbuft != NULL)); + } + else { + thisbuft = bufs[buft_count - 1]; + assert((thisbuft != NULL)); + /* Check for remaining space in the current buffer */ + /* If none, then create a new buffer */ + if (thisbuft->count == 0) { + bufs[buft_count++] = thisbuft = (buf_t *)MFU_CALLOC(1, sizeof(buf_t)); + } + } + if ((thisbuft->buf == NULL)) { + thisbuft->buf = MFU_MALLOC(BUFT_SIZE); + assert((thisbuft->buf != NULL)); + thisbuft->bufsize = BUFT_SIZE; + thisbuft->count = BUFT_SIZE; + thisbuft->dt = MPI_CHAR; + } + if (pipe(pipefd) == -1) { + perror("pipe"); + exit(EXIT_FAILURE); + } + pid = fork(); + if (pid == -1) { + perror("fork"); + exit(EXIT_FAILURE); + } + if (pid == 0) { + close(pipefd[0]); + dup2(pipefd[1], fileno(stdout)); + dup2(pipefd[1], fileno(stderr)); + execvp(argv[0], argv); + } + else { + int w_status; + size_t nbytes; + size_t read_bytes = 0; + uint64_t remaining, offset; + close(pipefd[1]); + buf = thisbuft->buf; + remaining = thisbuft->count; + offset = thisbuft->chars; + nbytes = strlen(cmdline); + /* Record the command line for the log! */ + if (nbytes < remaining) { + HDstrcpy(&buf[offset], cmdline); + thisbuft->chars += nbytes; + thisbuft->count -= nbytes; + remaining -= nbytes; + } + else { /* We're running out of space in the current buffer */ + char *nextpart; + strncpy(&buf[offset], cmdline, remaining); + nextpart = &cmdline[remaining + 1]; + thisbuft->count = 0; + thisbuft->chars += remaining; + + /* Create a new read buffer */ +#ifdef VERBOSE + HDprintf("[%d] Allocate-1 a new read buffer:: buft_count=%d\n", sg_mpi_rank, buft_count); +#endif + bufs[buft_count++] = thisbuft = (buf_t *)MFU_CALLOC(1, sizeof(buf_t)); + assert(thisbuft != NULL); + thisbuft->buf = MFU_MALLOC(BUFT_SIZE); + thisbuft->bufsize = BUFT_SIZE; + thisbuft->dt = MPI_CHAR; + /* Copy the remaining cmdline text into the new buffer */ + HDstrcpy(buf, nextpart); + /* And update our buffer info */ + // thisbuft->chars = strlen(nextpart) +1; + thisbuft->chars = strlen(nextpart); + thisbuft->count = BUFT_SIZE - thisbuft->chars; + } + offset = thisbuft->chars; + + do { + waitpid(pid, &w_status, WNOHANG); + if ((nbytes = (size_t)read(pipefd[0], &buf[offset], remaining)) > 0) { + offset += nbytes; + read_bytes += nbytes; + remaining -= nbytes; + if (remaining == 0) { + /* Update the current buffer prior to allocating the new one */ + thisbuft->count = 0; + thisbuft->chars += read_bytes; +#ifdef VERBOSE + HDprintf("[%d] Allocate-2 a new read buffer:: buft_count=%d\n", sg_mpi_rank, + buft_count); +#endif + bufs[buft_count++] = thisbuft = (buf_t *)MFU_CALLOC(1, sizeof(buf_t)); + assert(thisbuft != NULL); + thisbuft->buf = MFU_MALLOC(BUFT_SIZE); + thisbuft->bufsize = BUFT_SIZE; + thisbuft->dt = MPI_CHAR; + thisbuft->chars = BUFT_SIZE; + offset = 0; + remaining = BUFT_SIZE; + } + } + } while (!WIFEXITED(w_status)); + close(pipefd[0]); + wait(NULL); + + thisbuft->count = remaining; + thisbuft->chars = thisbuft->bufsize - remaining; + } + } + else if (log_stdout_in_file) { + int log_instance = -1; + pid_t pid; + size_t log_len; + char logpath[2048]; + char logErrors[2048]; + char current_dir[2048]; + char * logbase = HDstrdup(basename(filepath)); + char * thisapp = HDstrdup(basename(toolname)); + + if (processing_inputfile == 0) + log_instance = get_copy_count(logbase, thisapp); + + if (txtlog == NULL) { + if ((log_instance > 0) || processing_inputfile) { + if (processing_inputfile) + log_instance = current_input_index; + HDsprintf(logpath, "%s/%s_%s.log_%d", HDgetcwd(current_dir, sizeof(current_dir)), logbase, + thisapp, log_instance); + } + else { + HDsprintf(logpath, "%s/%s_%s.log", HDgetcwd(current_dir, sizeof(current_dir)), logbase, + thisapp); + } + } + else { + log_len = strlen(txtlog); + if ((log_instance > 0) || processing_inputfile) { + if (processing_inputfile) + log_instance = current_input_index; + if (txtlog[log_len - 1] == '/') + HDsprintf(logpath, "%s%s_%s.log_%d", txtlog, logbase, thisapp, log_instance); + else + HDsprintf(logpath, "%s/%s_%s.log_%d", txtlog, logbase, thisapp, log_instance); + } + else { + if (txtlog[log_len - 1] == '/') + HDsprintf(logpath, "%s%s_%s.log", txtlog, logbase, thisapp); + else + HDsprintf(logpath, "%s/%s_%s.log", txtlog, logbase, thisapp); + } + } + + if (log_errors_in_file) { + /* We co-locate the error logs in the same directories as the regular log files. + * The easiest way to do this is to simply replace the .log with .err in a + * copy of the logpath variable. + */ + log_len = strlen(logpath); + HDstrcpy(logErrors, logpath); + HDstrcpy(&logErrors[log_len - 3], "err"); + } + if (mfu_debug_level == MFU_LOG_VERBOSE) { + HDprintf("\tCreating logfile: %s\n", logpath); + fflush(stdout); + } + pid = fork(); + if (pid == 0) { + int efd; + int fd = open(logpath, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + dup2(fd, fileno(stdout)); + if (log_errors_in_file) { + efd = open(logErrors, O_RDWR | O_CREAT, S_IRUSR | S_IWUSR); + dup2(efd, fileno(stderr)); + close(efd); + } + else + dup2(fd, fileno(stderr)); + close(fd); + execvp(argv[0], argv); + } + int status; + pid = wait(&status); + if (logbase) + free(logbase); + if (thisapp) + free(thisapp); + } /* else if(log_stdout_in_file) */ +#endif /* #ifdef H5_HAVE_WINDOWS */ +} + +int MFU_PRED_EXEC(mfu_flist flist, uint64_t idx, void *arg); +int MFU_PRED_PRINT(mfu_flist flist, uint64_t idx, void *arg); + +int +MFU_PRED_EXEC(mfu_flist flist, uint64_t idx, void *arg) +{ + /* get file name for this item */ + int file_substituted = 0; + const char *fname = mfu_flist_file_get_name(flist, idx); + + char *toolname = NULL; + char filepath[1024]; + + size_t b_offset; + + /* get pointer to encoded argc count and argv array */ + int * count_ptr = arg; + char *buf = (char *)arg + sizeof(int); + + /* get number of argv parameters */ + int k = 0, count = *count_ptr; + toolname = buf; + + /* Get a copy of fname */ + HDstrcpy(filepath, fname); + + /* allocate a char* for each item in the argv array, + * plus one more for a trailing NULL + * 'count' in this case is the number of args, so + * so we add (+1) for the toolname and another (+1) + * for the trailing NULL to terminate the list + */ + + char cmdline[2048]; + char **argv = (char **)MFU_CALLOC((size_t)(count + 2), sizeof(char *)); + + argv[k++] = HDstrdup(toolname); + + HDmemset(cmdline, 0, sizeof(cmdline)); + buf += HDstrlen(toolname) + 1; + /* Reconstruct the command line that the user provided for the h5tool */ + for (k = 1; k < count; k++) { + if (buf[0] == '&') { + const char *fname_arg = NULL; + mfu_flist flist_arg; + void * check_ptr[2] = {NULL, NULL}; + + HDmemcpy(check_ptr, &buf[1], sizeof(void *)); + flist_arg = (mfu_flist)check_ptr[0]; + + /* +2 (see below) accounts for the '&' and the trailing zero pad */ + buf += sizeof(mfu_flist *) + 2; + fname_arg = mfu_flist_file_get_name(flist_arg, idx); + if (fname_arg == NULL) { + HDprintf("[%d] Warning: Unable to resolve file_substitution %d (idx=%ld)\n", sg_mpi_rank, + file_substituted, idx); + argv[k] = HDstrdup(fname); + } + else { + argv[k] = HDstrdup(fname_arg); + file_substituted++; + } + } + else { + argv[k] = HDstrdup(buf); + buf += HDstrlen(argv[k]) + 1; + } + } + + HDsprintf(cmdline, "\n---------\nCommand:"); + b_offset = strlen(cmdline); + for (k = 0; k < count; k++) { + HDsprintf(&cmdline[b_offset], " %s", argv[k]); + b_offset = strlen(cmdline); + } + HDsprintf(&cmdline[b_offset], "\n"); + run_command(count, argv, cmdline, fname); + + mfu_free(argv); + + return 0; +} + +int +MFU_PRED_PRINT(mfu_flist flist, uint64_t idx, void *arg __attribute__((unused))) +{ + const char *name = mfu_flist_file_get_name(flist, idx); + HDprintf("%s\n", name); + return 1; +} + +static void +pred_commit(mfu_pred *p) +{ + mfu_pred *cur = p; + while (cur) { + if (cur->f == MFU_PRED_PRINT || cur->f == MFU_PRED_EXEC) { + break; + } + cur = cur->next; + } +} + +static void +add_executable(int argc, char **argv, char *cmdstring, int *f_index, int f_count __attribute__((unused))) +{ + char cmdline[2048]; + HDsprintf(cmdline, "\n---------\nCommand: %s\n", cmdstring); + argv[argc] = NULL; + run_command(argc, argv, cmdline, argv[f_index[0]]); + return; +} + +static int +process_input_file(char *inputname, int myrank, int size) +{ + int index = 0; + char linebuf[PATH_MAX] = { + '\0', + }; + FILE * config = HDfopen(inputname, "r"); + mfu_flist flist1 = NULL; + + if (config == NULL) + return -1; + + flist1 = mfu_flist_new(); + + /* Flag the fact that we're processing an inputfile (script) + * so that we can generate a meaningful logfile name... + */ + processing_inputfile = 1; + + while (HDfgets(linebuf, sizeof(linebuf), config) != NULL) { + const char *delim = " \n"; + char * cmdline = NULL; + char * cmd = NULL; + char * arg = NULL; + char * argv[256]; + int fileindex[256]; + int filecount = 0; + int token = 0; + struct stat statbuf; + + char *eol = strchr(linebuf, '\n'); + if (eol) { + *eol = '\0'; + } + cmdline = HDstrdup(linebuf); + cmd = HDstrtok(linebuf, delim); + if (cmd) { + arg = cmd; + while (arg != NULL) { + char c = arg[0]; + if (token > 0) { + if ((c == '.') || (c == '/')) { + /* 'arg' looks to be a filepath */ + if (stat(arg, &statbuf) == 0) { + mfu_flist_insert_stat(flist1, arg, O_RDONLY, &statbuf); + } + fileindex[filecount++] = token; + } + } + argv[token++] = arg; + arg = strtok(NULL, delim); + } + + if (myrank == (index % size)) { + current_input_index = index; + add_executable(token, argv, cmdline, fileindex, filecount); + } + index++; + } + linebuf[0] = 0; + HDfree(cmdline); + } + + if (output_log_file) { + dh5tool_flist_write_text(output_log_file, flist1); + } + HDfclose(config); + + mfu_flist_free(&flist1); + return 0; +} + +int +main(int argc, const char *argv[]) +{ + int i; + int rc = 0; + + char *env_var = NULL; + + /* initialize MPI */ + MPI_Init(&argc, (char ***)&argv); + mfu_init(); + + /* Initialize h5tools lib */ + h5tools_init(); + + h5tools_setprogname(PROGRAMNAME); + h5tools_setstatus(EXIT_SUCCESS); + + /* get our rank and the size of comm_world */ + int rank, ranks; + MPI_Comm_rank(MPI_COMM_WORLD, &rank); + MPI_Comm_size(MPI_COMM_WORLD, &ranks); + + /* Assign the static global mpi_rank (for debugging) */ + sg_mpi_rank = rank; + +#if 0 + env_var = HDgetenv("HDF5_H5DWALK_PRINT_CMDLINE"); + if (env_var) { + int enable = HDatoi(env_var); + if (enable) { + + } + } +#endif + /* pointer to mfu_walk_opts */ + mfu_walk_opts_t *walk_opts = mfu_walk_opts_new(); + +#ifdef DAOS_SUPPORT + /* DAOS vars */ + daos_args_t *daos_args = daos_args_new(); +#endif + + int args_byte_length = -1; + int mfu_argc = argc; + char * args_buf = NULL; + char **h5tool_argv = copy_args(argc, argv, &mfu_argc, &args_byte_length); + + char *inputname = NULL; + char *outputname = NULL; + char *sortfields = NULL; + char *distribution = NULL; + + int text = 0; + int h5tool_argc = 0; + + mfu_debug_level = MFU_LOG_WARN; + h5tool_argv[argc] = 0; + + /* The struct option declaration can found in bits/getopt_ext.h + * I've reproduced it here: + * struct option { char * name; int has_arg; int *flag; int val}; + */ + int opt; + int tool_selected = 0; + int tool_args_start = -1; + int last_mfu_arg = 0; + + mfu_pred *pred_head = NULL; + + while (!tool_selected) { + opt = H5_get_option(argc, argv, s_opts, l_opts); + switch ((char)opt) { + default: + usage(); + h5dwalk_exit(EXIT_FAILURE); + break; + case 'i': + inputname = HDstrdup(H5_optarg); + last_mfu_arg = H5_optind; + if (inputname) + tool_selected = 1; + break; + case 'o': + outputname = HDstrdup(H5_optarg); + last_mfu_arg = H5_optind; + if (outputname) { + log_output_in_single_file = 1; + output_log_file = HDstrdup(H5_optarg); + text = 1; /* Format TXT, not HDF5 */ + } + break; + case 'E': + log_errors_in_file = 1; + errlog = HDstrdup(H5_optarg); + last_mfu_arg = H5_optind; + break; + case 'l': + log_stdout_in_file = 1; + if (H5_optarg) + txtlog = HDstrdup(H5_optarg); + break; + case 'T': + /* We need to stop parsing user options at this point. + * all remaining arguments should be utilized as the + * arguments to the selected HDF5 tools. + * We also want to avoid any misinterpretations if + * HDF5 tool options conflict with the MFU options. + */ + tool_selected = 1; + tool_args_start = H5_optind; + h5tool_argc = argc - mfu_argc; + last_mfu_arg = H5_optind; + /* Don't allow any further parsing of arguments */ + break; + case 'h': + usage(); + h5dwalk_exit(EXIT_SUCCESS); + break; + case '?': + usage(); + h5dwalk_exit(EXIT_SUCCESS); + break; + } + } + + if (inputname != NULL) { + if (tool_selected && (rank == 0)) { + if ((log_output_in_single_file == 0) && (log_stdout_in_file == 0)) + puts("WARNING: When utilizing --input, the only other supported " + "runtime argument is --output or -l"); + } + rc = process_input_file(inputname, rank, ranks); + mfu_finalize(); + h5dwalk_exit(rc); + } + + /**************************************************************/ + /* We might consider doing a tool specific argument checking */ + /* to prevent runtime errors. We would also like to allow */ + /* the same command line interface for parallel invocations */ + /* so that users don't get confused. Effectively, we should */ + /* strip out all MFU related arguments and retain copies of */ + /* everything else to pass into a serial instance of the tool */ + /* */ + /* As we move forward, we might allow the HDF5 tool to be */ + /* queried for an acceptable set set of runtime arguments. */ + /* This could be just a simple string to allow getopt_long */ + /* to be invoked on the remaing command line arguments. */ + /**************************************************************/ + + int *path_indices = NULL; + int numpaths = count_dirpaths(argc, tool_args_start, argv, &path_indices); + + const char **argpaths = NULL; + + /* store src and dest path strings */ + const char *path1 = NULL; + const char *path2 = NULL; + size_t pathlen_total = 0; + + if (numpaths && path_indices) { + argpaths = &argv[path_indices[0]]; + } + /* pointer to mfu_file src and dest objects */ + /* The dst object will only be used for tools which + * accept 2 (or more?) file arguments */ + mfu_file_t *mfu_src_file = NULL; + mfu_file_t *mfu_dst_file = NULL; + + /* first item is source and second is dest */ + mfu_param_path *srcpath = NULL; + mfu_param_path *destpath = NULL; + mfu_param_path *paths = NULL; + + mfu_flist flist1 = NULL; + mfu_flist flist2 = NULL; + + /* allocate structure to define walk options */ + if (use_config_file > 0) { + int count1 = 0, count2 = 0; + for (i = 0; i < use_config_file; i++) { + int index = config_index[i]; + const char *config_file = argv[index]; + if (i == 0) { + flist1 = mfu_flist_new(); + count1 = fill_file_list(flist1, config_file + 1, rank, ranks); + } + else if (i == 1) { + flist2 = mfu_flist_new(); + count2 = fill_file_list(flist2, config_file + 1, rank, ranks); + } + } + if (count1 != count2) { + HDprintf("config files have different file counts: (1) %d and (2) %d\n", count1, count2); + } + } + else if (numpaths > 0) { + + /* allocate space for each path */ + paths = (mfu_param_path *)MFU_MALLOC((size_t)numpaths * sizeof(mfu_param_path)); + mfu_src_file = mfu_file_new(); + + /* process each path */ + mfu_param_path_set_all((uint64_t)numpaths, (const char **)argpaths, paths, mfu_src_file, true); + + /* don't allow user to specify input file with walk */ + if (inputname != NULL) { + if (paths) { + mfu_free(&paths); + } + usage(); + h5dwalk_exit(EXIT_FAILURE); + } + } + else { + /* if we're not walking, we must be reading, + * and for that we need a file */ + if (inputname == NULL) { + if (rank == 0) { + MFU_LOG(MFU_LOG_ERR, "Either a or --input is required."); + } + usage(); + h5dwalk_exit(EXIT_FAILURE); + } + } + + if (numpaths > 0) { + flist1 = mfu_flist_new(); + srcpath = &paths[0]; + path1 = srcpath->path; + pathlen_total += strlen(path1); + mfu_flist_walk_param_paths(1, srcpath, walk_opts, flist1, mfu_src_file); + } + if (numpaths > 1) { + flist2 = mfu_flist_new(); + mfu_dst_file = mfu_file_new(); + destpath = &paths[1]; + path2 = destpath->path; + pathlen_total += HDstrlen(path2); + mfu_flist_walk_param_paths(1, destpath, walk_opts, flist2, mfu_dst_file); + } + + if (tool_selected && (args_byte_length > 0)) { + pred_head = mfu_pred_new(); + args_buf = (char *)HDmalloc((size_t)(args_byte_length + pathlen_total)); + } + + /* filter files to only include hdf5 files */ + if (flist1) { + filter_hdf_files(&flist1, NULL, 0, 0); + } + if (flist2) { + filter_hdf_files(&flist2, NULL, 0, 0); + } + + /* if (numpaths > 1) + * In a case where we requeire the list indices of files from multiple + * directories to match, we must utilize a mapping function. + * The question to answer is how does the mapping function work? + * The most probable is a sort function, e.g. + * 1) an alphabet sort? + * 2) sort by file size? + * 3) something else? + */ + if (args_buf != NULL) { + int k = 0; + char *ptr = args_buf + sizeof(int); + *(int *)args_buf = h5tool_argc; + for (i = tool_args_start - 1; i < argc; i++) { + int copy_flist = -1; + if (i == config_index[k]) { + copy_flist = k; + } + else if (path_indices && (i == path_indices[k])) { + copy_flist = k; + } + + /* Maybe copy one of the flist pointers */ + if (copy_flist >= 0) { + /* The '&' indicates that what follows is a pointer */ + *ptr++ = '&'; + /* Select which argument list should be used */ + if (k == 0) { + HDmemcpy(ptr, &flist1, sizeof(void *)); + } + if (k == 1) { + HDmemcpy(ptr, &flist2, sizeof(void *)); + } + ptr += sizeof(mfu_flist *); + k++; + } + else { + HDstrcpy(ptr, argv[i]); + ptr += HDstrlen(argv[i]); + } + *ptr++ = 0; + } + *ptr++ = 0; + + mfu_pred_add(pred_head, MFU_PRED_EXEC, (void *)args_buf); + pred_commit(pred_head); + } + + /* apply predicates to each item in list */ + mfu_flist flist3 = mfu_flist_filter_pred(flist1, pred_head); + + /* print summary statistics of flist */ + mfu_flist_print_summary(flist1); + + /* write data to cache file */ + if (outputname != NULL) { + if (!text) { + if (rank == 0) { + puts("ouput capture needs to be a text formated file"); + } + } + else { + dh5tool_flist_write_text(outputname, flist1); + } + } + +#ifdef DAOS_SUPPORT + daos_cleanup(daos_args, mfu_file, NULL); +#endif + + /* free users, groups, and files objects */ + mfu_flist_free(&flist1); + if (flist2) + mfu_flist_free(&flist2); + if (flist3) + mfu_flist_free(&flist3); + + /* free memory allocated for options */ + mfu_free(&distribution); + mfu_free(&sortfields); + mfu_free(&outputname); + mfu_free(&inputname); + + /* free the path parameters */ + mfu_param_path_free_all((uint64_t)numpaths, paths); + + /* free memory allocated to hold params */ + mfu_free(&paths); + + /* free the walk options */ + mfu_walk_opts_delete(&walk_opts); + + /* delete file object */ + mfu_file_delete(&mfu_src_file); + + h5tools_close(); + /* shut down MPI */ + mfu_finalize(); + MPI_Finalize(); + + return rc; +} + +/*------------------------------------------------------------------------- + * Function: h5dwalk_exit + * + * Purpose: close the tools library and exit + * + * Return: none + * + * Programmer: Albert Cheng + * Date: Feb 6, 2005 + * + * Comments: + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +H5_ATTR_NORETURN void +h5dwalk_exit(int status) +{ + int require_finalize = 0; + h5tools_close(); + mfu_finalize(); + + /* Check to see whether we need to call MPI_Finalize */ + MPI_Initialized(&require_finalize); + if (require_finalize) + MPI_Finalize(); + + HDexit(status); +} diff --git a/utils/tools/test/CMakeLists.txt b/utils/tools/test/CMakeLists.txt new file mode 100644 index 0000000..0f5335d --- /dev/null +++ b/utils/tools/test/CMakeLists.txt @@ -0,0 +1,8 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_TOOLS_TEST C) + +#-- Add the h5diff tests +if (HDF5_BUILD_PARALLEL_TOOLS) + add_subdirectory (h5dwalk) +endif() + diff --git a/utils/tools/test/Makefile.am b/utils/tools/test/Makefile.am new file mode 100644 index 0000000..88104f2 --- /dev/null +++ b/utils/tools/test/Makefile.am @@ -0,0 +1,32 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +## +# +# Tools HDF5 Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +if PARALLEL_TOOLS_CONDITIONAL + H5DWALK=h5dwalk +else + H5DWALK= +endif + +CONFIG=ordered + +# All subdirectories +SUBDIRS=$(H5DWALK) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/test/h5dwalk/CMakeLists.txt b/utils/tools/test/h5dwalk/CMakeLists.txt new file mode 100644 index 0000000..5f6c992 --- /dev/null +++ b/utils/tools/test/h5dwalk/CMakeLists.txt @@ -0,0 +1,15 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_TOOLS_TEST_H5DWALK) + +if (HDF5_BUILD_PARALLEL_TOOLS) + add_custom_command( + OUTPUT ${HDF5_TOOLS_DIR}/test/demo_destfiles.test + COMMAND bash -c ${HDF5_TOOLS_SRC_H5DWALK_SOURCE_DIR}/copy_demo_files.sh + ARGS ${HDF5_TOOLS_DIR}/test ${CMAKE_BINARY_DIR}/bin + DEPENDS ${HDF5_TOOLS_SRC_H5DWALK_SOURCE_DIR}/copy_demo_files.sh + ) +endif () + +if (HDF5_TEST_TOOLS AND HDF5_TEST_SERIAL) + include (CMakeTests.cmake) +endif () diff --git a/utils/tools/test/h5dwalk/CMakeTests.cmake b/utils/tools/test/h5dwalk/CMakeTests.cmake new file mode 100644 index 0000000..b9e52c5 --- /dev/null +++ b/utils/tools/test/h5dwalk/CMakeTests.cmake @@ -0,0 +1,56 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +############################################################################## +############################################################################## +### T E S T I N G ### +############################################################################## +############################################################################## + + file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + + +############################################################################## +############################################################################## +### T H E T E S T S M A C R O S ### +############################################################################## +############################################################################## + + macro (ADD_H5_TEST resultfile resultcode) + # If using memchecker add tests without using scripts + if (HDF5_ENABLE_USING_MEMCHECKER) + message("Entered ADD_H5_TEST - 0") + add_test (NAME H5DWALK-${resultfile} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ ${ARGN}) + set_tests_properties (H5DWALK-${resultfile} PROPERTIES + WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + if ("${resultcode}" STREQUAL "1") + set_tests_properties (H5DWALK-${resultfile} PROPERTIES WILL_FAIL "true") + endif () + else () + # Remove any output file left over from previous test run + add_test ( + NAME H5DWALK-${resultfile} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}" + -D "TEST_PROGRAM=$" + -D "TEST_ARGS=${ARGN}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}.h5dwalk" + -D "TEST_LIBRARY_DIRECTORY=${LL_PATH}" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + endif () + endmacro () + + ADD_H5_TEST(help-1 0 -h) diff --git a/utils/tools/test/h5dwalk/Makefile.am b/utils/tools/test/h5dwalk/Makefile.am new file mode 100644 index 0000000..c32dd0f --- /dev/null +++ b/utils/tools/test/h5dwalk/Makefile.am @@ -0,0 +1,43 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +# Include src directory +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib + +install-examples: + @echo "Creating demo files" && \ + . copy_demo_files.sh $(top_srcdir)/src + +bin_SCRIPTS:install-examples + +#test script and program +TEST_SCRIPT=testh5dwalk.sh copy_demo_files.sh +check_SCRIPTS=$(TEST_SCRIPT) +SCRIPT_DEPEND=../../h5dwalk/h5dwalk$(EXEEXT) + +# Tell automake to clean h5redeploy script +CLEANFILES= + +# These were generated by configure. Remove them only when distclean. +DISTCLEANFILES=testh5dwalk.sh copy_demo_files.sh + +# All programs rely on hdf5 library and h5tools library +LDADD=$(LIBH5TOOLS) $(LIBHDF5) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/tools/test/h5dwalk/copy_demo_files.sh.in b/utils/tools/test/h5dwalk/copy_demo_files.sh.in new file mode 100644 index 0000000..f20bf43 --- /dev/null +++ b/utils/tools/test/h5dwalk/copy_demo_files.sh.in @@ -0,0 +1,86 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +srcdir=@srcdir@ +TOP_BUILDDIR=.. + +# Determine if backward compatibility options enabled +DEPRECATED_SYMBOLS="yes" + +EXIT_SUCCESS=0 +EXIT_FAILURE=1 + +CP='cp' + +THIS_DIR=`pwd` +SRC_TOOLS_DIR=$srcdir/../../../../tools + +nerrors=0 +verbose=yes +exit_code=$EXIT_SUCCESS + + +# Add Testing files into the local testfiles directory:: +TESTDIR=./testfiles +test -d $TESTDIR || mkdir $TESTDIR + +echo "HDF5 \"$THIS_DIR/testfiles/h5diff_basic1.h5\" {" > "$THIS_DIR"/testfiles/h5diff_basic1.h5_h5dump.txt +echo "FILE_CONTENTS { + group / + group /g1 + dataset /g1/d1 + dataset /g1/d2 + dataset /g1/dset1 + dataset /g1/dset10 + dataset /g1/dset11 + dataset /g1/dset12 + dataset /g1/dset3 + dataset /g1/dset5 + dataset /g1/dset6 + dataset /g1/dset7 + dataset /g1/dset8 + dataset /g1/dset9 + dataset /g1/fp1 + dataset /g1/fp15 + dataset /g1/fp16 + dataset /g1/fp17 + dataset /g1/fp18 + dataset /g1/fp18_COPY + dataset /g1/fp19 + dataset /g1/fp19_COPY + dataset /g1/fp2 + dataset /g1/fp20 + dataset /g1/fp20_COPY + dataset /g1/ld + } +}" >> "$THIS_DIR"/testfiles/h5diff_basic1.h5_h5dump.txt + +# Create the help-1.txt output file for '-h' validation +echo " +Usage: h5dwalk [options] ... + +Options: + -i, --input - read list from file + -o, --output - write output summary to the named file. + -E, --error - write processed errors to file in text format + -l, --log_text - write individual tool outputs to a file. Logs can be written to an optional named directory. + -T, --tool - name of the HDF5 tool to invoke + -h, --help - print usage + +For more information see https://mpifileutils.readthedocs.io. +" > "$THIS_DIR"/testfiles/help-1.txt + +# Make a copy of the help-1.txt output file for --help validation +$CP "$srcdir"/help.h5dwalk "$THIS_DIR"/testfiles/help-1.txt +$CP "$srcdir"/help.h5dwalk "$THIS_DIR"/testfiles/help-2.txt +# Make a copy of a simple HDF5 datafile which will be used as input for h5dump -n (see the expected output above) +$CP "$SRC_TOOLS_DIR"/test/h5diff/testfiles/h5diff_basic1.h5 "$THIS_DIR"/testfiles diff --git a/utils/tools/test/h5dwalk/help.h5dwalk b/utils/tools/test/h5dwalk/help.h5dwalk new file mode 100644 index 0000000..986cbba --- /dev/null +++ b/utils/tools/test/h5dwalk/help.h5dwalk @@ -0,0 +1,13 @@ + +Usage: h5dwalk [options] ... + +Options: + -i, --input - read list from file + -o, --output - write output summary to the named file. + -E, --error - write processed errors to file in text format + -l, --log_text - write individual tool outputs to a file. Logs can be written to an optional named directory. + -T, --tool - name of the HDF5 tool to invoke + -h, --help - print usage + +For more information see https://mpifileutils.readthedocs.io. + diff --git a/utils/tools/test/h5dwalk/testh5dwalk.sh.in b/utils/tools/test/h5dwalk/testh5dwalk.sh.in new file mode 100644 index 0000000..4f6dbde --- /dev/null +++ b/utils/tools/test/h5dwalk/testh5dwalk.sh.in @@ -0,0 +1,249 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +# Tests for the h5dwalk tool + +# +# + +srcdir=@srcdir@ + +# Determine which filters are available +USE_FILTER_SZIP="@USE_FILTER_SZIP@" +USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@" + + +TESTNAME=h5dwalk + +EXIT_SUCCESS=0 +EXIT_FAILURE=1 + +THIS_DIR="`pwd`" +ROOTDIR="`cd ../../../..; pwd`" +cd "$THIS_DIR" +TOP_DIR="$ROOTDIR" + + +H5DWALK=../../h5dwalk/h5dwalk +H5DWALK_BIN="$TOP_DIR/utils/tools/h5dwalk/h5dwalk" + + +H5DUMP="$TOP_DIR/src/h5dump/h5dump" +H5DUMP_BIN="$TOP_DIR/tools/src/h5dump/h5dump" + +RM='rm -rf' +CMP='cmp -s' +DIFF='diff -c' +CP='cp' +DIRNAME='dirname' +LS='ls' +AWK='awk' +WC='wc' + +nerrors=0 +verbose=yes + +export LD_LIBRARY_PATH=@LL_PATH@ + +# source dirs +SRC_TOOLS="$TOP_DIR/tools/test" +SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles" + +# testfiles source dirs for tools +SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES" +SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES" +SRC_H5DIFF_TESTFILES="$SRC_TOOLS/h5diff/testfiles" +SRC_H5COPY_TESTFILES="$SRC_TOOLS/h5copy/testfiles" +SRC_H5REPACK_TESTFILES="$SRC_TOOLS/h5repack/testfiles" +SRC_H5JAM_TESTFILES="$SRC_TOOLS/h5jam/testfiles" +SRC_H5DWALK_TESTFILES="$SRC_TOOLS/h5dwalk/testfiles" +SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/h5import/testfiles" + +TESTDIR=./testfiles +test -d $TESTDIR || mkdir $TESTDIR + +echo "SRC_H5DIFF_TESTFILES = $SRC_H5DIFF_TESTFILES" +echo "Creating demo files" +. ./copy_demo_files.sh + + + +CLEAN_TESTFILES_AND_TESTDIR() +{ + echo "cleaning logfiles" + $RM $TESTDIR/*log* +} + +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Testing". +# +TESTING() { + SPACES=" " + echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012' +} + +# Run a test and print PASS or *FAIL*. If a test fails then increment +# the `nerrors' global variable and (if $verbose is set) display the +# difference between the actual output and the expected output. The +# expected output is given as the first argument to this function and +# the actual output file is calculated by replacing the `.ddl' with +# `.out'. The actual output is not removed if $HDF5_NOCLEANUP has a +# non-zero value. +# +TOOLTEST() { + expect="$TESTDIR/$1" + expect_err="$TESTDIR/`basename $1`.err" + actual="$TESTDIR/`basename $1`.out" + actual_err="$TESTDIR/`basename $1`.out.err" + actual_sav=${actual}-sav + actual_err_sav=${actual_err}-sav + shift + + # Run test. + + TESTING $H5DWALK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DWALK_BIN $@ + ) 1> $actual 2> $actual_err + + # save actual and actual_err in case they are needed later. + cp $actual $actual_sav + cp $actual_err $actual_err_sav + + if [ ! -f $expect ]; then + # Compare error files if the expect file doesn't exist. + if $CMP $expect_err $actual_err; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.err) differs from actual result (*.out.err)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect_err $actual_err |sed 's/^/ /' + fi + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err $actual_sav $actual_err_sav + fi +} + +TOOL_LOGTEST() { + expect="$TESTDIR/`basename $1`.txt" + expect_err="$TESTDIR/`basename $1`.err" + actual="$TESTDIR/`basename $1`.log" + actual_err="$TESTDIR/`basename $1`.out.err" + actual_sav=${actual}-sav + actual_err_sav=${actual_err}-sav + shift + + echo "running logtest" + + # Run test. + TESTING $H5DWALK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DWALK_BIN $@ + + ) 1> $actual 2> $actual_err + expect_len="`wc -l < $expect`" + + if [ ! -f $actual ]; then + echo "*FAILED*" + echo " The expected .log file is missing" + echo " Perhaps the test failed to run?" + else + actual_len="`wc -l < $actual`" + if [ $actual_len -eq $expect_len ]; then + echo " PASSED" + else + echo "*FAILED*" + echo " The generated .log file length does not match the expected length. $actual_len != $expected_len" + fi + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err $actual_sav $actual_err_sav + fi +} + +TOOL_CHK_LOGLEN() { + expect=$1 + shift + + echo "running tool_chk_loglen" + + # Run test. + TESTING $H5DWALK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DWALK_BIN $@ + ) + + expect_len="`wc -l < $expect`" + if [ "$expect_len" -gt 0 ]; then + echo " PASSED" + else + echo "*FAILED*" + echo " The generated .log file is empty!." + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $expect + fi +} + + +# Print a "SKIP" message +SKIP() { + TESTING $H5DWALK $@6 + echo " -SKIP-" +} + + + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + +TOOLTEST help-1.txt -h +TOOLTEST help-2.txt --help +TOOL_LOGTEST h5diff_basic1.h5_h5dump -l -T $H5DUMP_BIN -n ./h5diff_basic1.h5 +TOOL_CHK_LOGLEN showme-h5dump.log -o `pwd`/showme-h5dump.log -T $H5DUMP_BIN -n `pwd` + + +# +# +# Clean up temporary files/directories +CLEAN_TESTFILES_AND_TESTDIR + +if test $nerrors -eq 0 ; then + echo "All $TESTNAME tests passed." + exit $EXIT_SUCCESS +else + echo "$TESTNAME tests failed with $nerrors errors." + exit $EXIT_FAILURE +fi + -- cgit v0.12