diff options
187 files changed, 8039 insertions, 4081 deletions
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index b0ebed0..c3ed6b2 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -21,7 +21,7 @@ jobs: include: - name: "Windows Latest MSVC" artifact: "Windows-MSVC.tar.xz" - os: windows-latest + os: windows-2022 build_type: "Release" toolchain: "" cpp: ON @@ -30,7 +30,7 @@ jobs: ts: OFF hl: ON parallel: OFF - generator: "-G \"Visual Studio 16 2019\" -A x64" + generator: "-G \"Visual Studio 17 2022\" -A x64" - name: "Ubuntu Latest GCC" artifact: "Linux.tar.xz" os: ubuntu-latest @@ -82,7 +82,7 @@ jobs: # Threadsafe runs - name: "Windows TS MSVC" artifact: "Windows-MSVCTS.tar.xz" - os: windows-2016 + os: windows-2019 build_type: "Release" toolchain: "" cpp: OFF @@ -91,7 +91,7 @@ jobs: ts: ON hl: OFF parallel: OFF - generator: "-G \"Visual Studio 15 2017 Win64\"" + generator: "-G \"Visual Studio 16 2019\" -A x64" - name: "Ubuntu TS GCC" artifact: "LinuxTS.tar.xz" os: ubuntu-latest diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index d6081f4..33608b5 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -16,7 +16,7 @@ jobs: include: - name: "Windows Latest MSVC" artifact: "Windows-MSVC.tar.xz" - os: windows-latest + os: windows-2022 build_type: "Release" toolchain: "" cpp: ON @@ -25,7 +25,7 @@ jobs: ts: OFF hl: ON parallel: OFF - generator: "-G \"Visual Studio 16 2019\" -A x64" + generator: "-G \"Visual Studio 17 2022\" -A x64" - name: "Ubuntu Latest GCC" artifact: "Linux.tar.xz" os: ubuntu-latest @@ -77,7 +77,7 @@ jobs: # Threadsafe runs - name: "Windows TS MSVC" artifact: "Windows-MSVCTS.tar.xz" - os: windows-2016 + os: windows-2019 build_type: "Release" toolchain: "" cpp: OFF @@ -86,7 +86,7 @@ jobs: ts: ON hl: OFF parallel: OFF - generator: "-G \"Visual Studio 15 2017 Win64\"" + generator: "-G \"Visual Studio 16 2019\" -A x64" - name: "Ubuntu TS GCC" artifact: "LinuxTS.tar.xz" os: ubuntu-latest diff --git a/CMakeInstallation.cmake b/CMakeInstallation.cmake index d811ea5..b506f05 100644 --- a/CMakeInstallation.cmake +++ b/CMakeInstallation.cmake @@ -31,7 +31,7 @@ if (NOT HDF5_EXTERNALLY_CONFIGURED) if (HDF5_EXPORTED_TARGETS) install ( EXPORT ${HDF5_EXPORTED_TARGETS} - DESTINATION ${HDF5_INSTALL_CMAKE_DIR}/hdf5 + DESTINATION ${HDF5_INSTALL_CMAKE_DIR} FILE ${HDF5_PACKAGE}${HDF_PACKAGE_EXT}-targets.cmake NAMESPACE ${HDF_PACKAGE_NAMESPACE} COMPONENT configinstall @@ -72,7 +72,7 @@ set (CURRENT_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}" ) configure_package_config_file ( ${HDF_RESOURCES_DIR}/hdf5-config.cmake.in "${HDF5_BINARY_DIR}/${HDF5_PACKAGE}${HDF_PACKAGE_EXT}-config.cmake" - INSTALL_DESTINATION "${HDF5_INSTALL_CMAKE_DIR}/hdf5" + INSTALL_DESTINATION "${HDF5_INSTALL_CMAKE_DIR}" PATH_VARS INCLUDE_INSTALL_DIR SHARE_INSTALL_DIR CURRENT_BUILD_DIR INSTALL_PREFIX "${CMAKE_CURRENT_BINARY_DIR}" ) @@ -86,14 +86,14 @@ set (CURRENT_BUILD_DIR "${CMAKE_INSTALL_PREFIX}" ) configure_package_config_file ( ${HDF_RESOURCES_DIR}/hdf5-config.cmake.in "${HDF5_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${HDF5_PACKAGE}${HDF_PACKAGE_EXT}-config.cmake" - INSTALL_DESTINATION "${HDF5_INSTALL_CMAKE_DIR}/hdf5" + INSTALL_DESTINATION "${HDF5_INSTALL_CMAKE_DIR}" PATH_VARS INCLUDE_INSTALL_DIR SHARE_INSTALL_DIR CURRENT_BUILD_DIR ) if (NOT HDF5_EXTERNALLY_CONFIGURED) install ( FILES ${HDF5_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${HDF5_PACKAGE}${HDF_PACKAGE_EXT}-config.cmake - DESTINATION ${HDF5_INSTALL_CMAKE_DIR}/hdf5 + DESTINATION ${HDF5_INSTALL_CMAKE_DIR} COMPONENT configinstall ) endif () @@ -113,7 +113,7 @@ if (NOT HDF5_EXTERNALLY_CONFIGURED) #) install ( FILES ${HDF5_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/${HDF5_PACKAGE}${HDF_PACKAGE_EXT}-config-version.cmake - DESTINATION ${HDF5_INSTALL_CMAKE_DIR}/hdf5 + DESTINATION ${HDF5_INSTALL_CMAKE_DIR} COMPONENT configinstall ) endif () @@ -160,7 +160,6 @@ ./config/clang-warnings/error-general ./config/clang-warnings/general ./config/clang-warnings/no-developer-general -./config/clang-warnings/noerror-general ./config/gnu-warnings/4.8-4.last ./config/gnu-warnings/4.8 ./config/gnu-warnings/4.9 @@ -175,10 +174,9 @@ ./config/gnu-warnings/cxx-4.9 ./config/gnu-warnings/cxx-5 ./config/gnu-warnings/cxx-9 +./config/gnu-warnings/cxx-developer-4.8 ./config/gnu-warnings/cxx-error-5 ./config/gnu-warnings/cxx-error-general -./config/gnu-warnings/cxx-noerror-5 -./config/gnu-warnings/cxx-noerror-general ./config/gnu-warnings/developer-4.8 ./config/gnu-warnings/developer-7 ./config/gnu-warnings/developer-8 @@ -190,15 +188,13 @@ ./config/gnu-warnings/general ./config/gnu-warnings/gfort-general ./config/gnu-warnings/gfort-4.8 -./config/gnu-warnings/gfort-5 +./config/gnu-warnings/developer-gfort-5 ./config/gnu-warnings/gfort-6 ./config/gnu-warnings/gfort-8 +./config/gnu-warnings/no-cxx-developer-4.8 ./config/gnu-warnings/no-developer-4.8 ./config/gnu-warnings/no-developer-8 ./config/gnu-warnings/no-developer-general -./config/gnu-warnings/noerror-5 -./config/gnu-warnings/noerror-8 -./config/gnu-warnings/noerror-general ./config/intel-warnings/15 ./config/intel-warnings/18 ./config/intel-warnings/developer-general @@ -207,6 +203,10 @@ ./config/intel-warnings/win-general ./config/intel-warnings/ifort-general +./config/cmake/FindMFU.cmake +./config/cmake/FindDTCMP.cmake +./config/cmake/FindCIRCLE.cmake + ./config/site-specific/BlankForm ./doc/branches-explained.md @@ -219,7 +219,7 @@ ./doc/vfd-swmr-user-guide.md ./doc/code-conventions.md ./doc/contributing.md - +./doc/library-init-shutdown.md ./doxygen/aliases ./doxygen/CMakeLists.txt ./doxygen/Doxyfile.in @@ -228,11 +228,10 @@ ./doxygen/dox/Cookbook.dox ./doxygen/dox/DDLBNF110.dox ./doxygen/dox/DDLBNF112.dox -./doxygen/dox/FileFormatSpec.dox +./doxygen/dox/FTS.dox ./doxygen/dox/GettingStarted.dox ./doxygen/dox/H5AC_cache_config_t.dox ./doxygen/dox/MetadataCachingInHDF5.dox -./doxygen/dox/OtherSpecs.dox ./doxygen/dox/Overview.dox ./doxygen/dox/ReferenceManual.dox ./doxygen/dox/RFC.dox @@ -248,9 +247,12 @@ ./doxygen/dox/cookbook/Files.c ./doxygen/dox/cookbook/Files.dox ./doxygen/dox/cookbook/Performance.dox +./doxygen/examples/DebuggingHDF5Applications.html ./doxygen/examples/FF-IH_FileGroup.gif ./doxygen/examples/FF-IH_FileObject.gif +./doxygen/examples/FileFormat.html ./doxygen/examples/FileFormatSpecChunkDiagram.jpg +./doxygen/examples/Filters.html ./doxygen/examples/H5Pset_metadata_read_attempts.c ./doxygen/examples/H5Pset_object_flush_cb.c ./doxygen/examples/H5.format.1.0.html @@ -279,6 +281,7 @@ ./doxygen/examples/H5Z_examples.c ./doxygen/examples/H5_examples.c ./doxygen/examples/ImageSpec.html +./doxygen/examples/IOFlow.html ./doxygen/examples/PaletteExample1.gif ./doxygen/examples/Palettes.fm.anc.gif ./doxygen/examples/TableSpec.html @@ -294,6 +297,9 @@ ./doxygen/img/FF-IH_FileObject.gif ./doxygen/img/FileFormatSpecChunkDiagram.jpg ./doxygen/img/HDFG-logo.png +./doxygen/img/IOFlow.gif +./doxygen/img/IOFlow2.gif +./doxygen/img/IOFlow3.gif ./doxygen/img/PaletteExample1.gif ./doxygen/img/Palettes.fm.anc.gif ./doxygen/img/ftv2node.png @@ -3115,7 +3121,10 @@ ./hl/test/sepia.pal ./hl/test/test_ds.c ./hl/test/test_ds_be.h5 +./hl/test/test_ds_be_new_ref.h5 +./hl/test/test_ds_be_new_ref-32bit.h5 ./hl/test/test_ds_le.h5 +./hl/test/test_ds_le_new_ref.h5 ./hl/test/test_dset_append.c ./hl/test/test_file_image.c ./hl/test/test_h5do_compat.c @@ -3338,6 +3347,7 @@ ./java/src/hdf/hdf5lib/callbacks/H5P_prp_set_func_cb.java ./java/src/hdf/hdf5lib/callbacks/H5P_iterate_cb.java ./java/src/hdf/hdf5lib/callbacks/H5P_iterate_t.java +./java/src/hdf/hdf5lib/callbacks/package-info.java ./java/src/hdf/hdf5lib/exceptions/HDF5AttributeException.java ./java/src/hdf/hdf5lib/exceptions/HDF5BtreeException.java @@ -3363,6 +3373,7 @@ ./java/src/hdf/hdf5lib/exceptions/HDF5ReferenceException.java ./java/src/hdf/hdf5lib/exceptions/HDF5ResourceUnavailableException.java ./java/src/hdf/hdf5lib/exceptions/HDF5SymbolTableException.java +./java/src/hdf/hdf5lib/exceptions/package-info.java ./java/src/hdf/hdf5lib/structs/H5_ih_info_t.java ./java/src/hdf/hdf5lib/structs/H5A_info_t.java @@ -3377,12 +3388,14 @@ ./java/src/hdf/hdf5lib/structs/H5O_info_t.java ./java/src/hdf/hdf5lib/structs/H5O_native_info_t.java ./java/src/hdf/hdf5lib/structs/H5O_token_t.java +./java/src/hdf/hdf5lib/structs/package-info.java ./java/src/hdf/hdf5lib/H5.java ./java/src/hdf/hdf5lib/HDF5Constants.java ./java/src/hdf/hdf5lib/HDF5GroupInfo.java ./java/src/hdf/hdf5lib/HDFArray.java ./java/src/hdf/hdf5lib/HDFNativeData.java +./java/src/hdf/hdf5lib/package-info.java ./java/examples/Makefile.am ./java/examples/CMakeLists.txt @@ -3633,7 +3646,6 @@ ./config/cmake/HDF5UseFortran.cmake ./config/cmake/javaTargets.cmake.in ./config/cmake/jrunTest.cmake -./config/cmake/jvolTest.cmake ./config/cmake/libh5cc.in ./config/cmake/libhdf5.settings.cmake.in ./config/cmake/mccacheinit.cmake @@ -3875,6 +3887,7 @@ ./tools/src/h5import/Makefile.in ./tools/src/h5jam/Makefile.in ./tools/src/h5ls/Makefile.in +./tools/src/h5perf/Makefile.in ./tools/src/h5repack/Makefile.in ./tools/src/h5stat/Makefile.in ./tools/src/misc/Makefile.in diff --git a/Makefile.am b/Makefile.am index 8518114..2a544f4 100644 --- a/Makefile.am +++ b/Makefile.am @@ -88,9 +88,9 @@ else TOOLS_DIR= endif -SUBDIRS = src $(TESTSERIAL_DIR) $(TESTPARALLEL_DIR) bin utils $(TOOLS_DIR) . \ +SUBDIRS = src $(TESTSERIAL_DIR) $(TESTPARALLEL_DIR) bin $(TOOLS_DIR) utils . \ $(CXX_DIR) $(FORTRAN_DIR) $(JAVA_DIR) $(HDF5_HL_DIR) -DIST_SUBDIRS = src test testpar utils tools . c++ fortran hl examples java +DIST_SUBDIRS = src test testpar tools utils . c++ fortran hl examples java # Some files generated during configure that should be cleaned DISTCLEANFILES=config/stamp1 config/stamp2 @@ -1,4 +1,4 @@ -HDF5 version 1.13.0 currently under development +HDF5 version 1.13.1-1 currently under development ------------------------------------------------------------------------------ Please refer to the release_docs/INSTALL file for installation instructions. @@ -53,6 +53,7 @@ $Source = ""; "H5D_scatter_func_t" => "DS", "H5FD_mpio_xfer_t" => "Dt", "H5D_vds_view_t" => "Dv", + "H5FD_class_value_t" => "DV", "H5D_chunk_iter_op_t" => "x", "herr_t" => "e", "H5E_auto1_t" => "Ea", @@ -497,7 +498,7 @@ for $file (@ARGV) { $file_args = 0; # Ignore some files that do not need tracing macros - unless ($file eq "H5FDmulti.c" or $file eq "src/H5FDmulti.c" or $file eq "H5FDstdio.c" or $file eq "src/H5FDstdio.c" or $file eq "src/H5TS.c") { + unless ($file eq "H5FDmulti.c" or $file eq "src/H5FDmulti.c" or $file eq "H5FDstdio.c" or $file eq "src/H5FDstdio.c" or $file eq "src/H5TS.c" or $file eq "src/H5FDperform.c") { # Snarf up the entire file open SOURCE, $file or die "$file: $!\n"; diff --git a/c++/src/H5PredType.cpp b/c++/src/H5PredType.cpp index 72d560d..c407c33 100644 --- a/c++/src/H5PredType.cpp +++ b/c++/src/H5PredType.cpp @@ -406,31 +406,29 @@ PredType::makePredTypes() MIPS_F32_ = new PredType(H5T_MIPS_F32); MIPS_F64_ = new PredType(H5T_MIPS_F64); - NATIVE_CHAR_ = new PredType(H5T_NATIVE_CHAR); - NATIVE_INT_ = new PredType(H5T_NATIVE_INT); - NATIVE_FLOAT_ = new PredType(H5T_NATIVE_FLOAT); - NATIVE_SCHAR_ = new PredType(H5T_NATIVE_SCHAR); - NATIVE_UCHAR_ = new PredType(H5T_NATIVE_UCHAR); - NATIVE_SHORT_ = new PredType(H5T_NATIVE_SHORT); - NATIVE_USHORT_ = new PredType(H5T_NATIVE_USHORT); - NATIVE_UINT_ = new PredType(H5T_NATIVE_UINT); - NATIVE_LONG_ = new PredType(H5T_NATIVE_LONG); - NATIVE_ULONG_ = new PredType(H5T_NATIVE_ULONG); - NATIVE_LLONG_ = new PredType(H5T_NATIVE_LLONG); - NATIVE_ULLONG_ = new PredType(H5T_NATIVE_ULLONG); - NATIVE_DOUBLE_ = new PredType(H5T_NATIVE_DOUBLE); -#if H5_SIZEOF_LONG_DOUBLE != 0 + NATIVE_CHAR_ = new PredType(H5T_NATIVE_CHAR); + NATIVE_INT_ = new PredType(H5T_NATIVE_INT); + NATIVE_FLOAT_ = new PredType(H5T_NATIVE_FLOAT); + NATIVE_SCHAR_ = new PredType(H5T_NATIVE_SCHAR); + NATIVE_UCHAR_ = new PredType(H5T_NATIVE_UCHAR); + NATIVE_SHORT_ = new PredType(H5T_NATIVE_SHORT); + NATIVE_USHORT_ = new PredType(H5T_NATIVE_USHORT); + NATIVE_UINT_ = new PredType(H5T_NATIVE_UINT); + NATIVE_LONG_ = new PredType(H5T_NATIVE_LONG); + NATIVE_ULONG_ = new PredType(H5T_NATIVE_ULONG); + NATIVE_LLONG_ = new PredType(H5T_NATIVE_LLONG); + NATIVE_ULLONG_ = new PredType(H5T_NATIVE_ULLONG); + NATIVE_DOUBLE_ = new PredType(H5T_NATIVE_DOUBLE); NATIVE_LDOUBLE_ = new PredType(H5T_NATIVE_LDOUBLE); -#endif - NATIVE_B8_ = new PredType(H5T_NATIVE_B8); - NATIVE_B16_ = new PredType(H5T_NATIVE_B16); - NATIVE_B32_ = new PredType(H5T_NATIVE_B32); - NATIVE_B64_ = new PredType(H5T_NATIVE_B64); - NATIVE_OPAQUE_ = new PredType(H5T_NATIVE_OPAQUE); - NATIVE_HSIZE_ = new PredType(H5T_NATIVE_HSIZE); - NATIVE_HSSIZE_ = new PredType(H5T_NATIVE_HSSIZE); - NATIVE_HERR_ = new PredType(H5T_NATIVE_HERR); - NATIVE_HBOOL_ = new PredType(H5T_NATIVE_HBOOL); + NATIVE_B8_ = new PredType(H5T_NATIVE_B8); + NATIVE_B16_ = new PredType(H5T_NATIVE_B16); + NATIVE_B32_ = new PredType(H5T_NATIVE_B32); + NATIVE_B64_ = new PredType(H5T_NATIVE_B64); + NATIVE_OPAQUE_ = new PredType(H5T_NATIVE_OPAQUE); + NATIVE_HSIZE_ = new PredType(H5T_NATIVE_HSIZE); + NATIVE_HSSIZE_ = new PredType(H5T_NATIVE_HSSIZE); + NATIVE_HERR_ = new PredType(H5T_NATIVE_HERR); + NATIVE_HBOOL_ = new PredType(H5T_NATIVE_HBOOL); NATIVE_INT8_ = new PredType(H5T_NATIVE_INT8); NATIVE_UINT8_ = new PredType(H5T_NATIVE_UINT8); diff --git a/c++/src/cpp_doc_config b/c++/src/cpp_doc_config index 71db7f6..3eb7645 100644 --- a/c++/src/cpp_doc_config +++ b/c++/src/cpp_doc_config @@ -38,7 +38,7 @@ PROJECT_NAME = # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "1.13.0" +PROJECT_NUMBER = "1.13.1-1, currently under development" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/config/clang-cxxflags b/config/clang-cxxflags index c585ce7..76f095f 100644 --- a/config/clang-cxxflags +++ b/config/clang-cxxflags @@ -20,15 +20,15 @@ # # For now, do not promote any warnings to errors. # -PROMOTE_ERRORS_DFLT=no +WARNINGS_AS_ERRORS_DFLT=no # # This filter rewrites -Werror= as -W, in that way demoting warnings -# promoted to errors back to warnings, if PROMOTE_ERRORS is no. +# promoted to errors back to warnings, if WARNINGS_AS_ERRORS is no. # demote_errors() { - if [ ${PROMOTE_ERRORS:-${PROMOTE_ERRORS_DFLT}} = no ]; then + if [ ${WARNINGS_AS_ERRORS:-${WARNINGS_AS_ERRORS_DFLT}} = no ]; then sed 's,-Werror=,-W,g' else cat @@ -44,7 +44,7 @@ demote_errors() load_clang_arguments() { set -- $(for arg; do - sed 's,#.*$,,' $srcdir/config/clang-warnings/${arg} + sed 's,#.*$,,' $srcdir/config/clang-warnings/${arg} | demote_errors done) IFS=' ' echo "$*" } @@ -161,7 +161,6 @@ if test "X-clang" = "X-$cxx_vendor" -o "X-Apple LLVM" = "X-$cxx_vendor"; then H5_CXXFLAGS="$H5_CXXFLAGS $(load_clang_arguments general)" H5_ECXXFLAGS="$H5_ECXXFLAGS $(load_clang_arguments error-general)" - H5_NECXXFLAGS="$H5_NECXXFLAGS $(load_clang_arguments noerror-general)" ###################### # Developer warnings # diff --git a/config/clang-flags b/config/clang-flags index 92a8973..5c377ed 100644 --- a/config/clang-flags +++ b/config/clang-flags @@ -20,15 +20,15 @@ # # For now, do not promote any warnings to errors. # -PROMOTE_ERRORS_DFLT=no +WARNINGS_AS_ERRORS_DFLT=no # # This filter rewrites -Werror= as -W, in that way demoting warnings -# promoted to errors back to warnings, if PROMOTE_ERRORS is no. +# promoted to errors back to warnings, if WARNINGS_AS_ERRORS is no. # demote_errors() { - if [ ${PROMOTE_ERRORS:-${PROMOTE_ERRORS_DFLT}} = no ]; then + if [ ${WARNINGS_AS_ERRORS:-${WARNINGS_AS_ERRORS_DFLT}} = no ]; then sed 's,-Werror=,-W,g' else cat @@ -44,7 +44,7 @@ demote_errors() load_clang_arguments() { set -- $(for arg; do - sed 's,#.*$,,' $srcdir/config/clang-warnings/${arg} + sed 's,#.*$,,' $srcdir/config/clang-warnings/${arg} | demote_errors done) IFS=' ' echo "$*" } @@ -183,7 +183,6 @@ if test "X-clang" = "X-$cc_vendor" -o "X-Apple LLVM" = "X-$cc_vendor"; then H5_CFLAGS="$H5_CFLAGS $(load_clang_arguments general)" H5_ECFLAGS="$H5_ECFLAGS $(load_clang_arguments error-general)" - H5_NECFLAGS="$H5_NECFLAGS $(load_clang_arguments noerror-general)" ###################### # Developer warnings # diff --git a/config/clang-warnings/noerror-general b/config/clang-warnings/noerror-general deleted file mode 100644 index 4690ebd..0000000 --- a/config/clang-warnings/noerror-general +++ /dev/null @@ -1,82 +0,0 @@ -# -# These warnings will be treated as errors, using the error-general file, -# when HDF5_ENABLE_WARNINGS_AS_ERRORS is set to true for CMake or -# the --enable-warnings-as-errors option is specified for configure. -# Otherwise this file will be used to treat them as warnings. -# --Wbad-function-cast --Wimplicit-function-declaration --Wincompatible-pointer-types --Wmissing-declarations --Wpacked --Wshadow --Wswitch -# -# NOTE: Following files are not compatible with incompatible-pointer-types as errors -# src/H5Dchunk.c,src/H5Dint.c,src/H5Gint.c,src/H5HFcache.c,src/H5I.c,src/H5T.c --Wno-error=incompatible-pointer-types-discards-qualifiers -# -# -# NOTE: File Driver files are not compatible with these warnings as errors -# H5FDdirect.c,H5FDmpio.c,H5FDros3.c, -# -Werror=unused-function -# --Wunused-function -# -# H5FDdrvr_module.h -# -Werror=unused-variable -# --Wunused-variable -# -# H5VLpassthru.c -# -Werror=unused-parameter -# --Wunused-parameter -# -# -# -# NOTE: Tools files are not compatible with these warnings as errors -# lib/h5tools.c -# -Werror=cast-align -# --Wcast-align -# -# lib/h5tools_utils.c -# -Werror=unused-parameter -# -# -# NOTE: JNI files are not compatible with these warnings as errors -# jni/h5pDCPLImp.c,jni/nativeData.c,jni/h5util.c,jni/h5rImp.c -# jni/h5sImp.c,jni/h5tImp.c -# -Werror=cast-align -# jni/h5util.c -# -Werror=format(-overflow) -# --Wformat -# -# -#Examples and tests do not use the same set of extensive warning flags as libraries -# Here is a list of tests and examples that have issues with the stricter warnings as error -# -# NOTE: Test files are not compatible with these warnings as errors -# thread_id.c, -# -Werror=unused-function -# dsets.c -# -Werror=unused-parameter -# -# -# NOTE: Examples files are not compatible with these warnings as errors -# h5_vds-eiger.c,h5_vds-exclim.c,h5_vds.c,h5_vds-exc.c,h5_vds-percival-unlim-maxmin.c -# h5_vds-percival.c,h5_read.c,h5_rdwt.c,h5_mount.c,h5_extend.c,h5_extend_write.c -# h5_write.c,h5_vds-simpleIO.c,h5_ref2reg_deprec.c,h5_crtgrp.c,h5_select.c -# h5_vds-percival-unlim.c,h5_crtatt.c,h5_group.c,h5_attribute.c,h5_crtdat.c -# h5_reference_deprec.c -# h5_rdwt.c,h5_crtgrp.c,h5_crtatt.c,h5_crtdat.c -# -Werror=strict-prototypes -# h5_rdwt.c,h5_crtgrp.c,h5_crtatt.c,h5_crtdat.c -# -Werror=old-style-definition -# h5_vds-exclim.c,h5_vds.c,h5_vds-exc.c, -# -Werror=unused-variable -# h5_elink_unix2win.c,h5_extlink.c,h5_attribute.c -# -Werror=unused-parameter - diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 263cedf..483e551 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -195,7 +195,7 @@ endif () # Header-check flags set in config/cmake_ext_mod/ConfigureChecks.cmake # ---------------------------------------------------------------------- option (HDF5_ENABLE_MIRROR_VFD "Build the Mirror Virtual File Driver" OFF) -if (H5FD_ENABLE_MIRROR_VFD) +if (HDF5_ENABLE_MIRROR_VFD) if ( ${HDF_PREFIX}_HAVE_NETINET_IN_H AND ${HDF_PREFIX}_HAVE_NETDB_H AND ${HDF_PREFIX}_HAVE_ARPA_INET_H AND @@ -232,7 +232,7 @@ endif () # so this one is used. #----------------------------------------------------------------------------- set (RUN_OUTPUT_PATH_DEFAULT ${CMAKE_BINARY_DIR}) -macro (C_RUN FUNCTION_NAME SOURCE_CODE RETURN_VAR) +macro (C_RUN FUNCTION_NAME SOURCE_CODE RETURN_VAR RETURN_OUTPUT_VAR) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") message (VERBOSE "Detecting C ${FUNCTION_NAME}") endif () @@ -248,7 +248,7 @@ macro (C_RUN FUNCTION_NAME SOURCE_CODE RETURN_VAR) RUN_OUTPUT_VARIABLE OUTPUT_VAR ) - set (${RETURN_VAR} ${OUTPUT_VAR}) + set (${RETURN_OUTPUT_VAR} ${OUTPUT_VAR}) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") message (VERBOSE "* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * ") @@ -304,12 +304,11 @@ set (PROG_SRC #define C_LDBL_DIG DECIMAL_DIG\n\ #else\n\ #define C_LDBL_DIG LDBL_DIG\n\ -#endif\n\nint main() {\nFILE *pFile = fopen(\"pac_Cconftest.out\",\"w\")\\\;\nfprintf(pFile, \"\\%d\\\;\\%d\\\;\", C_LDBL_DIG, C_FLT128_DIG)\\\;\n\nreturn 0\\\;\n}\n +#endif\n\nint main() {\nprintf(\"\\%d\\\;\\%d\\\;\", C_LDBL_DIG, C_FLT128_DIG)\\\;\n\nreturn 0\\\;\n}\n " ) -C_RUN ("maximum decimal precision for C" ${PROG_SRC} PROG_RES) -file (READ "${RUN_OUTPUT_PATH_DEFAULT}/pac_Cconftest.out" PROG_OUTPUT4) +C_RUN ("maximum decimal precision for C" ${PROG_SRC} PROG_RES PROG_OUTPUT4) message (STATUS "Testing maximum decimal precision for C - ${PROG_OUTPUT4}") # dnl The output from the above program will be: diff --git a/config/cmake/FindCIRCLE.cmake b/config/cmake/FindCIRCLE.cmake new file mode 100644 index 0000000..b36d76c --- /dev/null +++ b/config/cmake/FindCIRCLE.cmake @@ -0,0 +1,49 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#[=======================================================================[.rst: +FindCIRCLE +-------- + +Find the native CIRCLE includes and library + +This module defines + +:: + + CIRCLE_INCLUDE_DIR, where to find CIRCLE.h, etc. + CIRCLE_LIBRARIES, the libraries required to use CIRCLE. + CIRCLE_FOUND, If false, do not try to use CIRCLE. + +also defined, but not for general use are + +:: + + CIRCLE_LIBRARY, where to find the CIRCLE library. +#]=======================================================================] + +if(DEFINED ENV{MFU_ROOT}) + set(ENV{MFU_INCLUDE} "$ENV{MFU_ROOT}/include") + set(ENV{MFU_LIB} "$ENV{MFU_ROOT}/lib") + set(ENV{MFU_LIB64} "$ENV{MFU_ROOT}/lib64") +else() + message("CIRCLE_LIBRARY: If you have problems building this library,\nconsider setting the MFU_ROOT environment variable to indicate\nwhere to find the support libraries and header files!") +endif() + +find_path(CIRCLE_INCLUDE_DIR + NAMES libcircle.h + HINTS ENV MFU_INCLUDE) + +find_library(CIRCLE_LIBRARY + NAMES circle + HINTS ENV MFU_LIB ENV MFU_LIB64 + ) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(CIRCLE REQUIRED_VARS CIRCLE_LIBRARY CIRCLE_INCLUDE_DIR) + +if(CIRCLE_FOUND) + set(CIRCLE_LIBRARIES ${CIRCLE_LIBRARY} ) +endif() + +mark_as_advanced(CIRCLE_INCLUDE_DIR CIRCLE_LIBRARY) diff --git a/config/cmake/FindDTCMP.cmake b/config/cmake/FindDTCMP.cmake new file mode 100644 index 0000000..b95ef20 --- /dev/null +++ b/config/cmake/FindDTCMP.cmake @@ -0,0 +1,48 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +#[=======================================================================[.rst: +FindDTCMP +-------- + +Find the native DTCMP includes and library + +This module defines + +:: + + DTCMP_INCLUDE_DIR, where to find DTCMP.h, etc. + DTCMP_LIBRARIES, the libraries required to use DTCMP. + DTCMP_FOUND, If false, do not try to use DTCMP. + +also defined, but not for general use are + +:: + + DTCMP_LIBRARY, where to find the DTCMP library. +#]=======================================================================] + +if(DEFINED ENV{MFU_ROOT}) + set(ENV{MFU_INCLUDE} "$ENV{MFU_ROOT}/include") + set(ENV{MFU_LIB} "$ENV{MFU_ROOT}/lib") + set(ENV{MFU_LIB64} "$ENV{MFU_ROOT}/lib64") +else() + message("DTCMP_LIBRARY: If you have problems building this library,\nconsider setting the MFU_ROOT environment variable to indicate\nwhere to find the support libraries and header files!") +endif() + +find_path(DTCMP_INCLUDE_DIR + NAMES dtcmp.h + HINTS ENV MFU_INCLUDE) + +find_library(DTCMP_LIBRARY + NAMES dtcmp + HINTS ENV MFU_LIB ENV MFU_LIB64) + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(DTCMP REQUIRED_VARS DTCMP_LIBRARY DTCMP_INCLUDE_DIR) + +if(DTCMP_FOUND) + set(DTCMP_LIBRARIES ${DTCMP_LIBRARY} ) +endif() + +mark_as_advanced(DTCMP_INCLUDE_DIR DTCMP_LIBRARY) diff --git a/config/cmake/FindMFU.cmake b/config/cmake/FindMFU.cmake new file mode 100644 index 0000000..37699e2 --- /dev/null +++ b/config/cmake/FindMFU.cmake @@ -0,0 +1,100 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +######################################################################### + +# - Derived from the FindTiff.cmake and FindJPEG.cmake that is included with cmake +# FindMFU + +# Find the native MFU includes and library + +# Imported targets +################## + +# This module defines the following :prop_tgt:`IMPORTED` targets: +# +# MFU::MFU +# The MFU library, if found. +# +# Result variables +################### + +# This module will set the following variables in your project: + +# MFU_FOUND, true if the MFU headers and libraries were found. +# MFU_INCLUDE_DIR, the directory containing the MFU headers. +# MFU_INCLUDE_DIRS, the directory containing the MFU headers. +# MFU_LIBRARIES, libraries to link against to use MFU. + +# Cache variables +################# + +# The following variables may also be set: + +# MFU_LIBRARY, where to find the MFU library. +# message (STATUS "Finding MFU library and headers..." ) +######################################################################### + + + +FIND_PATH(MFU_INCLUDE_DIR + NAMES mfu.h + HINTS "$ENV{MFU_ROOT}/include" +) +FIND_LIBRARY(MFU_LIBRARY + NAMES mfu + HINTS "$ENV{MFU_ROOT}/lib64" +) + +if(NOT MFU_LIBRARY) + set(mfu_names ${MFU_NAMES} mfu libmfu) + find_library(MFU_LIBRARY NAMES ${mfu_names}) + include(SelectLibraryConfigurations) + select_library_configurations(MFU) +endif() + +include(FindPackageHandleStandardArgs) +find_package_handle_standard_args(MFU + REQUIRED_VARS MFU_LIBRARY MFU_INCLUDE_DIR) + +if(MFU_FOUND) + set(MFU_LIBRARIES "${MFU_LIBRARY}") + set(MFU_INCLUDE_DIRS "${MFU_INCLUDE_DIR}") + set(LL_PATH "$ENV{MFU_ROOT}/lib64:$ENV{MFU_ROOT}/lib") + if(NOT TARGET MFU::MFU) + add_library(MFU::MFU UNKNOWN IMPORTED) + if(MFU_INCLUDE_DIRS) + set_target_properties(MFU::MFU PROPERTIES + INTERFACE_INCLUDE_DIRECTORIES "${MFU_INCLUDE_DIRS}") + endif() + if(EXISTS "${MFU_LIBRARY}") + set_target_properties(MFU::MFU PROPERTIES + IMPORTED_LINK_INTERFACE_LANGUAGES "C" + IMPORTED_LOCATION "${MFU_LIBRARY}") + endif() + endif() +endif() + +# Report the results. +if (NOT MFU_FOUND) + set (MFU_DIR_MESSAGE + "Mfu was not found. Make sure MFU_LIBRARY and MFU_INCLUDE_DIR are set or set the MFU_INSTALL environment variable." + ) + if (NOT MFU_FIND_QUIETLY) + if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") + message (VERBOSE "${MFU_DIR_MESSAGE}") + endif () + else () + if (MFU_FIND_REQUIRED) + message (FATAL_ERROR "Mfu was NOT found and is Required by this project") + endif () + endif () +endif () diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index 8fd1331..ae5a8c6 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -71,13 +71,13 @@ /* Define Fortran compiler ID */ #define H5_Fortran_COMPILER_ID @CMAKE_Fortran_COMPILER_ID@ +/* Define number of valid Fortran INTEGER KINDs (must be defined before F_IKIND)*/ +#cmakedefine H5_H5CONFIG_F_NUM_IKIND @H5_H5CONFIG_F_NUM_IKIND@ + /* Define valid Fortran INTEGER KINDs */ #cmakedefine H5_H5CONFIG_F_IKIND @H5_H5CONFIG_F_IKIND@ -/* Define number of valid Fortran INTEGER KINDs */ -#cmakedefine H5_H5CONFIG_F_NUM_IKIND @H5_H5CONFIG_F_NUM_IKIND@ - -/* Define number of valid Fortran REAL KINDs */ +/* Define number of valid Fortran REAL KINDs (must be defined before F_RKIND) */ #cmakedefine H5_H5CONFIG_F_NUM_RKIND @H5_H5CONFIG_F_NUM_RKIND@ /* Define valid Fortran REAL KINDs */ @@ -412,6 +412,9 @@ /* Define if the high-level library headers should be included in hdf5.h */ #cmakedefine H5_INCLUDE_HL @H5_INCLUDE_HL@ +/* Define if new-style references should be used with dimension scales */ +#cmakedefine H5_DIMENSION_SCALES_WITH_NEW_REF @H5_DIMENSION_SCALES_WITH_NEW_REF@ + /* Define if your system can convert long double to (unsigned) long long values correctly. */ #cmakedefine H5_LDOUBLE_TO_LLONG_ACCURATE @H5_LDOUBLE_TO_LLONG_ACCURATE@ diff --git a/config/cmake/HDF5UseFortran.cmake b/config/cmake/HDF5UseFortran.cmake index e192ec4..0d05e20 100644 --- a/config/cmake/HDF5UseFortran.cmake +++ b/config/cmake/HDF5UseFortran.cmake @@ -32,15 +32,10 @@ macro (READ_SOURCE SOURCE_START SOURCE_END RETURN_VAR) endmacro () set (RUN_OUTPUT_PATH_DEFAULT ${CMAKE_BINARY_DIR}) -if (NOT CMAKE_VERSION VERSION_LESS "3.14.0") - if (HDF5_REQUIRED_LIBRARIES) - set (CMAKE_REQUIRED_LIBRARIES "${HDF5_REQUIRED_LIBRARIES}") - endif () -else () # The provided CMake Fortran macros don't provide a general compile/run function # so this one is used. #----------------------------------------------------------------------------- -macro (FORTRAN_RUN FUNCTION_NAME SOURCE_CODE RUN_RESULT_VAR1 COMPILE_RESULT_VAR1 RETURN_VAR) +macro (FORTRAN_RUN FUNCTION_NAME SOURCE_CODE RUN_RESULT_VAR1 COMPILE_RESULT_VAR1 RETURN_VAR RETURN_OUTPUT_VAR) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") message (VERBOSE "Detecting Fortran ${FUNCTION_NAME}") endif () @@ -52,7 +47,9 @@ macro (FORTRAN_RUN FUNCTION_NAME SOURCE_CODE RUN_RESULT_VAR1 COMPILE_RESULT_VAR1 ${CMAKE_BINARY_DIR} ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeTmp/testFortranCompiler1.f90 LINK_LIBRARIES "${HDF5_REQUIRED_LIBRARIES}" + RUN_OUTPUT_VARIABLE OUTPUT_VAR ) + set (${RETURN_OUTPUT_VAR} ${OUTPUT_VAR}) if (${COMPILE_RESULT_VAR}) set(${RETURN_VAR} ${RUN_RESULT_VAR}) @@ -81,7 +78,6 @@ macro (FORTRAN_RUN FUNCTION_NAME SOURCE_CODE RUN_RESULT_VAR1 COMPILE_RESULT_VAR1 set(${RETURN_VAR} ${COMPILE_RESULT_VAR}) endif () endmacro () -endif () #----------------------------------------------------------------------------- # Check to see C_LONG_DOUBLE is available @@ -130,27 +126,22 @@ endif () #----------------------------------------------------------------------------- READ_SOURCE ("PROGRAM FC_AVAIL_KINDS" "END PROGRAM FC_AVAIL_KINDS" SOURCE_CODE) -if (NOT CMAKE_VERSION VERSION_LESS "3.14.0") - check_fortran_source_runs (${SOURCE_CODE} FC_AVAIL_KINDS_RESULT SRC_EXT f90) -else () FORTRAN_RUN ("REAL and INTEGER KINDs" "${SOURCE_CODE}" XX YY FC_AVAIL_KINDS_RESULT + PROG_OUTPUT ) -endif () - # dnl The output from the above program will be: # dnl -- LINE 1 -- valid integer kinds (comma seperated list) # dnl -- LINE 2 -- valid real kinds (comma seperated list) # dnl -- LINE 3 -- max decimal precision for reals # dnl -- LINE 4 -- number of valid integer kinds # dnl -- LINE 5 -- number of valid real kinds - -file (READ "${RUN_OUTPUT_PATH_DEFAULT}/pac_fconftest.out" PROG_OUTPUT) +# # Convert the string to a list of strings by replacing the carriage return with a semicolon -string (REGEX REPLACE "\n" ";" PROG_OUTPUT "${PROG_OUTPUT}") +string (REGEX REPLACE "[\r\n]+" ";" PROG_OUTPUT "${PROG_OUTPUT}") list (GET PROG_OUTPUT 0 pac_validIntKinds) list (GET PROG_OUTPUT 1 pac_validRealKinds) @@ -196,21 +187,15 @@ foreach (KIND ${VAR}) " PROGRAM main USE ISO_C_BINDING + USE, INTRINSIC :: ISO_FORTRAN_ENV, ONLY : stderr=>ERROR_UNIT IMPLICIT NONE INTEGER (KIND=${KIND}) a - OPEN(8,FILE='pac_validIntKinds.${KIND}.out',FORM='formatted') - WRITE(8,'(I0)') ${FC_SIZEOF_A} - CLOSE(8) + WRITE(stderr,'(I0)') ${FC_SIZEOF_A} END " ) - if (NOT CMAKE_VERSION VERSION_LESS "3.14.0") - check_fortran_source_runs (${PROG_SRC_${KIND}} VALIDINTKINDS_RESULT_${KIND} SRC_EXT f90) - else () - FORTRAN_RUN("INTEGER KIND SIZEOF" ${PROG_SRC_${KIND}} XX YY VALIDINTKINDS_RESULT_${KIND}) - endif () - file (READ "${RUN_OUTPUT_PATH_DEFAULT}/pac_validIntKinds.${KIND}.out" PROG_OUTPUT1) - string (REGEX REPLACE "\n" "" PROG_OUTPUT1 "${PROG_OUTPUT1}") + FORTRAN_RUN("INTEGER KIND SIZEOF" ${PROG_SRC_${KIND}} XX YY VALIDINTKINDS_RESULT_${KIND} PROG_OUTPUT1) + string (REGEX REPLACE "[\r\n]+" "" PROG_OUTPUT1 "${PROG_OUTPUT1}") set (pack_int_sizeof "${pack_int_sizeof} ${PROG_OUTPUT1},") endforeach () @@ -245,22 +230,16 @@ foreach (KIND ${VAR} ) " PROGRAM main USE ISO_C_BINDING + USE, INTRINSIC :: ISO_FORTRAN_ENV, ONLY : stderr=>ERROR_UNIT IMPLICIT NONE REAL (KIND=${KIND}) a - OPEN(8,FILE='pac_validRealKinds.${KIND}.out',FORM='formatted') - WRITE(8,'(I0)') ${FC_SIZEOF_A} - CLOSE(8) + WRITE(stderr,'(I0)') ${FC_SIZEOF_A} END " ) - if (NOT CMAKE_VERSION VERSION_LESS "3.14.0") - check_fortran_source_runs (${PROG_SRC2_${KIND}} VALIDREALKINDS_RESULT_${KIND} SRC_EXT f90) - else () - FORTRAN_RUN ("REAL KIND SIZEOF" ${PROG_SRC2_${KIND}} XX YY VALIDREALKINDS_RESULT_${KIND}) - endif () - file (READ "${RUN_OUTPUT_PATH_DEFAULT}/pac_validRealKinds.${KIND}.out" PROG_OUTPUT1) - string (REGEX REPLACE "\n" "" PROG_OUTPUT1 "${PROG_OUTPUT1}") - set (pack_real_sizeof "${pack_real_sizeof} ${PROG_OUTPUT1},") + FORTRAN_RUN ("REAL KIND SIZEOF" ${PROG_SRC2_${KIND}} XX YY VALIDREALKINDS_RESULT_${KIND} PROG_OUTPUT2) + string (REGEX REPLACE "[\r\n]+" "" PROG_OUTPUT2 "${PROG_OUTPUT2}") + set (pack_real_sizeof "${pack_real_sizeof} ${PROG_OUTPUT2},") endforeach () if (pack_real_sizeof STREQUAL "") @@ -293,27 +272,21 @@ set (PROG_SRC3 " PROGRAM main USE ISO_C_BINDING + USE, INTRINSIC :: ISO_FORTRAN_ENV, ONLY : stderr=>ERROR_UNIT IMPLICIT NONE INTEGER a REAL b DOUBLE PRECISION c - OPEN(8,FILE='pac_sizeof_native_kinds.out',FORM='formatted') - WRITE(8,*) ${FC_SIZEOF_A} - WRITE(8,*) kind(a) - WRITE(8,*) ${FC_SIZEOF_B} - WRITE(8,*) kind(b) - WRITE(8,*) ${FC_SIZEOF_C} - WRITE(8,*) kind(c) - CLOSE(8) + WRITE(stderr,*) ${FC_SIZEOF_A} + WRITE(stderr,*) kind(a) + WRITE(stderr,*) ${FC_SIZEOF_B} + WRITE(stderr,*) kind(b) + WRITE(stderr,*) ${FC_SIZEOF_C} + WRITE(stderr,*) kind(c) END " ) -if (NOT CMAKE_VERSION VERSION_LESS "3.14.0") - check_fortran_source_runs (${PROG_SRC3} PAC_SIZEOF_NATIVE_KINDS_RESULT SRC_EXT f90) -else () - FORTRAN_RUN ("SIZEOF NATIVE KINDs" ${PROG_SRC3} XX YY PAC_SIZEOF_NATIVE_KINDS_RESULT) -endif () -file (READ "${RUN_OUTPUT_PATH_DEFAULT}/pac_sizeof_native_kinds.out" PROG_OUTPUT3) +FORTRAN_RUN ("SIZEOF NATIVE KINDs" ${PROG_SRC3} XX YY PAC_SIZEOF_NATIVE_KINDS_RESULT PROG_OUTPUT3) # dnl The output from the above program will be: # dnl -- LINE 1 -- sizeof INTEGER # dnl -- LINE 2 -- kind of INTEGER @@ -321,9 +294,9 @@ file (READ "${RUN_OUTPUT_PATH_DEFAULT}/pac_sizeof_native_kinds.out" PROG_OUTPUT3 # dnl -- LINE 4 -- kind of REAL # dnl -- LINE 5 -- sizeof DOUBLE PRECISION # dnl -- LINE 6 -- kind of DOUBLE PRECISION - +# # Convert the string to a list of strings by replacing the carriage return with a semicolon -string (REGEX REPLACE "\n" ";" PROG_OUTPUT3 "${PROG_OUTPUT3}") +string (REGEX REPLACE "[\r\n]+" ";" PROG_OUTPUT3 "${PROG_OUTPUT3}") list (GET PROG_OUTPUT3 0 PAC_FORTRAN_NATIVE_INTEGER_SIZEOF) list (GET PROG_OUTPUT3 1 PAC_FORTRAN_NATIVE_INTEGER_KIND) diff --git a/config/cmake/HDF5_Examples.cmake.in b/config/cmake/HDF5_Examples.cmake.in index 2d8dc58..db638fd 100644 --- a/config/cmake/HDF5_Examples.cmake.in +++ b/config/cmake/HDF5_Examples.cmake.in @@ -83,7 +83,7 @@ set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSITE:STRING=${CTEST_SITE} -DBUILDN ############################################################################################################### if(WIN32) set(SITE_OS_NAME "Windows") - set(ENV{HDF5_DIR} "${INSTALLDIR}/cmake") + set(ENV{HDF5_DIR} "${INSTALLDIR}/share/cmake") set(CTEST_BINARY_NAME ${CTEST_SOURCE_NAME}\\build) set(CTEST_SOURCE_DIRECTORY "${CTEST_DASHBOARD_ROOT}\\${CTEST_SOURCE_NAME}") set(CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}\\${CTEST_BINARY_NAME}") @@ -94,6 +94,8 @@ else() set(CTEST_SOURCE_DIRECTORY "${CTEST_DASHBOARD_ROOT}/${CTEST_SOURCE_NAME}") set(CTEST_BINARY_DIRECTORY "${CTEST_DASHBOARD_ROOT}/${CTEST_BINARY_NAME}") endif() +### default HDF5_PLUGIN_PATH to where the filter libraries are located +set(ENV{HDF5_PLUGIN_PATH} "${INSTALLDIR}/lib/plugin") if(${CDASH_LOCAL}) set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCDASH_LOCAL:BOOL=ON") endif() diff --git a/config/cmake/HDFCXXCompilerFlags.cmake b/config/cmake/HDFCXXCompilerFlags.cmake index a121d0a..a5af43c 100644 --- a/config/cmake/HDFCXXCompilerFlags.cmake +++ b/config/cmake/HDFCXXCompilerFlags.cmake @@ -27,15 +27,14 @@ if (WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Intel") set (_INTEL_WINDOWS 1) endif () -if (WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" - AND "x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") +if (WIN32 AND CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang" AND "x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") set (_CLANG_MSVC_WINDOWS 1) endif() # MSVC 14.28 enables C5105, but the Windows SDK 10.0.18362.0 triggers it. -if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" AND CMAKE_CXX_COMPILER_LOADED) +if ((_CLANG_MSVC_WINDOWS OR CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") AND CMAKE_CXX_COMPILER_LOADED) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.28) + if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.28) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd5105") endif () endif () @@ -134,13 +133,9 @@ else () AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) # add the general CXX flags for g++ compiler versions 4.8 and above. ADD_H5_FLAGS (HDF5_CMAKE_CXX_FLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-general") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-error-general") - else () - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-noerror-general") - endif () + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-error-general") endif () - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + elseif (CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") ADD_H5_FLAGS (HDF5_CMAKE_CXX_FLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/general") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "PGI") list (APPEND HDF5_CMAKE_CXX_FLAGS "-Minform=inform") @@ -161,14 +156,14 @@ if (HDF5_ENABLE_DEV_WARNINGS) elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # Use the C warnings as CXX warnings are the same ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-general") - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + elseif (CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/developer-general") endif () else () if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") # Use the C warnings as CXX warnings are the same ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-general") - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + elseif (CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/no-developer-general") endif () endif () @@ -186,9 +181,9 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-4.8") if (HDF5_ENABLE_DEV_WARNINGS) # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-4.8") + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-developer-4.8") else () - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-4.8") + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-cxx-developer-4.8") endif () endif () @@ -202,11 +197,7 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) # autotools always add the C flags with the CXX flags ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-5") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-error-5") - else () - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-noerror-5") - endif () + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-error-5") endif () # Append more extra warning flags that only gcc 6.x+ knows about @@ -231,11 +222,7 @@ if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0) # Use the C warnings as CXX warnings are the same ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/8") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-8") - else () - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/noerror-8") - endif () + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-8") if (HDF5_ENABLE_DEV_WARNINGS) # Use the C warnings as CXX warnings are the same ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-8") diff --git a/config/cmake/HDFCompilerFlags.cmake b/config/cmake/HDFCompilerFlags.cmake index de5b563..eb43b73 100644 --- a/config/cmake/HDFCompilerFlags.cmake +++ b/config/cmake/HDFCompilerFlags.cmake @@ -24,8 +24,7 @@ if(WIN32 AND CMAKE_C_COMPILER_ID STREQUAL "Intel") set(_INTEL_WINDOWS 1) endif() -if(WIN32 AND CMAKE_C_COMPILER_ID STREQUAL "Clang" - AND "x${CMAKE_C_SIMULATE_ID}" STREQUAL "xMSVC") +if(WIN32 AND CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" AND "x${CMAKE_C_SIMULATE_ID}" STREQUAL "xMSVC") set(_CLANG_MSVC_WINDOWS 1) endif() @@ -154,22 +153,14 @@ else () # Add general CFlags for GCC versions 4.8 and above if (CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) ADD_H5_FLAGS (HDF5_CMAKE_C_FLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/general") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-general") - else () - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/noerror-general") - endif () + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-general") endif () # gcc automatically inlines based on the optimization level # this is just a failsafe list (APPEND H5_CFLAGS "-finline-functions") elseif (CMAKE_C_COMPILER_ID MATCHES "[Cc]lang") ADD_H5_FLAGS (HDF5_CMAKE_C_FLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/general") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/error-general") - else () - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/noerror-general") - endif () + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/error-general") elseif (CMAKE_C_COMPILER_ID STREQUAL "PGI") list (APPEND HDF5_CMAKE_C_FLAGS "-Minform=inform") endif () @@ -229,11 +220,7 @@ if (CMAKE_C_COMPILER_ID STREQUAL "GNU") # Append more extra warning flags that only gcc 5.x+ knows about if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 5.0) ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/5") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-5") - else () - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-5") - endif () + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-5") endif () # Append more extra warning flags that only gcc 6.x+ knows about @@ -254,9 +241,7 @@ if (CMAKE_C_COMPILER_ID STREQUAL "GNU") # Append more extra warning flags that only gcc 8.x+ knows about if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0) ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/8") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-8") - endif () + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-8") if (HDF5_ENABLE_DEV_WARNINGS) ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-8") else () diff --git a/config/cmake/HDFFortranCompilerFlags.cmake b/config/cmake/HDFFortranCompilerFlags.cmake index 94c40aa..754259e 100644 --- a/config/cmake/HDFFortranCompilerFlags.cmake +++ b/config/cmake/HDFFortranCompilerFlags.cmake @@ -79,7 +79,6 @@ if (NOT MSVC AND NOT MINGW) endif () if (CMAKE_Fortran_COMPILER_ID STREQUAL "GNU") - # Append more extra warning flags that only gcc 4.8+ knows about if (NOT CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 4.8) ADD_H5_FLAGS (HDF5_CMAKE_Fortran_FLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/gfort-4.8") @@ -91,10 +90,11 @@ if (NOT MSVC AND NOT MINGW) #endif () # Append more extra warning flags that only gcc 5.x+ knows about - # do not include -Wuse-without-only - #if (NOT CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 5.0) - # ADD_H5_FLAGS (HDF5_CMAKE_Fortran_FLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/gfort-5") - #endif () + if (NOT CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 5.0) + if (HDF5_ENABLE_DEV_WARNINGS) + ADD_H5_FLAGS (HDF5_CMAKE_Fortran_FLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-gfort-5") + endif () + endif () # Append more extra warning flags that only gcc 6.x+ knows about if (NOT CMAKE_Fortran_COMPILER_VERSION VERSION_LESS 6.0) diff --git a/config/cmake/hdf5-config.cmake.in b/config/cmake/hdf5-config.cmake.in index 8faa2fe..eab09ba 100644 --- a/config/cmake/hdf5-config.cmake.in +++ b/config/cmake/hdf5-config.cmake.in @@ -113,12 +113,12 @@ set (${HDF5_PACKAGE_NAME}_VERSION_MINOR @HDF5_VERSION_MINOR@) #----------------------------------------------------------------------------- if (NOT TARGET "@HDF5_PACKAGE@") if (${HDF5_PACKAGE_NAME}_ENABLE_Z_LIB_SUPPORT AND ${HDF5_PACKAGE_NAME}_PACKAGE_EXTLIBS) - include (@PACKAGE_SHARE_INSTALL_DIR@/@ZLIB_PACKAGE_NAME@/@ZLIB_PACKAGE_NAME@@HDF_PACKAGE_EXT@-targets.cmake) + include (@PACKAGE_SHARE_INSTALL_DIR@/@ZLIB_PACKAGE_NAME@@HDF_PACKAGE_EXT@-targets.cmake) endif () if (${HDF5_PACKAGE_NAME}_ENABLE_SZIP_SUPPORT AND ${HDF5_PACKAGE_NAME}_PACKAGE_EXTLIBS) - include (@PACKAGE_SHARE_INSTALL_DIR@/@SZ_PACKAGE_NAME@/@SZ_PACKAGE_NAME@@HDF_PACKAGE_EXT@-targets.cmake) + include (@PACKAGE_SHARE_INSTALL_DIR@/@SZ_PACKAGE_NAME@@HDF_PACKAGE_EXT@-targets.cmake) endif () - include (@PACKAGE_SHARE_INSTALL_DIR@/@HDF5_PACKAGE@/@HDF5_PACKAGE@@HDF_PACKAGE_EXT@-targets.cmake) + include (@PACKAGE_SHARE_INSTALL_DIR@/@HDF5_PACKAGE@@HDF_PACKAGE_EXT@-targets.cmake) endif () # Handle default component(static) : diff --git a/config/cmake/jrunTest.cmake b/config/cmake/jrunTest.cmake index 2ca1b75..13e50dd 100644 --- a/config/cmake/jrunTest.cmake +++ b/config/cmake/jrunTest.cmake @@ -47,7 +47,12 @@ else () set (LOG_LEVEL "${TEST_LOG_LEVEL}") endif () -message (STATUS "COMMAND: ${TEST_TESTER} -Xmx1024M -Dorg.slf4j.simpleLogger.defaultLog=${LOG_LEVEL} -Djava.library.path=\"${TEST_LIBRARY_DIRECTORY}\" -cp \"${TEST_CLASSPATH}\" ${TEST_ARGS} ${TEST_PROGRAM} ${ARGN}") +if (NOT TEST_VOL) + message (STATUS "COMMAND: ${TEST_TESTER} -Xmx1024M -Dorg.slf4j.simpleLogger.defaultLog=${LOG_LEVEL} -Djava.library.path=\"${TEST_LIBRARY_DIRECTORY}\" -cp \"${TEST_CLASSPATH}\" ${TEST_ARGS} ${TEST_PROGRAM} ${ARGN}") +else () + message (STATUS "USING ${TEST_VOL} ON COMMAND: ${TEST_TESTER} -Xmx1024M -Dorg.slf4j.simpleLogger.defaultLog=${LOG_LEVEL} -Djava.library.path=\"${TEST_LIBRARY_DIRECTORY}\" -cp \"${TEST_CLASSPATH}\" ${TEST_ARGS} ${TEST_PROGRAM} ${ARGN}") + set (ENV{HDF5_VOL_CONNECTOR} "${TEST_VOL}") +endif () if (WIN32) set (ENV{PATH} "$ENV{PATH}\\;${TEST_LIBRARY_DIRECTORY}") diff --git a/config/cmake/jvolTest.cmake b/config/cmake/jvolTest.cmake deleted file mode 100644 index b12c358..0000000 --- a/config/cmake/jvolTest.cmake +++ /dev/null @@ -1,301 +0,0 @@ -# -# Copyright by The HDF Group. -# All rights reserved. -# -# This file is part of HDF5. The full HDF5 copyright notice, including -# terms governing use, modification, and redistribution, is contained in -# the COPYING file, which can be found at the root of the source code -# distribution tree, or in https://www.hdfgroup.org/licenses. -# If you do not have access to either file, you may request a copy from -# help@hdfgroup.org. -# -# jrunTest.cmake executes a command and captures the output in a file. File is then compared -# against a reference file. Exit status of command can also be compared. -cmake_policy(SET CMP0007 NEW) - -# arguments checking -if (NOT TEST_TESTER) - message (FATAL_ERROR "Require TEST_TESTER to be defined") -endif () -if (NOT TEST_PROGRAM) - message (FATAL_ERROR "Require TEST_PROGRAM to be defined") -endif () -if (NOT TEST_LIBRARY_DIRECTORY) - message (STATUS "Require TEST_LIBRARY_DIRECTORY to be defined") -endif () -if (NOT TEST_FOLDER) - message (FATAL_ERROR "Require TEST_FOLDER to be defined") -endif () -if (NOT TEST_OUTPUT) - message (FATAL_ERROR "Require TEST_OUTPUT to be defined") -endif () -if (NOT TEST_CLASSPATH) - message (STATUS "Require TEST_CLASSPATH to be defined") -endif () -if (NOT TEST_VOL) - message (FATAL_ERROR "Require TEST_VOL to be defined") -endif () - -if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}") - file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT}) -endif () - -if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}.err") - file (REMOVE ${TEST_FOLDER}/${TEST_OUTPUT}.err) -endif () - -if (NOT TEST_LOG_LEVEL) - set (LOG_LEVEL "info") -else () - set (LOG_LEVEL "${TEST_LOG_LEVEL}") -endif () - -message (STATUS "USING ${TEST_VOL} ON COMMAND: ${TEST_TESTER} -Xmx1024M -Dorg.slf4j.simpleLogger.defaultLog=${LOG_LEVEL} -Djava.library.path=\"${TEST_LIBRARY_DIRECTORY}\" -cp \"${TEST_CLASSPATH}\" ${TEST_ARGS} ${TEST_PROGRAM} ${ARGN}") - -set (ENV{HDF5_VOL_CONNECTOR} "${TEST_VOL}") - -if (WIN32) - set (ENV{PATH} "$ENV{PATH}\\;${TEST_LIBRARY_DIRECTORY}") -else () - set (ENV{LD_LIBRARY_PATH} "$ENV{LD_LIBRARY_PATH}:${TEST_LIBRARY_DIRECTORY}") -endif () - -# run the test program, capture the stdout/stderr and the result var -execute_process ( - COMMAND ${TEST_TESTER} -Xmx1024M - -Dorg.slf4j.simpleLogger.defaultLogLevel=${LOG_LEVEL} - -Djava.library.path=${TEST_LIBRARY_DIRECTORY} - -cp "${TEST_CLASSPATH}" ${TEST_ARGS} ${TEST_PROGRAM} - ${ARGN} - WORKING_DIRECTORY ${TEST_FOLDER} - RESULT_VARIABLE TEST_RESULT - OUTPUT_FILE ${TEST_OUTPUT} - ERROR_FILE ${TEST_OUTPUT}.err - OUTPUT_VARIABLE TEST_OUT - ERROR_VARIABLE TEST_ERROR -) - -message (STATUS "COMMAND Result: ${TEST_RESULT}") - -# if the .err file exists and ERRROR_APPEND is enabled -if (EXISTS "${TEST_FOLDER}/${TEST_OUTPUT}.err") - file (READ ${TEST_FOLDER}/${TEST_OUTPUT}.err TEST_STREAM) - list(LENGTH TEST_STREAM test_len) - if (test_len GREATER 0) - if (TEST_MASK_FILE) - STRING(REGEX REPLACE "CurrentDir is [^\n]+\n" "CurrentDir is (dir name)\n" TEST_STREAM "${TEST_STREAM}") - endif () - - if (NOT ERROR_APPEND) - # write back to original .err file - file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT}.err "${TEST_STREAM}") - else () - # append error output to the stdout output file - file (APPEND ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}") - endif () - endif () -endif () - -# if the output file or the .err file needs to mask out error stack info -if (TEST_MASK_ERROR) - if (NOT TEST_ERRREF) - # the error stack has been appended to the output file - file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) - else () - # the error stack remains in the .err file - file (READ ${TEST_FOLDER}/${TEST_OUTPUT}.err TEST_STREAM) - endif () - string (REGEX REPLACE "Time:[^\n]+\n" "Time: XXXX\n" TEST_STREAM "${TEST_STREAM}") - string (REGEX REPLACE "thread [0-9]*:" "thread (IDs):" TEST_STREAM "${TEST_STREAM}") - string (REGEX REPLACE ": ([^\n]*)[.]c " ": (file name) " TEST_STREAM "${TEST_STREAM}") - string (REGEX REPLACE " line [0-9]*" " line (number)" TEST_STREAM "${TEST_STREAM}") - #string (REGEX REPLACE "v[1-9]*[.][0-9]*[.]" "version (number)." TEST_STREAM "${TEST_STREAM}") - string (REGEX REPLACE "HDF5 .[1-9]*[.][0-9]*[.][0-9]*[^)]*" "HDF5 (version (number)" TEST_STREAM "${TEST_STREAM}") - string (REGEX REPLACE "H5Eget_auto[1-2]*" "H5Eget_auto(1 or 2)" TEST_STREAM "${TEST_STREAM}") - string (REGEX REPLACE "H5Eset_auto[1-2]*" "H5Eset_auto(1 or 2)" TEST_STREAM "${TEST_STREAM}") - # write back the changes to the original files - if (NOT TEST_ERRREF) - file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT} "${TEST_STREAM}") - else () - file (WRITE ${TEST_FOLDER}/${TEST_OUTPUT}.err "${TEST_STREAM}") - endif () -endif () - -# if the return value is !=expected bail out -if (NOT TEST_RESULT EQUAL TEST_EXPECT) - message (STATUS "ERROR OUTPUT: ${TEST_STREAM}") - message (FATAL_ERROR "Failed: Test program ${TEST_PROGRAM} exited != 0.\n${TEST_ERROR}") -endif () - -message (STATUS "COMMAND Error: ${TEST_ERROR}") - -# compare output files to references unless this must be skipped -set (TEST_COMPARE_RESULT 0) -if (NOT TEST_SKIP_COMPARE) - if (EXISTS "${TEST_FOLDER}/${TEST_REFERENCE}") - file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM) - list(LENGTH TEST_STREAM test_len) - if (test_len GREATER 0) - if (WIN32) - configure_file(${TEST_FOLDER}/${TEST_REFERENCE} ${TEST_FOLDER}/${TEST_REFERENCE}.tmp NEWLINE_STYLE CRLF) - if (EXISTS "${TEST_FOLDER}/${TEST_REFERENCE}.tmp") - file(RENAME ${TEST_FOLDER}/${TEST_REFERENCE}.tmp ${TEST_FOLDER}/${TEST_REFERENCE}) - endif () - #file (READ ${TEST_FOLDER}/${TEST_REFERENCE} TEST_STREAM) - #file (WRITE ${TEST_FOLDER}/${TEST_REFERENCE} "${TEST_STREAM}") - endif () - - if (NOT TEST_SORT_COMPARE) - # now compare the output with the reference - execute_process ( - COMMAND ${CMAKE_COMMAND} -E compare_files ${TEST_FOLDER}/${TEST_OUTPUT} ${TEST_FOLDER}/${TEST_REFERENCE} - RESULT_VARIABLE TEST_COMPARE_RESULT - ) - else () - file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} v1) - file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} v2) - list (SORT v1) - list (SORT v2) - if (NOT v1 STREQUAL v2) - set(TEST_COMPARE_RESULT 1) - endif () - endif () - - if (TEST_COMPARE_RESULT) - set (TEST_COMPARE_RESULT 0) - file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT} test_act) - list (LENGTH test_act len_act) - file (STRINGS ${TEST_FOLDER}/${TEST_REFERENCE} test_ref) - list (LENGTH test_ref len_ref) - if (len_act GREATER 0 AND len_ref GREATER 0) - math (EXPR _FP_LEN "${len_ref} - 1") - foreach (line RANGE 0 ${_FP_LEN}) - list (GET test_act ${line} str_act) - list (GET test_ref ${line} str_ref) - if (NOT str_act STREQUAL str_ref) - if (str_act) - set (TEST_COMPARE_RESULT 1) - message (STATUS "line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n") - endif () - endif () - endforeach () - else () - if (len_act EQUAL 0) - message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_OUTPUT} is empty") - endif () - if (len_ref EQUAL 0) - message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_REFERENCE} is empty") - endif () - endif () - if (NOT len_act EQUAL len_ref) - set (TEST_COMPARE_RESULT 1) - endif () - endif () - endif () - - message (STATUS "COMPARE Result: ${TEST_COMPARE_RESULT}") - - # again, if return value is !=0 scream and shout - if (TEST_COMPARE_RESULT) - message (FATAL_ERROR "Failed: The output of ${TEST_OUTPUT} did not match ${TEST_REFERENCE}") - endif () - endif () - - # now compare the .err file with the error reference, if supplied - set (TEST_ERRREF_RESULT 0) - if (TEST_ERRREF) - file (READ ${TEST_FOLDER}/${TEST_ERRREF} TEST_STREAM) - list(LENGTH TEST_STREAM test_len) - if (test_len GREATER 0) - if (WIN32) - configure_file(${TEST_FOLDER}/${TEST_ERRREF} ${TEST_FOLDER}/${TEST_ERRREF}.tmp NEWLINE_STYLE CRLF) - if (EXISTS "${TEST_FOLDER}/${TEST_ERRREF}.tmp") - file(RENAME ${TEST_FOLDER}/${TEST_ERRREF}.tmp ${TEST_FOLDER}/${TEST_ERRREF}) - endif () - #file (READ ${TEST_FOLDER}/${TEST_ERRREF} TEST_STREAM) - #file (WRITE ${TEST_FOLDER}/${TEST_ERRREF} "${TEST_STREAM}") - endif () - - # now compare the error output with the error reference - execute_process ( - COMMAND ${CMAKE_COMMAND} -E compare_files ${TEST_FOLDER}/${TEST_OUTPUT}.err ${TEST_FOLDER}/${TEST_ERRREF} - RESULT_VARIABLE TEST_ERRREF_RESULT - ) - if (TEST_ERRREF_RESULT) - set (TEST_ERRREF_RESULT 0) - file (STRINGS ${TEST_FOLDER}/${TEST_OUTPUT}.err test_act) - list (LENGTH test_act len_act) - file (STRINGS ${TEST_FOLDER}/${TEST_ERRREF} test_ref) - list (LENGTH test_ref len_ref) - math (EXPR _FP_LEN "${len_ref} - 1") - if (len_act GREATER 0 AND len_ref GREATER 0) - math (EXPR _FP_LEN "${len_ref} - 1") - foreach (line RANGE 0 ${_FP_LEN}) - list (GET test_act ${line} str_act) - list (GET test_ref ${line} str_ref) - if (NOT str_act STREQUAL str_ref) - if (str_act) - set (TEST_ERRREF_RESULT 1) - message (STATUS "line = ${line}\n***ACTUAL: ${str_act}\n****REFER: ${str_ref}\n") - endif () - endif () - endforeach () - else () - if (len_act EQUAL 0) - message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_OUTPUT}.err is empty") - endif () - if (len_ref EQUAL 0) - message (STATUS "COMPARE Failed: ${TEST_FOLDER}/${TEST_ERRREF} is empty") - endif () - endif () - if (NOT len_act EQUAL len_ref) - set (TEST_ERRREF_RESULT 1) - endif () - endif () - endif () - - message (STATUS "COMPARE Result: ${TEST_ERRREF_RESULT}") - - # again, if return value is !=0 scream and shout - if (TEST_ERRREF_RESULT) - message (FATAL_ERROR "Failed: The error output of ${TEST_OUTPUT}.err did not match ${TEST_ERRREF}") - endif () - endif () -endif () - -set (TEST_GREP_RESULT 0) -if (TEST_GREP_COMPARE) - # now grep the output with the reference - file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) - list(LENGTH TEST_STREAM test_len) - if (test_len GREATER 0) - # TEST_REFERENCE should always be matched - string (REGEX MATCH "${TEST_REFERENCE}" TEST_MATCH ${TEST_STREAM}) - string (COMPARE EQUAL "${TEST_REFERENCE}" "${TEST_MATCH}" TEST_GREP_RESULT) - if (NOT TEST_GREP_RESULT) - message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did not contain ${TEST_REFERENCE}") - endif () - - string (REGEX MATCH "${TEST_FILTER}" TEST_MATCH ${TEST_STREAM}) - if (TEST_EXPECT) - # TEST_EXPECT (1) interprets TEST_FILTER as; NOT to match - string (LENGTH "${TEST_MATCH}" TEST_GREP_RESULT) - if (TEST_GREP_RESULT) - message (FATAL_ERROR "Failed: The output of ${TEST_PROGRAM} did contain ${TEST_FILTER}") - endif () - endif () - endif () -endif () - -# dump the output unless nodisplay option is set -if (TEST_SKIP_COMPARE AND NOT TEST_NO_DISPLAY) - file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) - execute_process ( - COMMAND ${CMAKE_COMMAND} -E echo ${TEST_STREAM} - RESULT_VARIABLE TEST_RESULT - ) -endif () - -# everything went fine... -message (STATUS "${TEST_PROGRAM} program used vol ${TEST_VOL} Passed") - diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in index eb83c3a..d80b0f8 100644 --- a/config/cmake/libhdf5.settings.cmake.in +++ b/config/cmake/libhdf5.settings.cmake.in @@ -64,28 +64,29 @@ Languages: Features: --------- - Parallel HDF5: @HDF5_ENABLE_PARALLEL@ -Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ - Large Parallel I/O: @LARGE_PARALLEL_IO@ - High-level library: @HDF5_BUILD_HL_LIB@ - Build HDF5 Tests: @BUILD_TESTING@ - Build HDF5 Tools: @HDF5_BUILD_TOOLS@ - Build High-level HDF5 Tools: @HDF5_BUILD_HL_TOOLS@ - Threadsafety: @HDF5_ENABLE_THREADSAFE@ - Default API mapping: @DEFAULT_API_VERSION@ - With deprecated public symbols: @HDF5_ENABLE_DEPRECATED_SYMBOLS@ - I/O filters (external): @EXTERNAL_FILTERS@ - MPE: @H5_HAVE_LIBLMPE@ - Direct VFD: @H5_HAVE_DIRECT@ - Mirror VFD: @H5_HAVE_MIRROR_VFD@ - (Read-Only) S3 VFD: @H5_HAVE_ROS3_VFD@ - (Read-Only) HDFS VFD: @H5_HAVE_LIBHDFS@ - dmalloc: @H5_HAVE_LIBDMALLOC@ - Packages w/ extra debug output: @INTERNAL_DEBUG_OUTPUT@ - API Tracing: @HDF5_ENABLE_TRACE@ - Using memory checker: @HDF5_ENABLE_USING_MEMCHECKER@ - Memory allocation sanity checks: @HDF5_MEMORY_ALLOC_SANITY_CHECK@ - Function Stack Tracing: @HDF5_ENABLE_CODESTACK@ - Use file locking: @HDF5_FILE_LOCKING_SETTING@ - Strict File Format Checks: @HDF5_STRICT_FORMAT_CHECKS@ - Optimization Instrumentation: @HDF5_Enable_Instrument@ + Parallel HDF5: @HDF5_ENABLE_PARALLEL@ + Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ + Large Parallel I/O: @LARGE_PARALLEL_IO@ + High-level library: @HDF5_BUILD_HL_LIB@ +Dimension scales w/ new references: @DIMENSION_SCALES_WITH_NEW_REF@ + Build HDF5 Tests: @BUILD_TESTING@ + Build HDF5 Tools: @HDF5_BUILD_TOOLS@ + Build High-level HDF5 Tools: @HDF5_BUILD_HL_TOOLS@ + Threadsafety: @HDF5_ENABLE_THREADSAFE@ + Default API mapping: @DEFAULT_API_VERSION@ + With deprecated public symbols: @HDF5_ENABLE_DEPRECATED_SYMBOLS@ + I/O filters (external): @EXTERNAL_FILTERS@ + MPE: @H5_HAVE_LIBLMPE@ + Direct VFD: @H5_HAVE_DIRECT@ + Mirror VFD: @H5_HAVE_MIRROR_VFD@ + (Read-Only) S3 VFD: @H5_HAVE_ROS3_VFD@ + (Read-Only) HDFS VFD: @H5_HAVE_LIBHDFS@ + dmalloc: @H5_HAVE_LIBDMALLOC@ + Packages w/ extra debug output: @INTERNAL_DEBUG_OUTPUT@ + API Tracing: @HDF5_ENABLE_TRACE@ + Using memory checker: @HDF5_ENABLE_USING_MEMCHECKER@ + Memory allocation sanity checks: @HDF5_MEMORY_ALLOC_SANITY_CHECK@ + Function Stack Tracing: @HDF5_ENABLE_CODESTACK@ + Use file locking: @HDF5_FILE_LOCKING_SETTING@ + Strict File Format Checks: @HDF5_STRICT_FORMAT_CHECKS@ + Optimization Instrumentation: @HDF5_Enable_Instrument@ diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake index 8df7dd4..2f5af77 100644 --- a/config/cmake/scripts/HDF5config.cmake +++ b/config/cmake/scripts/HDF5config.cmake @@ -37,8 +37,8 @@ cmake_minimum_required (VERSION 3.12) # CTEST_SOURCE_NAME - source folder ############################################################################## -set (CTEST_SOURCE_VERSION "1.13.0") -set (CTEST_SOURCE_VERSEXT "") +set (CTEST_SOURCE_VERSION "1.13.1") +set (CTEST_SOURCE_VERSEXT "-1") ############################################################################## # handle input parameters to script. diff --git a/config/cmake_ext_mod/HDFLibMacros.cmake b/config/cmake_ext_mod/HDFLibMacros.cmake index a60d09f..2af3229 100644 --- a/config/cmake_ext_mod/HDFLibMacros.cmake +++ b/config/cmake_ext_mod/HDFLibMacros.cmake @@ -158,7 +158,11 @@ macro (EXTERNAL_SZIP_LIBRARY compress_type encoding) set (SZIP_LIBRARIES ${SZIP_STATIC_LIBRARY}) set (SZIP_INCLUDE_DIR_GEN "${BINARY_DIR}") - set (SZIP_INCLUDE_DIR "${SOURCE_DIR}/src") + if (USE_LIBAEC) + set (SZIP_INCLUDE_DIR "${SOURCE_DIR}/include") + else () + set (SZIP_INCLUDE_DIR "${SOURCE_DIR}/src") + endif () set (SZIP_FOUND 1) set (SZIP_INCLUDE_DIRS ${SZIP_INCLUDE_DIR_GEN} ${SZIP_INCLUDE_DIR}) endmacro () @@ -167,6 +171,8 @@ endmacro () macro (PACKAGE_SZIP_LIBRARY compress_type) set (SZIP_HDR "SZconfig") if (USE_LIBAEC) + set (SZIP_HDR "aec_config") + else () set (SZIP_HDR "libaec_Export") endif () add_custom_target (SZIP-GenHeader-Copy ALL diff --git a/config/cmake_ext_mod/HDFMacros.cmake b/config/cmake_ext_mod/HDFMacros.cmake index ae0ebca..6da2b74 100644 --- a/config/cmake_ext_mod/HDFMacros.cmake +++ b/config/cmake_ext_mod/HDFMacros.cmake @@ -464,6 +464,9 @@ macro (ADD_H5_FLAGS h5_flag_var infile) list (GET TEST_FLAG_STREAM ${line} str_flag) string (REGEX REPLACE "^#.*" "" str_flag "${str_flag}") #message (TRACE "str_flag=${str_flag}") + if (NOT HDF5_ENABLE_WARNINGS_AS_ERRORS) + string (REGEX REPLACE "-Werror=" "-W" str_flag "${str_flag}") + endif () if (str_flag) list (APPEND ${h5_flag_var} "${str_flag}") endif () diff --git a/config/gnu-cxxflags b/config/gnu-cxxflags index 9f11406..56e89bc 100644 --- a/config/gnu-cxxflags +++ b/config/gnu-cxxflags @@ -20,15 +20,15 @@ # # For now, do not promote any warnings to errors. # -PROMOTE_ERRORS_DFLT=no +WARNINGS_AS_ERRORS_DFLT=no # # This filter rewrites -Werror= as -W, in that way demoting warnings -# promoted to errors back to warnings, if PROMOTE_ERRORS is no. +# promoted to errors back to warnings, if WARNINGS_AS_ERRORS is no. # demote_errors() { - if [ ${PROMOTE_ERRORS:-${PROMOTE_ERRORS_DFLT}} = no ]; then + if [ ${WARNINGS_AS_ERRORS:-${WARNINGS_AS_ERRORS_DFLT}} = no ]; then sed 's,-Werror=,-W,g' else cat @@ -44,7 +44,7 @@ demote_errors() load_gnu_arguments() { set -- $(for arg; do - sed 's,#.*$,,' $srcdir/config/gnu-warnings/${arg} + sed 's,#.*$,,' $srcdir/config/gnu-warnings/${arg} | demote_errors done) IFS=' ' echo "$*" } @@ -192,7 +192,6 @@ if test "X-g++" = "X-$cxx_vendor"; then if test $cxx_vers_major -eq 4 -a $cxx_vers_minor -ge 8 -o $cxx_vers_major -ge 5; then H5_CXXFLAGS="$H5_CXXFLAGS $(load_gnu_arguments cxx-general)" H5_ECXXFLAGS="$H5_ECXXFLAGS $(load_gnu_arguments cxx-error-general)" - H5_NECXXFLAGS="$H5_NECXXFLAGS $(load_gnu_arguments cxx-noerror-general)" ###################### # Developer warnings # @@ -222,8 +221,8 @@ if test "X-g++" = "X-$cxx_vendor"; then if test $cxx_vers_major -ge 5 -o $cxx_vers_major -eq 4 -a $cxx_vers_minor -ge 8; then H5_CXXFLAGS="$H5_CXXFLAGS $(load_gnu_arguments cxx-4.8)" # Use the C warnings as CXX warnings are the same - DEVELOPER_WARNING_CXXFLAGS="$DEVELOPER_WARNING_CXXFLAGS $(load_gnu_arguments developer-4.8)" - NO_DEVELOPER_WARNING_CXXFLAGS="$NO_DEVELOPER_WARNING_CXXFLAGS $(load_gnu_arguments no-developer-4.8)" + DEVELOPER_WARNING_CXXFLAGS="$DEVELOPER_WARNING_CXXFLAGS $(load_gnu_arguments cxx-developer-4.8)" + NO_DEVELOPER_WARNING_CXXFLAGS="$NO_DEVELOPER_WARNING_CXXFLAGS $(load_gnu_arguments no-cxx-developer-4.8)" fi # gcc >= 4.9 @@ -235,7 +234,6 @@ if test "X-g++" = "X-$cxx_vendor"; then if test $cxx_vers_major -ge 5; then H5_CXXFLAGS="$H5_CXXFLAGS $(load_gnu_arguments cxx-5)" H5_ECXXFLAGS="$H5_ECXXFLAGS $(load_gnu_arguments cxx-error-5)" - H5_NECXXFLAGS="$H5_NECXXFLAGS $(load_gnu_arguments cxx-noerror-5)" fi # gcc >= 6 @@ -256,7 +254,6 @@ if test "X-g++" = "X-$cxx_vendor"; then # Use the C warnings as CXX warnings are the same H5_CXXFLAGS="$H5_CXXFLAGS $(load_gnu_arguments 8)" #H5_ECXXFLAGS="$H5_ECXXFLAGS $(load_gnu_arguments error-8)" - #H5_NECXXFLAGS="$H5_NECXXFLAGS $(load_gnu_arguments noerror-8)" # Use the C warnings as CXX warnings are the same DEVELOPER_WARNING_CXXFLAGS="$DEVELOPER_WARNING_CXXFLAGS $(load_gnu_arguments developer-8)" NO_DEVELOPER_WARNING_CXXFLAGS="$NO_DEVELOPER_WARNING_CXXFLAGS $(load_gnu_arguments no-developer-8)" diff --git a/config/gnu-fflags b/config/gnu-fflags index ce12561..1333ce5 100644 --- a/config/gnu-fflags +++ b/config/gnu-fflags @@ -161,10 +161,10 @@ if test "X-gfortran" = "X-$f9x_vendor"; then # gfortran 4.9 (nothing new) - # gfortran >= 5 (do not include -Wuse-without-only) - #if test $f9x_vers_major -ge 5; then - # H5_FCFLAGS="$H5_FCFLAGS $(load_gnu_arguments gfort-5)" - #fi + # gfortran >= 5 + if test $f9x_vers_major -ge 5; then + DEVELOPER_WARNING_FCFLAGS="$DEVELOPER_WARNING_FCFLAGS $(load_gnu_arguments developer-gfort-5)" + fi # gfortran >= 6 if test $f9x_vers_major -ge 6; then diff --git a/config/gnu-flags b/config/gnu-flags index 3e4ceb2..89239df 100644 --- a/config/gnu-flags +++ b/config/gnu-flags @@ -20,15 +20,15 @@ # # For now, do not promote any warnings to errors. # -PROMOTE_ERRORS_DFLT=no +WARNINGS_AS_ERRORS_DFLT=no # # This filter rewrites -Werror= as -W, in that way demoting warnings -# promoted to errors back to warnings, if PROMOTE_ERRORS is no. +# promoted to errors back to warnings, if WARNINGS_AS_ERRORS is no. # demote_errors() { - if [ ${PROMOTE_ERRORS:-${PROMOTE_ERRORS_DFLT}} = no ]; then + if [ ${WARNINGS_AS_ERRORS:-${WARNINGS_AS_ERRORS_DFLT}} = no ]; then sed 's,-Werror=,-W,g' else cat @@ -44,7 +44,7 @@ demote_errors() load_gnu_arguments() { set -- $(for arg; do - sed 's,#.*$,,' $srcdir/config/gnu-warnings/${arg} + sed 's,#.*$,,' $srcdir/config/gnu-warnings/${arg} | demote_errors done) IFS=' ' echo "$*" } @@ -205,7 +205,6 @@ if test "X-gcc" = "X-$cc_vendor"; then if test $cc_vers_major -eq 4 -a $cc_vers_minor -ge 8 -o $cc_vers_major -gt 4; then H5_CFLAGS="$H5_CFLAGS $(load_gnu_arguments general)" H5_ECFLAGS="$H5_ECFLAGS $(load_gnu_arguments error-general)" - H5_NECFLAGS="$H5_NECFLAGS $(load_gnu_arguments noerror-general)" ###################### # Developer warnings # @@ -245,7 +244,6 @@ if test "X-gcc" = "X-$cc_vendor"; then if test $cc_vers_major -ge 5; then H5_CFLAGS="$H5_CFLAGS $(load_gnu_arguments 5)" H5_ECFLAGS="$H5_ECFLAGS $(load_gnu_arguments error-5)" - H5_NECFLAGS="$H5_NECFLAGS $(load_gnu_arguments noerror-5)" fi # gcc >= 6 @@ -263,7 +261,6 @@ if test "X-gcc" = "X-$cc_vendor"; then if test $cc_vers_major -ge 8; then H5_CFLAGS="$H5_CFLAGS $(load_gnu_arguments 8)" H5_ECFLAGS="$H5_ECFLAGS $(load_gnu_arguments error-8)" - H5_NECFLAGS="$H5_NECFLAGS $(load_gnu_arguments noerror-8)" DEVELOPER_WARNING_CFLAGS="$DEVELOPER_WARNING_CFLAGS $(load_gnu_arguments developer-8)" NO_DEVELOPER_WARNING_CFLAGS="$NO_DEVELOPER_WARNING_CFLAGS $(load_gnu_arguments no-developer-8)" fi diff --git a/config/gnu-warnings/cxx-developer-4.8 b/config/gnu-warnings/cxx-developer-4.8 new file mode 100644 index 0000000..e0d975f --- /dev/null +++ b/config/gnu-warnings/cxx-developer-4.8 @@ -0,0 +1,16 @@ +# developer warning flags added for GCC >= 4.5 +# +# developer warning flag added for GCC >= 4.6 +-Wsuggest-attribute=const + +# developer warning flags added for GCC >= 4.7 +-Wsuggest-attribute=noreturn +-Wsuggest-attribute=pure +# +# It's not clear that -Wvector-operation-performance warnings are +# actionable, so they are demoted to "developer" warnings. +# +-Wvector-operation-performance + +# developer warning flag added for GCC >= 4.8 +-Wsuggest-attribute=format diff --git a/config/gnu-warnings/cxx-noerror-5 b/config/gnu-warnings/cxx-noerror-5 deleted file mode 100644 index 7bce3e8..0000000 --- a/config/gnu-warnings/cxx-noerror-5 +++ /dev/null @@ -1,11 +0,0 @@ -# -# In GCC 4.4.7, the compiler gripes about shadowed global -# declarations when a local variable uses the name of a -# function that's in a system header file. For some reason, -# later versions of GCC (e.g., 5.2.0) don't complain about -# the shadowed globals. Maybe later versions are less fussy? -# Anyway, the shadowing seems to be harmless, and GCC 4.4.7 -# is not a supported compiler, so let us promote shadowed globals -# warnings to errors only for GCC 5 and later. -# --Wshadow diff --git a/config/gnu-warnings/cxx-noerror-general b/config/gnu-warnings/cxx-noerror-general deleted file mode 100644 index 8cc1d94..0000000 --- a/config/gnu-warnings/cxx-noerror-general +++ /dev/null @@ -1,32 +0,0 @@ -# -# HDF5 code should not trigger the following warnings under any -# circumstances, so ask the compiler to treat them as errors: -# -# NOTE: c++/test files are not compatible with these warnings as errors -# c++/test/tcompound.cpp -# -Werror=cast-align --Wcast-align -# NOTE: c++/src files are not compatible with these warnings as errors -# c++/src/H5Object.cpp -# -Werror=missing-declarations --Wmissing-declarations --Wpacked --Wredundant-decls --Wswitch -# NOTE: c++/test files are not compatible with these warnings as errors -# c++/test/tattr.cpp -# -Werror=unused-but-set-variable --Wunused-but-set-variable --Wunused-function --Wunused-variable -# NOTE: c++/src files are not compatible with these warnings as errors -# c++/src/H5Object.cpp,c++/src/H5StrType.cpp,c++/src/H5PredType.cpp -# -Werror=unused-parameter --Wunused-parameter -# -# Other files not compatible -# NOTE: c++/test files are not compatible with these warnings as errors -# c++/test/titerate.cpp,c++/test/tarray.cpp -# -Werror=missing-declarations -# c++/test/titerate.cpp,c++/test/tarray.cpp,c++/test/tlinks.cpp,c++/test/ttypes.cpp,c++/test/dsets.cpp -# -Werror=unused-parameter diff --git a/config/gnu-warnings/gfort-5 b/config/gnu-warnings/developer-gfort-5 index c5d3850..c5d3850 100644 --- a/config/gnu-warnings/gfort-5 +++ b/config/gnu-warnings/developer-gfort-5 diff --git a/config/gnu-warnings/error-5 b/config/gnu-warnings/error-5 index f7e1138..282ee3f 100644 --- a/config/gnu-warnings/error-5 +++ b/config/gnu-warnings/error-5 @@ -1,4 +1,5 @@ -Werror=incompatible-pointer-types +-Werror=int-conversion # # In GCC 4.4.7, the compiler gripes about shadowed global # declarations when a local variable uses the name of a diff --git a/config/gnu-warnings/error-8 b/config/gnu-warnings/error-8 index 2f54a4d..cbb25f6 100644 --- a/config/gnu-warnings/error-8 +++ b/config/gnu-warnings/error-8 @@ -1,7 +1,4 @@ -# NOTE: src/ files are not compatible with these warnings as errors -# src/H5Dchunk.c -# -Werror=cast-function-type --Wcast-function-type +-Werror=cast-function-type # # For GCC 8, promote maybe-initialized warnings to an error. GCC 8 # reports 0 maybe-uninitialized warnings where earlier versions diff --git a/config/gnu-warnings/error-general b/config/gnu-warnings/error-general index 8405c40..a66d284 100644 --- a/config/gnu-warnings/error-general +++ b/config/gnu-warnings/error-general @@ -11,6 +11,7 @@ -Werror=packed -Werror=pointer-sign -Werror=pointer-to-int-cast +-Werror=int-to-pointer-cast -Werror=redundant-decls -Werror=strict-prototypes -Werror=switch diff --git a/config/gnu-warnings/no-cxx-developer-4.8 b/config/gnu-warnings/no-cxx-developer-4.8 new file mode 100644 index 0000000..3c3de93 --- /dev/null +++ b/config/gnu-warnings/no-cxx-developer-4.8 @@ -0,0 +1,9 @@ +# no-developer warning flag added for GCC >= 4.6 +-Wno-suggest-attribute=const + +# no-developer warning flags added for GCC >= 4.7 +-Wno-suggest-attribute=noreturn +-Wno-suggest-attribute=pure + +# no-developer warning flag added for GCC >= 4.8 +-Wno-suggest-attribute=format diff --git a/config/gnu-warnings/noerror-5 b/config/gnu-warnings/noerror-5 deleted file mode 100644 index a7d40dd..0000000 --- a/config/gnu-warnings/noerror-5 +++ /dev/null @@ -1,12 +0,0 @@ --Wincompatible-pointer-types -# -# In GCC 4.4.7, the compiler gripes about shadowed global -# declarations when a local variable uses the name of a -# function that's in a system header file. For some reason, -# later versions of GCC (e.g., 5.2.0) don't complain about -# the shadowed globals. Maybe later versions are less fussy? -# Anyway, the shadowing seems to be harmless, and GCC 4.4.7 -# is not a supported compiler, so let us promote shadowed globals -# warnings to errors only for GCC 5 and later. -# --Wshadow diff --git a/config/gnu-warnings/noerror-8 b/config/gnu-warnings/noerror-8 deleted file mode 100644 index 2f54a4d..0000000 --- a/config/gnu-warnings/noerror-8 +++ /dev/null @@ -1,25 +0,0 @@ -# NOTE: src/ files are not compatible with these warnings as errors -# src/H5Dchunk.c -# -Werror=cast-function-type --Wcast-function-type -# -# For GCC 8, promote maybe-initialized warnings to an error. GCC 8 -# reports 0 maybe-uninitialized warnings where earlier versions -# make many false reports. GCC 8 seems to analyze calls to static -# in order to detect initializations that occur there. It's possible -# that GCC 8 only performs that analysis at -O3, though. -# -# -# NOTE: File Driver files are not compatible with these warnings as errors -# H5FDlog.c, -# -Werror=maybe-uninitialized --Wmaybe-uninitialized -# NOTE: src/ files are not compatible with these warnings as errors -# src/H5Shyper.c,src/H5SL.c,src/H5Shyper.c -# -Werror=maybe-uninitialized -# NOTE: Test files are not compatible with these warnings as errors -# test/cache_common.c, -# -Werror=maybe-uninitialized -# NOTE: hl/src/ files are not compatible with these warnings as errors -# hl/src/H5DS.c, -# -Werror=maybe-uninitialized diff --git a/config/gnu-warnings/noerror-general b/config/gnu-warnings/noerror-general deleted file mode 100644 index f49d89a..0000000 --- a/config/gnu-warnings/noerror-general +++ /dev/null @@ -1,92 +0,0 @@ -# -# These warnings will be treated as errors, using the error-general file, -# when HDF5_ENABLE_WARNINGS_AS_ERRORS is set to true for CMake or -# the --enable-warnings-as-errors option is specified for configure. -# Otherwise this file will be used to treat them as warnings. -# --Wbad-function-cast --Wimplicit-function-declaration --Wmissing-declarations --Wmissing-prototypes --Wnested-externs --Wold-style-definition --Wpacked --Wpointer-sign --Wpointer-to-int-cast --Wredundant-decls --Wstrict-prototypes --Wswitch -# -#-Werror=discarded-qualifiers -# -# -# NOTE: File Driver files are not compatible with these warnings as errors -# H5FDdirect.c,H5FDmpio.c,H5FDros3.c, -# -Werror=unused-function -# --Wunused-function -# -# H5FDdrvr_module.h -# -Werror=unused-variable -# --Wunused-variable -# -# H5VLpassthru.c -# -Werror=unused-parameter -# --Wunused-parameter -# -# -# -# NOTE: Tools files are not compatible with these warnings as errors -# lib/h5tools.c -# -Werror=cast-align -# --Wcast-align -# -# lib/h5diff_array.c -# -Werror=unused-but-set-variable -# --Wunused-but-set-variable -# -# lib/h5tools_utils.c -# -Werror=unused-parameter -# -# -# NOTE: JNI files are not compatible with these warnings as errors -# jni/h5pDCPLImp.c,jni/nativeData.c,jni/h5util.c,jni/h5rImp.c -# jni/h5sImp.c,jni/h5tImp.c -# -Werror=cast-align -# jni/h5util.c -# -Werror=format(-overflow) -# --Wformat -# -# -#Examples and tests do not use the same set of extensive warning flags as libraries -# Here is a list of tests and examples that have issues with the stricter warnings as error -# -# NOTE: Test files are not compatible with these warnings as errors -# thread_id.c, -# -Werror=unused-function -# dsets.c -# -Werror=unused-parameter -# external.c,perform/sio_engine.c -# -Werror=format(-truncation) -# -# -# NOTE: Examples files are not compatible with these warnings as errors -# h5_vds-eiger.c,h5_vds-exclim.c,h5_vds.c,h5_vds-exc.c,h5_vds-percival-unlim-maxmin.c -# h5_vds-percival.c,h5_read.c,h5_rdwt.c,h5_mount.c,h5_extend.c,h5_extend_write.c -# h5_write.c,h5_vds-simpleIO.c,h5_ref2reg_deprec.c,h5_crtgrp.c,h5_select.c -# h5_vds-percival-unlim.c,h5_crtatt.c,h5_group.c,h5_attribute.c,h5_crtdat.c -# h5_reference_deprec.c -# h5_rdwt.c,h5_crtgrp.c,h5_crtatt.c,h5_crtdat.c -# -Werror=strict-prototypes -# h5_rdwt.c,h5_crtgrp.c,h5_crtatt.c,h5_crtdat.c -# -Werror=old-style-definition -# h5_vds-exclim.c,h5_vds.c,h5_vds-exc.c, -# -Werror=unused-variable -# h5_elink_unix2win.c,h5_extlink.c,h5_attribute.c -# -Werror=unused-parameter - diff --git a/config/sanitizer/code-coverage.cmake b/config/sanitizer/code-coverage.cmake index 8d765f7..c79aeac 100644 --- a/config/sanitizer/code-coverage.cmake +++ b/config/sanitizer/code-coverage.cmake @@ -99,8 +99,7 @@ if(CODE_COVERAGE AND NOT CODE_COVERAGE_ADDED) ${CMAKE_COVERAGE_OUTPUT_DIRECTORY} DEPENDS ccov-clean) - if(CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" - OR CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") + if(CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" OR CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") # Messages message(STATUS "Building with llvm Code Coverage Tools") @@ -206,8 +205,7 @@ function(target_code_coverage TARGET_NAME) if(CODE_COVERAGE) # Add code coverage instrumentation to the target's linker command - if(CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" - OR CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") + if(CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" OR CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") target_compile_options(${TARGET_NAME} PRIVATE -fprofile-instr-generate -fcoverage-mapping) set_property( @@ -229,8 +227,7 @@ function(target_code_coverage TARGET_NAME) # Add shared library to processing for 'all' targets if(target_type STREQUAL "SHARED_LIBRARY" AND target_code_coverage_ALL) - if(CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" - OR CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") + if(CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" OR CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") add_custom_target( ccov-run-${TARGET_NAME} COMMAND echo "-object=$<TARGET_FILE:${TARGET_NAME}>" >> @@ -250,8 +247,7 @@ function(target_code_coverage TARGET_NAME) # For executables add targets to run and produce output if(target_type STREQUAL "EXECUTABLE") - if(CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" - OR CMAKE_CXX_COMPILER_ID MATCHES "(Apple)?Cc]lang") + if(CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" OR CMAKE_CXX_COMPILER_ID MATCHES "(Apple)?Cc]lang") # If there are shared objects to also work with, generate the string to # add them here @@ -409,8 +405,7 @@ endfunction() # any subdirectories. To add coverage instrumentation to only specific targets, # use `target_code_coverage`. function(add_code_coverage) - if(CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" - OR CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") + if(CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" OR CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") add_compile_options(-fprofile-instr-generate -fcoverage-mapping) add_link_options(-fprofile-instr-generate -fcoverage-mapping) elseif(CMAKE_C_COMPILER_ID MATCHES "GNU" OR CMAKE_CXX_COMPILER_ID MATCHES "GNU") @@ -435,8 +430,7 @@ function(add_code_coverage_all_targets) "${multi_value_keywords}" ${ARGN}) if(CODE_COVERAGE) - if(CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" - OR CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") + if(CMAKE_C_COMPILER_ID MATCHES "[Cc]lang" OR CMAKE_CXX_COMPILER_ID MATCHES "[Cc]lang") # Merge the profile data for all of the run executables add_custom_target( diff --git a/config/sanitizer/sanitizers.cmake b/config/sanitizer/sanitizers.cmake index 58c4050..8bf1d5b 100644 --- a/config/sanitizer/sanitizers.cmake +++ b/config/sanitizer/sanitizers.cmake @@ -30,7 +30,7 @@ endfunction() message(STATUS "USE_SANITIZER=${USE_SANITIZER}, CMAKE_C_COMPILER_ID=${CMAKE_C_COMPILER_ID}") if(USE_SANITIZER) - if(INTEL_CLANG OR "${CMAKE_C_COMPILER_ID}" MATCHES "Clang") + if(INTEL_CLANG OR CMAKE_C_COMPILER_ID MATCHES "[Cc]lang") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) if(UNIX) diff --git a/configure.ac b/configure.ac index 3290de4..ab55f3e 100644 --- a/configure.ac +++ b/configure.ac @@ -23,7 +23,7 @@ AC_PREREQ([2.69]) ## NOTE: Do not forget to change the version number here when we do a ## release!!! ## -AC_INIT([HDF5], [1.13.0], [help@hdfgroup.org]) +AC_INIT([HDF5], [1.13.1-1], [help@hdfgroup.org]) AC_CONFIG_SRCDIR([src/H5.c]) AC_CONFIG_HEADERS([src/H5config.h]) @@ -237,6 +237,41 @@ elif test $hdf5_cv_host != $host; then fi ## ---------------------------------------------------------------------- +## Check if we should consider certain compiler warnings as errors +## +## We have to set WARNINGS_AS_ERRORS before sourcing a $host_config +## file, below. +## +## These should NOT be on by default as the risk of breakage is high +## when compiling HDF5 on new (or new versions) of platforms and +## compilers. It can also cause failures when header files we have no +## control over (e.g. MPI, HDFS) raise warnings. +## +AC_MSG_CHECKING([enable warnings as errors]) +AC_ARG_ENABLE([warnings-as-errors], + [AS_HELP_STRING([--enable-warnings-as-errors], + [Determines whether certain warnings will be + considered errors. This is mainly for use + by HDF5 library developers. + [default=no] + ])], + [WARNINGS_AS_ERRORS=$enableval]) + +## Set default +if test "X-$WARNINGS_AS_ERRORS" = X- ; then + WARNINGS_AS_ERRORS=no +fi + +case "X-$WARNINGS_AS_ERRORS" in + X-yes|X-no) + AC_MSG_RESULT([$WARNINGS_AS_ERRORS]) + ;; + *) + AC_MSG_ERROR([Unrecognized value: $WARNINGS_AS_ERRORS]) + ;; +esac + +## ---------------------------------------------------------------------- ## Source any special files that we need. These files normally aren't ## present but can be used by the maintainers to fine tune things like ## turning on debug or profiling flags for the compiler. The search order @@ -788,11 +823,14 @@ AC_LANG_POP(C++) AC_SUBST([HDF5_HL]) AC_SUBST([HDF5_HL_TOOLS]) -## The high-level library is enabled unless the build mode is clean. +## The high-level library and high-level tools are enabled unless the build mode +## is clean. if test "X-$BUILD_MODE" = "X-clean" ; then HDF5_HL=no + HDF5_HL_TOOLS=no else HDF5_HL=yes + HDF5_HL_TOOLS=yes fi ## high-level library directories (set when needed, blank until then) @@ -841,6 +879,38 @@ fi ## ---------------------------------------------------------------------- +## Enable new references for dimension scales +## +AC_SUBST([DIMENSION_SCALES_WITH_NEW_REF]) +AC_MSG_CHECKING([whether to use new references with dimension scales]); +AC_ARG_ENABLE([dimension-scales-with-new-ref], + [AS_HELP_STRING([--enable-dimension-scales-with-new-ref], + [Use new references when creating dimension scales. + [default=no] + ])], + [DIMENSION_SCALES_WITH_NEW_REF=$enableval]) + +## Set the default value to use old references. +if test "X-$DIMENSION_SCALES_WITH_NEW_REF" = X- ; then + DIMENSION_SCALES_WITH_NEW_REF=no +fi + +case "X-$DIMENSION_SCALES_WITH_NEW_REF" in + X-yes) + AC_MSG_RESULT([yes]) + AC_DEFINE([DIMENSION_SCALES_WITH_NEW_REF], [1], + [Define if new references for dimension scales were requested]) + + ;; + X-no) + AC_MSG_RESULT([no]) + ;; + *) + AC_MSG_ERROR([Unrecognized value: $DIMENSION_SCALES_WITH_NEW_REF]) + ;; +esac + +## ---------------------------------------------------------------------- ## Check which archiving tool to use. This needs to be done before ## the AM_PROG_LIBTOOL macro. ## @@ -1126,6 +1196,7 @@ if test "X$HDF5_DOXYGEN" = "Xyes"; then AC_SUBST([DOXYGEN_EXTERNAL_SEARCH]) AC_SUBST([DOXYGEN_SEARCHENGINE_URL]) AC_SUBST([DOXYGEN_STRIP_FROM_PATH]) + AC_SUBST([DOXYGEN_STRIP_FROM_INC_PATH]) AC_SUBST([DOXYGEN_PREDEFINED]) # SRCDIR Environment variables used inside doxygen macro for the source location: @@ -1133,7 +1204,7 @@ if test "X$HDF5_DOXYGEN" = "Xyes"; then DOXYGEN_VERSION_STRING=${PACKAGE_VERSION} DOXYGEN_INCLUDE_ALIASES='$(SRCDIR)/doxygen/aliases' DOXYGEN_PROJECT_LOGO='$(SRCDIR)/doxygen/img/HDFG-logo.png' - DOXYGEN_PROJECT_BRIEF='C-API Reference' + DOXYGEN_PROJECT_BRIEF='' DOXYGEN_INPUT_DIRECTORY='$(SRCDIR) $(SRCDIR)/doxygen/dox' DOXYGEN_OPTIMIZE_OUTPUT_FOR_C=YES DOXYGEN_MACRO_EXPANSION=YES @@ -1149,6 +1220,7 @@ if test "X$HDF5_DOXYGEN" = "Xyes"; then DOXYGEN_EXTERNAL_SEARCH=NO DOXYGEN_SEARCHENGINE_URL= DOXYGEN_STRIP_FROM_PATH='$(SRCDIR)' + DOXYGEN_STRIP_FROM_INC_PATH='$(SRCDIR)' DOXYGEN_PREDEFINED='H5_HAVE_DIRECT H5_HAVE_LIBHDFS H5_HAVE_MAP_API H5_HAVE_PARALLEL H5_HAVE_ROS3_VFD' DX_INIT_DOXYGEN([HDF5], [./doxygen/Doxyfile], [hdf5lib_docs]) @@ -1505,6 +1577,7 @@ case "X-$withval" in ;; esac + ## ---------------------------------------------------------------------- ## Make the external filters list available to *.in files ## At this point it's unset (no external filters by default) but it @@ -2230,6 +2303,13 @@ case "X-$ASSERTS" in ;; esac +## Incorporate the potentially warning-to-error promoting flags after +## feature tests. Some of the tests (e.g., `off_t`) generate compilation +## errors with the flags we chose for the HDF5 library and tools. +## +H5_CFLAGS="$H5_CFLAGS $H5_ECFLAGS" +H5_CXXFLAGS="$H5_CXXFLAGS $H5_ECXXFLAGS" + ## ---------------------------------------------------------------------- ## Check if developer warnings should be turned on ## These are warnings that provide suggestions like gcc's -Wsuggest-attribute. @@ -2257,6 +2337,7 @@ fi case "X-$DEV_WARNINGS" in X-yes) H5_CFLAGS="$H5_CFLAGS $DEVELOPER_WARNING_CFLAGS" + H5_FCFLAGS="$H5_FCFLAGS $DEVELOPER_WARNING_FCFLAGS" AC_MSG_RESULT([yes]) ;; X-no) @@ -2269,45 +2350,6 @@ case "X-$DEV_WARNINGS" in esac ## ---------------------------------------------------------------------- -## Check if we should consider certain compiler warnings as errors -## -## These should NOT be on by default as the risk of breakage is high -## when compiling HDF5 on new (or new versions) of platforms and -## compilers. It can also cause failures when header files we have no -## control over (e.g. MPI, HDFS) raise warnings. -## -AC_MSG_CHECKING([enable warnings as errors]) -AC_ARG_ENABLE([warnings-as-errors], - [AS_HELP_STRING([--enable-warnings-as-errors], - [Determines whether certain warnings will be - considered errors. This is mainly for use - by HDF5 library developers. - [default=no] - ])], - [WARNINGS_AS_ERRORS=$enableval]) - -## Set default -if test "X-$WARNINGS_AS_ERRORS" = X- ; then - WARNINGS_AS_ERRORS=no -fi - -case "X-$WARNINGS_AS_ERRORS" in - X-yes) - H5_CFLAGS="$H5_CFLAGS $H5_ECFLAGS" - H5_CXXFLAGS="$H5_CXXFLAGS $H5_ECXXFLAGS" - AC_MSG_RESULT([yes]) - ;; - X-no) - H5_CFLAGS="$H5_CFLAGS $H5_NECFLAGS" - H5_CXXFLAGS="$H5_CXXFLAGS $H5_NECXXFLAGS" - AC_MSG_RESULT([no]) - ;; - *) - AC_MSG_ERROR([Unrecognized value: $WARNINGS_AS_ERRORS]) - ;; -esac - -## ---------------------------------------------------------------------- ## Check if the compiler should use profiling flags/settings ## AC_MSG_CHECKING([profiling]) diff --git a/doc/library-init-shutdown.md b/doc/library-init-shutdown.md new file mode 100644 index 0000000..917d213 --- /dev/null +++ b/doc/library-init-shutdown.md @@ -0,0 +1,56 @@ +# HDF5 Library initialization and shutdown + +## Application perspective + +### Implicit initialization and shutdown + +When a developer exports a new symbol as part of the HDF5 library, +they should make sure that an application cannot enter the library in an +uninitialized state through a new API function, or read an uninitialized +value from a non-function HDF5 symbol. + +The HDF5 library initializes itself when an application either enters +the library through an API function call such as `H5Fopen`, or when +an application evaluates an HDF5 symbol that represents either a +property-list identifier such as `H5F_ACC_RDONLY` or `H5F_ACC_RDWR`, +a property-list class identifier such as `H5P_FILE_ACCESS`, a VFD +identifier such as `H5FD_FAMILY` or `H5FD_SEC2`, or a type identifier +such as `H5T_NATIVE_INT64`. + +The library sets a flag when initialization occurs and as long as the +flag is set, skips initialization. + +The library provides a couple of macros that initialize the library +as necessary. The library is initialized as a side-effect of the +`FUNC_ENTER_API*` macros used at the top of most API functions. HDF5 +library symbols other than functions are provided through `#define`s +that use `H5OPEN` to introduce a library-initialization call (`H5open`) +at each site where a non-function symbol is used. + +Ordinarily the library registers an `atexit(3)` handler to shut itself +down when the application exits. + +### Explicit initialization and shutdown + +An application may use an API call, `H5open`, to explicitly initialize +the library. `H5close` explicitly shuts down the library. + +## Library internals perspective + +No matter how library initializion begins, eventually the internal +function `H5_init_library` will be called. `H5_init_library` is +responsible for calling the initializers for every internal HDF5 +library module (aka "package") in the correct order so that no module is +initialized before its prerequisite modules. A table in `H5_init_library` +establishes the order of initialization. If a developer adds a +module to the library that it is appropriate to initialize with the rest +of the library, then they should insert its initializer into the right +place in the table. + +`H5_term_library` drives library shutdown. Library shutdown is +table-driven, too. If a developer adds a module that needs to release +resources during library shutdown, then they should add a call at the +right place to the shutdown table. Note that some entries in the shutdown +table are marked as "barriers," and if a new module should only be +shutdown *strictly after* the preceding modules, then it should be marked +as a barrier. See the comments in `H5_term_library` for more information. diff --git a/doxygen/CMakeLists.txt b/doxygen/CMakeLists.txt index 36ce590..3462d50 100644 --- a/doxygen/CMakeLists.txt +++ b/doxygen/CMakeLists.txt @@ -27,6 +27,7 @@ if (DOXYGEN_FOUND) set (DOXYGEN_EXTERNAL_SEARCH NO) set (DOXYGEN_SEARCHENGINE_URL) set (DOXYGEN_STRIP_FROM_PATH ${HDF5_SOURCE_DIR}) + set (DOXYGEN_STRIP_FROM_INC_PATH ${HDF5_SOURCE_DIR}) set (DOXYGEN_PREDEFINED "H5_HAVE_DIRECT H5_HAVE_LIBHDFS H5_HAVE_MAP_API H5_HAVE_PARALLEL H5_HAVE_ROS3_VFD") # This configure and individual custom targets work together diff --git a/doxygen/Doxyfile.in b/doxygen/Doxyfile.in index 44d9974..8c871de 100644 --- a/doxygen/Doxyfile.in +++ b/doxygen/Doxyfile.in @@ -179,7 +179,7 @@ STRIP_FROM_PATH = @DOXYGEN_STRIP_FROM_PATH@ # specify the list of include paths that are normally passed to the compiler # using the -I flag. -STRIP_FROM_INC_PATH = +STRIP_FROM_INC_PATH = @DOXYGEN_STRIP_FROM_INC_PATH@ # If the SHORT_NAMES tag is set to YES, doxygen will generate much shorter (but # less readable) file names. This can be useful is your file systems doesn't @@ -856,6 +856,7 @@ INPUT_ENCODING = UTF-8 FILE_PATTERNS = H5*public.h \ H5*module.h \ H5FDcore.h \ + H5FDdevelop.h \ H5FDdirect.h \ H5FDfamily.h \ H5FDhdfs.h \ diff --git a/doxygen/aliases b/doxygen/aliases index 06c3445..f83a875 100644 --- a/doxygen/aliases +++ b/doxygen/aliases @@ -24,6 +24,7 @@ ALIASES += htri_t="Returns zero (false), a positive (true) or a negative (failur ALIASES += api_vers_2{3}="\1() is a macro that is mapped to either \2() or \3().\n\see \ref api-compat-macros" ALIASES += api_vers_3{4}="\1() is a macro that is mapped to either \2() or \3() or \4().\n\see \ref api-compat-macros" +ALIASES += api_vers_4{5}="\1() is a macro that is mapped to either \2() or \3() or \4() or \5().\n\see \ref api-compat-macros" ALIASES += deprecation_note{1}="\deprecated Superseded by \1." @@ -252,10 +253,17 @@ ALIASES += ref_vol_doc="VOL documentation" ################################################################################ ALIASES += ref_rfc20210528="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_multi_thread.pdf\">Multi-Thread HDF5</a>" +ALIASES += ref_rfc20210219="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/selection_io_RFC_210610.pdf\">Selection I/O</a>" +ALIASES += ref_rfc20200213="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_VFD_subfiling_200424.pdf\">VFD Sub-filing</a>" +ALIASES += ref_rfc20200210="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/Onion_VFD_RFC_211122.pdf\">Onion VFD</a>" ALIASES += ref_rfc20190923="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/2019-09-23-RFC_VOL_feature_flags.pdf\">Virtual Object Layer (VOL) API Compatibility</a>" +ALIASES += ref_rfc20190715="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/var_len_data_sketch_design_190715.pdf\">Variable-Length Data in HDF5 Sketch Design</a>" ALIASES += ref_rfc20190410="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_VFD_Plugin.docx.pdf\">A Plugin Interface for HDF5 Virtual File Drivers</a>" ALIASES += ref_rfc20181231="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_Min_Obj_Headers_181231.pdf\">Dataset Object Header Size</a>" ALIASES += ref_rfc20181220="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/3.2.1_3.2.2_deliverable_181220_v4.pdf\">MS 3.2 – Addressing Scalability: Scalability of open, close, flush CASE STUDY: CGNS Hotspot analysis of CGNS cgp_open</a>" +ALIASES += ref_rfc20180830="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_Sparse_Chunks180830.pdf\">Sparse Chunks</a>" +ALIASES += ref_rfc20180829="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/mirror_VFD_RFC_2018-10-05.pdf\">H5FD_MIRROR Virtual File Driver</a>" +ALIASES += ref_rfc20180815="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/splitter_VFD_RFC_180830.pdf\">Splitter_VFD</a>" ALIASES += ref_rfc20180620="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC-Chunking%20Functions-2018-06-20-v3.docx.pdf\">Chunk query functionality in HDF5</a>" ALIASES += ref_rfc20180610="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/VFD_SWMR_RFC_200916.pdf\">VFD SWMR</a>" ALIASES += ref_rfc20180321="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC-API_Contexts-2018-03-21.docx.pdf\">API Contexts</a>" @@ -298,7 +306,7 @@ ALIASES += ref_rfc20120305="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC%20P ALIASES += ref_rfc20120220="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/h5repack_improve_hyperslab_over_chunked_dataset_v1.pdf\"><tt>h5repack</tt>: Improved Hyperslab selections for Large Chunked Datasets</a>" ALIASES += ref_rfc20120120="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/2012-1-25-Maintainers-guide-for-datatype.docx.pdf\">A Maintainer’s Guide for the Datatype Module in HDF5 Library</a>" ALIASES += ref_rfc20120104="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_actual_io_v4-1_done.docx.pdf\">Actual I/O Mode</a>" -ALIASES += ref_rfc20111119="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC-H5Ocompare-review_v6.pdf\">New public functions to handle comparison</a>" +ALIASES += ref_rfc20111119="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC-H5Ocompare-review_v6.pdf\">New public functions to handle comparison</a>" ALIASES += ref_rfc20110825="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/2011-08-31-RFC_H5Ocopy_Named_DT_v2.docx.pdf\">Merging Named Datatypes in H5Ocopy()</a>" ALIASES += ref_rfc20110811="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_Enhancement_Hyperslab_Selection-1.4.docx.pdf\">Expanding the HDF5 Hyperslab Selection Interface</a>" ALIASES += ref_rfc20110726="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/metadata_aggregation_RFC_v03.docx.pdf\">HDF5 File Space Allocation and Aggregation</a>" @@ -318,8 +326,12 @@ ALIASES += ref_rfc20091218="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RCF_h5d ALIASES += ref_rfc20090907="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_Tools_Lib_v2.pdf\">HDF5 Tools Library Functions</a>" ALIASES += ref_rfc20090612="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_h5diff_default_epsilon.pdf\">Default EPSILON values for comparing floating point data</a>" ALIASES += ref_rfc20081218="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_h5diff_NonComparable.pdf\">Reporting of Non-Comparable Datasets by <tt>h5diff</tt></a>" +ALIASES += ref_rfc20081205="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_elink_callback.pdf\">External Link Traversal Callback</a>" +ALIASES += ref_rfc20081030="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_chunk_cache_functions.pdf\">Setting Raw Data Chunk Cache Parameters in HDF5</a>" ALIASES += ref_rfc20080915="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/FileFreeSpacePerformance.pdf\">Performance Report for Free-space Manager</a>" ALIASES += ref_rfc20080904="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/ExternalLinkFileAccessProperty.pdf\">Setting File Access Property List for accessing External Link</a>" +ALIASES += ref_rfc20080728="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_Native_Time_Types.pdf\">Native Time Types in HDF5</a>" +ALIASES += ref_rfc20080723="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_Special_Values_in_HDF5.pdf\">Special Values in HDF5</a>" ALIASES += ref_rfc20080301="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/DynamicTransformations_RFC.pdf\">Dynamic Transformations to HDF5 Data</a>" ALIASES += ref_rfc20080209="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC-Using-SVN-branching-Feb9.pdf\">Using SVN branching to improve software development process at THG</a>" ALIASES += ref_rfc20080206="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC-HIS-REL-1.8_Feb6.pdf\">Maintaining the <tt>HISTORY.txt</tt> and <tt>RELEASE.txt</tt> files in HDF5</a>" @@ -327,7 +339,7 @@ ALIASES += ref_rfc20071111="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/AURA-co ALIASES += ref_rfc20071018="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/RFC_NaNsHDF5.pdf\"><tt>NaN</tt> detection in HDF5</a>" ALIASES += ref_rfc20070801="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/Metadata_Journaling_RFC.pdf\">Metadata Journaling to Improve Crash Survivability</a>" ALIASES += ref_rfc20070413="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/API_Compatibility_RFC.txt.pdf\">API Compatibility Strategies for HDF5</a>" -ALIASES += ref_rfc20070115="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/PrivateHeap.pdf\">A 'Private' Heap for HDF5</a>" +ALIASES += ref_rfc20070115="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/PrivateHeap.pdf\">A \"Private\" Heap for HDF5</a>" ALIASES += ref_rfc20060623="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/coll_ind_dd6.pdf\">Performance Comparison of Collective I/O and Independent I/O with Derived Datatypes</a>" ALIASES += ref_rfc20060604="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/h5stat_spec_v3_2006-06-04.pdf\"><tt>h5stat</tt> tool</a>" ALIASES += ref_rfc20060505="<a href=\"https://docs.hdfgroup.org/hdf5/rfc/Simple%20Performance%20Test%20on%20Fletcher32%20Filter.pdf\">Simple Performance Test on Fletcher32 Filter</a>" diff --git a/doxygen/dox/About.dox b/doxygen/dox/About.dox index d315566..9000a78 100644 --- a/doxygen/dox/About.dox +++ b/doxygen/dox/About.dox @@ -12,12 +12,118 @@ of <em>documentation done right</em>. \section about_documentation Documentation about Documentation -\li \todo Describe how to add a reference or a new RFC -\li \todo Describe how to add an example -\li \todo Describe how to include plain HTML -\li \todo Describe how to add an API macro -\li \todo Describe the custom commands -\li \todo Describe the S3 bucket layout and update routine -\li \todo Link the RM template +In this section, we describe common documentation maintenance tasks. + +\subsection plain_html Including Plain HTML Pages + +The most common use case for this is the inclusion of older documentation. +New documentation should, whenever possible, be created using Doxygen markdown! + +Use Doxygen's <a href="https://www.doxygen.nl/manual/commands.html#cmdhtmlinclude"><code>htmlinclude</code></a> +special command to include existing plain HTML pages. + +An example from this documentation set can be seen +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/dox/FileFormatSpec.dox">here</a>. + +\subsection new_rm_entry Creating a New Reference Manual Entry + +Please refer to the \ref RMT for guidance on how to create a new reference manual entry. + +\subsubsection new_example Adding and Referencing API Examples + +For each HDF5 module, such as \Code{H5F}, there is an examples source file called +\Code{H5*_examples.c}. For example, the \Code{H5F} API examples are located in +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/examples/H5F_examples.c"> +<code>H5F_examples.c</code></a>. Examples are code blocks marked as Doxygen +<a href="https://www.doxygen.nl/manual/commands.html#cmdsnippet">snippets</a>. +For example, the source code for the H5Fcreate() API sample is located between +the +\verbatim +//! <!-- [create] --> +... +//! <!-- [create] --> +\endverbatim +comments in +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/examples/H5F_examples.c"> +<code>H5F_examples.c</code></a>. + +Add a new API example by adding a new code block enclosed between matching +snippet tags. The name of the tag is usually the function name stripped of +the module prefix. + +The inclusion of such a block of code can then be triggered via Doxygen's +<a href="https://www.doxygen.nl/manual/commands.html#cmdsnippet"><code>snippet</code></a> +special command. For example, the following markup +\verbatim +* \snippet H5F_examples.c create +\endverbatim +yields +\snippet H5F_examples.c create + +\subsubsection api_macro Adding an API Macro + +API macros are handled by the <code>api_vers_2, api_vers_3, api_vers_4</code> +custom commands. The numbers indicate the number of potential API function +mappings. For example, H5Acreate() has two potential mappings, H5Acreate1() and +H5Acreate2(). To trigger the creation of a reference manual entry for H5Acreate() +use the following markup: +\verbatim +\api_vers_2{H5Acreate,H5Acreate1,H5Acreate2} +\endverbatim +This yields: + +\api_vers_2{H5Acreate,H5Acreate1,H5Acreate2} + +\subsection custom_commands Creating Custom Commands + +See Doxygen's <a href="https://www.doxygen.nl/manual/custcmd.html">Custom Commands documentation</a> +as a general reference. + +All custom commands for this project are located in the +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/aliases"><tt>aliases</tt></a> +file in the <a href="https://github.com/HDFGroup/hdf5/tree/develop/doxygen"><tt>doxygen</tt></a> +subdirectory of the <a href="https://github.com/HDFGroup/hdf5">main HDF5 repo</a>. + +The custom commands are grouped in sections. Find a suitable section for your command or +ask for help if unsure! + +\subsection new_rfc Adding a New RFC or Referencing an Existing RFC + +For ease of reference, we define custom commands for each RFC in the <tt>RFCs</tt> section +of the +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/aliases"><tt>aliases</tt></a> +file. For example the custom command \Code{ref_rfc20141210} can be used to insert a +reference to "RFC: Virtual Object Layer". In other words, the markup +\verbatim +\ref_rfc20141210 +\endverbatim +yields a clickable link: + +\ref_rfc20141210 + +To add a new RFC, add a custom command for the RFC to the +<a href="https://github.com/HDFGroup/hdf5/blob/develop/doxygen/aliases"><tt>aliases</tt></a> +file. The naming convention for the custom command is \Code{ref_rfcYYYYMMDD}, +where \Code{YYYYMMDD} is the ID of the RFC. The URL is composed of the prefix +\verbatim +https://docs.hdfgroup.org/hdf5/rfc/ +\endverbatim +and the name of your RFC file, typically, a PDF file, i.e., the full URL would +be +\verbatim +https://docs.hdfgroup.org/hdf5/rfc/my_great_rfc_name.pdf +\endverbatim + +\subsection hosting How Do Updates and Changes Get Published? + +Currently, the files underlying this documentation website are stored in an +bucket on AWS S3. The top-level bucket is <pre>s3://docs.hdfgroup.org/hdf5/</pre> +There are folders for the <tt>development</tt> branch and all supported release +version. + +Talk to your friendly IT-team if you need write access, or you need someone to +push an updated version for you! + +\todo Make the publication a GitHub action! */ diff --git a/doxygen/dox/FTS.dox b/doxygen/dox/FTS.dox new file mode 100644 index 0000000..9dae7c1 --- /dev/null +++ b/doxygen/dox/FTS.dox @@ -0,0 +1,8 @@ +/** \page FTS Full-Text Search + +\htmlonly +<script async src="https://cse.google.com/cse.js?cx=75c754173f9b90804"></script> +<div class="gcse-search"></div> +\endhtmlonly + +*/
\ No newline at end of file diff --git a/doxygen/dox/FileFormatSpec.dox b/doxygen/dox/FileFormatSpec.dox deleted file mode 100644 index fc10574..0000000 --- a/doxygen/dox/FileFormatSpec.dox +++ /dev/null @@ -1,23 +0,0 @@ -/** \page FMT3 HDF5 File Format Specification Version 3.0 - -\htmlinclude H5.format.html - -*/ - -/** \page FMT2 HDF5 File Format Specification Version 2.0 - -\htmlinclude H5.format.2.0.html - -*/ - -/** \page FMT11 HDF5 File Format Specification Version 1.1 - -\htmlinclude H5.format.1.1.html - -*/ - -/** \page FMT1 HDF5 File Format Specification Version 1.0 - -\htmlinclude H5.format.1.0.html - -*/
\ No newline at end of file diff --git a/doxygen/dox/OtherSpecs.dox b/doxygen/dox/OtherSpecs.dox deleted file mode 100644 index e53f26e..0000000 --- a/doxygen/dox/OtherSpecs.dox +++ /dev/null @@ -1,11 +0,0 @@ -/** \page IMG HDF5 Image and Palette Specification Version 1.2 - -\htmlinclude ImageSpec.html - -*/ - -/** \page TBL HDF5 Table Specification Version 1.0 - -\htmlinclude TableSpec.html - -*/ diff --git a/doxygen/dox/RFC.dox b/doxygen/dox/RFC.dox index c16dcea..c2562b0 100644 --- a/doxygen/dox/RFC.dox +++ b/doxygen/dox/RFC.dox @@ -3,10 +3,17 @@ <table> <tr><th>RFC ID</th><th>Title</th><th>Comments</th></tr> <tr> <td>2021-05-28</td> <td>\ref_rfc20210528</td> <td></td></tr> +<tr> <td>2021-02-19</td> <td>\ref_rfc20210219</td> <td></td></tr> +<tr> <td>2020-02-13</td> <td>\ref_rfc20200213</td> <td></td></tr> +<tr> <td>2020-02-10</td> <td>\ref_rfc20200210</td> <td></td></tr> <tr> <td>2019-09-23</td> <td>\ref_rfc20190923</td> <td></td></tr> +<tr> <td>2019-07-15</td> <td>\ref_rfc20190715</td> <td></td> </tr> <tr> <td>2019-04-10</td> <td>\ref_rfc20190410</td> <td></td> </tr> <tr> <td>2018-12-31</td> <td>\ref_rfc20181231</td> <td></td> </tr> <tr> <td>2018-12-20</td> <td>\ref_rfc20181220</td> <td></td> </tr> +<tr> <td>2018-08-30</td> <td>\ref_rfc20180830</td> <td></td> </tr> +<tr> <td>2018-08-29</td> <td>\ref_rfc20180829</td> <td></td> </tr> +<tr> <td>2018-08-15</td> <td>\ref_rfc20180815</td> <td></td> </tr> <tr> <td>2018-06-20</td> <td>\ref_rfc20180620</td> <td></td> </tr> <tr> <td>2018-06-10</td> <td>\ref_rfc20180610</td> <td></td> </tr> <tr> <td>2018-03-21</td> <td>\ref_rfc20180321</td> <td></td> </tr> @@ -69,8 +76,12 @@ <tr> <td>2009-09-07</td> <td>\ref_rfc20090907</td> <td></td> </tr> <tr> <td>2009-06-12</td> <td>\ref_rfc20090612</td> <td></td> </tr> <tr> <td>2008-12-18</td> <td>\ref_rfc20081218</td> <td></td> </tr> +<tr> <td>2008-12-05</td> <td>\ref_rfc20081205</td> <td></td> </tr> +<tr> <td>2008-10-30</td> <td>\ref_rfc20081030</td> <td></td> </tr> <tr> <td>2008-09-15</td> <td>\ref_rfc20080915</td> <td></td> </tr> <tr> <td>2008-09-04</td> <td>\ref_rfc20080904</td> <td></td> </tr> +<tr> <td>2008-07-28</td> <td>\ref_rfc20080728</td> <td></td> </tr> +<tr> <td>2008-07-23</td> <td>\ref_rfc20080723</td> <td></td> </tr> <tr> <td>2008-03-01</td> <td>\ref_rfc20080301</td> <td></td> </tr> <tr> <td>2008-02-09</td> <td>\ref_rfc20080209</td> <td></td> </tr> <tr> <td>2008-02-06</td> <td>\ref_rfc20080206</td> <td></td> </tr> diff --git a/doxygen/dox/Specifications.dox b/doxygen/dox/Specifications.dox index 4ae48d0..5a36d61 100644 --- a/doxygen/dox/Specifications.dox +++ b/doxygen/dox/Specifications.dox @@ -19,4 +19,40 @@ \li <a href="https://support.hdfgroup.org/HDF5/doc/HL/H5DS_Spec.pdf"> HDF5 Dimension Scale Specification</a> -*/
\ No newline at end of file +*/ + +/** \page FMT3 HDF5 File Format Specification Version 3.0 + +\htmlinclude H5.format.html + +*/ + +/** \page FMT2 HDF5 File Format Specification Version 2.0 + +\htmlinclude H5.format.2.0.html + +*/ + +/** \page FMT11 HDF5 File Format Specification Version 1.1 + +\htmlinclude H5.format.1.1.html + +*/ + +/** \page FMT1 HDF5 File Format Specification Version 1.0 + +\htmlinclude H5.format.1.0.html + +*/ + +/** \page IMG HDF5 Image and Palette Specification Version 1.2 + +\htmlinclude ImageSpec.html + +*/ + +/** \page TBL HDF5 Table Specification Version 1.0 + +\htmlinclude TableSpec.html + +*/ diff --git a/doxygen/dox/TechnicalNotes.dox b/doxygen/dox/TechnicalNotes.dox index 2bda175..9bd2802 100644 --- a/doxygen/dox/TechnicalNotes.dox +++ b/doxygen/dox/TechnicalNotes.dox @@ -1,6 +1,10 @@ /** \page TN Technical Notes \li \link api-compat-macros API Compatibility Macros \endlink +\li \ref APPDBG "Debugging HDF5 Applications" +\li \ref FMTDISC "File Format Walkthrough" +\li \ref FILTER "Filters" +\li \ref IOFLOW "HDF5 Raw I/O Flow Notes" \li \ref TNMDC "Metadata Caching in HDF5" \li \ref MT "Thread Safe library" \li \ref VFL "Virtual File Layer" @@ -13,8 +17,32 @@ */ +/** \page IOFLOW HDF5 Raw I/O Flow Notes + +\htmlinclude IOFlow.html + +*/ + /** \page VFL HDF5 Virtual File Layer \htmlinclude VFL.html */ + +/** \page FMTDISC HDF5 File Format Discussion + +\htmlinclude FileFormat.html + +*/ + +/** \page FILTER HDF5 Filters + +\htmlinclude Filters.html + +*/ + +/** \page APPDBG Debugging HDF5 Applications + +\htmlinclude DebuggingHDF5Applications.html + +*/
\ No newline at end of file diff --git a/doxygen/examples/DebuggingHDF5Applications.html b/doxygen/examples/DebuggingHDF5Applications.html new file mode 100644 index 0000000..c6aaf74 --- /dev/null +++ b/doxygen/examples/DebuggingHDF5Applications.html @@ -0,0 +1,392 @@ +<html> + <head> + <title>Debugging HDF5 Applications</title> + + <h2>Introduction</h2> + + <p>The HDF5 library contains a number of debugging features to + make programmers' lives easier including the ability to print + detailed error messages, check invariant conditions, display + timings and other statistics, and trace API function calls and + return values. + + </p><dl> + <dt><b>Error Messages</b> + </dt><dd>Error messages are normally displayed automatically on the + standard error stream and include a stack trace of the library + including file names, line numbers, and function names. The + application has complete control over how error messages are + displayed and can disable the display on a permanent or + temporary basis. Refer to the documentation for the H5E error + handling package. + + <br><br> + </dd><dt><b>Invariant Conditions</b> + </dt><dd>Unless <code>NDEBUG</code> is defined during compiling, the + library will include code to verify that invariant conditions + have the expected values. When a problem is detected the + library will display the file and line number within the + library and the invariant condition that failed. A core dump + may be generated for post mortem debugging. The code to + perform these checks can be included on a per-package bases. + + <br><br> + </dd><dt><b>Timings and Statistics</b> + </dt><dd>The library can be configured to accumulate certain + statistics about things like cache performance, datatype + conversion, data space conversion, and data filters. The code + is included on a per-package basis and enabled at runtime by + an environment variable. + + <br><br> + </dd><dt><b>API Tracing</b> + </dt><dd>All API calls made by an application can be displayed and + include formal argument names and actual values and the + function return value. This code is also conditionally + included at compile time and enabled at runtime. + </dd></dl> + + <p>The statistics and tracing can be displayed on any output + stream (including streams opened by the shell) with output from + different packages even going to different streams. + + </p><h2>Error Messages</h2> + + <p>By default any API function that fails will print an error + stack to the standard error stream. + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <tbody><tr> + <td> + <p><code></code></p><pre><code> +HDF5-DIAG: Error detected in thread 0. Back trace follows. + #000: H5F.c line 1245 in H5Fopen(): unable to open file + major(04): File interface + minor(10): Unable to open file + #001: H5F.c line 846 in H5F_open(): file does not exist + major(04): File interface + minor(10): Unable to open file + </code></pre> + </td> + </tr> + </tbody></table> + </center> + + <p>The error handling package (H5E) is described + <a href="./group___h5_e.html">elsewhere</a>. + + </p><h2>Invariant Conditions</h2> + + <p>To include checks for invariant conditions the library should + be configured with <code>--disable-production</code>, the + default for versions before 1.2. The library designers have made + every attempt to handle error conditions gracefully but an + invariant condition assertion may fail in certain cases. The + output from a failure usually looks something like this: + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <tbody><tr> + <td> + <p><code></code></p><pre><code> +Assertion failed: H5.c:123: i<NELMTS(H5_debug_g) +IOT Trap, core dumped. + </code></pre> + </td> + </tr> + </tbody></table> + </center> + + <h2>Timings and Statistics</h2> + + <p>Code to accumulate statistics is included at compile time by + using the <code>--enable-debug</code> configure switch. The + switch can be followed by an equal sign and a comma-separated + list of package names or else a default list is used. + + </p><p> + </p><center> + <table border="" align="center" width="80%"> + <tbody><tr> + <th>Name</th> + <th>Default</th> + <th>Description</th> + </tr> + <tr> + <td align="center">a</td> + <td align="center">No</td> + <td>Attributes</td> + </tr> + <tr> + <td align="center">ac</td> + <td align="center">Yes</td> + <td>Meta data cache</td> + </tr> + <tr> + <td align="center">b</td> + <td align="center">Yes</td> + <td>B-Trees</td> + </tr> + <tr> + <td align="center">d</td> + <td align="center">Yes</td> + <td>Datasets</td> + </tr> + <tr> + <td align="center">e</td> + <td align="center">Yes</td> + <td>Error handling</td> + </tr> + <tr> + <td align="center">f</td> + <td align="center">Yes</td> + <td>Files</td> + </tr> + <tr> + <td align="center">g</td> + <td align="center">Yes</td> + <td>Groups</td> + </tr> + <tr> + <td align="center">hg</td> + <td align="center">Yes</td> + <td>Global heap</td> + </tr> + <tr> + <td align="center">hl</td> + <td align="center">No</td> + <td>Local heaps</td> + </tr> + <tr> + <td align="center">i</td> + <td align="center">Yes</td> + <td>Interface abstraction</td> + </tr> + <tr> + <td align="center">mf</td> + <td align="center">No</td> + <td>File memory management</td> + </tr> + <tr> + <td align="center">mm</td> + <td align="center">Yes</td> + <td>Library memory managment</td> + </tr> + <tr> + <td align="center">o</td> + <td align="center">No</td> + <td>Object headers and messages</td> + </tr> + <tr> + <td align="center">p</td> + <td align="center">Yes</td> + <td>Property lists</td> + </tr> + <tr> + <td align="center">s</td> + <td align="center">Yes</td> + <td>Data spaces</td> + </tr> + <tr> + <td align="center">t</td> + <td align="center">Yes</td> + <td>Datatypes</td> + </tr> + <tr> + <td align="center">v</td> + <td align="center">Yes</td> + <td>Vectors</td> + </tr> + <tr> + <td align="center">z</td> + <td align="center">Yes</td> + <td>Raw data filters</td> + </tr> + </tbody></table> + </center> + + <p>In addition to including the code at compile time the + application must enable each package at runtime. This is done + by listing the package names in the <code>HDF5_DEBUG</code> + environment variable. That variable may also contain file + descriptor numbers (the default is `2') which control the output + for all following packages up to the next file number. The + word <code>all</code> refers to all packages. Any word my be + preceded by a minus sign to turn debugging off for the package. + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <caption align="top"><b>Sample debug specifications</b></caption> + <tbody><tr valign="top"> + <td><code>all</code></td> + <td>This causes debugging output from all packages to be + sent to the standard error stream.</td> + </tr> + <tr valign="top"> + <td><code>all -t -s</code></td> + <td>Debugging output for all packages except datatypes + and data spaces will appear on the standard error + stream.</td> + </tr> + <tr valign="top"> + <td><code>-all ac 255 t,s</code></td> + <td>This disables all debugging even if the default was to + debug something, then output from the meta data cache is + send to the standard error stream and output from data + types and spaces is sent to file descriptor 255 which + should be redirected by the shell.</td> + </tr> + </tbody></table> + </center> + + <p>The components of the <code>HDF5_DEBUG</code> value may be + separated by any non-lowercase letter. + + </p><h2>API Tracing</h2> + + <p>The HDF5 library can trace API calls by printing the + function name, the argument names and their values, and the + return value. Some people like to see lots of output during + program execution instead of using a good symbolic debugger, and + this feature is intended for their consumption. For example, + the output from <code>h5ls foo</code> after turning on tracing, + includes: + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <tbody><tr> + <td> + <code><pre> +H5Tcopy(type=184549388) = 184549419 (type); +H5Tcopy(type=184549392) = 184549424 (type); +H5Tlock(type=184549424) = SUCCEED; +H5Tcopy(type=184549393) = 184549425 (type); +H5Tlock(type=184549425) = SUCCEED; +H5Fopen(filename="foo", flags=0, access=H5P_DEFAULT) = FAIL; +HDF5-DIAG: Error detected in thread 0. Back trace follows. + #000: H5F.c line 1245 in H5Fopen(): unable to open file + major(04): File interface + minor(10): Unable to open file + #001: H5F.c line 846 in H5F_open(): file does not exist + major(04): File interface + minor(10): Unable to open file + </pre></code> + </td> + </tr> + </tbody></table> + </center> + + <p>The code that performs the tracing must be included in the + library by specifying the <code>--enable-trace</code> + configuration switch (the default for versions before 1.2). Then + the word <code>trace</code> must appear in the value of the + <code>HDF5_DEBUG</code> variable. The output will appear on the + last file descriptor before the word <code>trace</code> or two + (standard error) by default. + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <tbody><tr> + <td>To display the trace on the standard error stream: + <code><pre>$ env HDF5_DEBUG=trace a.out + </pre></code> + </td> + </tr> + <tr> + <td>To send the trace to a file: + <code><pre>$ env HDF5_DEBUG="55 trace" a.out 55>trace-output + </pre></code> + </td> + </tr> + </tbody></table> + </center> + + <h3>Performance</h3> + + <p>If the library was not configured for tracing then there is no + unnecessary overhead since all tracing code is excluded. + However, if tracing is enabled but not used there is a small + penalty. First, code size is larger because of extra + statically-declared character strings used to store argument + types and names and extra auto variable pointer in each + function. Also, execution is slower because each function sets + and tests a local variable and each API function calls the + <code>H5_trace()</code> function. + + </p><p>If tracing is enabled and turned on then the penalties from the + previous paragraph apply plus the time required to format each + line of tracing information. There is also an extra call to + H5_trace() for each API function to print the return value. + + </p><h3>Safety</h3> + + <p>The tracing mechanism is invoked for each API function before + arguments are checked for validity. If bad arguments are passed + to an API function it could result in a segmentation fault. + However, the tracing output is line-buffered so all previous + output will appear. + + </p><h3>Completeness</h3> + + <p>There are two API functions that don't participate in + tracing. They are <code>H5Eprint()</code> and + <code>H5Eprint_cb()</code> because their participation would + mess up output during automatic error reporting. + + </p><p>On the other hand, a number of API functions are called during + library initialization and they print tracing information. + + </p><h3>Implementation</h3> + + <p>For those interested in the implementation here is a + description. Each API function should have a call to one of the + <code>H5TRACE()</code> macros immediately after the + <code>FUNC_ENTER()</code> macro. The first argument is the + return type encoded as a string. The second argument is the + types of all the function arguments encoded as a string. The + remaining arguments are the function arguments. This macro was + designed to be as terse and unobtrousive as possible. + + </p><p>In order to keep the <code>H5TRACE()</code> calls synchronized + with the source code we've written a perl script which gets + called automatically just before Makefile dependencies are + calculated for the file. However, this only works when one is + using GNU make. To reinstrument the tracing explicitly, invoke + the <code>trace</code> program from the hdf5 bin directory with + the names of the source files that need to be updated. If any + file needs to be modified then a backup is created by appending + a tilde to the file name. + + </p><p> + </p><center> + <table border="" align="center" width="100%"> + <caption align="top"><b>Explicit Instrumentation</b></caption> + <tbody><tr> + <td> + <code><pre> +$ ../bin/trace *.c +H5E.c: in function `H5Ewalk_cb': +H5E.c:336: warning: trace info was not inserted + </pre></code> + </td> + </tr> + </tbody></table> + </center> + + <p>Note: The warning message is the result of a comment of the + form <code>/*NO TRACE*/</code> somewhere in the function + body. Tracing information will not be updated or inserted if + such a comment exists. + + </p><p>Error messages have the same format as a compiler so that they + can be parsed from program development environments like + Emacs. Any function which generates an error will not be + modified.</p> + +</body></html> diff --git a/doxygen/examples/FileFormat.html b/doxygen/examples/FileFormat.html new file mode 100644 index 0000000..fc35357 --- /dev/null +++ b/doxygen/examples/FileFormat.html @@ -0,0 +1,1275 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<!-- saved from url=(0072)https://gamma.hdfgroup.org/papers/HISS/030515.FileFormat/FileFormat.html --> +<html><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> + <title>HDF5 File Format Discussion</title> + + <meta name="author" content="Quincey Koziol"> +</head> + +<body text="#000000" bgcolor="#FFFFFF"> + +<style type="text/css"> +OL.loweralpha { list-style-type: lower-alpha } +OL.upperroman { list-style-type: upper-roman } +</style> + +<style type="text/css"> +TD CAPTION EM { color: red } +TD EM { color: blue } +TABLE CAPTION STRONG { font-size: larger } +</style> + +<center><h1>HDF5 File Format Discussion</h1></center> +<center><h3>Quincey Koziol<br> + koziol@ncsa.uiuc.edu<br> + May 15, 2003 +</h3></center> + +<ol class="upperroman"> + +<li><h3><u>Document's Audience:</u></h3> + +<ul> + <li>Current H5 library designers and knowledgable external developers.</li> +</ul> + +</li><li><h3><u>Background Reading:</u></h3> + +<dl> + <dt><a href="https://docs.hdfgroup.org/hdf5/develop/_s_p_e_c.html">HDF5 File Format Specification</a> + </dt><dd>This describes the current HDF5 file format. +</dd></dl> + +</li><li><h3><u>Introduction:</u></h3> + +<dl> + <dt><strong>What is this document about?</strong></dt> + <dd>This document attempts to explain the HDF5 file format + specification with a few examples and describes some potential + improvements to the format specification. + </dd> <br> +</dl> + +</li><li><h3><u>File Format Examples:</u></h3> + +<p>This section has several small programs and describes the format of a file +created with each of them. +</p> + +<p>Example program one - <em>Create an empty file</em>: +</p><pre> <code> +#include "hdf5.h" +#include <assert.h> + +int main() +{ + hid_t fid; /* File ID */ + herr_t ret; /* Generic return value */ + + /* Create the file */ + fid=H5Fcreate("example1.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert(fid>=0); + + /* Close the file */ + ret=H5Fclose(fid); + assert(ret>=0); + + return(0); +} +</assert.h></code> </pre> + + <center> + <table border="" align="center" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Super Block</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td>\211</td> + <td>'H'</td> + <td>'D'</td> + <td>'F'</td> + </tr> + + <tr align="center"> + <td>\r</td> + <td>\n</td> + <td>\032</td> + <td>\n</td> + </tr> + + <tr align="center"> + <td>0</td> + <td>0</td> + <td>0</td> + <td>0</td> + </tr> + + <tr align="center"> + <td>0</td> + <td>8</td> + <td>8</td> + <td>0</td> + </tr> + + <tr align="center"> + <td colspan="2">4</td> + <td colspan="2">16</td> + </tr> + + <tr align="center"> + <td colspan="4">0x00000003</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">?</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>928<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">H5G_CACHED_STAB (1)</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4"><br>384<br><br></td> + </tr> + <tr align="center"> + <td colspan="4"><br>96<br><br></td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example1.h5 + +Reading signature at address 0 (rel) +File Super Block... +File name: example1.h5 +File access flags 0x00000000 +File open reference count: 1 +Address of super block: 0 (abs) +Size of user block: 0 bytes +Super block version number: 0 +Free list version number: 0 +Root group symbol table entry version number: 0 +Shared header version number: 0 +Size of file offsets (haddr_t type): 8 bytes +Size of file lengths (hsize_t type): 8 bytes +Symbol table leaf node 1/2 rank: 4 +Symbol table internal node 1/2 rank: 16 +File consistency flags: 0x00000003 +Base address: 0 (abs) +Free list address: UNDEF (rel) +Address of driver information block: UNDEF (rel) +Root group symbol table entry: + Name offset into private heap: 0 + Object header address: 928 + Dirty: Yes + Cache info type: Symbol Table + Cached information: + B-tree address: 384 + Heap address: 96 +</code> </pre> + + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group Object Header</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="1" width="25%">1</td> + <td colspan="1" width="25%">0</td> + <td colspan="2" width="50%">2</td> + </tr> + <tr align="center"> + <td colspan="4">1</td> + </tr> + <tr align="center"> + <td colspan="4">32</td> + </tr> + <tr align="center"> + <td colspan="2">0x0011</td> + <td colspan="2">16</td> + </tr> + <tr align="center"> + <td>0x01</td> + <td colspan="3">0</td> + </tr> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4"><br>384<br><br></td> + </tr> + <tr align="center"> + <td colspan="4"><br>96<br><br></td> + </tr> + </tbody></table> + </td> + </tr> + <tr align="center"> + <td colspan="2">0</td> + <td colspan="2">0</td> + </tr> + <tr align="center"> + <td>0x00</td> + <td colspan="3">0</td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example1.h5 928 + +New address: 928 +Reading signature at address 928 (rel) +Object Header... +Dirty: 0 +Version: 1 +Header size (in bytes): 16 +Number of links: 1 +Number of messages (allocated): 2 (32) +Number of chunks (allocated): 1 (8) +Chunk 0... + Dirty: 0 + Address: 944 + Size in bytes: 32 +Message 0... + Message ID (sequence number): 0x0011 stab(0) + Shared message: No + Constant: Yes + Raw size in obj header: 16 bytes + Chunk number: 0 + Message Information: + B-tree address: 384 + Name heap address: 96 +Message 1... + Message ID (sequence number): 0x0000 null(0) + Shared message: No + Constant: No + Raw size in obj header: 0 bytes + Chunk number: 0 + Message Information: + <no info="" for="" this="" message=""> +</no></code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group Local Heap</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td>'H'</td> + <td>'E'</td> + <td>'A'</td> + <td>'P'</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4">256</td> + </tr> + + <tr align="center"> + <td colspan="4">8</td> + </tr> + + <tr align="center"> + <td colspan="4">128</td> + </tr> + </tbody></table> + </center> +<br> + +<pre> <code> +%h5debug example1.h5 96 + +New address: 96 +Reading signature at address 96 (rel) +Local Heap... +Dirty: 0 +Header size (in bytes): 32 +Address of heap data: 128 +Data bytes allocated on disk: 256 +Data bytes allocated in core: 256 +Free Blocks (offset, size): + Block #0: 8, 248 +Percent of heap used: 3.12% +Data follows (`__' indicates free region)... + 0: 00 00 00 00 00 00 00 00 __ __ __ __ __ __ __ __ ........ + 16: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 32: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 48: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 64: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 80: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 96: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 112: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 128: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 144: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 160: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 176: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 192: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 208: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 224: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 240: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + +</code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group B-tree</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + + </tr><tr align="center"> + <td>'T'</td> + <td>'R'</td> + <td>'E'</td> + <td>'E'</td> + + </tr><tr align="center"> + <td>0</td> + <td>0</td> + <td colspan="2">0</td> + + </tr><tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + + </tr></tbody></table> + </center> +<br> +<pre> <code> +%h5debug example1.h5 384 96 + +New address: 384 +Reading signature at address 384 (rel) +Tree type ID: H5B_SNODE_ID +Size of node: 544 +Size of raw (disk) key: 8 +Dirty flag: False +Number of initial dirty children: 0 +Level: 0 +Address of left sibling: UNDEF +Address of right sibling: UNDEF +Number of children (max): 0 (32) + +</code> </pre> + +<p></p> + +<p>Example program two - <em>Create a file with a single dataset in it</em>: +</p><pre> <code> +#include "hdf5.h" +#include <assert.h> + +int main() +{ + hid_t fid; /* File ID */ + hid_t sid; /* Dataspace ID */ + hid_t did; /* Dataset ID */ + herr_t ret; /* Generic return value */ + + /* Create the file */ + fid=H5Fcreate("example2.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert(fid>=0); + + /* Create a scalar dataspace for the dataset */ + sid=H5Screate(H5S_SCALAR); + assert(sid>=0); + + /* Create a trivial dataset */ + did=H5Dcreate(fid, "Dataset", H5T_NATIVE_INT, sid, H5P_DEFAULT); + assert(did>=0); + + /* Close the dataset */ + ret=H5Dclose(did); + assert(ret>=0); + + /* Close the dataspace */ + ret=H5Sclose(sid); + assert(ret>=0); + + /* Close the file */ + ret=H5Fclose(fid); + assert(ret>=0); + + return(0); +} +</assert.h></code> </pre> + + <center> + <table border="" align="center" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Super Block</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td>\211</td> + <td>'H'</td> + <td>'D'</td> + <td>'F'</td> + </tr> + + <tr align="center"> + <td>\r</td> + <td>\n</td> + <td>\032</td> + <td>\n</td> + </tr> + + <tr align="center"> + <td>0</td> + <td>0</td> + <td>0</td> + <td>0</td> + </tr> + + <tr align="center"> + <td>0</td> + <td>8</td> + <td>8</td> + <td>0</td> + </tr> + + <tr align="center"> + <td colspan="2">4</td> + <td colspan="2">16</td> + </tr> + + <tr align="center"> + <td colspan="4">0x00000003</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">?</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>928<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">H5G_CACHED_STAB (1)</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4"><br>384<br><br></td> + </tr> + <tr align="center"> + <td colspan="4"><br>96<br><br></td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 + +Reading signature at address 0 (rel) +File Super Block... +File name: example2.h5 +File access flags 0x00000000 +File open reference count: 1 +Address of super block: 0 (abs) +Size of user block: 0 bytes +Super block version number: 0 +Free list version number: 0 +Root group symbol table entry version number: 0 +Shared header version number: 0 +Size of file offsets (haddr_t type): 8 bytes +Size of file lengths (hsize_t type): 8 bytes +Symbol table leaf node 1/2 rank: 4 +Symbol table internal node 1/2 rank: 16 +File consistency flags: 0x00000003 +Base address: 0 (abs) +Free list address: UNDEF (rel) +Address of driver information block: UNDEF (rel) +Root group symbol table entry: + Name offset into private heap: 0 + Object header address: 928 + Dirty: Yes + Cache info type: Symbol Table + Cached entry information: + B-tree address: 384 + Heap address: 96 +</code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group Object Header</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="1" width="25%">1</td> + <td colspan="1" width="25%">0</td> + <td colspan="2" width="50%">2</td> + </tr> + <tr align="center"> + <td colspan="4">1</td> + </tr> + <tr align="center"> + <td colspan="4">32</td> + </tr> + <tr align="center"> + <td colspan="2">0x0011</td> + <td colspan="2">16</td> + </tr> + <tr align="center"> + <td>0x01</td> + <td colspan="3">0</td> + </tr> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4"><br>384<br><br></td> + </tr> + <tr align="center"> + <td colspan="4"><br>96<br><br></td> + </tr> + </tbody></table> + </td> + </tr> + <tr align="center"> + <td colspan="2">0</td> + <td colspan="2">0</td> + </tr> + <tr align="center"> + <td>0x00</td> + <td colspan="3">0</td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 928 + +New address: 928 +Reading signature at address 928 (rel) +Object Header... +Dirty: 0 +Version: 1 +Header size (in bytes): 16 +Number of links: 1 +Number of messages (allocated): 2 (32) +Number of chunks (allocated): 1 (8) +Chunk 0... + Dirty: 0 + Address: 944 + Size in bytes: 32 +Message 0... + Message ID: 0x0011 stab(0) + Shared message: No + Constant: Yes + Raw size in obj header: 16 bytes + Chunk number: 0 + Message Information: + B-tree address: 384 + Name heap address: 96 +Message 1... + Message ID: 0x0000 null(0) + Shared message: No + Constant: No + Raw size in obj header: 0 bytes + Chunk number: 0 + Message Information: + <no info="" for="" this="" message=""> +</no></code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group Local Heap</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td>'H'</td> + <td>'E'</td> + <td>'A'</td> + <td>'P'</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4">256</td> + </tr> + + <tr align="center"> + <td colspan="4">16</td> + </tr> + + <tr align="center"> + <td colspan="4">128</td> + </tr> + </tbody></table> + </center> +<br> + +<pre> <code> +%h5debug example2.h5 96 + +New address: 96 +Reading signature at address 96 (rel) +Local Heap... +Dirty: 0 +Header size (in bytes): 32 +Address of heap data: 128 +Data bytes allocated on disk: 256 +Data bytes allocated in core: 256 +Free Blocks (offset, size): + Block #0: 16, 240 +Percent of heap used: 6.25% +Data follows (`__' indicates free region)... + 0: 00 00 00 00 00 00 00 00 44 61 74 61 73 65 74 00 ........Dataset. + 16: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 32: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 48: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 64: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 80: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 96: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 112: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 128: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 144: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 160: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 176: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 192: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 208: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 224: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ + 240: __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ __ +</code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group B-tree</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + + </tr><tr align="center"> + <td>'T'</td> + <td>'R'</td> + <td>'E'</td> + <td>'E'</td> + + </tr><tr align="center"> + <td>0</td> + <td>0</td> + <td colspan="2">1</td> + + </tr><tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>0xffffffffffffffff<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>0<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>1248<br><br></td> + + </tr><tr align="center"> + <td colspan="4"><br>8<br><br></td> + + </tr></tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 384 96 + +New address: 384 +Reading signature at address 384 (rel) +Tree type ID: H5B_SNODE_ID +Size of node: 544 +Size of raw (disk) key: 8 +Dirty flag: False +Number of initial dirty children: 0 +Level: 0 +Address of left sibling: UNDEF +Address of right sibling: UNDEF +Number of children (max): 1 (32) +Child 0... + Address: 1248 + Left Key: + Heap offset: 0 + Name : + Right Key: + Heap offset: 8 + Name : Dataset +</code> </pre> + + <center> + <table border="" cellpadding="4" width="80%"> + <caption align="top"> + <strong>Root Group B-tree Symbol Table Node</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + + </tr><tr align="center"> + <td>'S'</td> + <td>'N'</td> + <td>'O'</td> + <td>'D'</td> + + </tr><tr align="center"> + <td>1</td> + <td>0</td> + <td colspan="2">1</td> + + </tr><tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td colspan="4">8</td> + </tr> + + <tr align="center"> + <td colspan="4"><br>976<br><br></td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4">0</td> + </tr> + + <tr align="center"> + <td colspan="4"><br><br>0<br><br><br></td> + </tr> + </tbody></table> + </td> + + </tr></tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 1248 96 + +New address: 1248 +Reading signature at address 1248 (rel) +Symbol Table Node... +Dirty: No +Size of Node (in bytes): 328 +Number of Symbols: 1 of 8 +Symbol 0: + Name: `Dataset' + Name offset into private heap: 8 + Object header address: 976 + Dirty: No + Cache info type: Nothing Cached +</code> </pre> + + <center> + <table border="" cellpadding="4" width="90%"> + <caption align="top"> + <strong>'/Dataset' Object Header</strong> + </caption> + + <tbody><tr align="center"> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + <th width="25%">byte</th> + </tr> + + <tr align="center"> + <td colspan="1"><em>Version:</em> 1</td> + <td colspan="1"><em>Reserved:</em> 0</td> + <td colspan="2"><em>Number of Header Messages:</em> 6</td> + </tr> + <tr align="center"> + <td colspan="4"><em>Object Reference Count:</em> 1</td> + </tr> + <tr align="center"> + <td colspan="4"><em>Total Object Header Size:</em> 256</td> + </tr> +<!-- Fill Value Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top" color="#80FFFF"> + <em>Fill Value Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0005</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 8</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x01</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Version --> + <td colspan="1"><em>Version:</em> 1</td> + <!-- Space Allocation Time --> + <td colspan="1"><em>Space Allocation Time:</em> 2 (Late)</td> + <!-- Fill Value Writing Time --> + <td colspan="1"><em>Fill Value Writing Time:</em> 0 (At allocation)</td> + <!-- Fill Value Defined --> + <td colspan="1"><em>Fill Value Defined:</em> 0 (Undefined)</td> + </tr> + <tr align="center"> + <!-- Fill Value Datatype Size --> + <td colspan="4"><em>Fill Value Datatype Size:</em> 0 (Use dataset's datatype for fill-value datatype)</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Datatype Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Datatype Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0003</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 16</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x01</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Type Class and Version --> + <td colspan="1"> + <table border="" width="100%"> + <tbody><tr align="center"> + <td width="50%"><em>Version:</em> 0x1</td> + <td><em>Class:</em> 0x0 (Fixed-Point)</td> + </tr> + </tbody></table> + </td> + <!-- Class Bit Field --> + <td colspan="3"><em>Fixed-Point Bit-Field:</em> 0x08 (Little-endian, No padding, Signed)</td> + </tr> + <tr align="center"> + <!-- Type Size (in bytes) --> + <td colspan="4"><em>Size:</em> 4</td> + </tr> + <tr align="center"> + <!-- Bit Offset --> + <td colspan="2"><em>Bit Offset:</em> 0</td> + <!-- Bit Precision --> + <td colspan="2"><em>Bit Precision:</em> 32</td> + </tr> + <tr align="center"> + <!-- Message alignment filler --> + <td colspan="4"><em>Message Alignment Filler:</em> -</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Dataspace Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Dataspace Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0001</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 8</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x00</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Version --> + <td colspan="1"><em>Version:</em> 1</td> + <!-- Rank --> + <td colspan="1"><em>Rank:</em> 0 (Scalar)</td> + <!-- Flags --> + <td colspan="1"><em>Flags:</em> 0x00 (No maximum dimensions, no permutation information)</td> + <!-- Reserved --> + <td colspan="1"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Reserved --> + <td colspan="4"><em>Reserved:</em> 0</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Layout Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Layout Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0008</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 24</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x00</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Version --> + <td colspan="1"><em>Version:</em> 1</td> + <!-- Rank --> + <td colspan="1"><em>Rank:</em> 1 (Dataspace rank+1)</td> + <!-- Class --> + <td colspan="1"><em>Class:</em> 1 (Contiguous)</td> + <!-- Reserved --> + <td colspan="1"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Reserved --> + <td colspan="4"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Address --> + <td colspan="4"><br><em>Address:</em> 0xffffffffffffffff (Undefined)<br><br></td> + </tr> + <tr align="center"> + <!-- Layout Dimensions --> + <td colspan="4"><em>Dimension 0 Size:</em> 4 (Datatype size)</td> + </tr> + <tr align="center"> + <!-- Message alignment filler --> + <td colspan="4"><em>Message Alignment Filler:</em> -</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Modification Date & Time Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Modification Date & Time Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <!-- Object Header Message Type --> + <td colspan="2"><em>Message Type:</em> 0x0012</td> + <!-- Object Header Message Length --> + <td colspan="2"><em>Message Data Size:</em> 8</td> + </tr> + <tr align="center"> + <!-- Object Header Message Flags --> + <td colspan="1"><em>Flags:</em> 0x00</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Version --> + <td colspan="1"><em>Version:</em> 1</td> + <!-- Reserved --> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + <tr align="center"> + <!-- Modification time --> + <td colspan="4"><em>Seconds Since Epoch:</em> 1052401700 (2003-05-08 08:48:20 CDT)</td> + </tr> + </tbody></table> + </td> + </tr> +<!-- Null Header Message --> + <tr align="center"> + <td colspan="4"> + <table border="" width="100%"> + <caption align="top"> + <em>Null Header Message</em> + </caption> + <tbody><tr align="center"> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + <th width="25%"></th> + </tr> + + <tr align="center"> + <td colspan="2"><em>Message Type:</em> 0x0000</td> + <td colspan="2"><em>Message Data Size:</em> 144</td> + </tr> + <tr align="center"> + <td colspan="1"><em>Flags:</em> 0x00</td> + <td colspan="3"><em>Reserved:</em> 0</td> + </tr> + </tbody></table> + </td> + </tr> + </tbody></table> + </center> +<br> +<pre> <code> +%h5debug example2.h5 976 + +New address: 976 +Reading signature at address 976 (rel) +Object Header... +Dirty: 0 +Version: 1 +Header size (in bytes): 16 +Number of links: 1 +Number of messages (allocated): 6 (32) +Number of chunks (allocated): 1 (8) +Chunk 0... + Dirty: 0 + Address: 992 + Size in bytes: 256 +Message 0... + Message ID (sequence number): 0x0005 `fill_new' (0) + Shared: No + Constant: Yes + Raw size in obj header: 8 bytes + Chunk number: 0 + Message Information: + Version: 1 + Space Allocation Time: Late + Fill Time: On Allocation + Fill Value Defined: Undefined + Size: 0 + Data type: <dataset type=""> +Message 1... + Message ID (sequence number): 0x0003 data_type(0) + Shared message: No + Constant: Yes + Raw size in obj header: 16 bytes + Chunk number: 0 + Message Information: + Type class: integer + Size: 4 bytes + Byte order: little endian + Precision: 32 bits + Offset: 0 bits + Low pad type: zero + High pad type: zero + Sign scheme: 2's comp +Message 2... + Message ID (sequence number): 0x0001 simple_dspace(0) + Shared message: No + Constant: No + Raw size in obj header: 8 bytes + Chunk number: 0 + Message Information: + Rank: 0 +Message 3... + Message ID (sequence number): 0x0008 layout(0) + Shared message: No + Constant: No + Raw size in obj header: 24 bytes + Chunk number: 0 + Message Information: + Data address: UNDEF + Number of dimensions: 1 + Size: {4} +Message 4... + Message ID (sequence number): 0x0012 mtime_new(0) + Shared message: No + Constant: No + Raw size in obj header: 8 bytes + Chunk number: 0 + Message Information: + Time: 2003-03-05 14:52:00 CST +Message 5... + Message ID (sequence number): 0x0000 null(0) + Shared message: No + Constant: No + Raw size in obj header: 144 bytes + Chunk number: 0 + Message Information: + <no info="" for="" this="" message=""> +</no></dataset></code> </pre> + +<p></p> + +</li></ol> + + + +</body></html> diff --git a/doxygen/examples/Filters.html b/doxygen/examples/Filters.html new file mode 100644 index 0000000..2d5bc5e --- /dev/null +++ b/doxygen/examples/Filters.html @@ -0,0 +1,450 @@ +<html> + <head> + <title>Filters</title> + <h1>Filters in HDF5</h1> + + <b>Note: Transient pipelines described in this document have not + been implemented.</b> + + <h2>Introduction</h2> + + <p>HDF5 allows chunked data to pass through user-defined filters + on the way to or from disk. The filters operate on chunks of an + <code>H5D_CHUNKED</code> dataset can be arranged in a pipeline + so output of one filter becomes the input of the next filter. + + </p><p>Each filter has a two-byte identification number (type + <code>H5Z_filter_t</code>) allocated by The HDF Group and can also be + passed application-defined integer resources to control its + behavior. Each filter also has an optional ASCII comment + string. + + </p> + <table> + <tbody><tr> + <th>Values for <code>H5Z_filter_t</code></th> + <th>Description</th> + </tr> + + <tr valign="top"> + <td><code>0-255</code></td> + <td>These values are reserved for filters predefined and + registered by the HDF5 library and of use to the general + public. They are described in a separate section + below.</td> + </tr> + + <tr valign="top"> + <td><code>256-511</code></td> + <td>Filter numbers in this range are used for testing only + and can be used temporarily by any organization. No + attempt is made to resolve numbering conflicts since all + definitions are by nature temporary.</td> + </tr> + + <tr valign="top"> + <td><code>512-65535</code></td> + <td>Reserved for future assignment. Please contact the + <a href="mailto:help@hdfgroup.org">HDF5 development team</a> + to reserve a value or range of values for + use by your filters.</td> + </tr></tbody></table> + + <h2>Defining and Querying the Filter Pipeline</h2> + + <p>Two types of filters can be applied to raw data I/O: permanent + filters and transient filters. The permanent filter pipeline is + defned when the dataset is created while the transient pipeline + is defined for each I/O operation. During an + <code>H5Dwrite()</code> the transient filters are applied first + in the order defined and then the permanent filters are applied + in the order defined. For an <code>H5Dread()</code> the + opposite order is used: permanent filters in reverse order, then + transient filters in reverse order. An <code>H5Dread()</code> + must result in the same amount of data for a chunk as the + original <code>H5Dwrite()</code>. + + </p><p>The permanent filter pipeline is defined by calling + <code>H5Pset_filter()</code> for a dataset creation property + list while the transient filter pipeline is defined by calling + that function for a dataset transfer property list. + + </p><dl> + <dt><code>herr_t H5Pset_filter (hid_t <em>plist</em>, + H5Z_filter_t <em>filter</em>, unsigned int <em>flags</em>, + size_t <em>cd_nelmts</em>, const unsigned int + <em>cd_values</em>[])</code> + </dt><dd>This function adds the specified <em>filter</em> and + corresponding properties to the end of the transient or + permanent output filter pipeline (depending on whether + <em>plist</em> is a dataset creation or dataset transfer + property list). The <em>flags</em> argument specifies certain + general properties of the filter and is documented below. The + <em>cd_values</em> is an array of <em>cd_nelmts</em> integers + which are auxiliary data for the filter. The integer values + will be stored in the dataset object header as part of the + filter information. + </dd><dt><code>int H5Pget_nfilters (hid_t <em>plist</em>)</code> + </dt><dd>This function returns the number of filters defined in the + permanent or transient filter pipeline depending on whether + <em>plist</em> is a dataset creation or dataset transfer + property list. In each pipeline the filters are numbered from + 0 through <em>N</em>-1 where <em>N</em> is the value returned + by this function. During output to the file the filters of a + pipeline are applied in increasing order (the inverse is true + for input). Zero is returned if there are no filters in the + pipeline and a negative value is returned for errors. + </dd><dt><code>H5Z_filter_t H5Pget_filter (hid_t <em>plist</em>, + int <em>filter_number</em>, unsigned int *<em>flags</em>, + size_t *<em>cd_nelmts</em>, unsigned int + *<em>cd_values</em>, size_t namelen, char name[])</code> + </dt><dd>This is the query counterpart of + <code>H5Pset_filter()</code> and returns information about a + particular filter number in a permanent or transient pipeline + depending on whether <em>plist</em> is a dataset creation or + dataset transfer property list. On input, <em>cd_nelmts</em> + indicates the number of entries in the <em>cd_values</em> + array allocated by the caller while on exit it contains the + number of values defined by the filter. The + <em>filter_number</em> should be a value between zero and + <em>N</em>-1 as described for <code>H5Pget_nfilters()</code> + and the function will return failure (a negative value) if the + filter number is out of range. If <em>name</em> is a pointer + to an array of at least <em>namelen</em> bytes then the filter + name will be copied into that array. The name will be null + terminated if the <em>namelen</em> is large enough. The + filter name returned will be the name appearing in the file or + else the name registered for the filter or else an empty string. + </dd></dl> + + <p>The flags argument to the functions above is a bit vector of + the following fields: + + </p> + <table> + <tbody><tr> + <th>Values for <em>flags</em></th> + <th>Description</th> + </tr> + + <tr valign="top"> + <td><code>H5Z_FLAG_OPTIONAL</code></td> + <td>If this bit is set then the filter is optional. If + the filter fails (see below) during an + <code>H5Dwrite()</code> operation then the filter is + just excluded from the pipeline for the chunk for which + it failed; the filter will not participate in the + pipeline during an <code>H5Dread()</code> of the chunk. + This is commonly used for compression filters: if the + compression result would be larger than the input then + the compression filter returns failure and the + uncompressed data is stored in the file. If this bit is + clear and a filter fails then the + <code>H5Dwrite()</code> or <code>H5Dread()</code> also + fails.</td> + </tr> + </tbody></table> + + <h2>Defining Filters</h2> + + <p>Each filter is bidirectional, handling both input and output to + the file, and a flag is passed to the filter to indicate the + direction. In either case the filter reads a chunk of data from + a buffer, usually performs some sort of transformation on the + data, places the result in the same or new buffer, and returns + the buffer pointer and size to the caller. If something goes + wrong the filter should return zero to indicate a failure. + + </p><p>During output, a filter that fails or isn't defined and is + marked as optional is silently excluded from the pipeline and + will not be used when reading that chunk of data. A required + filter that fails or isn't defined causes the entire output + operation to fail. During input, any filter that has not been + excluded from the pipeline during output and fails or is not + defined will cause the entire input operation to fail. + + </p><p>Filters are defined in two phases. The first phase is to + define a function to act as the filter and link the function + into the application. The second phase is to register the + function, associating the function with an + <code>H5Z_filter_t</code> identification number and a comment. + + </p><dl> + <dt><code>typedef size_t (*H5Z_func_t)(unsigned int + <em>flags</em>, size_t <em>cd_nelmts</em>, const unsigned int + <em>cd_values</em>[], size_t <em>nbytes</em>, size_t + *<em>buf_size</em>, void **<em>buf</em>)</code> + </dt><dd>The <em>flags</em>, <em>cd_nelmts</em>, and + <em>cd_values</em> are the same as for the + <code>H5Pset_filter()</code> function with the additional flag + <code>H5Z_FLAG_REVERSE</code> which is set when the filter is + called as part of the input pipeline. The input buffer is + pointed to by <em>*buf</em> and has a total size of + <em>*buf_size</em> bytes but only <em>nbytes</em> are valid + data. The filter should perform the transformation in place if + possible and return the number of valid bytes or zero for + failure. If the transformation cannot be done in place then + the filter should allocate a new buffer with + <code>malloc()</code> and assign it to <em>*buf</em>, + assigning the allocated size of that buffer to + <em>*buf_size</em>. The old buffer should be freed + by calling <code>free()</code>. + + <br><br> + </dd><dt><code>herr_t H5Zregister (H5Z_filter_t <em>filter_id</em>, + const char *<em>comment</em>, H5Z_func_t + <em>filter</em>)</code> + </dt><dd>The <em>filter</em> function is associated with a filter + number and a short ASCII comment which will be stored in the + hdf5 file if the filter is used as part of a permanent + pipeline during dataset creation. + </dd></dl> + + <h2>Predefined Filters</h2> + + <p>If <code>zlib</code> version 1.1.2 or later was found + during configuration then the library will define a filter whose + <code>H5Z_filter_t</code> number is + <code>H5Z_FILTER_DEFLATE</code>. Since this compression method + has the potential for generating compressed data which is larger + than the original, the <code>H5Z_FLAG_OPTIONAL</code> flag + should be turned on so such cases can be handled gracefully by + storing the original data instead of the compressed data. The + <em>cd_nvalues</em> should be one with <em>cd_value[0]</em> + being a compression agression level between zero and nine, + inclusive (zero is the fastest compression while nine results in + the best compression ratio). + + </p><p>A convenience function for adding the + <code>H5Z_FILTER_DEFLATE</code> filter to a pipeline is: + + </p><dl> + <dt><code>herr_t H5Pset_deflate (hid_t <em>plist</em>, unsigned + <em>aggression</em>)</code> + </dt><dd>The deflate compression method is added to the end of the + permanent or transient filter pipeline depending on whether + <em>plist</em> is a dataset creation or dataset transfer + property list. The <em>aggression</em> is a number between + zero and nine (inclusive) to indicate the tradeoff between + speed and compression ratio (zero is fastest, nine is best + ratio). + </dd></dl> + + <p>Even if the <code>zlib</code> isn't detected during + configuration the application can define + <code>H5Z_FILTER_DEFLATE</code> as a permanent filter. If the + filter is marked as optional (as with + <code>H5Pset_deflate()</code>) then it will always fail and be + automatically removed from the pipeline. Applications that read + data will fail only if the data is actually compressed; they + won't fail if <code>H5Z_FILTER_DEFLATE</code> was part of the + permanent output pipeline but was automatically excluded because + it didn't exist when the data was written. + + </p><p><code>zlib</code> can be acquired from + <code><a href="http://www.cdrom.com/pub/infozip/zlib/"> + http://www.cdrom.com/pub/infozip/zlib/</a></code>. + + </p><h2>Example</h2> + + <p>This example shows how to define and register a simple filter + that adds a checksum capability to the data stream. + + </p><p>The function that acts as the filter always returns zero + (failure) if the <code>md5()</code> function was not detected at + configuration time (left as an excercise for the reader). + Otherwise the function is broken down to an input and output + half. The output half calculates a checksum, increases the size + of the output buffer if necessary, and appends the checksum to + the end of the buffer. The input half calculates the checksum + on the first part of the buffer and compares it to the checksum + already stored at the end of the buffer. If the two differ then + zero (failure) is returned, otherwise the buffer size is reduced + to exclude the checksum. + + </p> + <table> + <tbody><tr> + <td> + <p><code></code></p><pre><code> + size_t + md5_filter(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf) + { + #ifdef HAVE_MD5 + unsigned char cksum[16]; + + if (flags & H5Z_REVERSE) { + /* Input */ + assert(nbytes>=16); + md5(nbytes-16, *buf, cksum); + + /* Compare */ + if (memcmp(cksum, (char*)(*buf)+nbytes-16, 16)) { + return 0; /*fail*/ + } + + /* Strip off checksum */ + return nbytes-16; + + } else { + /* Output */ + md5(nbytes, *buf, cksum); + + /* Increase buffer size if necessary */ + if (nbytes+16>*buf_size) { + *buf_size = nbytes + 16; + *buf = realloc(*buf, *buf_size); + } + + /* Append checksum */ + memcpy((char*)(*buf)+nbytes, cksum, 16); + return nbytes+16; + } + #else + return 0; /*fail*/ + #endif + } + </code></pre> + </td> + </tr> + </tbody></table> + + <p>Once the filter function is defined it must be registered so + the HDF5 library knows about it. Since we're testing this + filter we choose one of the <code>H5Z_filter_t</code> numbers + from the reserved range. We'll randomly choose 305. + + </p><p> + </p> + <table> + <tbody><tr> + <td> + <p><code></code></p><pre><code> + #define FILTER_MD5 305 + herr_t status = H5Zregister(FILTER_MD5, "md5 checksum", md5_filter); + </code></pre> + </td> + </tr> + </tbody></table> + + <p>Now we can use the filter in a pipeline. We could have added + the filter to the pipeline before defining or registering the + filter as long as the filter was defined and registered by time + we tried to use it (if the filter is marked as optional then we + could have used it without defining it and the library would + have automatically removed it from the pipeline for each chunk + written before the filter was defined and registered). + + </p><p> + </p> + <table> + <tbody><tr> + <td> + <p><code></code></p><pre><code> + hid_t dcpl = H5Pcreate(H5P_DATASET_CREATE); + hsize_t chunk_size[3] = {10,10,10}; + H5Pset_chunk(dcpl, 3, chunk_size); + H5Pset_filter(dcpl, FILTER_MD5, 0, 0, NULL); + hid_t dset = H5Dcreate(file, "dset", H5T_NATIVE_DOUBLE, space, dcpl); + </code></pre> + </td> + </tr> + </tbody></table> + + <h2>6. Filter Diagnostics</h2> + + <p>If the library is compiled with debugging turned on for the H5Z + layer (usually as a result of <code>configure + --enable-debug=z</code>) then filter statistics are printed when + the application exits normally or the library is closed. The + statistics are written to the standard error stream and include + two lines for each filter that was used: one for input and one + for output. The following fields are displayed: + + </p><p> + </p> + <table> + <tbody><tr> + <th>Field Name</th> + <th>Description</th> + </tr> + + <tr valign="top"> + <td>Method</td> + <td>This is the name of the method as defined with + <code>H5Zregister()</code> with the charaters + "< or ">" prepended to indicate + input or output.</td> + </tr> + + <tr valign="top"> + <td>Total</td> + <td>The total number of bytes processed by the filter + including errors. This is the maximum of the + <em>nbytes</em> argument or the return value. + </td></tr> + + <tr valign="top"> + <td>Errors</td> + <td>This field shows the number of bytes of the Total + column which can be attributed to errors.</td> + </tr> + + <tr valign="top"> + <td>User, System, Elapsed</td> + <td>These are the amount of user time, system time, and + elapsed time in seconds spent in the filter function. + Elapsed time is sensitive to system load. These times + may be zero on operating systems that don't support the + required operations.</td> + </tr> + + <tr valign="top"> + <td>Bandwidth</td> + <td>This is the filter bandwidth which is the total + number of bytes processed divided by elapsed time. + Since elapsed time is subject to system load the + bandwidth numbers cannot always be trusted. + Furthermore, the bandwidth includes bytes attributed to + errors which may significanly taint the value if the + function is able to detect errors without much + expense.</td> + </tr> + </tbody></table> + + <p> + </p> + <table> + <caption align="bottom"> + <b>Example: Filter Statistics</b> + </caption> + <tbody><tr> + <td> + <p><code></code></p><pre><code>H5Z: filter statistics accumulated ov= + er life of library: + Method Total Errors User System Elapsed Bandwidth + ------ ----- ------ ---- ------ ------- --------- + >deflate 160000 40000 0.62 0.74 1.33 117.5 kBs + <deflate 120000 0 0.11 0.00 0.12 1.000 MBs + </code></pre> + </td> + </tr> + </tbody></table> + + <hr> + + + <p><a name="fn1">Footnote 1:</a> Dataset chunks can be compressed + through the use of filters. Developers should be aware that + reading and rewriting compressed chunked data can result in holes + in an HDF5 file. In time, enough such holes can increase the + file size enough to impair application or library performance + when working with that file. See + <a href="https://support.hdfgroup.org/HDF5/doc1.6/Performance.html#Freespace"> + Freespace Management</a> + in the chapter + <a href="https://support.hdfgroup.org/HDF5/doc1.6/Performance.html"> + Performance Analysis and Issues</a>.</p> +</html> diff --git a/doxygen/examples/IOFlow.html b/doxygen/examples/IOFlow.html new file mode 100644 index 0000000..6b2c27e --- /dev/null +++ b/doxygen/examples/IOFlow.html @@ -0,0 +1,137 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN"> +<!-- saved from url=(0064)https://gamma.hdfgroup.org/papers/HISS/030821.IOFlow/IOFlow.html --> +<html><head><meta http-equiv="Content-Type" content="text/html; charset=UTF-8"> + <title>HDF5 Raw I/O Flow Notes</title> + + <meta name="author" content="Quincey Koziol"> +</head> + +<body text="#000000" bgcolor="#FFFFFF"> + +<style type="text/css"> +OL.loweralpha { list-style-type: lower-alpha } +OL.upperroman { list-style-type: upper-roman } +</style> + +<center><h1>HDF5 Raw I/O Flow Notes</h1></center> +<center><h3>Quincey Koziol<br> + koziol@ncsa.uiuc.edu<br> + August 20, 2003 +</h3></center> + +<ol class="upperroman"> + +<li><h3><u>Document's Audience:</u></h3> + +<ul> + <li>Current H5 library designers and knowledgable external developers.</li> +</ul> + +</li><li><h3><u>Background Reading:</u></h3> + +</li><li><h3><u>Introduction:</u></h3> + +<dl> + <dt><strong>What is this document about?</strong></dt> + <dd>This document attempts to supplement the flow charts describing + the flow of control for raw data I/O in the library. + </dd> <br> +</dl> + +</li><li><h3><u>Figures:</u></h3> +<p>The following figures provide the main information:</p> + <table> + <tr><td><img src="IOFlow.gif" alt="High-Level View of Writing Raw Data" style="height:50%;"></td></tr> + <tr><td><img src="IOFlow2.gif" alt="Perform Serial or Parallel I/O" style="height:50%;"></td></tr> + <tr><td><img src="IOFlow3.gif" alt="Gather/Convert/Scatter" style="height:50%;"></td></tr> + </table> + +</li><li><h3><u>Notes From Accompanying Figures:</u></h3> + +<p>This section provides notes to augment the information in the accompanying + figures. +</p> + +<ol> + <li><b>Validate Parameters</b> - Resolve any H5S_ALL parameters + for dataspace selections to actual dataspaces, allocate + conversion buffers, etc. + </li> + + <li><b>Space Allocated in File?</b> - Space may not have been allocated + in the file to store the dataset data, if "late allocation" was chosen + for the allocation time when the dataset was created. + </li> + + <li><b>Allocate & Fill Space</b> - These operations allocate both contiguous + and chunked dataset's space in the file. The chunked dataset space + allocation iterates through all the chunks in the file and allocates + both the B-tree information and the raw data in the file. Because of + the way filters work, fill-values are written out for chunked datasets + as they are allocated, instead of as a separate step. + In parallel + I/O, the chunked dataset allocation can potentially be time-consuming, + since all the raw data in the dataset is allocated from one process. + </li> + + <li><b>Datatype Conversion Needed?</b> - This currently is the deciding + factor between doing "direct I/O" (in serial or parallel) and needing + to perform gather/convert/scatter operations. I believe that MPI + is capable of performing a limited range of type conversions and if so, + we should add support to detect when they can be used. This will + allow more I/O operations to be performed collectively. + </li> + + <li><b>Collective I/O Requested/Allowed?</b> - A user has to both request + that collective I/O occur and also their I/O operation must meet the + requirements that the library sets for supporting collective parallel + I/O: + <ul> + <li>The dataspace must be scalar or simple (which is a no-op really, + since we don't support "complex" dataspaces in the library + currently). + </li> + <li>The selection must be regular. "all" selections + and hyperslab selections that were + made with only one call to H5Sselect_hyperslab() (i.e. not a + hyperslab selection that has been aggregated over multiple + selection calls) are regular. Supporting point and + irregular hyperslab selections are on the "to do" list. + </li> + <li>The dataset must be stored contiguously on disk (as shown in the + figure also). Supporting chunked dataset storage is also + on the "to do" list. + </li> + </ul> + </li> + + <li><b>Build "chunk map"</b> - This step still has some scalability issues + as it creates a data structure that is proportional to the number of + chunks which will be written to, which could potentially be very large. + Building the "chunk map" information incrementally is on the "to do" + list also. + </li> + + <li><b>Perform Chunked I/O</b> - As the figure shows, there is no support + for collective parallel I/O on chunked datasets currently. As noted + earlier, this is on the "to do" list. + </li> + + <li><b>Perform "Direct" Serial I/O</b> - "Direct" serial I/O writes data + from the application's buffer, without any intervening buffer or memory + copies. For maximum efficiency and performance, the elements in the + selections should be adjoining. + </li> + + <li><b>Perform Collective Parallel I/O</b> - This step also writes data + directly from an application buffer, but additionally uses collective + MPI I/O operations to combine the data from each process in the parallel + application in an efficient manner. + </li> +</ol> + +</li></ol> + + + +</body></html> diff --git a/doxygen/hdf5doxy_layout.xml b/doxygen/hdf5doxy_layout.xml index fc20aa1..6efa690 100644 --- a/doxygen/hdf5doxy_layout.xml +++ b/doxygen/hdf5doxy_layout.xml @@ -12,6 +12,7 @@ <tab type="user" url="@ref TN" title="Technical Notes" /> <tab type="user" url="@ref SPEC" title="Specifications" /> <tab type="user" url="@ref RFC" title="RFCs" /> + <tab type="user" url="@ref FTS" title="Full-Text Search" /> <tab type="user" url="@ref About" title="About" /> </navindex> diff --git a/doxygen/img/IOFlow.gif b/doxygen/img/IOFlow.gif Binary files differnew file mode 100644 index 0000000..3e79030 --- /dev/null +++ b/doxygen/img/IOFlow.gif diff --git a/doxygen/img/IOFlow2.gif b/doxygen/img/IOFlow2.gif Binary files differnew file mode 100644 index 0000000..c75ca79 --- /dev/null +++ b/doxygen/img/IOFlow2.gif diff --git a/doxygen/img/IOFlow3.gif b/doxygen/img/IOFlow3.gif Binary files differnew file mode 100644 index 0000000..316cd1e --- /dev/null +++ b/doxygen/img/IOFlow3.gif diff --git a/fortran/src/H5_f.c b/fortran/src/H5_f.c index d245cae..0277e5c 100644 --- a/fortran/src/H5_f.c +++ b/fortran/src/H5_f.c @@ -118,24 +118,20 @@ h5init_types_c(hid_t_f *types, hid_t_f *floatingtypes, hid_t_f *integertypes) if ((types[6] = (hid_t_f)H5Tcopy(H5T_NATIVE_DOUBLE)) < 0) return ret_value; } /* end if */ -#if H5_SIZEOF_LONG_DOUBLE != 0 else if (sizeof(real_f) == sizeof(long double)) { if ((types[6] = (hid_t_f)H5Tcopy(H5T_NATIVE_LDOUBLE)) < 0) return ret_value; } /* end else */ -#endif /* Find appropriate size to store Fortran DOUBLE */ if (sizeof(double_f) == sizeof(double)) { if ((types[7] = (hid_t_f)H5Tcopy(H5T_NATIVE_DOUBLE)) < 0) return ret_value; } /*end if */ -#if H5_SIZEOF_LONG_DOUBLE != 0 else if (sizeof(double_f) == sizeof(long double)) { if ((types[7] = (hid_t_f)H5Tcopy(H5T_NATIVE_LDOUBLE)) < 0) return ret_value; } /*end else */ -#endif #ifdef H5_HAVE_FLOAT128 else if (sizeof(double_f) == sizeof(__float128)) { if ((types[7] = H5Tcopy(H5T_NATIVE_FLOAT)) < 0) @@ -169,12 +165,10 @@ h5init_types_c(hid_t_f *types, hid_t_f *floatingtypes, hid_t_f *integertypes) if ((types[11] = (hid_t_f)H5Tcopy(H5T_NATIVE_DOUBLE)) < 0) return ret_value; } /*end if */ -#if H5_SIZEOF_LONG_DOUBLE != 0 else if (sizeof(real_C_FLOAT_f) == sizeof(long double)) { if ((types[11] = (hid_t_f)H5Tcopy(H5T_NATIVE_LDOUBLE)) < 0) return ret_value; } /*end else */ -#endif /* * FIND H5T_NATIVE_REAL_C_DOUBLE */ @@ -186,12 +180,10 @@ h5init_types_c(hid_t_f *types, hid_t_f *floatingtypes, hid_t_f *integertypes) if ((types[12] = (hid_t_f)H5Tcopy(H5T_NATIVE_DOUBLE)) < 0) return ret_value; } /*end if */ -#if H5_SIZEOF_LONG_DOUBLE != 0 else if (sizeof(real_C_DOUBLE_f) == sizeof(long double)) { if ((types[12] = (hid_t_f)H5Tcopy(H5T_NATIVE_LDOUBLE)) < 0) return ret_value; } /*end else */ -#endif /* * FIND H5T_NATIVE_REAL_C_LONG_DOUBLE */ diff --git a/fortran/test/tH5A.F90 b/fortran/test/tH5A.F90 index 4d56bed..d5ce9a2 100644 --- a/fortran/test/tH5A.F90 +++ b/fortran/test/tH5A.F90 @@ -408,13 +408,13 @@ CONTAINS !open the INTEGER attrbute by name ! CALL h5aopen_name_f(dset_id, aname5, attr5_id, error) - CALL check("h5aopen_idx_f",error,total_error) + CALL check("h5aopen_name_f",error,total_error) ! !open the NULL attrbute by name ! CALL h5aopen_name_f(dset_id, aname6, attr6_id, error) - CALL check("h5aopen_idx_f",error,total_error) + CALL check("h5aopen_name_f",error,total_error) ! !get the attrbute name diff --git a/hl/CMakeLists.txt b/hl/CMakeLists.txt index 5061c6c..9391231 100644 --- a/hl/CMakeLists.txt +++ b/hl/CMakeLists.txt @@ -2,6 +2,14 @@ cmake_minimum_required (VERSION 3.12) project (HDF5_HL C) #----------------------------------------------------------------------------- +# Option to use new-style references with dimension scale APIs +#----------------------------------------------------------------------------- +option (HDF5_DIMENSION_SCALES_NEW_REF "Use new-style references with dimension scale APIs" OFF) +if (HDF5_DIMENSION_SCALES_NEW_REF) + set (H5_DIMENSION_SCALES_WITH_NEW_REF 1) +endif () + +#----------------------------------------------------------------------------- # List Source files #----------------------------------------------------------------------------- diff --git a/hl/fortran/src/H5LTfc.c b/hl/fortran/src/H5LTfc.c index e87a9d3..4cb9265 100644 --- a/hl/fortran/src/H5LTfc.c +++ b/hl/fortran/src/H5LTfc.c @@ -319,11 +319,9 @@ h5ltset_attribute_c(hid_t_f *loc_id, size_t_f *namelen, _fcd dsetname, size_t_f else if ((size_t)*sizeof_val == sizeof(double)) ret = H5LT_set_attribute_numerical(c_loc_id, c_name, c_attrname, c_size, H5T_NATIVE_DOUBLE, (const double *)buf); -#if H5_SIZEOF_LONG_DOUBLE != 0 else if ((size_t)*sizeof_val == sizeof(long double)) ret = H5LT_set_attribute_numerical(c_loc_id, c_name, c_attrname, c_size, H5T_NATIVE_LDOUBLE, (const long double *)buf); -#endif else goto done; } @@ -413,10 +411,8 @@ h5ltget_attribute_c(hid_t_f *loc_id, size_t_f *namelen, _fcd dsetname, size_t_f ret = H5LTget_attribute(c_loc_id, c_name, c_attrname, H5T_NATIVE_FLOAT, buf); else if ((size_t)*sizeof_val == sizeof(double)) ret = H5LTget_attribute(c_loc_id, c_name, c_attrname, H5T_NATIVE_DOUBLE, buf); -#if H5_SIZEOF_LONG_DOUBLE != 0 else if ((size_t)*sizeof_val == sizeof(long double)) ret = H5LTget_attribute(c_loc_id, c_name, c_attrname, H5T_NATIVE_LDOUBLE, buf); -#endif else goto done; } diff --git a/hl/src/H5DS.c b/hl/src/H5DS.c index 2bd4046..c947d16 100644 --- a/hl/src/H5DS.c +++ b/hl/src/H5DS.c @@ -18,7 +18,40 @@ /* Local routines */ static herr_t H5DS_is_reserved(hid_t did); -static hid_t H5DS_get_REFLIST_type(void); + +/*------------------------------------------------------------------------- + * Function: H5DSwith_new_ref + * + * Purpose: Determines if new references are used with dimension scales. + * The function H5DSwith_new_ref takes any object identifier and checks + * if new references are used for dimension scales. Currently, + * new references are used when non-native VOL connector is used or when + * H5_DIMENSION_SCALES_WITH_NEW_REF is set up via configure option. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5DSwith_new_ref(hid_t obj_id, hbool_t *with_new_ref) +{ + hbool_t config_flag = FALSE; + hbool_t native = FALSE; + + if (!with_new_ref) + return FAIL; + + if (H5VLobject_is_native(obj_id, &native) < 0) + return FAIL; + +#ifdef H5_DIMENSION_SCALES_WITH_NEW_REF + config_flag = TRUE; +#endif + + *with_new_ref = (config_flag || !native); + + return SUCCEED; +} /*------------------------------------------------------------------------- * Function: H5DSset_scale @@ -106,22 +139,34 @@ H5DSset_scale(hid_t dsid, const char *dimname) herr_t H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) { - int has_dimlist; - int has_reflist; - int is_ds; - hssize_t nelmts; - hid_t sid; /* space ID */ - hid_t tid = -1; /* attribute type ID */ - hid_t ntid = -1; /* attribute native type ID */ - hid_t aid = -1; /* attribute ID */ - int rank; /* rank of dataset */ - hsize_t dims[1]; /* dimension of the "REFERENCE_LIST" array */ - ds_list_t dsl; /* attribute data in the DS pointing to the dataset */ - ds_list_t * dsbuf = NULL; /* array of attribute data in the DS pointing to the dataset */ - hobj_ref_t ref_to_ds; /* reference to the DS */ - hobj_ref_t ref_j; /* iterator reference */ - hvl_t * buf = NULL; /* VL buffer to store in the attribute */ - hid_t dsid_j; /* DS dataset ID in DIMENSION_LIST */ + int has_dimlist; + int has_reflist; + int is_ds; + hssize_t nelmts; + hid_t sid, sid_w; /* space ID */ + hid_t tid = H5I_INVALID_HID; /* attribute type ID */ + hid_t ntid = H5I_INVALID_HID; /* attribute native type ID */ + hid_t aid = H5I_INVALID_HID; /* attribute ID */ + int rank; /* rank of dataset */ + hsize_t dims[1]; /* dimension of the "REFERENCE_LIST" array */ + + ds_list_t dsl; /* attribute data in the DS pointing to the dataset */ + ds_list_t *dsbuf = NULL; /* array of attribute data in the DS pointing to the dataset */ + ds_list_t *dsbuf_w = + NULL; /* array of "REFERENCE_LIST" attribute data to write when adding new reference to a dataset */ + hobj_ref_t ref_to_ds; /* reference to the DS */ + hobj_ref_t ref_j; /* iterator reference */ + + /* Variables to be used when new references are used */ + nds_list_t ndsl; + nds_list_t *ndsbuf = NULL; + nds_list_t *ndsbuf_w = NULL; + H5R_ref_t nref_to_ds; + H5R_ref_t nref_j; + hbool_t is_new_ref; + + hvl_t * buf = NULL; /* VL buffer to store in the attribute */ + hid_t dsid_j; /* DS dataset ID in DIMENSION_LIST */ H5O_info2_t oi1, oi2; H5I_type_t it1, it2; int i; @@ -159,6 +204,14 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) return FAIL; } /* end if */ + /*------------------------------------------------------------------------- + * determine if old or new references should be used + *------------------------------------------------------------------------- + */ + + if (H5DSwith_new_ref(did, &is_new_ref) < 0) + return FAIL; + /* get ID type */ if ((it1 = H5Iget_type(did)) < 0) return FAIL; @@ -209,14 +262,23 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) * and one to the dataset, saved in "REFERENCE_LIST" *------------------------------------------------------------------------- */ - /* create a reference for the >>DS<< dataset */ - if (H5Rcreate(&ref_to_ds, dsid, ".", H5R_OBJECT, (hid_t)-1) < 0) - return FAIL; - - /* create a reference for the >>data<< dataset */ - if (H5Rcreate(&dsl.ref, did, ".", H5R_OBJECT, (hid_t)-1) < 0) - return FAIL; + if (is_new_ref) { + /* create a reference for the >>DS<< dataset */ + if (H5Rcreate_object(dsid, ".", H5P_DEFAULT, &nref_to_ds) < 0) + return FAIL; + /* create a reference for the >>data<< dataset */ + if (H5Rcreate_object(did, ".", H5P_DEFAULT, &ndsl.ref) < 0) + return FAIL; + } + else { + /* create a reference for the >>DS<< dataset */ + if (H5Rcreate(&ref_to_ds, dsid, ".", H5R_OBJECT, (hid_t)-1) < 0) + return FAIL; + /* create a reference for the >>data<< dataset */ + if (H5Rcreate(&dsl.ref, did, ".", H5R_OBJECT, (hid_t)-1) < 0) + return FAIL; + } /* try to find the attribute "DIMENSION_LIST" on the >>data<< dataset */ if ((has_dimlist = H5LT_find_attribute(did, DIMENSION_LIST)) < 0) return FAIL; @@ -234,9 +296,14 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) return FAIL; /* create the type for the attribute "DIMENSION_LIST" */ - if ((tid = H5Tvlen_create(H5T_STD_REF_OBJ)) < 0) - goto out; - + if (is_new_ref) { + if ((tid = H5Tvlen_create(H5T_STD_REF)) < 0) + goto out; + } + else { + if ((tid = H5Tvlen_create(H5T_STD_REF_OBJ)) < 0) + goto out; + } /* create the attribute */ if ((aid = H5Acreate2(did, DIMENSION_LIST, tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto out; @@ -252,24 +319,32 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) } /* store the REF information in the index of the dataset that has the DS */ - buf[idx].len = 1; - buf[idx].p = HDmalloc(1 * sizeof(hobj_ref_t)); - ((hobj_ref_t *)buf[idx].p)[0] = ref_to_ds; - + buf[idx].len = 1; + if (is_new_ref) { + buf[idx].p = HDmalloc(1 * sizeof(H5R_ref_t)); + ((H5R_ref_t *)buf[idx].p)[0] = nref_to_ds; + } + else { + buf[idx].p = HDmalloc(1 * sizeof(hobj_ref_t)); + ((hobj_ref_t *)buf[idx].p)[0] = ref_to_ds; + } /* write the attribute with the reference */ if (H5Awrite(aid, tid, buf) < 0) goto out; /* close */ - if (H5Treclaim(tid, sid, H5P_DEFAULT, buf) < 0) - goto out; + if (is_new_ref) { + if (H5Rdestroy(&nref_to_ds) < 0) + goto out; + } if (H5Sclose(sid) < 0) goto out; if (H5Tclose(tid) < 0) goto out; if (H5Aclose(aid) < 0) goto out; - + HDfree(buf[idx].p); + buf[idx].p = NULL; HDfree(buf); buf = NULL; } @@ -302,12 +377,20 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) /* iterate all the REFs in this dimension IDX */ for (i = 0; i < (int)buf[idx].len; i++) { /* get the reference */ - ref_j = ((hobj_ref_t *)buf[idx].p)[i]; + if (is_new_ref) { + nref_j = ((H5R_ref_t *)buf[idx].p)[i]; - /* get the scale id for this REF */ - if ((dsid_j = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref_j)) < 0) - goto out; + /* get the scale id for this REF */ + if ((dsid_j = H5Ropen_object(&nref_j, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto out; + } + else { + ref_j = ((hobj_ref_t *)buf[idx].p)[i]; + /* get the scale id for this REF */ + if ((dsid_j = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref_j)) < 0) + goto out; + } /* get info for DS in the parameter list */ if (H5Oget_info3(dsid, &oi1, H5O_INFO_BASIC) < 0) goto out; @@ -335,17 +418,33 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) /* we are adding one more DS to this dimension */ if (buf[idx].len > 0) { buf[idx].len++; - len = buf[idx].len; - buf[idx].p = HDrealloc(buf[idx].p, len * sizeof(hobj_ref_t)); - ((hobj_ref_t *)buf[idx].p)[len - 1] = ref_to_ds; + len = buf[idx].len; + if (is_new_ref) { + buf[idx].p = HDrealloc(buf[idx].p, len * sizeof(H5R_ref_t)); + ((H5R_ref_t *)buf[idx].p)[len - 1] = nref_to_ds; + } + else { + buf[idx].p = HDrealloc(buf[idx].p, len * sizeof(hobj_ref_t)); + ((hobj_ref_t *)buf[idx].p)[len - 1] = ref_to_ds; + } } /* end if */ else { /* store the REF information in the index of the dataset that has the DS */ - buf[idx].len = 1; - buf[idx].p = HDmalloc(sizeof(hobj_ref_t)); - ((hobj_ref_t *)buf[idx].p)[0] = ref_to_ds; + buf[idx].len = 1; + if (is_new_ref) { + buf[idx].p = HDmalloc(sizeof(H5R_ref_t)); + ((H5R_ref_t *)buf[idx].p)[0] = nref_to_ds; + } + else { + buf[idx].p = HDmalloc(sizeof(hobj_ref_t)); + ((hobj_ref_t *)buf[idx].p)[0] = ref_to_ds; + } } /* end else */ } /* end if */ + else { + if (is_new_ref && H5Rdestroy(&nref_to_ds) < 0) + goto out; + } /* write the attribute with the new references */ if (H5Awrite(aid, tid, buf) < 0) @@ -385,29 +484,40 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) goto out; /* create the compound datatype for the attribute "REFERENCE_LIST" */ - if ((tid = H5Tcreate(H5T_COMPOUND, sizeof(ds_list_t))) < 0) - goto out; - - /* insert reference field */ - if (H5Tinsert(tid, "dataset", HOFFSET(ds_list_t, ref), H5T_STD_REF_OBJ) < 0) - goto out; - - /* insert dimension idx of the dataset field */ - if (H5Tinsert(tid, "dimension", HOFFSET(ds_list_t, dim_idx), H5T_NATIVE_INT) < 0) - goto out; + if (is_new_ref) { + if ((tid = H5Tcreate(H5T_COMPOUND, sizeof(nds_list_t))) < 0) + goto out; + if (H5Tinsert(tid, "dataset", HOFFSET(nds_list_t, ref), H5T_STD_REF) < 0) + goto out; + if (H5Tinsert(tid, "dimension", HOFFSET(nds_list_t, dim_idx), H5T_NATIVE_UINT) < 0) + goto out; + } + else { + if ((tid = H5Tcreate(H5T_COMPOUND, sizeof(ds_list_t))) < 0) + goto out; + if (H5Tinsert(tid, "dataset", HOFFSET(ds_list_t, ref), H5T_STD_REF_OBJ) < 0) + goto out; + if (H5Tinsert(tid, "dimension", HOFFSET(ds_list_t, dim_idx), H5T_NATIVE_UINT) < 0) + goto out; + } /* create the attribute */ if ((aid = H5Acreate2(dsid, REFERENCE_LIST, tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto out; /* store the IDX information */ - dsl.dim_idx = idx; - - /* write the attribute with the reference */ - if (H5Awrite(aid, tid, &dsl) < 0) - goto out; - - /* close */ + if (is_new_ref) { + ndsl.dim_idx = idx; + if (H5Awrite(aid, tid, &ndsl) < 0) + goto out; + if (H5Rdestroy(&ndsl.ref) < 0) + goto out; + } + else { + dsl.dim_idx = idx; + if (H5Awrite(aid, tid, &dsl) < 0) + goto out; + } if (H5Sclose(sid) < 0) goto out; if (H5Tclose(tid) < 0) @@ -421,6 +531,9 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) *------------------------------------------------------------------------- */ else if (has_reflist == 1) { + hid_t tmp_id; /* Temporary DS dataset ID to recreate reference */ + int j; + if ((aid = H5Aopen(dsid, REFERENCE_LIST, H5P_DEFAULT)) < 0) goto out; @@ -428,7 +541,7 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) goto out; /* get native type to read attribute REFERENCE_LIST */ - if ((ntid = H5DS_get_REFLIST_type()) < 0) + if ((ntid = H5Tget_native_type(tid, H5T_DIR_ASCEND)) < 0) goto out; /* get and save the old reference(s) */ @@ -439,17 +552,22 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) goto out; nelmts++; - - dsbuf = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t)); - if (dsbuf == NULL) - goto out; - - if (H5Aread(aid, ntid, dsbuf) < 0) - goto out; + if (is_new_ref) { + ndsbuf = (nds_list_t *)HDmalloc((size_t)nelmts * sizeof(nds_list_t)); + if (ndsbuf == NULL) + goto out; + if (H5Aread(aid, ntid, ndsbuf) < 0) + goto out; + } + else { + dsbuf = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t)); + if (dsbuf == NULL) + goto out; + if (H5Aread(aid, ntid, dsbuf) < 0) + goto out; + } /* close */ - if (H5Sclose(sid) < 0) - goto out; if (H5Aclose(aid) < 0) goto out; @@ -458,40 +576,93 @@ H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx) *------------------------------------------------------------------------- */ + /* Allocate new buffer to copy old references and add new one */ + + if (is_new_ref) { + ndsbuf_w = (nds_list_t *)HDmalloc((size_t)nelmts * sizeof(nds_list_t)); + if (ndsbuf_w == NULL) + goto out; + } + else { + dsbuf_w = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t)); + if (dsbuf_w == NULL) + goto out; + } + /* Recreate the references we read from the existing "REFERENCE_LIST" attribute */ + for (j = 0; j < nelmts - 1; j++) { + if (is_new_ref) { + ndsbuf_w[j].dim_idx = ndsbuf[j].dim_idx; + tmp_id = H5Ropen_object(&ndsbuf[j].ref, H5P_DEFAULT, H5P_DEFAULT); + if (tmp_id < 0) + goto out; + if (H5Rcreate_object(tmp_id, ".", H5P_DEFAULT, &ndsbuf_w[j].ref) < 0) { + H5Dclose(tmp_id); + goto out; + } + } + else { + dsbuf_w[j] = dsbuf[j]; + } + } + /* store the IDX information (index of the dataset that has the DS) */ + if (is_new_ref) { + ndsl.dim_idx = idx; + ndsbuf_w[nelmts - 1] = ndsl; + } + else { + dsl.dim_idx = idx; + dsbuf_w[nelmts - 1] = dsl; + } + /* the attribute must be deleted, in order to the new one can reflect the changes*/ if (H5Adelete(dsid, REFERENCE_LIST) < 0) goto out; - /* store the IDX information (index of the dataset that has the DS) */ - dsl.dim_idx = idx; - dsbuf[nelmts - 1] = dsl; - /* create a new data space for the new references array */ dims[0] = (hsize_t)nelmts; - if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + if ((sid_w = H5Screate_simple(1, dims, NULL)) < 0) goto out; /* create the attribute again with the changes of space */ - if ((aid = H5Acreate2(dsid, REFERENCE_LIST, tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) + if ((aid = H5Acreate2(dsid, REFERENCE_LIST, tid, sid_w, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto out; /* write the attribute with the new references */ - if (H5Awrite(aid, ntid, dsbuf) < 0) - goto out; - - /* close */ + if (is_new_ref) { + if (H5Awrite(aid, ntid, ndsbuf_w) < 0) + goto out; + if (H5Treclaim(tid, sid, H5P_DEFAULT, ndsbuf_w) < 0) + goto out; + } + else { + if (H5Awrite(aid, ntid, dsbuf_w) < 0) + goto out; + if (H5Treclaim(tid, sid, H5P_DEFAULT, dsbuf_w) < 0) + goto out; + } if (H5Sclose(sid) < 0) goto out; + if (H5Sclose(sid_w) < 0) + goto out; if (H5Tclose(tid) < 0) goto out; if (H5Aclose(aid) < 0) goto out; if (H5Tclose(ntid) < 0) goto out; - - HDfree(dsbuf); - dsbuf = NULL; + if (is_new_ref) { + HDfree(ndsbuf); + dsbuf = NULL; + HDfree(ndsbuf_w); + dsbuf = NULL; + } + else { + HDfree(dsbuf); + dsbuf = NULL; + HDfree(dsbuf_w); + dsbuf = NULL; + } } /* has_reflist */ /*------------------------------------------------------------------------- @@ -515,6 +686,8 @@ out: HDfree(buf); if (dsbuf) HDfree(dsbuf); + if (dsbuf_w) + HDfree(dsbuf_w); H5E_BEGIN_TRY { @@ -554,27 +727,35 @@ out: herr_t H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx) { - int has_dimlist; - int has_reflist; - hssize_t nelmts; - hid_t dsid_j; /* DS dataset ID in DIMENSION_LIST */ - hid_t did_i; /* dataset ID in REFERENCE_LIST */ - hid_t sid; /* space ID */ - hid_t tid = -1; /* attribute type ID */ - hid_t ntid = -1; /* attribute native type ID */ - hid_t aid = -1; /* attribute ID */ - int rank; /* rank of dataset */ - ds_list_t * dsbuf = NULL; /* array of attribute data in the DS pointing to the dataset */ - hsize_t dims[1]; /* dimension of the "REFERENCE_LIST" array */ - hobj_ref_t ref; /* reference to the DS */ - hvl_t * buf = NULL; /* VL buffer to store in the attribute */ - int i; - size_t j; - hssize_t ii; - H5O_info2_t did_oi, dsid_oi, tmp_oi; - int found_dset = 0, found_ds = 0; - int have_ds = 0; - htri_t is_scale; + int has_dimlist; + int has_reflist; + hssize_t nelmts; + hid_t dsid_j; /* DS dataset ID in DIMENSION_LIST */ + hid_t did_i; /* dataset ID in REFERENCE_LIST */ + hid_t sid = H5I_INVALID_HID; /* space ID */ + hid_t sid_w = H5I_INVALID_HID; /* space ID */ + hid_t tid = H5I_INVALID_HID; /* attribute type ID */ + hid_t ntid = H5I_INVALID_HID; /* attribute native type ID */ + hid_t aid = H5I_INVALID_HID; /* attribute ID */ + int rank; /* rank of dataset */ + nds_list_t * ndsbuf = NULL; /* array of attribute data in the DS pointing to the dataset */ + nds_list_t * ndsbuf_w = NULL; /* array of attribute data in the DS pointing to the dataset to write*/ + ds_list_t * dsbuf = NULL; /* array of attribute data in the DS pointing to the dataset */ + ds_list_t * dsbuf_w = NULL; /* array of attribute data in the DS pointing to the dataset to write*/ + hsize_t dims[1]; /* dimension of the "REFERENCE_LIST" array */ + H5R_ref_t nref; + hobj_ref_t ref; /* reference to the DS */ + hvl_t * buf = NULL; /* VL buffer to store in the attribute */ + int i; + size_t j; + hssize_t ii; + H5O_info2_t did_oi, dsid_oi, tmp_oi; + int found_dset = 0, found_ds = 0; + int have_ds = 0; + htri_t is_scale; + hbool_t is_new_ref; + unsigned int tmp_idx; + hid_t tmp_id; /*------------------------------------------------------------------------- * parameter checking @@ -612,9 +793,17 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx) } /* end if */ /*------------------------------------------------------------------------- - * Find "DIMENSION_LIST" + * determine if old or new references should be used + *------------------------------------------------------------------------- + */ + if (H5DSwith_new_ref(did, &is_new_ref) < 0) + return FAIL; + + /*------------------------------------------------------------------------- + * find "DIMENSION_LIST" *------------------------------------------------------------------------- */ + /* try to find the attribute "DIMENSION_LIST" on the >>data<< dataset */ if ((has_dimlist = H5LT_find_attribute(did, DIMENSION_LIST)) < 0) return FAIL; @@ -676,13 +865,22 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx) /* reset */ if (buf[idx].len > 0) { for (j = 0; j < buf[idx].len; j++) { - /* get the reference */ - ref = ((hobj_ref_t *)buf[idx].p)[j]; + if (is_new_ref) { + /* get the reference */ + nref = ((H5R_ref_t *)buf[idx].p)[j]; - /* get the DS id */ - if ((dsid_j = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0) - goto out; + /* get the scale id for this REF */ + if ((dsid_j = H5Ropen_object(&nref, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto out; + } + else { + /* get the reference */ + ref = ((hobj_ref_t *)buf[idx].p)[j]; + /* get the DS id */ + if ((dsid_j = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0) + goto out; + } /* get info for this DS */ if (H5Oget_info3(dsid_j, &tmp_oi, H5O_INFO_BASIC) < 0) goto out; @@ -708,8 +906,14 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx) size_t len = buf[idx].len; - if (j < len - 1) - ((hobj_ref_t *)buf[idx].p)[j] = ((hobj_ref_t *)buf[idx].p)[len - 1]; + if (j < len - 1) { + if (is_new_ref) { + ((H5R_ref_t *)buf[idx].p)[j] = ((H5R_ref_t *)buf[idx].p)[len - 1]; + } + else { + ((hobj_ref_t *)buf[idx].p)[j] = ((hobj_ref_t *)buf[idx].p)[len - 1]; + } + } len = --buf[idx].len; if (len == 0) { HDfree(buf[idx].p); @@ -770,7 +974,7 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx) goto out; /* get native type to read attribute REFERENCE_LIST */ - if ((ntid = H5DS_get_REFLIST_type()) < 0) + if ((ntid = H5Tget_native_type(tid, H5T_DIR_ASCEND)) < 0) goto out; /* get and save the old reference(s) */ @@ -780,22 +984,65 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx) if ((nelmts = H5Sget_simple_extent_npoints(sid)) < 0) goto out; - dsbuf = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t)); - if (dsbuf == NULL) - goto out; - - if (H5Aread(aid, ntid, dsbuf) < 0) - goto out; - + if (is_new_ref) { + ndsbuf = (nds_list_t *)HDmalloc((size_t)nelmts * sizeof(nds_list_t)); + if (ndsbuf == NULL) + goto out; + if (H5Aread(aid, ntid, ndsbuf) < 0) + goto out; + ndsbuf_w = (nds_list_t *)HDmalloc((size_t)nelmts * sizeof(nds_list_t)); + if (ndsbuf_w == NULL) + goto out; + } + else { + dsbuf = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t)); + if (dsbuf == NULL) + goto out; + if (H5Aread(aid, ntid, dsbuf) < 0) + goto out; + dsbuf_w = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t)); + if (dsbuf_w == NULL) + goto out; + } + /* Recreate the references we read from the existing "REFERENCE_LIST" attribute */ + for (i = 0; i < nelmts; i++) { + if (is_new_ref) { + ndsbuf_w[i].dim_idx = ndsbuf[i].dim_idx; + tmp_id = H5Ropen_object(&ndsbuf[i].ref, H5P_DEFAULT, H5P_DEFAULT); + if (tmp_id < 0) + goto out; + if (H5Rcreate_object(tmp_id, ".", H5P_DEFAULT, &ndsbuf_w[i].ref) < 0) { + H5Dclose(tmp_id); + goto out; + } + H5Dclose(tmp_id); + } + else { + dsbuf_w[i] = dsbuf[i]; + } + } for (ii = 0; ii < nelmts; ii++) { /* First check if we have the same dimension index */ - if (idx == dsbuf[ii].dim_idx) { + if (is_new_ref) { + tmp_idx = ndsbuf_w[ii].dim_idx; + } + else { + tmp_idx = dsbuf_w[ii].dim_idx; + } + if (idx == tmp_idx) { /* get the reference to the dataset */ - ref = dsbuf[ii].ref; - - /* get the dataset id */ - if ((did_i = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0) - goto out; + if (is_new_ref) { + /* get the dataset id */ + nref = ndsbuf_w[ii].ref; + if ((did_i = H5Ropen_object(&nref, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto out; + } + else { + /* get the dataset id */ + ref = dsbuf_w[ii].ref; + if ((did_i = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0) + goto out; + } /* get info for this dataset */ if (H5Oget_info3(did_i, &tmp_oi, H5O_INFO_BASIC) < 0) @@ -813,7 +1060,12 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx) goto out; if (!token_cmp) { /* copy the last one to replace the one which is found */ - dsbuf[ii] = dsbuf[nelmts - 1]; + if (is_new_ref) { + ndsbuf_w[ii] = ndsbuf_w[nelmts - 1]; + } + else { + dsbuf_w[ii] = dsbuf_w[nelmts - 1]; + } nelmts--; found_dset = 1; break; @@ -822,9 +1074,7 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx) } /* if we have the same dimension index */ } /* ii */ - /* close space and attribute */ - if (H5Sclose(sid) < 0) - goto out; + /* close attribute */ if (H5Aclose(aid) < 0) goto out; @@ -851,32 +1101,69 @@ H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx) /* create a new data space for the new references array */ dims[0] = (hsize_t)nelmts; - if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + if ((sid_w = H5Screate_simple(1, dims, NULL)) < 0) goto out; /* create the attribute again with the changes of space */ - if ((aid = H5Acreate2(dsid, REFERENCE_LIST, tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) + if ((aid = H5Acreate2(dsid, REFERENCE_LIST, tid, sid_w, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto out; /* write the new attribute with the new references */ - if (H5Awrite(aid, ntid, dsbuf) < 0) - goto out; + if (is_new_ref) { + if (H5Awrite(aid, ntid, ndsbuf_w) < 0) + goto out; + } + else { + if (H5Awrite(aid, ntid, dsbuf_w) < 0) + goto out; + } - /* close space and attribute */ - if (H5Sclose(sid) < 0) - goto out; if (H5Aclose(aid) < 0) goto out; } /* nelmts */ + /* Free refrences */ + if (is_new_ref) { + if (H5Treclaim(tid, sid, H5P_DEFAULT, ndsbuf) < 0) + goto out; + if (H5Sclose(sid) < 0) + goto out; + if (sid_w > 0) { + if (H5Treclaim(tid, sid_w, H5P_DEFAULT, ndsbuf_w) < 0) + goto out; + if (H5Sclose(sid_w) < 0) + goto out; + } + } + else { + if (H5Treclaim(tid, sid, H5P_DEFAULT, dsbuf) < 0) + goto out; + if (H5Sclose(sid) < 0) + goto out; + if (sid_w > 0) { + if (H5Treclaim(tid, sid_w, H5P_DEFAULT, dsbuf_w) < 0) + goto out; + if (H5Sclose(sid_w) < 0) + goto out; + } + } /* close type */ if (H5Tclose(tid) < 0) goto out; if (H5Tclose(ntid) < 0) goto out; - - HDfree(dsbuf); - dsbuf = NULL; + if (is_new_ref) { + HDfree(ndsbuf); + HDfree(ndsbuf_w); + ndsbuf = NULL; + ndsbuf_w = NULL; + } + else { + HDfree(dsbuf); + HDfree(dsbuf_w); + dsbuf = NULL; + dsbuf_w = NULL; + } return SUCCEED; @@ -889,6 +1176,14 @@ out: H5Tclose(ntid); H5Tclose(tid); + if (ndsbuf) { + HDfree(ndsbuf); + ndsbuf = NULL; + } + if (ndsbuf_w) { + HDfree(ndsbuf_w); + ndsbuf_w = NULL; + } if (dsbuf) { HDfree(dsbuf); dsbuf = NULL; @@ -936,21 +1231,24 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx) int has_dimlist; int has_reflist; hssize_t nelmts; - hid_t sid; /* space ID */ - hid_t tid = -1; /* attribute type ID */ - hid_t ntid = -1; /* attribute native type ID */ - hid_t aid = -1; /* attribute ID */ - int rank; /* rank of dataset */ - ds_list_t * dsbuf = NULL; /* array of attribute data in the DS pointing to the dataset */ - hobj_ref_t ref; /* reference to the DS */ - hvl_t * buf = NULL; /* VL buffer to store in the attribute */ - hid_t dsid_j; /* DS dataset ID in DIMENSION_LIST */ - hid_t did_i; /* dataset ID in REFERENCE_LIST */ + hid_t sid; /* space ID */ + hid_t tid = H5I_INVALID_HID; /* attribute type ID */ + hid_t ntid = H5I_INVALID_HID; /* attribute native type ID */ + hid_t aid = H5I_INVALID_HID; /* attribute ID */ + int rank; /* rank of dataset */ + nds_list_t *ndsbuf = NULL; /* array of attribute data in the DS pointing to the dataset */ + ds_list_t * dsbuf = NULL; /* array of attribute data in the DS pointing to the dataset */ + H5R_ref_t nref; /* reference to the DS */ + hobj_ref_t ref; /* reference to the DS */ + hvl_t * buf = NULL; /* VL buffer to store in the attribute */ + hid_t dsid_j; /* DS dataset ID in DIMENSION_LIST */ + hid_t did_i; /* dataset ID in REFERENCE_LIST */ H5O_info2_t oi1, oi2, oi3, oi4; H5I_type_t it1, it2; int i; int found_dset = 0, found_ds = 0; htri_t is_scale; + hbool_t is_new_ref; /*------------------------------------------------------------------------- * parameter checking @@ -982,6 +1280,14 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx) return FAIL; } /* end if */ + /*------------------------------------------------------------------------- + * determine if old or new references should be used + *------------------------------------------------------------------------- + */ + + if (H5DSwith_new_ref(did, &is_new_ref) < 0) + return FAIL; + /* get ID type */ if ((it1 = H5Iget_type(did)) < 0) return FAIL; @@ -1042,12 +1348,22 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx) /* iterate all the REFs in this dimension IDX */ for (i = 0; i < (int)buf[idx].len; i++) { - /* get the reference */ - ref = ((hobj_ref_t *)buf[idx].p)[i]; + if (is_new_ref) { + /* get the reference */ + nref = ((H5R_ref_t *)buf[idx].p)[i]; - /* get the scale id for this REF */ - if ((dsid_j = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0) - goto out; + /* get the scale id for this REF */ + if ((dsid_j = H5Ropen_object(&nref, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto out; + } + else { + /* get the reference */ + ref = ((hobj_ref_t *)buf[idx].p)[i]; + + /* get the scale id for this REF */ + if ((dsid_j = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0) + goto out; + } /* get info for DS in the parameter list */ if (H5Oget_info3(dsid, &oi1, H5O_INFO_BASIC) < 0) @@ -1107,7 +1423,7 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx) goto out; /* get native type to read REFERENCE_LIST attribute */ - if ((ntid = H5DS_get_REFLIST_type()) < 0) + if ((ntid = H5Tget_native_type(tid, H5T_DIR_ASCEND)) < 0) goto out; /* get and save the old reference(s) */ @@ -1117,13 +1433,20 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx) if ((nelmts = H5Sget_simple_extent_npoints(sid)) < 0) goto out; - dsbuf = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t)); - - if (dsbuf == NULL) - goto out; - - if (H5Aread(aid, ntid, dsbuf) < 0) - goto out; + if (is_new_ref) { + ndsbuf = (nds_list_t *)HDmalloc((size_t)nelmts * sizeof(nds_list_t)); + if (ndsbuf == NULL) + goto out; + if (H5Aread(aid, ntid, ndsbuf) < 0) + goto out; + } + else { + dsbuf = (ds_list_t *)HDmalloc((size_t)nelmts * sizeof(ds_list_t)); + if (dsbuf == NULL) + goto out; + if (H5Aread(aid, ntid, dsbuf) < 0) + goto out; + } /*------------------------------------------------------------------------- * iterate @@ -1131,40 +1454,58 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx) */ for (i = 0; i < nelmts; i++) { - /* get the reference */ - ref = dsbuf[i].ref; - /* the reference was not deleted */ - if (ref) { + if (is_new_ref) { + nref = ndsbuf[i].ref; + /* get the dataset id */ + if ((did_i = H5Ropen_object(&nref, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto out; + } + else { + ref = dsbuf[i].ref; /* get the dataset id */ if ((did_i = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0) goto out; + } - /* get info for dataset in the parameter list */ - if (H5Oget_info3(did, &oi3, H5O_INFO_BASIC) < 0) - goto out; + /* get info for dataset in the parameter list */ + if (H5Oget_info3(did, &oi3, H5O_INFO_BASIC) < 0) + goto out; - /* get info for this dataset */ - if (H5Oget_info3(did_i, &oi4, H5O_INFO_BASIC) < 0) - goto out; + /* get info for this dataset */ + if (H5Oget_info3(did_i, &oi4, H5O_INFO_BASIC) < 0) + goto out; - /* same object */ - if (oi3.fileno == oi4.fileno) { - int token_cmp; + /* same object */ + if (oi3.fileno == oi4.fileno) { + int token_cmp; - if (H5Otoken_cmp(did, &oi3.token, &oi4.token, &token_cmp) < 0) - goto out; + if (H5Otoken_cmp(did, &oi3.token, &oi4.token, &token_cmp) < 0) + goto out; + if (is_new_ref) { + if (!token_cmp && (idx == ndsbuf[i].dim_idx)) + found_dset = 1; + } + else { if (!token_cmp && (idx == dsbuf[i].dim_idx)) found_dset = 1; - } /* end if */ + } + } /* end if */ - /* close the dereferenced dataset */ - if (H5Dclose(did_i) < 0) - goto out; - } /* if */ - } /* i */ + /* close the dereferenced dataset */ + if (H5Dclose(did_i) < 0) + goto out; + } /* for */ /* close */ + if (is_new_ref) { + if (H5Treclaim(ntid, sid, H5P_DEFAULT, ndsbuf) < 0) + goto out; + } + else { + if (H5Treclaim(ntid, sid, H5P_DEFAULT, dsbuf) < 0) + goto out; + } if (H5Sclose(sid) < 0) goto out; if (H5Tclose(ntid) < 0) @@ -1174,8 +1515,14 @@ H5DSis_attached(hid_t did, hid_t dsid, unsigned int idx) if (H5Aclose(aid) < 0) goto out; - HDfree(dsbuf); - dsbuf = NULL; + if (ndsbuf) { + HDfree(ndsbuf); + ndsbuf = NULL; + } + if (dsbuf) { + HDfree(dsbuf); + dsbuf = NULL; + } } /* has_reflist */ if (found_ds && found_dset) @@ -1198,6 +1545,10 @@ out: HDfree(buf); buf = NULL; } + if (ndsbuf) { + HDfree(ndsbuf); + ndsbuf = NULL; + } if (dsbuf) { HDfree(dsbuf); dsbuf = NULL; @@ -1249,17 +1600,19 @@ H5DSiterate_scales(hid_t did, unsigned int dim, int *ds_idx, H5DS_iterate_t visi { hid_t scale_id; int rank; - hobj_ref_t ref; /* reference to the DS */ - hid_t sid; /* space ID */ - hid_t tid = -1; /* attribute type ID */ - hid_t aid = -1; /* attribute ID */ - hvl_t * buf = NULL; /* VL buffer to store in the attribute */ - H5I_type_t it; /* ID type */ + H5R_ref_t nref; /* reference to the DS */ + hobj_ref_t ref; /* reference to the DS */ + hid_t sid; /* space ID */ + hid_t tid = H5I_INVALID_HID; /* attribute type ID */ + hid_t aid = H5I_INVALID_HID; /* attribute ID */ + hvl_t * buf = NULL; /* VL buffer to store in the attribute */ + H5I_type_t it; /* ID type */ herr_t ret_value = 0; int j_idx; int nscales; int has_dimlist; int i; + hbool_t is_new_ref; /*------------------------------------------------------------------------- * parameter checking @@ -1272,6 +1625,14 @@ H5DSiterate_scales(hid_t did, unsigned int dim, int *ds_idx, H5DS_iterate_t visi if (H5I_DATASET != it) return FAIL; + /*------------------------------------------------------------------------- + * determine if old or new references should be used + *------------------------------------------------------------------------- + */ + + if (H5DSwith_new_ref(did, &is_new_ref) < 0) + return FAIL; + /* get the number of scales assotiated with this DIM */ if ((nscales = H5DSget_num_scales(did, dim)) < 0) return FAIL; @@ -1330,17 +1691,32 @@ H5DSiterate_scales(hid_t did, unsigned int dim, int *ds_idx, H5DS_iterate_t visi /* iterate */ for (i = j_idx; i < nscales; i++) { - /* get the reference */ - ref = ((hobj_ref_t *)buf[dim].p)[i]; - - /* disable error reporting, the ID might refer to a deleted dataset */ - H5E_BEGIN_TRY - { - /* get the DS id */ - if ((scale_id = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0) - goto out; + if (is_new_ref) { + /* get the reference */ + nref = ((H5R_ref_t *)buf[dim].p)[i]; + + /* disable error reporting, the ID might refer to a deleted dataset */ + H5E_BEGIN_TRY + { + /* get the DS id */ + if ((scale_id = H5Ropen_object(&nref, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto out; + } + H5E_END_TRY; + } + else { + /* get the reference */ + ref = ((hobj_ref_t *)buf[dim].p)[i]; + + /* disable error reporting, the ID might refer to a deleted dataset */ + H5E_BEGIN_TRY + { + /* get the DS id */ + if ((scale_id = H5Rdereference2(did, H5P_DEFAULT, H5R_OBJECT, &ref)) < 0) + goto out; + } + H5E_END_TRY; } - H5E_END_TRY; /* set the return IDX OUT value at current scale index */ if (ds_idx != NULL) { @@ -1413,12 +1789,12 @@ herr_t H5DSset_label(hid_t did, unsigned int idx, const char *label) { int has_labels; - hid_t sid = -1; /* space ID */ - hid_t tid = -1; /* attribute type ID */ - hid_t aid = -1; /* attribute ID */ - int rank; /* rank of dataset */ - hsize_t dims[1]; /* dimensions of dataset */ - H5I_type_t it; /* ID type */ + hid_t sid = H5I_INVALID_HID; /* space ID */ + hid_t tid = H5I_INVALID_HID; /* attribute type ID */ + hid_t aid = H5I_INVALID_HID; /* attribute ID */ + int rank; /* rank of dataset */ + hsize_t dims[1]; /* dimensions of dataset */ + H5I_type_t it; /* ID type */ unsigned int i; union { /* union is needed to eliminate compiler warnings about */ char ** buf; /* discarding the 'const' qualifier in the free */ @@ -1616,12 +1992,12 @@ ssize_t H5DSget_label(hid_t did, unsigned int idx, char *label, size_t size) { int has_labels; - hid_t sid = -1; /* space ID */ - hid_t tid = -1; /* attribute type ID */ - hid_t aid = -1; /* attribute ID */ - int rank; /* rank of dataset */ - char ** buf = NULL; /* buffer to store in the attribute */ - H5I_type_t it; /* ID type */ + hid_t sid = H5I_INVALID_HID; /* space ID */ + hid_t tid = H5I_INVALID_HID; /* attribute type ID */ + hid_t aid = H5I_INVALID_HID; /* attribute ID */ + int rank; /* rank of dataset */ + char ** buf = NULL; /* buffer to store in the attribute */ + H5I_type_t it; /* ID type */ size_t nbytes = 0; size_t copy_len; int i; @@ -1764,10 +2140,10 @@ out: ssize_t H5DSget_scale_name(hid_t did, char *name, size_t size) { - hid_t aid; /* attribute ID */ - hid_t tid = -1; /* attribute type ID */ - hid_t sid; /* space ID */ - H5I_type_t it; /* ID type */ + hid_t aid = H5I_INVALID_HID; /* attribute ID */ + hid_t tid = H5I_INVALID_HID; /* attribute type ID */ + hid_t sid = H5I_INVALID_HID; /* space ID */ + H5I_type_t it; /* ID type */ size_t nbytes; size_t copy_len; int has_name; @@ -1881,13 +2257,13 @@ out: htri_t H5DSis_scale(hid_t did) { - hid_t tid = -1; /* attribute type ID */ - hid_t aid = -1; /* attribute ID */ - herr_t attr_class; /* has the "CLASS" attribute */ - htri_t is_ds = -1; /* set to "not a dimension scale" */ - H5I_type_t it; /* type of identifier */ - char * buf = NULL; /* buffer to read name of attribute */ - size_t string_size; /* size of storage for the attribute */ + hid_t tid = H5I_INVALID_HID; /* attribute type ID */ + hid_t aid = H5I_INVALID_HID; /* attribute ID */ + herr_t attr_class; /* has the "CLASS" attribute */ + htri_t is_ds = -1; /* set to "not a dimension scale" */ + H5I_type_t it; /* type of identifier */ + char * buf = NULL; /* buffer to read name of attribute */ + size_t string_size; /* size of storage for the attribute */ H5T_class_t type_class; H5T_str_t strpad; @@ -1995,12 +2371,12 @@ int H5DSget_num_scales(hid_t did, unsigned int idx) { int has_dimlist; - hid_t sid; /* space ID */ - hid_t tid = -1; /* attribute type ID */ - hid_t aid = -1; /* attribute ID */ - int rank; /* rank of dataset */ - hvl_t * buf = NULL; /* VL buffer to store in the attribute */ - H5I_type_t it; /* ID type */ + hid_t sid; /* space ID */ + hid_t tid = H5I_INVALID_HID; /* attribute type ID */ + hid_t aid = H5I_INVALID_HID; /* attribute ID */ + int rank; /* rank of dataset */ + hvl_t * buf = NULL; /* VL buffer to store in the attribute */ + H5I_type_t it; /* ID type */ int nscales; /*------------------------------------------------------------------------- @@ -2112,12 +2488,12 @@ out: static herr_t H5DS_is_reserved(hid_t did) { - int has_class; - hid_t tid = -1; - hid_t aid = -1; - char * buf; /* Name of attribute */ - hsize_t storage_size; /* Size of storage for attribute */ - herr_t ret; + int has_class; + hid_t tid = H5I_INVALID_HID; + hid_t aid = H5I_INVALID_HID; + char * buf = NULL; /* Name of attribute */ + size_t string_size; /* Size of storage for attribute */ + herr_t ret; /* try to find the attribute "CLASS" on the dataset */ if ((has_class = H5LT_find_attribute(did, "CLASS")) < 0) @@ -2142,10 +2518,10 @@ H5DS_is_reserved(hid_t did) goto out; /* allocate buffer large enough to hold string */ - if ((storage_size = H5Aget_storage_size(aid)) == 0) + if ((string_size = H5Tget_size(tid)) == 0) goto out; - buf = (char *)HDmalloc((size_t)storage_size * sizeof(char) + 1); + buf = (char *)HDmalloc((size_t)string_size * sizeof(char)); if (buf == NULL) goto out; @@ -2174,51 +2550,11 @@ H5DS_is_reserved(hid_t did) out: H5E_BEGIN_TRY { + if (buf) + HDfree(buf); H5Tclose(tid); H5Aclose(aid); } H5E_END_TRY; return FAIL; } - -/*------------------------------------------------------------------------- - * Function: H5DS_get_REFLIST_type - * - * Purpose: This is a helper function to return a native type for - * the REFERENCE_LIST attribute. - * - * Return: Type identifier on success and negative on failure - * - * Programmer: Elena Pourmal - * - * Date: May 22, 2010 - * - *------------------------------------------------------------------------- - */ -static hid_t -H5DS_get_REFLIST_type(void) -{ - hid_t ntid_t = -1; - - /* Build native type that corresponds to compound datatype - used to store ds_list_t structure in the REFERENCE_LIST - attribute */ - - if ((ntid_t = H5Tcreate(H5T_COMPOUND, sizeof(ds_list_t))) < 0) - goto out; - - if (H5Tinsert(ntid_t, "dataset", HOFFSET(ds_list_t, ref), H5T_STD_REF_OBJ) < 0) - goto out; - - if (H5Tinsert(ntid_t, "dimension", HOFFSET(ds_list_t, dim_idx), H5T_NATIVE_INT) < 0) - goto out; - - return ntid_t; -out: - H5E_BEGIN_TRY - { - H5Tclose(ntid_t); - } - H5E_END_TRY; - return FAIL; -} diff --git a/hl/src/H5DSprivate.h b/hl/src/H5DSprivate.h index 0403a4c..ede0209 100644 --- a/hl/src/H5DSprivate.h +++ b/hl/src/H5DSprivate.h @@ -20,12 +20,18 @@ /* public LT prototypes */ #include "H5DSpublic.h" -/* attribute type of a DS dataset */ +/* attribute type of a DS dataset when old references are used*/ typedef struct ds_list_t { hobj_ref_t ref; /* object reference */ unsigned int dim_idx; /* dimension index of the dataset */ } ds_list_t; +/* attribute type of a DS dataset when new references are used*/ +typedef struct nds_list_t { + H5R_ref_t ref; + unsigned int dim_idx; /* dimension index of the dataset */ +} nds_list_t; + /*------------------------------------------------------------------------- * private functions *------------------------------------------------------------------------- diff --git a/hl/src/H5DSpublic.h b/hl/src/H5DSpublic.h index 7306cbc..979a173 100644 --- a/hl/src/H5DSpublic.h +++ b/hl/src/H5DSpublic.h @@ -25,6 +25,8 @@ typedef herr_t (*H5DS_iterate_t)(hid_t dset, unsigned dim, hid_t scale, void *vi extern "C" { #endif +H5_HLDLL herr_t H5DSwith_new_ref(hid_t obj_id, hbool_t *with_new_ref); + H5_HLDLL herr_t H5DSattach_scale(hid_t did, hid_t dsid, unsigned int idx); H5_HLDLL herr_t H5DSdetach_scale(hid_t did, hid_t dsid, unsigned int idx); diff --git a/hl/src/H5LD.c b/hl/src/H5LD.c index 8bfd0d6..2368127 100644 --- a/hl/src/H5LD.c +++ b/hl/src/H5LD.c @@ -29,9 +29,9 @@ static herr_t H5LD_get_dset_elmts(hid_t did, const hsize_t *prev_dims, const hsi * Function: H5LD_clean_vector * * Purpose: Process the vector of info: - * 1) free the array of pointers to member names in listv[n] - * 2) close the type id of the last member in listv[n] - * 3) free the H5LD_memb_t structure itself as pointed to by listv[n] + * 1) free the array of pointers to member names in listv[n] + * 2) close the type id of the last member in listv[n] + * 3) free the H5LD_memb_t structure itself as pointed to by listv[n] * * Return: void * @@ -69,12 +69,12 @@ H5LD_clean_vector(H5LD_memb_t *listv[]) * Function: H5LD_construct_info() * * Purpose: Get the remaining info for a field: - * 1) Get the type id of the last member in the field - * 2) Get the total offset of all the members in the field - * 3) Get the type size of the last member in the field + * 1) Get the type id of the last member in the field + * 2) Get the total offset of all the members in the field + * 3) Get the type size of the last member in the field * * Return: Success: 0 - * Failure: negative + * Failure: negative * * Programmer: Vailin Choi; Aug 2010 * @@ -129,24 +129,24 @@ done: * Function: H5LD_construct_vector * * Purpose: Process the comma-separated list of fields in "fields" as follows: - * Example: - * "fields": "a.b.c,d" - * listv[0]->tot_offset = total offset of "a" & "b" & "c" - * listv[0]->last_tid = type id of "c" - * listv[0]->last_tsize = type size of "c" - * listv[0]->names[0] = "a" - * listv[0]->names[1] = "b" - * listv[0]->names[2] = "c" - * listv[0]->names[3] = NULL - * - * listv[1]->tot_offset = offset of "d" - * listv[1]->last_tid = type id of "d" - * listv[1]->last_tsize = type size of "d" - * listv[1]->names[0] = "d" - * listv[1]->names[1] = NULL + * Example: + * "fields": "a.b.c,d" + * listv[0]->tot_offset = total offset of "a" & "b" & "c" + * listv[0]->last_tid = type id of "c" + * listv[0]->last_tsize = type size of "c" + * listv[0]->names[0] = "a" + * listv[0]->names[1] = "b" + * listv[0]->names[2] = "c" + * listv[0]->names[3] = NULL + * + * listv[1]->tot_offset = offset of "d" + * listv[1]->last_tid = type id of "d" + * listv[1]->last_tsize = type size of "d" + * listv[1]->names[0] = "d" + * listv[1]->names[1] = NULL * * Return: Success: # of comma-separated fields in "fields" - * Failure: negative value + * Failure: negative value * * Programmer: Vailin Choi; Aug 2010 * @@ -251,8 +251,10 @@ H5LD_construct_vector(char *fields, H5LD_memb_t *listv[] /*OUT*/, hid_t par_tid) goto done; } /* end if */ else { - if (memb) + if (memb) { + HDfree(memb->names); HDfree(memb); + } goto done; } /* end else */ } /* while !end_of_fields */ @@ -272,10 +274,10 @@ done: * Function: H5LD_get_dset_dims * * Purpose: To return the current size for each dimension of the - * dataset's dataspace + * dataset's dataspace * * Return: Success: 0 - * Failure: negative value + * Failure: negative value * * Programmer: Vailin Choi; March 2010 * @@ -316,12 +318,12 @@ done: * Function: H5LD_get_dset_type_size * * Purpose: To return the size of the dataset's datatype in bytes - * null "fields": return the size of the dataset's datatype - * non-null "fields": return the size of the dataset's datatype - * with respect to the selection in "fields" + * null "fields": return the size of the dataset's datatype + * non-null "fields": return the size of the dataset's datatype + * with respect to the selection in "fields" * * Return: Success: size of the dataset's datatype - * Failure: 0 (valid datatypes are never zero size) + * Failure: 0 (valid datatypes are never zero size) * * Programmer: Vailin Choi; March 2010 * @@ -402,7 +404,7 @@ done: * Purpose: To retrieve selected data from the dataset * * Return: Success: 0 - * Failure: negative + * Failure: negative * * Programmer: Vailin Choi; August 2010 * @@ -578,7 +580,7 @@ done: * Purpose: To retrieve the current dimension sizes for a dataset * * Return: Success: 0 - * Failure: negative value + * Failure: negative value * * Programmer: Vailin Choi; March 2010 * @@ -596,7 +598,7 @@ H5LDget_dset_dims(hid_t did, hsize_t *cur_dims) * Purpose: To return the size in bytes of the datatype for the dataset * * Return: Success: size in bytes of the dataset's datatype - * Failure: 0 (valid datatypes are never zero size) + * Failure: 0 (valid datatypes are never zero size) * * Programmer: Vailin Choi; March 2010 * @@ -614,7 +616,7 @@ H5LDget_dset_type_size(hid_t did, const char *fields) * Purpose: To retrieve selected data from the dataset * * Return: Success: 0 - * Failure: negative value + * Failure: negative value * * Programmer: Vailin Choi; March 2010 * diff --git a/hl/src/H5LT.c b/hl/src/H5LT.c index 8f2b33f..238bbf2 100644 --- a/hl/src/H5LT.c +++ b/hl/src/H5LT.c @@ -1856,46 +1856,6 @@ H5LTset_attribute_double(hid_t loc_id, const char *obj_name, const char *attr_na } /*------------------------------------------------------------------------- - * Function: find_attr - * - * Purpose: operator function used by H5LT_find_attribute - * - * Programmer: Pedro Vicente - * - * Date: June 21, 2001 - * - * Comments: - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -static herr_t -find_attr(H5_ATTR_UNUSED hid_t loc_id, const char *name, H5_ATTR_UNUSED const H5A_info_t *ainfo, - void *op_data) -{ - int ret = H5_ITER_CONT; - - /* check the arguments */ - if (name == NULL) - return H5_ITER_CONT; - - /* Shut compiler up */ - (void)loc_id; - (void)ainfo; - - /* Define a positive value for return value if the attribute was found. This will - * cause the iterator to immediately return that positive value, - * indicating short-circuit success - */ - - if (HDstrncmp(name, (char *)op_data, MAX(HDstrlen((char *)op_data), HDstrlen(name))) == 0) - ret = H5_ITER_STOP; - - return ret; -} - -/*------------------------------------------------------------------------- * Function: H5LTfind_attribute * * Purpose: Inquires if an attribute named attr_name exists attached to @@ -1926,32 +1886,22 @@ H5LTfind_attribute(hid_t loc_id, const char *attr_name) * * Date: June 21, 2001 * - * Comments: - * The function uses H5Aiterate2 with the operator function find_attr - * * Return: - * Success: The return value of the first operator that - * returns non-zero, or zero if all members were - * processed with no operator returning non-zero. + * Success: Positive if the attribute exists attached to the + * object loc_id. Zero if the attribute does not + * exist attached to the object loc_id. * * Failure: Negative if something goes wrong within the - * library, or the negative value returned by one - * of the operators. + * library. * *------------------------------------------------------------------------- */ -/* H5Aiterate wants a non-const pointer but we have a const pointer in the API - * call. It's safe to ignore this because we control the callback, don't - * modify the op_data buffer (i.e.: attr_name) during the traversal, and the - * library never modifies that buffer. - */ -H5_GCC_CLANG_DIAG_OFF("cast-qual") herr_t H5LT_find_attribute(hid_t loc_id, const char *attr_name) { - return H5Aiterate2(loc_id, H5_INDEX_NAME, H5_ITER_INC, NULL, find_attr, (void *)attr_name); + htri_t attr_exists = H5Aexists(loc_id, attr_name); + return (attr_exists < 0) ? (herr_t)-1 : (attr_exists) ? (herr_t)1 : (herr_t)0; } -H5_GCC_CLANG_DIAG_ON("cast-qual") /*------------------------------------------------------------------------- * Function: H5LTget_attribute_ndims @@ -2579,11 +2529,9 @@ H5LT_dtype_to_text(hid_t dtype, char *dt_str, H5LT_lang_t lang, size_t *slen, hb } else if (H5Tequal(dtype, H5T_NATIVE_DOUBLE)) { HDsnprintf(dt_str, *slen, "H5T_NATIVE_DOUBLE"); -#if H5_SIZEOF_LONG_DOUBLE != 0 } else if (H5Tequal(dtype, H5T_NATIVE_LDOUBLE)) { HDsnprintf(dt_str, *slen, "H5T_NATIVE_LDOUBLE"); -#endif } else { HDsnprintf(dt_str, *slen, "undefined float"); diff --git a/hl/test/CMakeTests.cmake b/hl/test/CMakeTests.cmake index 44208e3..e532253 100644 --- a/hl/test/CMakeTests.cmake +++ b/hl/test/CMakeTests.cmake @@ -31,7 +31,9 @@ set (HL_REFERENCE_TEST_FILES dslat.txt dslon.txt test_ds_be.h5 + test_ds_be_new_ref.h5 test_ds_le.h5 + test_ds_le_new_ref.h5 test_ld.h5 ) diff --git a/hl/test/gen_test_ds.c b/hl/test/gen_test_ds.c index a56e6cf..d7a9f44 100644 --- a/hl/test/gen_test_ds.c +++ b/hl/test/gen_test_ds.c @@ -63,7 +63,11 @@ static int test_foreign_scaleattached(const char *filename); #define SCALE_4_NAME "scalename_4_" #define FILENAME "test_ds_" -#define FILEEXT ".h5" +#ifdef H5_DIMENSION_SCALES_WITH_NEW_REF +#define FILEEXT "_new_ref.h5" +#else +#define FILEEXT ".h5" +#endif /*------------------------------------------------------------------------- * the main program diff --git a/hl/test/test_ds.c b/hl/test/test_ds.c index 964e13f..c66ef49 100644 --- a/hl/test/test_ds.c +++ b/hl/test/test_ds.c @@ -137,10 +137,15 @@ static int test_attach_detach(void); #define DIM0_LABEL "Latitude" #define DIM1_LABEL "Longitude" +#ifdef H5_DIMENSION_SCALES_WITH_NEW_REF +#define FOREIGN_FILE1 "test_ds_le_new_ref.h5" +#define FOREIGN_FILE2 "test_ds_be_new_ref.h5" +#else #define FOREIGN_FILE1 "test_ds_le.h5" #define FOREIGN_FILE2 "test_ds_be.h5" -#define FILENAME "test_ds" -#define FILEEXT ".h5" +#endif +#define FILENAME "test_ds" +#define FILEEXT ".h5" #define FILE1 "test_ds3.h5" #define FILE2 "test_ds4.h5" @@ -161,10 +166,15 @@ static int test_attach_detach(void); int main(void) { - int nerrors = 0; + hid_t file_id = H5I_INVALID_HID; + int nerrors = 0; /* create file to be used in following tests */ - if (create_test_file("1") < 0) { + if ((file_id = create_test_file("1")) < 0) { + nerrors = 1; + goto error; + } + if (H5Fclose(file_id) < 0) { nerrors = 1; goto error; } @@ -179,7 +189,11 @@ main(void) nerrors += test_long_scalenames("1") < 0 ? 1 : 0; nerrors += test_float_scalenames("1") < 0 ? 1 : 0; nerrors += test_numberofscales("1") < 0 ? 1 : 0; - if (create_test_file("2") < 0) { + if ((file_id = create_test_file("2")) < 0) { + nerrors = 1; + goto error; + } + if (H5Fclose(file_id) < 0) { nerrors = 1; goto error; } @@ -254,10 +268,9 @@ open_test_file(const char *fileext) herr_t create_char_dataset(hid_t fid, const char *dsidx, int fulldims) { - int rank = 3; - int rankds = 1; - hsize_t dims[3] = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE}; - char buf[DIM1_SIZE * DIM2_SIZE * DIM3_SIZE]; + int rank = 3; + int rankds = 1; + hsize_t dims[3] = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE}; hsize_t s1_dim[1] = {DIM1_SIZE}; hsize_t s2_dim[1] = {DIM2_SIZE}; hsize_t s3_dim[1] = {DIM3_SIZE}; @@ -275,7 +288,7 @@ create_char_dataset(hid_t fid, const char *dsidx, int fulldims) HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx); /* make a dataset */ - if (H5LTmake_dataset_char(fid, name, rank, dims, buf) >= 0) { + if (H5LTmake_dataset_char(fid, name, rank, dims, NULL) >= 0) { if (fulldims == 0) { /* make a DS dataset for the first dimension */ if (create_DS1_char_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0) @@ -309,10 +322,9 @@ create_char_dataset(hid_t fid, const char *dsidx, int fulldims) herr_t create_short_dataset(hid_t fid, const char *dsidx, int fulldims) { - int rank = 3; - int rankds = 1; - hsize_t dims[3] = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE}; - short buf[DIM1_SIZE * DIM2_SIZE * DIM3_SIZE]; + int rank = 3; + int rankds = 1; + hsize_t dims[3] = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE}; hsize_t s1_dim[1] = {DIM1_SIZE}; hsize_t s2_dim[1] = {DIM2_SIZE}; hsize_t s3_dim[1] = {DIM3_SIZE}; @@ -330,7 +342,7 @@ create_short_dataset(hid_t fid, const char *dsidx, int fulldims) HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx); /* make a dataset */ - if (H5LTmake_dataset_short(fid, name, rank, dims, buf) >= 0) { + if (H5LTmake_dataset_short(fid, name, rank, dims, NULL) >= 0) { if (fulldims == 0) { /* make a DS dataset for the first dimension */ if (create_DS1_short_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0) @@ -364,10 +376,9 @@ create_short_dataset(hid_t fid, const char *dsidx, int fulldims) herr_t create_int_dataset(hid_t fid, const char *dsidx, int fulldims) { - int rank = RANK; - int rankds = 1; - hsize_t dims[RANK] = {DIM1_SIZE, DIM2_SIZE}; - int buf[DIM1_SIZE * DIM2_SIZE]; + int rank = RANK; + int rankds = 1; + hsize_t dims[RANK] = {DIM1_SIZE, DIM2_SIZE}; hsize_t s1_dim[1] = {DIM1_SIZE}; hsize_t s2_dim[1] = {DIM2_SIZE}; int s1_wbuf[DIM1_SIZE] = {10, 20, 30}; @@ -380,7 +391,7 @@ create_int_dataset(hid_t fid, const char *dsidx, int fulldims) HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx); /* make a dataset */ - if (H5LTmake_dataset_int(fid, name, rank, dims, buf) >= 0) { + if (H5LTmake_dataset_int(fid, name, rank, dims, NULL) >= 0) { if (fulldims == 0) { /* make a DS dataset for the first dimension */ if (create_DS1_int_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0) @@ -409,7 +420,6 @@ create_long_dataset(hid_t fid, const char *dsname, const char *dsidx, int fulldi int rank = 4; int rankds = 1; hsize_t dims[4] = {DIM1_SIZE, DIM2_SIZE, DIM3_SIZE, DIM4_SIZE}; - long * buf = NULL; hsize_t s1_dim[1] = {DIM1_SIZE}; hsize_t s2_dim[1] = {DIM2_SIZE}; hsize_t s3_dim[1] = {DIM3_SIZE}; @@ -429,12 +439,8 @@ create_long_dataset(hid_t fid, const char *dsname, const char *dsidx, int fulldi long s43_wbuf[DIM4_SIZE] = {180, 180}; long s44_wbuf[DIM4_SIZE] = {280, 280}; - /* Allocate buffer */ - if (NULL == (buf = (long *)HDmalloc(sizeof(long) * DIM1_SIZE * DIM2_SIZE * DIM3_SIZE * DIM4_SIZE))) - goto error; - /* make a dataset */ - if (H5LTmake_dataset_long(fid, dsname, rank, dims, buf) >= 0) { + if (H5LTmake_dataset_long(fid, dsname, rank, dims, NULL) >= 0) { if (fulldims == 0) { /* make a DS dataset for the first dimension */ if (create_DS1_long_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0) @@ -471,23 +477,18 @@ create_long_dataset(hid_t fid, const char *dsname, const char *dsidx, int fulldi else goto error; - HDfree(buf); - return SUCCEED; error: - HDfree(buf); - return FAIL; } herr_t create_float_dataset(hid_t fid, const char *dsidx, int fulldims) { - int rank = RANK; - int rankds = 1; - hsize_t dims[RANK] = {DIM1_SIZE, DIM2_SIZE}; - float buf[DIM1_SIZE * DIM2_SIZE]; + int rank = RANK; + int rankds = 1; + hsize_t dims[RANK] = {DIM1_SIZE, DIM2_SIZE}; hsize_t s1_dim[1] = {DIM1_SIZE}; hsize_t s2_dim[1] = {DIM2_SIZE}; float s1_wbuf[DIM1_SIZE] = {10, 20, 30}; @@ -500,7 +501,7 @@ create_float_dataset(hid_t fid, const char *dsidx, int fulldims) HDsnprintf(name, sizeof(name), "%s%s", DATASET_NAME, dsidx); /* make a dataset */ - if (H5LTmake_dataset_float(fid, name, rank, dims, buf) >= 0) { + if (H5LTmake_dataset_float(fid, name, rank, dims, NULL) >= 0) { if (fulldims == 0) { /* make a DS dataset for the first dimension */ if (create_DS1_float_datasets(fid, dsidx, rankds, s1_dim, s1_wbuf, NULL) < 0) diff --git a/hl/test/test_ds_be_new_ref-32bit.h5 b/hl/test/test_ds_be_new_ref-32bit.h5 Binary files differnew file mode 100644 index 0000000..ee327e9 --- /dev/null +++ b/hl/test/test_ds_be_new_ref-32bit.h5 diff --git a/hl/test/test_ds_be_new_ref.h5 b/hl/test/test_ds_be_new_ref.h5 Binary files differnew file mode 100644 index 0000000..eddfa02 --- /dev/null +++ b/hl/test/test_ds_be_new_ref.h5 diff --git a/hl/test/test_ds_le_new_ref.h5 b/hl/test/test_ds_le_new_ref.h5 Binary files differnew file mode 100644 index 0000000..8625d77 --- /dev/null +++ b/hl/test/test_ds_le_new_ref.h5 diff --git a/hl/test/test_packet_vlen.c b/hl/test/test_packet_vlen.c index c668a07..6d6bf34 100644 --- a/hl/test/test_packet_vlen.c +++ b/hl/test/test_packet_vlen.c @@ -539,8 +539,8 @@ test_VLof_VLtype(void) HDfprintf(stderr, "Cannot allocate memory for VL data! uu=%u\n", uu); goto error; } - t1->len = vv * 1; - for (ww = 0; ww < (vv * 1); ww++) + t1->len = vv + 1; + for (ww = 0; ww < (vv + 1); ww++) ((unsigned int *)t1->p)[ww] = uu * 100 + vv * 10 + ww; } /* end for */ } /* end for */ diff --git a/java/src/hdf/hdf5lib/CMakeLists.txt b/java/src/hdf/hdf5lib/CMakeLists.txt index 5651031..a26e117 100644 --- a/java/src/hdf/hdf5lib/CMakeLists.txt +++ b/java/src/hdf/hdf5lib/CMakeLists.txt @@ -40,6 +40,11 @@ set (HDF5_JAVA_HDF_HDF5_CALLBACKS_SOURCES callbacks/Callbacks.java ) +set (HDF5_JAVADOC_HDF_HDF5_CALLBACKS_SOURCES + ${HDF5_JAVA_HDF_HDF5_CALLBACKS_SOURCES} + callbacks/package-info.java +) + set (HDF5_JAVA_HDF_HDF5_EXCEPTIONS_SOURCES exceptions/HDF5Exception.java exceptions/HDF5IdException.java @@ -67,6 +72,11 @@ set (HDF5_JAVA_HDF_HDF5_EXCEPTIONS_SOURCES exceptions/HDF5SymbolTableException.java ) +set (HDF5_JAVADOC_HDF_HDF5_EXCEPTIONS_SOURCES + ${HDF5_JAVA_HDF_HDF5_EXCEPTIONS_SOURCES} + exceptions/package-info.java +) + set (HDF5_JAVA_HDF_HDF5_STRUCTS_SOURCES structs/H5_ih_info_t.java structs/H5A_info_t.java @@ -83,6 +93,11 @@ set (HDF5_JAVA_HDF_HDF5_STRUCTS_SOURCES structs/H5O_token_t.java ) +set (HDF5_JAVADOC_HDF_HDF5_STRUCTS_SOURCES + ${HDF5_JAVA_HDF_HDF5_STRUCTS_SOURCES} + structs/package-info.java +) + set (HDF5_JAVA_HDF_HDF5_SOURCES HDFArray.java HDF5Constants.java @@ -91,6 +106,11 @@ set (HDF5_JAVA_HDF_HDF5_SOURCES H5.java ) +set (HDF5_JAVADOC_HDF_HDF5_SOURCES + ${HDF5_JAVA_HDF_HDF5_SOURCES} + package-info.java +) + set (CMAKE_JNI_TARGET TRUE) file (WRITE ${PROJECT_BINARY_DIR}/Manifest.txt @@ -114,7 +134,7 @@ add_dependencies (${HDF5_JAVA_HDF5_LIB_TARGET} ${HDF5_JAVA_JNI_LIB_TARGET}) set_target_properties (${HDF5_JAVA_HDF5_LIB_TARGET} PROPERTIES FOLDER libraries/java) create_javadoc(hdf5_java_doc - FILES ${HDF5_JAVA_HDF_HDF5_CALLBACKS_SOURCES} ${HDF5_JAVA_HDF_HDF5_EXCEPTIONS_SOURCES} ${HDF5_JAVA_HDF_HDF5_STRUCTS_SOURCES} ${HDF5_JAVA_HDF_HDF5_SOURCES} + FILES ${HDF5_JAVADOC_HDF_HDF5_CALLBACKS_SOURCES} ${HDF5_JAVADOC_HDF_HDF5_EXCEPTIONS_SOURCES} ${HDF5_JAVADOC_HDF_HDF5_STRUCTS_SOURCES} ${HDF5_JAVADOC_HDF_HDF5_SOURCES} OVERVIEW ${HDF5_JAVA_HDF5_SRC_DIR}/overview.html CLASSPATH ${CMAKE_JAVA_INCLUDE_PATH} WINDOWTITLE "HDF5 Java" diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 63547df..4129f50 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -64,8 +64,8 @@ import hdf.hdf5lib.structs.H5O_token_t; * This code is the called by Java programs to access the entry points of the HDF5 library. Each routine wraps a single * HDF5 entry point, generally with the arguments and return codes analogous to the C interface. * <p> - * For details of the HDF5 library, see the HDF5 Documentation at: <a - * href="http://hdfgroup.org/HDF5/">http://hdfgroup.org/HDF5/</a> + * For details of the HDF5 library, see the HDF5 Documentation at: + * <a href="http://hdfgroup.org/HDF5/">http://hdfgroup.org/HDF5/</a> * <hr> * <p> * <b>Mapping of arguments for Java</b> @@ -162,8 +162,8 @@ import hdf.hdf5lib.structs.H5O_token_t; * <p> * For Java, this ``ANY'' is a problem, as the type of data must always be declared. Furthermore, multidimensional * arrays are definitely <i>not</i> layed out contiguously in memory. It would be infeasible to declare a separate - * routine for every combination of number type and dimensionality. For that reason, the <a - * href="./hdf.hdf5lib.HDFArray.html"><b>HDFArray</b></a> class is used to discover the type, shape, and size of the + * routine for every combination of number type and dimensionality. For that reason, the + * <a href="./hdf.hdf5lib.HDFArray.html"><b>HDFArray</b></a> class is used to discover the type, shape, and size of the * data array at run time, and to convert to and from a contiguous array of bytes in synchronized static native C order. * <p> * The upshot is that any Java array of numbers (either primitive or sub-classes of type <b>Number</b>) can be passed as @@ -187,8 +187,8 @@ import hdf.hdf5lib.structs.H5O_token_t; * <b><i>H5F_ACC_RDWR</i></b> and <b><i>H5P_DEFAULT</i></b>. * <p> * The HDF-5 API defines a set of values that describe number types and sizes, such as "H5T_NATIVE_INT" and "hsize_t". - * These values are determined at run time by the HDF-5 C library. To support these parameters, the Java class <a - * href="./hdf.hdf5lib.HDF5CDataTypes.html"> <b>HDF5CDataTypes</b></a> looks up the values when initiated. The values + * These values are determined at run time by the HDF-5 C library. To support these parameters, the Java class + * <a href="./hdf.hdf5lib.HDF5CDataTypes.html"> <b>HDF5CDataTypes</b></a> looks up the values when initiated. The values * can be accessed as public variables of the Java class, such as: * * <pre> @@ -204,8 +204,8 @@ import hdf.hdf5lib.structs.H5O_token_t; * JHI5. Errors are converted into Java exceptions. This is totally different from the C interface, but is very natural * for Java programming. * <p> - * The exceptions of the JHI5 are organized as sub-classes of the class <a - * href="./hdf.hdf5lib.exceptions.HDF5Exception.html"> <b>HDF5Exception</b></a>. There are two subclasses of + * The exceptions of the JHI5 are organized as sub-classes of the class + * <a href="./hdf.hdf5lib.exceptions.HDF5Exception.html"> <b>HDF5Exception</b></a>. There are two subclasses of * <b>HDF5Exception</b>, <a href="./hdf.hdf5lib.exceptions.HDF5LibraryException.html"> <b>HDF5LibraryException</b></a> * and <a href="./hdf.hdf5lib.exceptions.HDF5JavaException.html"> <b>HDF5JavaException</b></a>. The sub-classes of the * former represent errors from the HDF-5 C library, while sub-classes of the latter represent errors in the JHI5 @@ -216,7 +216,7 @@ import hdf.hdf5lib.structs.H5O_token_t; * exception handlers to print out the HDF-5 error stack. * <hr> * - * @version HDF5 1.13.0 <BR> + * @version HDF5 1.13.1 <BR> * <b>See also: <a href ="./hdf.hdf5lib.HDFArray.html"> hdf.hdf5lib.HDFArray</a> </b><BR> * <a href ="./hdf.hdf5lib.HDF5Constants.html"> hdf.hdf5lib.HDF5Constants</a><BR> * <a href ="./hdf.hdf5lib.HDF5CDataTypes.html"> hdf.hdf5lib.HDF5CDataTypes</a><BR> @@ -239,7 +239,7 @@ public class H5 implements java.io.Serializable { * * Make sure to update the versions number when a different library is used. */ - public final static int LIB_VERSION[] = { 1, 13, 0 }; + public final static int LIB_VERSION[] = { 1, 13, 1 }; /** * add system property to load library by path @@ -12776,8 +12776,11 @@ public class H5 implements java.io.Serializable { * IN: Field name of the field index to retrieve. * * @return if field is defined, the index; else negative. + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. **/ - public synchronized static native int H5Tget_member_index(long type_id, String field_name); + public synchronized static native int H5Tget_member_index(long type_id, String field_name) throws HDF5LibraryException; /** * H5Tget_member_name retrieves the name of a field of a compound datatype or an element of an enumeration datatype. @@ -12788,8 +12791,11 @@ public class H5 implements java.io.Serializable { * IN: Field index (0-based) of the field name to retrieve. * * @return a valid pointer to the name if successful; otherwise null. + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. **/ - public synchronized static native String H5Tget_member_name(long type_id, int field_idx); + public synchronized static native String H5Tget_member_name(long type_id, int field_idx) throws HDF5LibraryException; /** * H5Tget_member_offset returns the byte offset of the specified member of the compound datatype. This is the byte @@ -12801,11 +12807,8 @@ public class H5 implements java.io.Serializable { * IN: Field index (0-based) of the field type to retrieve. * * @return the offset of the member. - * - * @exception HDF5LibraryException - * - Error from the HDF-5 Library. **/ - public synchronized static native long H5Tget_member_offset(long type_id, int membno) throws HDF5LibraryException; + public synchronized static native long H5Tget_member_offset(long type_id, int membno); /** * H5Tget_member_type returns the datatype of the specified member. @@ -13460,7 +13463,7 @@ public class H5 implements java.io.Serializable { * @exception HDF5LibraryException * - Error from the HDF-5 Library. **/ - public synchronized static native long H5VLregister_connector_by_name(String connector_name, long vipl_id); + public synchronized static native long H5VLregister_connector_by_name(String connector_name, long vipl_id) throws HDF5LibraryException; /** * H5VLregister_connector_by_value registers a new VOL connector as a member of the virtual object layer class. * @@ -13475,7 +13478,7 @@ public class H5 implements java.io.Serializable { * @exception HDF5LibraryException * - Error from the HDF-5 Library. **/ - public synchronized static native long H5VLregister_connector_by_value(int connector_value, long vipl_id); + public synchronized static native long H5VLregister_connector_by_value(int connector_value, long vipl_id) throws HDF5LibraryException; /** * H5VLis_connector_registered_by_name tests whether a VOL class has been registered. * @@ -13487,7 +13490,7 @@ public class H5 implements java.io.Serializable { * @exception HDF5LibraryException * - Error from the HDF-5 Library. **/ - public synchronized static native boolean H5VLis_connector_registered_by_name(String name); + public synchronized static native boolean H5VLis_connector_registered_by_name(String name) throws HDF5LibraryException; /** * H5VLis_connector_registered_by_value tests whether a VOL class has been registered. * @@ -13499,7 +13502,7 @@ public class H5 implements java.io.Serializable { * @exception HDF5LibraryException * - Error from the HDF-5 Library. **/ - public synchronized static native boolean H5VLis_connector_registered_by_value(int connector_value); + public synchronized static native boolean H5VLis_connector_registered_by_value(int connector_value) throws HDF5LibraryException; /** * H5VLget_connector_id retrieves the ID for a registered VOL connector for a given object. * @@ -13511,7 +13514,7 @@ public class H5 implements java.io.Serializable { * @exception HDF5LibraryException * - Error from the HDF-5 Library. **/ - public synchronized static native long H5VLget_connector_id(long object_id); + public synchronized static native long H5VLget_connector_id(long object_id) throws HDF5LibraryException; /** * H5VLget_connector_id_by_name retrieves the ID for a registered VOL connector. * @@ -13523,7 +13526,7 @@ public class H5 implements java.io.Serializable { * @exception HDF5LibraryException * - Error from the HDF-5 Library. **/ - public synchronized static native long H5VLget_connector_id_by_name(String name); + public synchronized static native long H5VLget_connector_id_by_name(String name) throws HDF5LibraryException; /** * H5VLget_connector_id_by_value retrieves the ID for a registered VOL connector. * @@ -13535,7 +13538,7 @@ public class H5 implements java.io.Serializable { * @exception HDF5LibraryException * - Error from the HDF-5 Library. **/ - public synchronized static native long H5VLget_connector_id_by_value(int connector_value); + public synchronized static native long H5VLget_connector_id_by_value(int connector_value) throws HDF5LibraryException; /** * H5VLget_connector_name returns the connector name for the VOL associated with the * object or file ID. @@ -13548,7 +13551,7 @@ public class H5 implements java.io.Serializable { * @exception HDF5LibraryException * - Error from the HDF-5 Library. **/ - public synchronized static native String H5VLget_connector_name(long object_id); + public synchronized static native String H5VLget_connector_name(long object_id) throws HDF5LibraryException; /** * H5VLclose closes a VOL connector ID. * @@ -13558,7 +13561,7 @@ public class H5 implements java.io.Serializable { * @exception HDF5LibraryException * - Error from the HDF-5 Library. **/ - public synchronized static native void H5VLclose(long connector_id); + public synchronized static native void H5VLclose(long connector_id) throws HDF5LibraryException; /** * H5VLunregister_connector removes a VOL connector ID from the library. * @@ -13568,7 +13571,7 @@ public class H5 implements java.io.Serializable { * @exception HDF5LibraryException * - Error from the HDF-5 Library. **/ - public synchronized static native void H5VLunregister_connector(long connector_id); + public synchronized static native void H5VLunregister_connector(long connector_id) throws HDF5LibraryException; // /////// unimplemented //////// // hid_t H5VLregister_connector(const H5VL_class_t *cls, hid_t vipl_id); diff --git a/java/src/hdf/hdf5lib/callbacks/package-info.java b/java/src/hdf/hdf5lib/callbacks/package-info.java new file mode 100644 index 0000000..0833162 --- /dev/null +++ b/java/src/hdf/hdf5lib/callbacks/package-info.java @@ -0,0 +1,27 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/** All callback definitions must derive from the Callbacks interface. Any + * derived interfaces must define a single public method named "callback". + * You are responsible for deregistering your callback (if necessary) + * in its {@link Object#finalize} method. If native code attempts to call + * a callback which has been GC'd, you will likely crash the VM. If + * there is no method to deregister the callback (e.g. <code>atexit</code> + * in the C library), you must ensure that you always keep a live reference + * to the callback object.<p> + * A callback should generally never throw an exception, since it doesn't + * necessarily have an encompassing Java environment to catch it. Any + * exceptions thrown will be passed to the default callback exception + * handler. + */ +package hdf.hdf5lib.callbacks; diff --git a/java/src/hdf/hdf5lib/exceptions/package-info.java b/java/src/hdf/hdf5lib/exceptions/package-info.java new file mode 100644 index 0000000..8640ccb --- /dev/null +++ b/java/src/hdf/hdf5lib/exceptions/package-info.java @@ -0,0 +1,31 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/** + * <p> + * The package exceptions contains error classes for the Java HDF5 Interface. + * <p> + * There are two sub-classes of exceptions defined: + * <ol> + * <li> + * HDF5LibraryException -- errors raised the HDF5 library code + * <li> + * HDF5JavaException -- errors raised the HDF5 Java wrapper code + * </ol> + * <p> + * The HDF5LibraryException is the base class for the classes that represent specific error conditions. + * In particular, HDF5LibraryException has a sub-class for each major + * error code returned by the HDF5 library. + * + */ +package hdf.hdf5lib.exceptions;
\ No newline at end of file diff --git a/java/src/hdf/hdf5lib/package-info.java b/java/src/hdf/hdf5lib/package-info.java new file mode 100644 index 0000000..838a266 --- /dev/null +++ b/java/src/hdf/hdf5lib/package-info.java @@ -0,0 +1,174 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + + +/** + * This package is the Java interface for the HDF5 library. + * <p> + * This code is the called by Java programs to access the entry points of the HDF5 library. Each routine wraps a single + * HDF5 entry point, generally with the arguments and return codes analogous to the C interface. + * <p> + * For details of the HDF5 library, see the HDF5 Documentation at: + * <a href="http://hdfgroup.org/HDF5/">http://hdfgroup.org/HDF5/</a> + * <hr> + * <p> + * <b>Mapping of arguments for Java</b> + * + * <p> + * In general, arguments to the HDF Java API are straightforward translations from the 'C' API described in the HDF + * Reference Manual. + * + * <table border=1> + * <caption><b>HDF-5 C types to Java types</b> </caption> + * <tr> + * <td><b>HDF-5</b></td> + * <td><b>Java</b></td> + * </tr> + * <tr> + * <td>H5T_NATIVE_INT</td> + * <td>int, Integer</td> + * </tr> + * <tr> + * <td>H5T_NATIVE_SHORT</td> + * <td>short, Short</td> + * </tr> + * <tr> + * <td>H5T_NATIVE_FLOAT</td> + * <td>float, Float</td> + * </tr> + * <tr> + * <td>H5T_NATIVE_DOUBLE</td> + * <td>double, Double</td> + * </tr> + * <tr> + * <td>H5T_NATIVE_CHAR</td> + * <td>byte, Byte</td> + * </tr> + * <tr> + * <td>H5T_C_S1</td> + * <td>java.lang.String</td> + * </tr> + * <tr> + * <td>void * <BR> + * (i.e., pointer to `Any')</td> + * <td>Special -- see HDFArray</td> + * </tr> + * </table> + * <b>General Rules for Passing Arguments and Results</b> + * <p> + * In general, arguments passed <b>IN</b> to Java are the analogous basic types, as above. The exception is for arrays, + * which are discussed below. + * <p> + * The <i>return value</i> of Java methods is also the analogous type, as above. A major exception to that rule is that + * all HDF functions that return SUCCEED/FAIL are declared <i>boolean</i> in the Java version, rather than <i>int</i> as + * in the C. Functions that return a value or else FAIL are declared the equivalent to the C function. However, in most + * cases the Java method will raise an exception instead of returning an error code. + * See <a href="#ERRORS">Errors and Exceptions</a> below. + * <p> + * Java does not support pass by reference of arguments, so arguments that are returned through <b>OUT</b> parameters + * must be wrapped in an object or array. The Java API for HDF consistently wraps arguments in arrays. + * <p> + * For instance, a function that returns two integers is declared: + * + * <pre> + * h_err_t HDF5dummy( int *a1, int *a2) + * </pre> + * + * For the Java interface, this would be declared: + * + * <pre> + * public synchronized static native void HDF5dummy(int args[]); + * </pre> + * + * where <i>a1</i> is <i>args[0]</i> and <i>a2</i> is <i>args[1]</i>, and would be invoked: + * + * <pre> + * H5.HDF5dummy(a); + * </pre> + * + * <p> + * All the routines where this convention is used will have specific documentation of the details, given below. + * <p> + * <b>Arrays</b> + * <p> + * HDF5 needs to read and write multi-dimensional arrays of any number type (and records). The HDF5 API describes the + * layout of the source and destination, and the data for the array passed as a block of bytes, for instance, + * + * <pre> + * herr_t H5Dread(long fid, long filetype, long memtype, long memspace, void *data); + * </pre> + * + * <p> + * where ``void *'' means that the data may be any valid numeric type, and is a contiguous block of bytes that is the + * data for a multi-dimensional array. The other parameters describe the dimensions, rank, and datatype of the array on + * disk (source) and in memory (destination). + * <p> + * For Java, this ``ANY'' is a problem, as the type of data must always be declared. Furthermore, multidimensional + * arrays are definitely <i>not</i> layed out contiguously in memory. It would be infeasible to declare a separate + * routine for every combination of number type and dimensionality. For that reason, the + * <a href="./hdf.hdf5lib.HDFArray.html"><b>HDFArray</b></a> class is used to discover the type, shape, and size of the + * data array at run time, and to convert to and from a contiguous array of bytes in synchronized static native C order. + * <p> + * The upshot is that any Java array of numbers (either primitive or sub-classes of type <b>Number</b>) can be passed as + * an ``Object'', and the Java API will translate to and from the appropriate packed array of bytes needed by the C + * library. So the function above would be declared: + * + * <pre> + * public synchronized static native int H5Dread(long fid, long filetype, long memtype, long memspace, Object data); + * </pre> + * OPEN_IDS.addElement(id); + + * and the parameter <i>data</i> can be any multi-dimensional array of numbers, such as float[][], or int[][][], or + * Double[][]. + * <p> + * <b>HDF-5 Constants</b> + * <p> + * The HDF-5 API defines a set of constants and enumerated values. Most of these values are available to Java programs + * via the class <a href="./hdf.hdf5lib.HDF5Constants.html"> <b>HDF5Constants</b></a>. For example, the parameters for + * the h5open() call include two numeric values, <b><i>HDFConstants.H5F_ACC_RDWR</i></b> and + * <b><i>HDF5Constants.H5P_DEFAULT</i></b>. As would be expected, these numbers correspond to the C constants + * <b><i>H5F_ACC_RDWR</i></b> and <b><i>H5P_DEFAULT</i></b>. + * <p> + * The HDF-5 API defines a set of values that describe number types and sizes, such as "H5T_NATIVE_INT" and "hsize_t". + * These values are determined at run time by the HDF-5 C library. To support these parameters, the Java class + * <a href="./hdf.hdf5lib.HDF5CDataTypes.html"> <b>HDF5CDataTypes</b></a> looks up the values when initiated. The values + * can be accessed as public variables of the Java class, such as: + * + * <pre> + * long data_type = HDF5CDataTypes.JH5T_NATIVE_INT; + * </pre> + * + * The Java application uses both types of constants the same way, the only difference is that the + * <b><i>HDF5CDataTypes</i></b> may have different values on different platforms. + * <p> + * <b>Error handling and Exceptions</b> + * <p> + * The HDF5 error API (H5E) manages the behavior of the error stack in the HDF-5 library. This API is available from the + * JHI5. Errors are converted into Java exceptions. This is totally different from the C interface, but is very natural + * for Java programming. + * <p> + * The exceptions of the JHI5 are organized as sub-classes of the class + * <a href="./hdf.hdf5lib.exceptions.HDF5Exception.html"> <b>HDF5Exception</b></a>. There are two subclasses of + * <b>HDF5Exception</b>, <a href="./hdf.hdf5lib.exceptions.HDF5LibraryException.html"> <b>HDF5LibraryException</b></a> + * and <a href="./hdf.hdf5lib.exceptions.HDF5JavaException.html"> <b>HDF5JavaException</b></a>. The sub-classes of the + * former represent errors from the HDF-5 C library, while sub-classes of the latter represent errors in the JHI5 + * wrapper and support code. + * <p> + * The super-class <b><i>HDF5LibraryException</i></b> implements the method '<b><i>printStackTrace()</i></b>', which + * prints out the HDF-5 error stack, as described in the HDF-5 C API <i><b>H5Eprint()</b>.</i> This may be used by Java + * exception handlers to print out the HDF-5 error stack. + * <hr> + * + * <b>See also: <a href="http://hdfgroup.org/HDF5/"> http://hdfgroup.org/HDF5"</a></b> + **/ +package hdf.hdf5lib;
\ No newline at end of file diff --git a/java/src/hdf/hdf5lib/structs/package-info.java b/java/src/hdf/hdf5lib/structs/package-info.java new file mode 100644 index 0000000..8a9d97d --- /dev/null +++ b/java/src/hdf/hdf5lib/structs/package-info.java @@ -0,0 +1,17 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/** All structure definitions define java equivalents of the C structures needed + * by the C API calls. See the C API for information about the structures. + */ +package hdf.hdf5lib.structs; diff --git a/java/src/jni/h5util.c b/java/src/jni/h5util.c index 4272205..7313c28 100644 --- a/java/src/jni/h5util.c +++ b/java/src/jni/h5util.c @@ -237,7 +237,7 @@ h5str_convert(JNIEnv *env, char **in_str, hid_t container, hid_t tid, void *out_ HDmemcpy(cptr, &tmp_double, sizeof(double)); break; } -#if H5_SIZEOF_LONG_DOUBLE != 0 && H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE case sizeof(long double): { long double tmp_ldouble = 0.0; @@ -658,8 +658,6 @@ h5str_sprint_reference(JNIEnv *env, h5str_t *out_str, void *ref_p) int ret_value = FAIL; - if (!h5str_append(out_str, " \"")) - H5_ASSERTION_ERROR(ENVONLY, "Unable to append string."); buf_size = H5Rget_file_name(ref_vp, NULL, 0); if (buf_size) { ref_name = (char *)HDmalloc(sizeof(char) * (size_t)buf_size + 1); @@ -697,8 +695,6 @@ h5str_sprint_reference(JNIEnv *env, h5str_t *out_str, void *ref_p) ref_name = NULL; } } - if (!h5str_append(out_str, "\"")) - H5_ASSERTION_ERROR(ENVONLY, "Unable to append string."); ret_value = SUCCEED; done: @@ -725,6 +721,10 @@ h5str_region_dataset(JNIEnv *env, h5str_t *out_str, H5R_ref_t *ref_vp, int expan if ((new_obj_sid = H5Ropen_region(ref_vp, H5P_DEFAULT, H5P_DEFAULT)) < 0) H5_LIBRARY_ERROR(ENVONLY); + if (expand_data == 0) + if (h5str_sprint_reference(ENVONLY, out_str, ref_vp) < 0) + CHECK_JNI_EXCEPTION(ENVONLY, JNI_FALSE); + if ((region_type = H5Sget_select_type(new_obj_sid)) > H5S_SEL_ERROR) { if (H5S_SEL_POINTS == region_type) { if (h5str_dump_region_points(ENVONLY, out_str, new_obj_sid, new_obj_id, expand_data) < 0) @@ -816,7 +816,7 @@ h5str_sprintf(JNIEnv *env, h5str_t *out_str, hid_t container, hid_t tid, void *i break; } -#if H5_SIZEOF_LONG_DOUBLE != 0 && H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE case sizeof(long double): { long double tmp_ldouble = 0.0; diff --git a/java/test/CMakeLists.txt b/java/test/CMakeLists.txt index ffaf5f5..c6cf607 100644 --- a/java/test/CMakeLists.txt +++ b/java/test/CMakeLists.txt @@ -113,14 +113,14 @@ endif () get_property (target_name TARGET ${HDF5_JAVA_JNI_LIB_TARGET} PROPERTY OUTPUT_NAME) set (CMD_ARGS "-Dhdf.hdf5lib.H5.loadLibraryName=${target_name}$<$<CONFIG:Debug>:${CMAKE_DEBUG_POSTFIX}>;") +set (CMAKE_JAVA_CLASSPATH ".") +foreach (CMAKE_INCLUDE_PATH ${CMAKE_JAVA_INCLUDE_PATH}) + set (CMAKE_JAVA_CLASSPATH "${CMAKE_JAVA_CLASSPATH}${CMAKE_JAVA_INCLUDE_FLAG_SEP}${CMAKE_INCLUDE_PATH}") +endforeach () if (HDF5_TEST_JAVA AND HDF5_TEST_SERIAL) foreach (test_file ${HDF5_JAVA_TEST_SOURCES}) - set (CMAKE_JAVA_CLASSPATH ".") - foreach (CMAKE_INCLUDE_PATH ${CMAKE_JAVA_INCLUDE_PATH}) - set (CMAKE_JAVA_CLASSPATH "${CMAKE_JAVA_CLASSPATH}${CMAKE_JAVA_INCLUDE_FLAG_SEP}${CMAKE_INCLUDE_PATH}") - endforeach () - set (CMAKE_JAVA_CLASSPATH "${CMAKE_JAVA_CLASSPATH}${CMAKE_JAVA_INCLUDE_FLAG_SEP}${${HDF5_JAVA_TEST_LIB_TARGET}_${test_file}_JAR_FILE}") + set (TEST_JAVA_CLASSPATH "${CMAKE_JAVA_CLASSPATH}${CMAKE_JAVA_INCLUDE_FLAG_SEP}${${HDF5_JAVA_TEST_LIB_TARGET}_${test_file}_JAR_FILE}") add_test ( NAME JUnit-${test_file}-clearall-objects @@ -132,7 +132,7 @@ if (HDF5_TEST_JAVA AND HDF5_TEST_SERIAL) NAME JUnit-${test_file} COMMAND "${CMAKE_COMMAND}" -D "TEST_TESTER=${CMAKE_Java_RUNTIME};${CMAKE_Java_RUNTIME_FLAGS}" - -D "TEST_CLASSPATH:STRING=${CMAKE_JAVA_CLASSPATH}" + -D "TEST_CLASSPATH:STRING=${TEST_JAVA_CLASSPATH}" -D "TEST_ARGS:STRING=${CMD_ARGS}-ea;org.junit.runner.JUnitCore" -D "TEST_PROGRAM=test.${test_file}" -D "TEST_LIBRARY_DIRECTORY=${CMAKE_TEST_OUTPUT_DIRECTORY}" @@ -184,7 +184,7 @@ if (HDF5_TEST_JAVA AND HDF5_TEST_SERIAL) -D "TEST_MASK_ERROR=TRUE" # -D "TEST_FILTER:STRING=${testfilter}" -D "TEST_REFERENCE=JUnit-${volname}-${voltest}.txt" - -P "${HDF_RESOURCES_DIR}/jvolTest.cmake" + -P "${HDF_RESOURCES_DIR}/jrunTest.cmake" ) set_tests_properties (JUnit-VOL-${volname}-${voltest} PROPERTIES ENVIRONMENT "HDF5_PLUGIN_PATH=${CMAKE_BINARY_DIR}/testdir2" @@ -228,12 +228,8 @@ if (HDF5_TEST_JAVA AND HDF5_TEST_SERIAL) foreach (volinfo IN LISTS ${voltest}) foreach (h5_file ${HDF5_JAVA_TEST_SOURCES}) - set (CMAKE_JAVA_CLASSPATH ".") - foreach (CMAKE_INCLUDE_PATH ${CMAKE_JAVA_INCLUDE_PATH}) - set (CMAKE_JAVA_CLASSPATH "${CMAKE_JAVA_CLASSPATH}${CMAKE_JAVA_INCLUDE_FLAG_SEP}${CMAKE_INCLUDE_PATH}") - endforeach () - set (CMAKE_JAVA_CLASSPATH "${CMAKE_JAVA_CLASSPATH}${CMAKE_JAVA_INCLUDE_FLAG_SEP}${${HDF5_JAVA_TEST_LIB_TARGET}_${h5_file}_JAR_FILE}") - DO_VOL_TEST (${h5_file} ${voltest} "${volinfo}" ${CMAKE_JAVA_CLASSPATH}) + set (VOL_JAVA_CLASSPATH "${CMAKE_JAVA_CLASSPATH}${CMAKE_JAVA_INCLUDE_FLAG_SEP}${${HDF5_JAVA_TEST_LIB_TARGET}_${h5_file}_JAR_FILE}") + DO_VOL_TEST (${h5_file} ${voltest} "${volinfo}" "${VOL_JAVA_CLASSPATH}") endforeach () endforeach () endforeach () diff --git a/java/test/TestH5.java b/java/test/TestH5.java index 1f298f3..1f81f09 100644 --- a/java/test/TestH5.java +++ b/java/test/TestH5.java @@ -287,7 +287,7 @@ public class TestH5 { */ @Test public void testH5get_libversion() { - int libversion[] = { 1, 13, 0 }; + int libversion[] = { 1, 13, 1 }; try { H5.H5get_libversion(libversion); @@ -326,7 +326,7 @@ public class TestH5 { */ @Test public void testH5check_version() { - int majnum = 1, minnum = 13, relnum = 0; + int majnum = 1, minnum = 13, relnum = 1; try { H5.H5check_version(majnum, minnum, relnum); diff --git a/m4/aclocal_fc.f90 b/m4/aclocal_fc.f90 index e9a11c0..240a768 100644 --- a/m4/aclocal_fc.f90 +++ b/m4/aclocal_fc.f90 @@ -82,6 +82,7 @@ END PROGRAM PROG_FC_C_LONG_DOUBLE_EQ_C_DOUBLE !---- START ----- Determine the available KINDs for REALs and INTEGERs PROGRAM FC_AVAIL_KINDS + USE, INTRINSIC :: ISO_FORTRAN_ENV, ONLY : stderr=>ERROR_UNIT IMPLICIT NONE INTEGER :: ik, jk, k, kk, max_decimal_prec INTEGER :: prev_rkind, num_rkinds = 1, num_ikinds = 1 @@ -89,8 +90,6 @@ PROGRAM FC_AVAIL_KINDS INTEGER, DIMENSION(1:10) :: list_rkinds = -1 LOGICAL :: new_kind - OPEN(8, FILE='pac_fconftest.out', FORM='formatted') - ! Find integer KINDs list_ikinds(num_ikinds)=SELECTED_INT_KIND(1) DO ik = 2, 36 @@ -103,11 +102,11 @@ PROGRAM FC_AVAIL_KINDS ENDDO DO k = 1, num_ikinds - WRITE(8,'(I0)', ADVANCE='NO') list_ikinds(k) + WRITE(stderr,'(I0)', ADVANCE='NO') list_ikinds(k) IF(k.NE.num_ikinds)THEN - WRITE(8,'(A)',ADVANCE='NO') ',' + WRITE(stderr,'(A)',ADVANCE='NO') ',' ELSE - WRITE(8,'()') + WRITE(stderr,'()') ENDIF ENDDO @@ -140,17 +139,17 @@ PROGRAM FC_AVAIL_KINDS ENDDO prec DO k = 1, num_rkinds - WRITE(8,'(I0)', ADVANCE='NO') list_rkinds(k) + WRITE(stderr,'(I0)', ADVANCE='NO') list_rkinds(k) IF(k.NE.num_rkinds)THEN - WRITE(8,'(A)',ADVANCE='NO') ',' + WRITE(stderr,'(A)',ADVANCE='NO') ',' ELSE - WRITE(8,'()') + WRITE(stderr,'()') ENDIF ENDDO - WRITE(8,'(I0)') max_decimal_prec - WRITE(8,'(I0)') num_ikinds - WRITE(8,'(I0)') num_rkinds + WRITE(stderr,'(I0)') max_decimal_prec + WRITE(stderr,'(I0)') num_ikinds + WRITE(stderr,'(I0)') num_rkinds END PROGRAM FC_AVAIL_KINDS !---- END ----- Determine the available KINDs for REALs and INTEGERs diff --git a/m4/aclocal_fc.m4 b/m4/aclocal_fc.m4 index e5d664d..49ff485 100644 --- a/m4/aclocal_fc.m4 +++ b/m4/aclocal_fc.m4 @@ -68,7 +68,7 @@ AC_DEFUN([PAC_PROG_FC_ISO_FORTRAN_ENV],[ AC_MSG_CHECKING([if Fortran compiler supports intrinsic module ISO_FORTRAN_ENV]) TEST_SRC="`sed -n '/PROGRAM PROG_FC_ISO_FORTRAN_ENV/,/END PROGRAM PROG_FC_ISO_FORTRAN_ENV/p' $srcdir/m4/aclocal_fc.f90`" AC_LINK_IFELSE([$TEST_SRC],[AC_MSG_RESULT([yes]) - HAVE_ISO_FORTRAN_ENV="yes"], + HAVE_ISO_FORTRAN_ENV="yes"], [AC_MSG_RESULT([no])]) ]) @@ -79,7 +79,7 @@ AC_DEFUN([PAC_PROG_FC_SIZEOF],[ AC_MSG_CHECKING([if Fortran compiler supports intrinsic SIZEOF]) TEST_SRC="`sed -n '/PROGRAM PROG_FC_SIZEOF/,/END PROGRAM PROG_FC_SIZEOF/p' $srcdir/m4/aclocal_fc.f90`" AC_LINK_IFELSE([$TEST_SRC],[AC_MSG_RESULT([yes]) - HAVE_SIZEOF_FORTRAN="yes"], + HAVE_SIZEOF_FORTRAN="yes"], [AC_MSG_RESULT([no])]) ]) @@ -90,7 +90,7 @@ AC_DEFUN([PAC_PROG_FC_C_SIZEOF],[ AC_MSG_CHECKING([if Fortran compiler supports intrinsic C_SIZEOF]) TEST_SRC="`sed -n '/PROGRAM PROG_FC_C_SIZEOF/,/END PROGRAM PROG_FC_C_SIZEOF/p' $srcdir/m4/aclocal_fc.f90`" AC_LINK_IFELSE([$TEST_SRC], [AC_MSG_RESULT([yes]) - HAVE_C_SIZEOF_FORTRAN="yes"], + HAVE_C_SIZEOF_FORTRAN="yes"], [AC_MSG_RESULT([no])]) ]) @@ -101,7 +101,7 @@ AC_DEFUN([PAC_PROG_FC_STORAGE_SIZE],[ AC_MSG_CHECKING([if Fortran compiler supports intrinsic STORAGE_SIZE]) TEST_SRC="`sed -ne '/PROGRAM PROG_FC_STORAGE_SIZE/,/END PROGRAM PROG_FC_STORAGE_SIZE/p' $srcdir/m4/aclocal_fc.f90`" AC_LINK_IFELSE([$TEST_SRC], [AC_MSG_RESULT([yes]) - HAVE_STORAGE_SIZE_FORTRAN="yes"], + HAVE_STORAGE_SIZE_FORTRAN="yes"], [AC_MSG_RESULT([no])]) ]) @@ -114,7 +114,7 @@ AC_DEFUN([PAC_PROG_FC_HAVE_C_LONG_DOUBLE],[ TEST_SRC="" TEST_SRC="`sed -n '/PROGRAM PROG_FC_HAVE_C_LONG_DOUBLE/,/END PROGRAM PROG_FC_HAVE_C_LONG_DOUBLE/p' $srcdir/m4/aclocal_fc.f90`" AC_LINK_IFELSE([$TEST_SRC], [AC_MSG_RESULT([yes]) - HAVE_C_LONG_DOUBLE_FORTRAN="yes"], + HAVE_C_LONG_DOUBLE_FORTRAN="yes"], [AC_MSG_RESULT([no])]) ]) @@ -146,8 +146,8 @@ AC_DEFUN([PAC_PROG_FC_HAVE_F2003_REQUIREMENTS],[ dnl ------------------------------------------------------------------------- dnl AC_F9X_MODS() dnl -dnl Check how F9X handles modules. This macro also checks which -dnl command-line option to use to include the module once it's built. +dnl Check how F9X handles modules. This macro also checks which +dnl command-line option to use to include the module once it's built. dnl AC_DEFUN([AC_F9X_MODS], [AC_MSG_CHECKING(what $FC does with modules) @@ -223,7 +223,6 @@ else fi AC_SUBST(F9XMODFLAG) AC_SUBST(F9XMODEXT) -rm -rf conftest* AC_LANG_POP(Fortran) ]) @@ -241,9 +240,9 @@ dnl Change to the Fortran 90 language dnl Try link a simple MPI program. AC_MSG_CHECKING([whether a simple MPI-IO Fortran program can be linked]) AC_LINK_IFELSE([$TEST_SRC], - [AC_MSG_RESULT([yes])], - [AC_MSG_RESULT([no]) - AC_MSG_ERROR([unable to link a simple MPI-IO Fortran program])]) + [AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no]) + AC_MSG_ERROR([unable to link a simple MPI-IO Fortran program])]) dnl Change to the C language AC_LANG_POP(Fortran) @@ -257,51 +256,43 @@ dnl This is a runtime test. dnl AC_DEFUN([PAC_FC_AVAIL_KINDS],[ AC_LANG_PUSH([Fortran]) -rm -f pac_fconftest.out TEST_SRC="`sed -n '/PROGRAM FC_AVAIL_KINDS/,/END PROGRAM FC_AVAIL_KINDS/p' $srcdir/m4/aclocal_fc.f90`" AC_RUN_IFELSE([$TEST_SRC], [ - if test -s pac_fconftest.out ; then - - dnl The output from the above program will be: - dnl -- LINE 1 -- valid integer kinds (comma seperated list) - dnl -- LINE 2 -- valid real kinds (comma seperated list) - dnl -- LINE 3 -- max decimal precision for reals - dnl -- LINE 4 -- number of valid integer kinds - dnl -- LINE 5 -- number of valid real kinds - - pac_validIntKinds="`sed -n '1p' pac_fconftest.out`" - pac_validRealKinds="`sed -n '2p' pac_fconftest.out`" - PAC_FC_MAX_REAL_PRECISION="`sed -n '3p' pac_fconftest.out`" + dnl The output from the above program will be: + dnl -- LINE 1 -- valid integer kinds (comma seperated list) + dnl -- LINE 2 -- valid real kinds (comma seperated list) + dnl -- LINE 3 -- max decimal precision for reals + dnl -- LINE 4 -- number of valid integer kinds + dnl -- LINE 5 -- number of valid real kinds + + pac_validIntKinds=$(./conftest$EXEEXT 2>&1 | sed -n '1p') + pac_validRealKinds=$(./conftest$EXEEXT 2>&1 | sed -n '2p') + PAC_FC_MAX_REAL_PRECISION=$(./conftest$EXEEXT 2>&1 | sed -n '3p') AC_DEFINE_UNQUOTED([PAC_FC_MAX_REAL_PRECISION], $PAC_FC_MAX_REAL_PRECISION, [Define Fortran Maximum Real Decimal Precision]) PAC_FC_ALL_INTEGER_KINDS="{`echo $pac_validIntKinds`}" PAC_FC_ALL_REAL_KINDS="{`echo $pac_validRealKinds`}" - PAC_FORTRAN_NUM_INTEGER_KINDS="`sed -n '4p' pac_fconftest.out`" - H5CONFIG_F_NUM_IKIND="INTEGER, PARAMETER :: num_ikinds = `echo $PAC_FORTRAN_NUM_INTEGER_KINDS`" - H5CONFIG_F_IKIND="INTEGER, DIMENSION(1:num_ikinds) :: ikind = (/`echo $pac_validIntKinds`/)" - H5CONFIG_F_NUM_RKIND="INTEGER, PARAMETER :: num_rkinds = `sed -n '5p' pac_fconftest.out`" - H5CONFIG_F_RKIND="INTEGER, DIMENSION(1:num_rkinds) :: rkind = (/`echo $pac_validRealKinds`/)" + PAC_FORTRAN_NUM_INTEGER_KINDS=$(./conftest$EXEEXT 2>&1 | sed -n '4p') + H5CONFIG_F_NUM_IKIND="INTEGER, PARAMETER :: num_ikinds = `echo $PAC_FORTRAN_NUM_INTEGER_KINDS`" + H5CONFIG_F_IKIND="INTEGER, DIMENSION(1:num_ikinds) :: ikind = (/`echo $pac_validIntKinds`/)" + H5CONFIG_F_NUM_RKIND="INTEGER, PARAMETER :: num_rkinds = $(./conftest$EXEEXT 2>&1 | sed -n '5p')" + H5CONFIG_F_RKIND="INTEGER, DIMENSION(1:num_rkinds) :: rkind = (/`echo $pac_validRealKinds`/)" - AC_DEFINE_UNQUOTED([H5CONFIG_F_NUM_RKIND], $H5CONFIG_F_NUM_RKIND, [Define number of valid Fortran REAL KINDs]) - AC_DEFINE_UNQUOTED([H5CONFIG_F_NUM_IKIND], $H5CONFIG_F_NUM_IKIND, [Define number of valid Fortran INTEGER KINDs]) - AC_DEFINE_UNQUOTED([H5CONFIG_F_RKIND], $H5CONFIG_F_RKIND, [Define valid Fortran REAL KINDs]) - AC_DEFINE_UNQUOTED([H5CONFIG_F_IKIND], $H5CONFIG_F_IKIND, [Define valid Fortran INTEGER KINDs]) + AC_DEFINE_UNQUOTED([H5CONFIG_F_NUM_RKIND], $H5CONFIG_F_NUM_RKIND, [Define number of valid Fortran REAL KINDs]) + AC_DEFINE_UNQUOTED([H5CONFIG_F_NUM_IKIND], $H5CONFIG_F_NUM_IKIND, [Define number of valid Fortran INTEGER KINDs]) + AC_DEFINE_UNQUOTED([H5CONFIG_F_RKIND], $H5CONFIG_F_RKIND, [Define valid Fortran REAL KINDs]) + AC_DEFINE_UNQUOTED([H5CONFIG_F_IKIND], $H5CONFIG_F_IKIND, [Define valid Fortran INTEGER KINDs]) AC_MSG_CHECKING([for Number of Fortran INTEGER KINDs]) AC_MSG_RESULT([$PAC_FORTRAN_NUM_INTEGER_KINDS]) AC_MSG_CHECKING([for Fortran INTEGER KINDs]) AC_MSG_RESULT([$PAC_FC_ALL_INTEGER_KINDS]) - AC_MSG_CHECKING([for Fortran REAL KINDs]) - AC_MSG_RESULT([$PAC_FC_ALL_REAL_KINDS]) - AC_MSG_CHECKING([for Fortran REALs maximum decimal precision]) - AC_MSG_RESULT([$PAC_FC_MAX_REAL_PRECISION]) - else - AC_MSG_RESULT([Error]) - AC_MSG_ERROR([No output from Fortran test program!]) - fi - rm -f pac_fconftest.out + AC_MSG_CHECKING([for Fortran REAL KINDs]) + AC_MSG_RESULT([$PAC_FC_ALL_REAL_KINDS]) + AC_MSG_CHECKING([for Fortran REALs maximum decimal precision]) + AC_MSG_RESULT([$PAC_FC_MAX_REAL_PRECISION]) ],[ AC_MSG_RESULT([Error]) AC_MSG_ERROR([Failed to run Fortran program to determine available KINDs]) @@ -314,29 +305,22 @@ AC_REQUIRE([PAC_FC_AVAIL_KINDS]) AC_MSG_CHECKING([sizeof of available INTEGER KINDs]) AC_LANG_PUSH([Fortran]) pack_int_sizeof="" -rm -f pac_fconftest.out for kind in `echo $pac_validIntKinds | sed -e 's/,/ /g'`; do AC_LANG_CONFTEST([ AC_LANG_SOURCE([ PROGRAM main USE ISO_C_BINDING + USE ISO_FORTRAN_ENV, ONLY : stderr=>ERROR_UNIT IMPLICIT NONE INTEGER (KIND=$kind) a - OPEN(8, FILE='pac_fconftest.out', FORM='formatted') - WRITE(8,'(I0)') $FC_SIZEOF_A - CLOSE(8) + WRITE(stderr,'(I0)') $FC_SIZEOF_A END ]) ]) AC_RUN_IFELSE([],[ - if test -s pac_fconftest.out ; then - sizes="`cat pac_fconftest.out`" - pack_int_sizeof="$pack_int_sizeof $sizes," - else - AC_MSG_ERROR([No output from Fortran test program!]) - fi - rm -f pac_fconftest.out + sizes=$(./conftest$EXEEXT 2>&1) + pack_int_sizeof="$pack_int_sizeof $sizes," ],[ AC_MSG_ERROR([Fortran program fails to build or run!]) ],[ @@ -359,22 +343,16 @@ for kind in `echo $pac_validRealKinds | sed -e 's/,/ /g'`; do AC_LANG_SOURCE([ PROGRAM main USE ISO_C_BINDING + USE ISO_FORTRAN_ENV, ONLY : stderr=>ERROR_UNIT IMPLICIT NONE REAL (KIND=$kind) :: a - OPEN(8, FILE='pac_fconftest.out', FORM='formatted') - WRITE(8,'(I0)') $FC_SIZEOF_A - CLOSE(8) + WRITE(stderr,'(I0)') $FC_SIZEOF_A END ]) ]) AC_RUN_IFELSE([],[ - if test -s pac_fconftest.out ; then - sizes="`cat pac_fconftest.out`" - pack_real_sizeof="$pack_real_sizeof $sizes," - else - AC_MSG_ERROR([No output from Fortran test program!]) - fi - rm -f pac_fconftest.out + sizes=$(./conftest$EXEEXT 2>&1) + pack_real_sizeof="$pack_real_sizeof $sizes," ],[ AC_MSG_ERROR([Fortran program fails to build or run!]) ],[ @@ -396,33 +374,27 @@ rm -f pac_fconftest.out AC_LANG_SOURCE([ PROGRAM main USE ISO_C_BINDING + USE ISO_FORTRAN_ENV, ONLY : stderr=>ERROR_UNIT IMPLICIT NONE INTEGER a REAL b DOUBLE PRECISION c - OPEN(8, FILE='pac_fconftest.out', FORM='formatted') - WRITE(8,*) $FC_SIZEOF_A - WRITE(8,*) KIND(a) - WRITE(8,*) $FC_SIZEOF_B - WRITE(8,*) KIND(b) - WRITE(8,*) $FC_SIZEOF_C - WRITE(8,*) KIND(c) - CLOSE(8) + WRITE(stderr,*) $FC_SIZEOF_A + WRITE(stderr,*) KIND(a) + WRITE(stderr,*) $FC_SIZEOF_B + WRITE(stderr,*) KIND(b) + WRITE(stderr,*) $FC_SIZEOF_C + WRITE(stderr,*) KIND(c) END ]) ]) AC_RUN_IFELSE([],[ - if test -s pac_fconftest.out ; then - PAC_FORTRAN_NATIVE_INTEGER_SIZEOF="`sed -n '1p' pac_fconftest.out`" - PAC_FORTRAN_NATIVE_INTEGER_KIND="`sed -n '2p' pac_fconftest.out`" - PAC_FORTRAN_NATIVE_REAL_SIZEOF="`sed -n '3p' pac_fconftest.out`" - PAC_FORTRAN_NATIVE_REAL_KIND="`sed -n '4p' pac_fconftest.out`" - PAC_FORTRAN_NATIVE_DOUBLE_SIZEOF="`sed -n '5p' pac_fconftest.out`" - PAC_FORTRAN_NATIVE_DOUBLE_KIND="`sed -n '6p' pac_fconftest.out`" - else - AC_MSG_ERROR([No output from Fortran test program!]) - fi - rm -f pac_fconftest.out + PAC_FORTRAN_NATIVE_INTEGER_SIZEOF=$(./conftest$EXEEXT 2>&1 | sed -n '1p') + PAC_FORTRAN_NATIVE_INTEGER_KIND=$(./conftest$EXEEXT 2>&1 | sed -n '2p') + PAC_FORTRAN_NATIVE_REAL_SIZEOF=$(./conftest$EXEEXT 2>&1 | sed -n '3p') + PAC_FORTRAN_NATIVE_REAL_KIND=$(./conftest$EXEEXT 2>&1 | sed -n '4p') + PAC_FORTRAN_NATIVE_DOUBLE_SIZEOF=$(./conftest$EXEEXT 2>&1 | sed -n '5p') + PAC_FORTRAN_NATIVE_DOUBLE_KIND=$(./conftest$EXEEXT 2>&1 | sed -n '6p') ],[ AC_MSG_ERROR([Fortran program fails to build or run!]) ],[ @@ -434,7 +406,6 @@ AC_LANG_POP([Fortran]) AC_DEFUN([PAC_FC_LDBL_DIG],[ AC_MSG_CHECKING([maximum decimal precision for C]) -rm -f pac_Cconftest.out AC_LANG_CONFTEST([ AC_LANG_PROGRAM([ #include <float.h> @@ -458,19 +429,12 @@ rm -f pac_Cconftest.out #define C_LDBL_DIG LDBL_DIG #endif ],[[ - FILE * pFile; - pFile = fopen("pac_Cconftest.out","w"); - fprintf(pFile, "%d\n%d\n", C_LDBL_DIG, C_FLT128_DIG); + fprintf(stderr, "%d\n%d\n", C_LDBL_DIG, C_FLT128_DIG); ]]) ]) AC_RUN_IFELSE([],[ - if test -s pac_Cconftest.out ; then - LDBL_DIG="`sed -n '1p' pac_Cconftest.out`" - FLT128_DIG="`sed -n '2p' pac_Cconftest.out`" - else - AC_MSG_ERROR([No output from C decimal precision program!]) - fi - rm -f pac_Cconftest.out + LDBL_DIG=$(./conftest$EXEEXT 2>&1 | sed -n '1p') + FLT128_DIG=$(./conftest$EXEEXT 2>&1 | sed -n '2p') ],[ AC_MSG_ERROR([C program fails to build or run!]) ],[]) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index ed12c5e..c21217f 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -1,4 +1,4 @@ -HDF5 version 1.13.0 currently under development +HDF5 version 1.13.1-1 currently under development ================================================================================ @@ -37,9 +37,7 @@ CONTENTS - New Features - Support for new platforms and languages - Bug Fixes since HDF5-1.12.0 -- Supported Platforms -- Tested Configuration Features Summary -- More Tested Platforms +- Platforms Tested - Known Problems - CMake vs. Autotools installations @@ -49,11 +47,48 @@ New Features Configuration: ------------- + - Added new configure option to support building parallel tools. + See Tools below (autotools - CMake): + --enable-parallel-tools HDF5_BUILD_PARALLEL_TOOLS + + (RAW - 2021/10/25) + + - Added new configure options to enable dimension scales APIs (H5DS*) to + use new object references with the native VOL connector (aka native HDF5 + library). New references are always used for non-native terminal VOL + connectors (e.g., DAOS). + + Autotools --enable-dimension-scales-with-new-ref + CMake HDF5_DIMENSION_SCALES_NEW_REF=ON + + (EIP - 2021/10/25, HDFFV-11180) + + - Refactored the utils folder. + + Added subfolder test and moved the 'swmr_check_compat_vfd.c file' + from test into utils/test. Deleted the duplicate swmr_check_compat_vfd.c + file in hl/tools/h5watch folder. Also fixed vfd check options. + + (ADB - 2021/10/18) + + - Changed autotools and CMake configurations to derive both + compilation warnings-as-errors and warnings-only-warn configurations + from the same files, 'config/*/*error*'. Removed redundant files + 'config/*/*noerror*'. + + (DCY - 2021/09/29) + - Added new option to control the build of High-Level tools that default ON/enabled. Add configure options (autotools - CMake): - enable-hltools HDF5_BUILD_HL_TOOLS + --enable-hltools HDF5_BUILD_HL_TOOLS + + Disabling this option prevents building the gif tool which + contains the following CVEs: + HDFFV-10592 CVE-2018-17433 + HDFFV-10593 CVE-2018-17436 + HDFFV-11048 CVE-2020-10809 (ADB - 2021/09/16, HDFFV-11266) @@ -164,19 +199,19 @@ New Features - CMake option to build the HDF filter plugins project as an external project - The HDF filter plugins project is a collection of registered compression - filters that can be dynamically loaded when needed to access data stored - in a hdf5 file. This CMake-only option allows the plugins to be built and - distributed with the hdf5 library and tools. Like the options for szip and - zlib, either a tgz file or a git repository can be specified for the source. + The HDF filter plugins project is a collection of registered compression + filters that can be dynamically loaded when needed to access data stored + in a hdf5 file. This CMake-only option allows the plugins to be built and + distributed with the hdf5 library and tools. Like the options for szip and + zlib, either a tgz file or a git repository can be specified for the source. - The option was refactored to use the CMake FetchContent process. This allows - more control over the filter targets, but required external project command - options to be moved to a CMake include file, HDF5PluginCache.cmake. Also - enabled the filter examples to be used as tests for operation of the - filter plugins. + The option was refactored to use the CMake FetchContent process. This allows + more control over the filter targets, but required external project command + options to be moved to a CMake include file, HDF5PluginCache.cmake. Also + enabled the filter examples to be used as tests for operation of the + filter plugins. - (ADB - 2020/12/10, OESS-98) + (ADB - 2020/12/10, OESS-98) - FreeBSD Autotools configuration now defaults to 'cc' and 'c++' compilers @@ -229,244 +264,244 @@ New Features - Autotools and CMake target added to produce doxygen generated documentation - The default is OFF or disabled. - Autoconf option is '--enable-doxygen' - autotools make target is 'doxygen' and will build all doxygen targets - CMake configure option is 'HDF5_BUILD_DOC'. - CMake target is 'doxygen' for all available doxygen targets - CMake target is 'hdf5lib_doc' for the src subdirectory + The default is OFF or disabled. + Autoconf option is '--enable-doxygen' + autotools make target is 'doxygen' and will build all doxygen targets + CMake configure option is 'HDF5_BUILD_DOC'. + CMake target is 'doxygen' for all available doxygen targets + CMake target is 'hdf5lib_doc' for the src subdirectory - (ADB - 2020/11/03) + (ADB - 2020/11/03) - CMake option to use MSVC naming conventions with MinGW - HDF5_MSVC_NAMING_CONVENTION option enable to use MSVC naming conventions - when using a MinGW toolchain + HDF5_MSVC_NAMING_CONVENTION option enable to use MSVC naming conventions + when using a MinGW toolchain - (xan - 2020/10/30) + (xan - 2020/10/30) - CMake option to statically link gcc libs with MinGW - HDF5_MINGW_STATIC_GCC_LIBS allows to statically link libg/libstdc++ - with the MinGW toolchain + HDF5_MINGW_STATIC_GCC_LIBS allows to statically link libg/libstdc++ + with the MinGW toolchain - (xan - 2020/10/30) + (xan - 2020/10/30) - CMake option to build the HDF filter plugins project as an external project - The HDF filter plugins project is a collection of registered compression - filters that can be dynamically loaded when needed to access data stored - in a hdf5 file. This CMake-only option allows the plugins to be built and - distributed with the hdf5 library and tools. Like the options for szip and - zlib, either a tgz file or a git repository can be specified for the source. + The HDF filter plugins project is a collection of registered compression + filters that can be dynamically loaded when needed to access data stored + in a hdf5 file. This CMake-only option allows the plugins to be built and + distributed with the hdf5 library and tools. Like the options for szip and + zlib, either a tgz file or a git repository can be specified for the source. - The necessary options are (see the INSTALL_CMake.txt file): - HDF5_ENABLE_PLUGIN_SUPPORT - PLUGIN_TGZ_NAME or PLUGIN_GIT_URL - There are more options necessary for various filters and the plugin project - documents should be referenced. + The necessary options are (see the INSTALL_CMake.txt file): + HDF5_ENABLE_PLUGIN_SUPPORT + PLUGIN_TGZ_NAME or PLUGIN_GIT_URL + There are more options necessary for various filters and the plugin project + documents should be referenced. - (ADB - 2020/09/27, OESS-98) + (ADB - 2020/09/27, OESS-98) - Added CMake option to format source files - HDF5_ENABLE_FORMATTERS option will enable creation of targets using the - pattern - HDF5_*_SRC_FORMAT - where * corresponds to the source folder - or tool folder. All sources can be formatted by executing the format target; - make format + HDF5_ENABLE_FORMATTERS option will enable creation of targets using the + pattern - HDF5_*_SRC_FORMAT - where * corresponds to the source folder + or tool folder. All sources can be formatted by executing the format target; + make format - (ADB - 2020/08/24) + (ADB - 2020/08/24) - Add file locking configure and CMake options - HDF5 1.10.0 introduced a file locking scheme, primarily to help - enforce SWMR setup. Formerly, the only user-level control of the scheme - was via the HDF5_USE_FILE_LOCKING environment variable. + HDF5 1.10.0 introduced a file locking scheme, primarily to help + enforce SWMR setup. Formerly, the only user-level control of the scheme + was via the HDF5_USE_FILE_LOCKING environment variable. - This change introduces configure-time options that control whether - or not file locking will be used and whether or not the library - ignores errors when locking has been disabled on the file system - (useful on some HPC Lustre installations). + This change introduces configure-time options that control whether + or not file locking will be used and whether or not the library + ignores errors when locking has been disabled on the file system + (useful on some HPC Lustre installations). - In both the Autotools and CMake, the settings have the effect of changing - the default property list settings (see the H5Pset/get_file_locking() - entry, below). + In both the Autotools and CMake, the settings have the effect of changing + the default property list settings (see the H5Pset/get_file_locking() + entry, below). - The yes/no/best-effort file locking configure setting has also been - added to the libhdf5.settings file. + The yes/no/best-effort file locking configure setting has also been + added to the libhdf5.settings file. - Autotools: + Autotools: - An --enable-file-locking=(yes|no|best-effort) option has been added. + An --enable-file-locking=(yes|no|best-effort) option has been added. - yes: Use file locking. - no: Do not use file locking. - best-effort: Use file locking and ignore "disabled" errors. + yes: Use file locking. + no: Do not use file locking. + best-effort: Use file locking and ignore "disabled" errors. - CMake: + CMake: - Two self-explanatory options have been added: + Two self-explanatory options have been added: - HDF5_USE_FILE_LOCKING - HDF5_IGNORE_DISABLED_FILE_LOCKS + HDF5_USE_FILE_LOCKING + HDF5_IGNORE_DISABLED_FILE_LOCKS - Setting both of these to ON is the equivalent to the Autotools' - best-effort setting. + Setting both of these to ON is the equivalent to the Autotools' + best-effort setting. - NOTE: - The precedence order of the various file locking control mechanisms is: + NOTE: + The precedence order of the various file locking control mechanisms is: - 1) HDF5_USE_FILE_LOCKING environment variable (highest) + 1) HDF5_USE_FILE_LOCKING environment variable (highest) - 2) H5Pset_file_locking() + 2) H5Pset_file_locking() - 3) configure/CMake options (which set the property list defaults) + 3) configure/CMake options (which set the property list defaults) - 4) library defaults (currently best-effort) + 4) library defaults (currently best-effort) - (DER - 2020/07/30, HDFFV-11092) + (DER - 2020/07/30, HDFFV-11092) - CMake option to link the generated Fortran MOD files into the include directory. - The Fortran generation of MOD files by a Fortran compile can produce - different binary files between SHARED and STATIC compiles with different - compilers and/or different platforms. Note that it has been found that - different versions of Fortran compilers will produce incompatible MOD - files. Currently, CMake will locate these MOD files in subfolders of - the include directory and add that path to the Fortran library target - in the CMake config file, which can be used by the CMake find library - process. For other build systems using the binary from a CMake install, - a new CMake configuration can be used to copy the pre-chosen version - of the Fortran MOD files into the install include directory. - - The default will depend on the configuration of - BUILD_STATIC_LIBS and BUILD_SHARED_LIBS: - YES YES Default to SHARED - YES NO Default to STATIC - NO YES Default to SHARED - NO NO Default to SHARED - The defaults can be overridden by setting the config option - HDF5_INSTALL_MOD_FORTRAN to one of NO, SHARED, or STATIC - - (ADB - 2020/07/09, HDFFV-11116) + The Fortran generation of MOD files by a Fortran compile can produce + different binary files between SHARED and STATIC compiles with different + compilers and/or different platforms. Note that it has been found that + different versions of Fortran compilers will produce incompatible MOD + files. Currently, CMake will locate these MOD files in subfolders of + the include directory and add that path to the Fortran library target + in the CMake config file, which can be used by the CMake find library + process. For other build systems using the binary from a CMake install, + a new CMake configuration can be used to copy the pre-chosen version + of the Fortran MOD files into the install include directory. + + The default will depend on the configuration of + BUILD_STATIC_LIBS and BUILD_SHARED_LIBS: + YES YES Default to SHARED + YES NO Default to STATIC + NO YES Default to SHARED + NO NO Default to SHARED + The defaults can be overridden by setting the config option + HDF5_INSTALL_MOD_FORTRAN to one of NO, SHARED, or STATIC + + (ADB - 2020/07/09, HDFFV-11116) - CMake option to use AEC (open source SZip) library instead of SZip - The open source AEC library is a replacement library for SZip. In - order to use it for hdf5 the libaec CMake source was changed to add - "-fPIC" and exclude test files. Autotools does not build the - compression libraries within hdf5 builds. New option USE_LIBAEC is - required to compensate for the different files produced by AEC build. + The open source AEC library is a replacement library for SZip. In + order to use it for hdf5 the libaec CMake source was changed to add + "-fPIC" and exclude test files. Autotools does not build the + compression libraries within hdf5 builds. New option USE_LIBAEC is + required to compensate for the different files produced by AEC build. - (ADB - 2020/04/22, OESS-65) + (ADB - 2020/04/22, OESS-65) - CMake ConfigureChecks.cmake file now uses CHECK_STRUCT_HAS_MEMBER - Some handcrafted tests in HDFTests.c has been removed and the CMake - CHECK_STRUCT_HAS_MEMBER module has been used. + Some handcrafted tests in HDFTests.c has been removed and the CMake + CHECK_STRUCT_HAS_MEMBER module has been used. - (ADB - 2020/03/24, TRILAB-24) + (ADB - 2020/03/24, TRILAB-24) - Both build systems use same set of warnings flags - GNU C, C++ and gfortran warnings flags were moved to files in a config - sub-folder named gnu-warnings. Flags that only are available for a specific - version of the compiler are in files named with that version. - Clang C warnings flags were moved to files in a config sub-folder - named clang-warnings. - Intel C, Fortran warnings flags were moved to files in a config sub-folder - named intel-warnings. + GNU C, C++ and gfortran warnings flags were moved to files in a config + sub-folder named gnu-warnings. Flags that only are available for a specific + version of the compiler are in files named with that version. + Clang C warnings flags were moved to files in a config sub-folder + named clang-warnings. + Intel C, Fortran warnings flags were moved to files in a config sub-folder + named intel-warnings. - There are flags in named "error-xxx" files with warnings that may - be promoted to errors. Some source files may still need fixes. + There are flags in named "error-xxx" files with warnings that may + be promoted to errors. Some source files may still need fixes. - There are also pairs of files named "developer-xxx" and "no-developer-xxx" - that are chosen by the CMake option:HDF5_ENABLE_DEV_WARNINGS or the - configure option:--enable-developer-warnings. + There are also pairs of files named "developer-xxx" and "no-developer-xxx" + that are chosen by the CMake option:HDF5_ENABLE_DEV_WARNINGS or the + configure option:--enable-developer-warnings. - In addition, CMake no longer applies these warnings for examples. + In addition, CMake no longer applies these warnings for examples. - (ADB - 2020/03/24, TRILAB-192) + (ADB - 2020/03/24, TRILAB-192) - Added test script for file size compare - If CMake minimum version is at least 3.14, the fileCompareTest.cmake - script will compare file sizes. + If CMake minimum version is at least 3.14, the fileCompareTest.cmake + script will compare file sizes. - (ADB - 2020/02/24, HDFFV-11036) + (ADB - 2020/02/24, HDFFV-11036) - Update CMake minimum version to 3.12 - Updated CMake minimum version to 3.12 and added version checks - for Windows features. + Updated CMake minimum version to 3.12 and added version checks + for Windows features. - (ADB - 2020/02/05, TRILABS-142) + (ADB - 2020/02/05, TRILABS-142) - Fixed CMake include properties for Fortran libraries - Corrected the library properties for Fortran to use the - correct path for the Fortran module files. + Corrected the library properties for Fortran to use the + correct path for the Fortran module files. - (ADB - 2020/02/04, HDFFV-11012) + (ADB - 2020/02/04, HDFFV-11012) - Added common warnings files for gnu and intel - Added warnings files to use one common set of flags - during configure for both autotools and CMake build - systems. The initial implementation only affects a - general set of flags for gnu and intel compilers. + Added warnings files to use one common set of flags + during configure for both autotools and CMake build + systems. The initial implementation only affects a + general set of flags for gnu and intel compilers. - (ADB - 2020/01/17) + (ADB - 2020/01/17) - Added new options to CMake for control of testing - Added CMake options (default ON); - HDF5_TEST_SERIAL AND/OR HDF5_TEST_PARALLEL - combined with: - HDF5_TEST_TOOLS - HDF5_TEST_EXAMPLES - HDF5_TEST_SWMR - HDF5_TEST_FORTRAN - HDF5_TEST_CPP - HDF5_TEST_JAVA + Added CMake options (default ON); + HDF5_TEST_SERIAL AND/OR HDF5_TEST_PARALLEL + combined with: + HDF5_TEST_TOOLS + HDF5_TEST_EXAMPLES + HDF5_TEST_SWMR + HDF5_TEST_FORTRAN + HDF5_TEST_CPP + HDF5_TEST_JAVA - (ADB - 2020/01/15, HDFFV-11001) + (ADB - 2020/01/15, HDFFV-11001) - Added Clang sanitizers to CMake for analyzer support if compiler is clang. - Added CMake code and files to execute the Clang sanitizers if - HDF5_ENABLE_SANITIZERS is enabled and the USE_SANITIZER option - is set to one of the following: - Address - Memory - MemoryWithOrigins - Undefined - Thread - Leak - 'Address;Undefined' + Added CMake code and files to execute the Clang sanitizers if + HDF5_ENABLE_SANITIZERS is enabled and the USE_SANITIZER option + is set to one of the following: + Address + Memory + MemoryWithOrigins + Undefined + Thread + Leak + 'Address;Undefined' - (ADB - 2019/12/12, TRILAB-135) + (ADB - 2019/12/12, TRILAB-135) - Update CMake for VS2019 support - CMake added support for VS2019 in version 3.15. Changes to the CMake - generator setting required changes to scripts. Also updated version - references in CMake files as necessary. + CMake added support for VS2019 in version 3.15. Changes to the CMake + generator setting required changes to scripts. Also updated version + references in CMake files as necessary. - (ADB - 2019/11/18, HDFFV-10962) + (ADB - 2019/11/18, HDFFV-10962) - Update CMake options to match new autotools options - Add configure options (autotools - CMake): - enable-asserts HDF5_ENABLE_ASSERTS - enable-symbols HDF5_ENABLE_SYMBOLS - enable-profiling HDF5_ENABLE_PROFILING - enable-optimization HDF5_ENABLE_OPTIMIZATION - In addition NDEBUG is no longer forced defined and relies on the CMake - process. + Add configure options (autotools - CMake): + enable-asserts HDF5_ENABLE_ASSERTS + enable-symbols HDF5_ENABLE_SYMBOLS + enable-profiling HDF5_ENABLE_PROFILING + enable-optimization HDF5_ENABLE_OPTIMIZATION + In addition NDEBUG is no longer forced defined and relies on the CMake + process. - (ADB - 2019/10/07, HDFFV-100901, HDFFV-10637, TRILAB-97) + (ADB - 2019/10/07, HDFFV-100901, HDFFV-10637, TRILAB-97) Library: @@ -488,7 +523,7 @@ New Features which case the release part of version, in major.minor.release, must be exact. An environment variable still controls the logic. - (ADB - 2021/07/27) + (ADB - 2021/07/27) - gcc warning suppression macros were moved out of H5public.h @@ -501,7 +536,7 @@ New Features VFD refactoring, the macros have been duplicated in H5FDmulti.c to suppress the format string warnings there. - (DER - 2021/06/03) + (DER - 2021/06/03) - H5Gcreate1() now rejects size_hint parameters larger than UINT32_MAX @@ -518,7 +553,7 @@ New Features The Doxygen documentation has been updated and passing values larger than UINT32_MAX for size_hint will now produce a normal HDF5 error. - (DER - 2021/04/29, HDFFV-11241) + (DER - 2021/04/29, HDFFV-11241) - H5Pset_fapl_log() no longer crashes when passed an invalid fapl ID @@ -532,7 +567,7 @@ New Features The pointer is now correctly initialized and the API call now produces a normal HDF5 error when fed an invalid fapl ID. - (DER - 2021/04/28, HDFFV-11240) + (DER - 2021/04/28, HDFFV-11240) - Fixes a segfault when H5Pset_mdc_log_options() is called multiple times @@ -544,7 +579,7 @@ New Features The string is now handled properly and the segfault no longer occurs. - (DER - 2021/04/27, HDFFV-11239) + (DER - 2021/04/27, HDFFV-11239) - HSYS_GOTO_ERROR now emits the results of GetLastError() on Windows @@ -563,15 +598,15 @@ New Features The format string on Windows has been changed from: - "%s, errno = %d, error message = '%s'" + "%s, errno = %d, error message = '%s'" to: - "%s, errno = %d, error message = '%s', Win32 GetLastError() = %"PRIu32"" + "%s, errno = %d, error message = '%s', Win32 GetLastError() = %"PRIu32"" for those inclined to parse it for error values. - (DER - 2021/03/21) + (DER - 2021/03/21) - File locking now works on Windows @@ -587,26 +622,26 @@ New Features same scheme as POSIX systems. We lock the entire file when we set up the locks (by passing DWORDMAX as both size parameters to LockFileEx()). - (DER - 2021/03/19, HDFFV-10191) + (DER - 2021/03/19, HDFFV-10191) - H5Epush_ret() now requires a trailing semicolon - H5Epush_ret() is a function-like macro that has been changed to - contain a `do {} while(0)` loop. Consequently, a trailing semicolon - is now required to end the `while` statement. Previously, a trailing - semi would work, but was not mandatory. This change was made to allow - clang-format to correctly format the source code. + H5Epush_ret() is a function-like macro that has been changed to + contain a `do {} while(0)` loop. Consequently, a trailing semicolon + is now required to end the `while` statement. Previously, a trailing + semi would work, but was not mandatory. This change was made to allow + clang-format to correctly format the source code. - (SAM - 2021/03/03) + (SAM - 2021/03/03) - Improved performance of H5Sget_select_elem_pointlist - Modified library to cache the point after the last block of points - retrieved by H5Sget_select_elem_pointlist, so a subsequent call to the - same function to retrieve the next block of points from the list can - proceed immediately without needing to iterate over the point list. + Modified library to cache the point after the last block of points + retrieved by H5Sget_select_elem_pointlist, so a subsequent call to the + same function to retrieve the next block of points from the list can + proceed immediately without needing to iterate over the point list. - (NAF - 2021/01/19) + (NAF - 2021/01/19) - Replaced H5E_ATOM with H5E_ID in H5Epubgen.h @@ -620,130 +655,130 @@ New Features - Add a new public function H5Ssel_iter_reset - This function resets a dataspace selection iterator back to an - initial state so that it may be used for iteration once more. - This can be useful when needing to iterate over a selection - multiple times without having to repeatedly create/destroy - a selection iterator for that dataspace selection. + This function resets a dataspace selection iterator back to an + initial state so that it may be used for iteration once more. + This can be useful when needing to iterate over a selection + multiple times without having to repeatedly create/destroy + a selection iterator for that dataspace selection. - (JTH - 2020/09/18) + (JTH - 2020/09/18) - Remove HDFS VFD stubs - The original implementation of the HDFS VFD included non-functional - versions of the following public API calls when the HDFS VFD is - not built as a part of the HDF5 library: + The original implementation of the HDFS VFD included non-functional + versions of the following public API calls when the HDFS VFD is + not built as a part of the HDF5 library: - * H5FD_hdfs_init() - * H5Pget_fapl_hdfs() - * H5Pset_fapl_hdfs() + * H5FD_hdfs_init() + * H5Pget_fapl_hdfs() + * H5Pset_fapl_hdfs() - They will remain present in HDF5 1.10 and HDF5 1.12 releases - for binary compatibility purposes but have been removed as of 1.14.0. + They will remain present in HDF5 1.10 and HDF5 1.12 releases + for binary compatibility purposes but have been removed as of 1.14.0. - Note that this has nothing to do with the real HDFS VFD API calls - that are fully functional when the HDFS VFD is configured and built. + Note that this has nothing to do with the real HDFS VFD API calls + that are fully functional when the HDFS VFD is configured and built. - We simply changed: + We simply changed: - #ifdef LIBHDFS - <real API call> - #else - <useless stub> - #endif + #ifdef LIBHDFS + <real API call> + #else + <useless stub> + #endif - to: + to: - #ifdef LIBHDFS - <real API call> - #endif + #ifdef LIBHDFS + <real API call> + #endif - Which is how the other optional VFDs are handled. + Which is how the other optional VFDs are handled. - (DER - 2020/08/27) + (DER - 2020/08/27) - Add Mirror VFD - Use TCP/IP sockets to perform write-only (W/O) file I/O on a remote - machine. Must be used in conjunction with the Splitter VFD. + Use TCP/IP sockets to perform write-only (W/O) file I/O on a remote + machine. Must be used in conjunction with the Splitter VFD. - (JOS - 2020/03/13, TBD) + (JOS - 2020/03/13, TBD) - Add Splitter VFD - Maintain separate R/W and W/O channels for "concurrent" file writes - to two files using a single HDF5 file handle. + Maintain separate R/W and W/O channels for "concurrent" file writes + to two files using a single HDF5 file handle. - (JOS - 2020/03/13, TBD) + (JOS - 2020/03/13, TBD) - Refactored public exposure of haddr_t type in favor of "object tokens" - To better accommodate HDF5 VOL connectors where "object addresses in a file" - may not make much sense, the following changes were made to the library: - - * Introduced new H5O_token_t "object token" type, which represents a - unique and permanent identifier for referencing an HDF5 object within - a container; these "object tokens" are meant to replace object addresses. - Along with the new type, a new H5Oopen_by_token API call was introduced - to open an object by a token, similar to how object addresses were - previously used with H5Oopen_by_addr. - - * Introduced new H5Lget_info2, H5Lget_info_by_idx2, H5Literate2, H5Literate_by_name2, - H5Lvisit2 and H5Lvisit_by_name2 API calls, along with their associated H5L_info2_t - struct and H5L_iterate2_t callback function, which work with the newly-introduced - object tokens, instead of object addresses. The original functions have been - renamed to version 1 functions and are deprecated in favor of the new version 2 - functions. The H5L_info_t and H5L_iterate_t types have been renamed to version 1 - types and are now deprecated in favor of their version 2 counterparts. For each of - the functions and types, compatibility macros take place of the original symbols. - - * Introduced new H5Oget_info3, H5Oget_info_by_name3, H5Oget_info_by_idx3, - H5Ovisit3 and H5Ovisit_by_name3 API calls, along with their associated H5O_info2_t - struct and H5O_iterate2_t callback function, which work with the newly-introduced - object tokens, instead of object addresses. The version 2 functions are now - deprecated in favor of the version 3 functions. The H5O_info_t and H5O_iterate_t - types have been renamed to version 1 types and are now deprecated in favor of their - version 2 counterparts. For each, compatibility macros take place of the original - symbols. - - * Introduced new H5Oget_native_info, H5Oget_native_info_by_name and - H5Oget_native_info_by_idx API calls, along with their associated H5O_native_info_t - struct, which are used to retrieve the native HDF5 file format-specific information - about an object. This information (such as object header info and B-tree/heap info) - has been removed from the new H5O_info2_t struct so that the more generic - H5Oget_info(_by_name/_by_idx)3 routines will not try to retrieve it for non-native - VOL connectors. - - * Added new H5Otoken_cmp, H5Otoken_to_str and H5Otoken_from_str routines to compare - two object tokens, convert an object token into a nicely-readable string format and - to convert an object token string back into a real object token, respectively. - - (DER, QAK, JTH - 2020/01/16) + To better accommodate HDF5 VOL connectors where "object addresses in a file" + may not make much sense, the following changes were made to the library: + + * Introduced new H5O_token_t "object token" type, which represents a + unique and permanent identifier for referencing an HDF5 object within + a container; these "object tokens" are meant to replace object addresses. + Along with the new type, a new H5Oopen_by_token API call was introduced + to open an object by a token, similar to how object addresses were + previously used with H5Oopen_by_addr. + + * Introduced new H5Lget_info2, H5Lget_info_by_idx2, H5Literate2, H5Literate_by_name2, + H5Lvisit2 and H5Lvisit_by_name2 API calls, along with their associated H5L_info2_t + struct and H5L_iterate2_t callback function, which work with the newly-introduced + object tokens, instead of object addresses. The original functions have been + renamed to version 1 functions and are deprecated in favor of the new version 2 + functions. The H5L_info_t and H5L_iterate_t types have been renamed to version 1 + types and are now deprecated in favor of their version 2 counterparts. For each of + the functions and types, compatibility macros take place of the original symbols. + + * Introduced new H5Oget_info3, H5Oget_info_by_name3, H5Oget_info_by_idx3, + H5Ovisit3 and H5Ovisit_by_name3 API calls, along with their associated H5O_info2_t + struct and H5O_iterate2_t callback function, which work with the newly-introduced + object tokens, instead of object addresses. The version 2 functions are now + deprecated in favor of the version 3 functions. The H5O_info_t and H5O_iterate_t + types have been renamed to version 1 types and are now deprecated in favor of their + version 2 counterparts. For each, compatibility macros take place of the original + symbols. + + * Introduced new H5Oget_native_info, H5Oget_native_info_by_name and + H5Oget_native_info_by_idx API calls, along with their associated H5O_native_info_t + struct, which are used to retrieve the native HDF5 file format-specific information + about an object. This information (such as object header info and B-tree/heap info) + has been removed from the new H5O_info2_t struct so that the more generic + H5Oget_info(_by_name/_by_idx)3 routines will not try to retrieve it for non-native + VOL connectors. + + * Added new H5Otoken_cmp, H5Otoken_to_str and H5Otoken_from_str routines to compare + two object tokens, convert an object token into a nicely-readable string format and + to convert an object token string back into a real object token, respectively. + + (DER, QAK, JTH - 2020/01/16) - Add new public function H5Sselect_adjust. - This function shifts a dataspace selection by a specified logical offset - within the dataspace extent. This can be useful for VOL developers to - implement chunked datasets. + This function shifts a dataspace selection by a specified logical offset + within the dataspace extent. This can be useful for VOL developers to + implement chunked datasets. - (NAF - 2019/11/18) + (NAF - 2019/11/18) - Add new public function H5Sselect_project_intersection. - This function computes the intersection between two dataspace selections - and projects that intersection into a third selection. This can be useful - for VOL developers to implement chunked or virtual datasets. + This function computes the intersection between two dataspace selections + and projects that intersection into a third selection. This can be useful + for VOL developers to implement chunked or virtual datasets. - (NAF - 2019/11/13, ID-148) + (NAF - 2019/11/13, ID-148) - Add new public function H5VLget_file_type. - This function returns a datatype equivalent to the supplied datatype but - with the location set to be in the file. This datatype can then be used - with H5Tconvert to convert data between file and in-memory representation. - This function is intended for use only by VOL connector developers. + This function returns a datatype equivalent to the supplied datatype but + with the location set to be in the file. This datatype can then be used + with H5Tconvert to convert data between file and in-memory representation. + This function is intended for use only by VOL connector developers. - (NAF - 2019/11/08, ID-127) + (NAF - 2019/11/08, ID-127) Parallel Library: @@ -763,10 +798,10 @@ New Features h5pget_file_locking_f() h5pset_file_locking_f() - See the configure option discussion for HDFFV-11092 (above) for more - information on the file locking feature and how it's controlled. + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. - (DER - 2020/07/30, HDFFV-11092) + (DER - 2020/07/30, HDFFV-11092) C++ Library: ------------ @@ -775,10 +810,10 @@ New Features FileAccPropList::setFileLocking() FileAccPropList::getFileLocking() - See the configure option discussion for HDFFV-11092 (above) for more - information on the file locking feature and how it's controlled. + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. - (DER - 2020/07/30, HDFFV-11092) + (DER - 2020/07/30, HDFFV-11092) Java Library: @@ -795,12 +830,12 @@ New Features - Added new H5S functions. - H5Sselect_copy, H5Sselect_shape_same, H5Sselect_adjust, - H5Sselect_intersect_block, H5Sselect_project_intersection, - H5Scombine_hyperslab, H5Smodify_select, H5Scombine_select - wrapper functions added. + H5Sselect_copy, H5Sselect_shape_same, H5Sselect_adjust, + H5Sselect_intersect_block, H5Sselect_project_intersection, + H5Scombine_hyperslab, H5Smodify_select, H5Scombine_select + wrapper functions added. - (ADB - 2020/10/27, HDFFV-10868) + (ADB - 2020/10/27, HDFFV-10868) - Add wrappers for H5Pset/get_file_locking() API calls @@ -808,120 +843,120 @@ New Features H5Pget_use_file_locking() H5Pget_ignore_disabled_file_locking() - Unlike the C++ and Fortran wrappers, there are separate getters for the - two file locking settings, each of which returns a boolean value. + Unlike the C++ and Fortran wrappers, there are separate getters for the + two file locking settings, each of which returns a boolean value. - See the configure option discussion for HDFFV-11092 (above) for more - information on the file locking feature and how it's controlled. + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. - (DER - 2020/07/30, HDFFV-11092) + (DER - 2020/07/30, HDFFV-11092) - Added ability to test java library with VOLs. - Created a new CMake script that combines the java and vol test scripts. + Created a new CMake script that combines the java and vol test scripts. - (ADB - 2020/02/03, HDFFV-10996) + (ADB - 2020/02/03, HDFFV-10996) - Tests fail for non-English locales. - In the JUnit tests with a non-English locale, only the part before - the decimal comma is replaced by XXXX and this leads to a comparison - error. Changed the regex for the Time substitution. + In the JUnit tests with a non-English locale, only the part before + the decimal comma is replaced by XXXX and this leads to a comparison + error. Changed the regex for the Time substitution. - (ADB - 2020/01/09, HDFFV-10995) + (ADB - 2020/01/09, HDFFV-10995) Tools: ------ - Refactored the perform tools and removed depends on test library. - Moved the perf and h5perf tools from tools/test/perform to - tools/src/h5perf so that they can be installed. This required - that the test library dependency be removed by copying the - needed functions from h5test.c. - The standalone scripts and other perform tools remain in the - tools/test/perform folder. + Moved the perf and h5perf tools from tools/test/perform to + tools/src/h5perf so that they can be installed. This required + that the test library dependency be removed by copying the + needed functions from h5test.c. + The standalone scripts and other perform tools remain in the + tools/test/perform folder. - (ADB - 2021/08/10) + (ADB - 2021/08/10) - Removed partial long exceptions - Some of the tools accepted shortened versions of the long options - (ex: --datas instead of --dataset). These were implemented inconsistently, - are difficult to maintian, and occasionally block useful long option - names. These partial long options have been removed from all the tools. + Some of the tools accepted shortened versions of the long options + (ex: --datas instead of --dataset). These were implemented inconsistently, + are difficult to maintian, and occasionally block useful long option + names. These partial long options have been removed from all the tools. - (DER - 2021/08/03) + (DER - 2021/08/03) - h5repack added help text for user-defined filters. - Added help text line that states the valid values of the filter flag - for user-defined filters; - filter_flag: 1 is OPTIONAL or 0 is MANDATORY + Added help text line that states the valid values of the filter flag + for user-defined filters; + filter_flag: 1 is OPTIONAL or 0 is MANDATORY - (ADB - 2021/01/14, HDFFV-11099) + (ADB - 2021/01/14, HDFFV-11099) - Added h5delete tool - Deleting HDF5 storage when using the VOL can be tricky when the VOL - does not create files. The h5delete tool is a simple wrapper around - the H5Fdelete() API call that uses the VOL specified in the - HDF5_VOL_CONNECTOR environment variable to delete a "file". If - the call to H5Fdelete() fails, the tool will attempt to use - the POSIX remove(3) call to remove the file. + Deleting HDF5 storage when using the VOL can be tricky when the VOL + does not create files. The h5delete tool is a simple wrapper around + the H5Fdelete() API call that uses the VOL specified in the + HDF5_VOL_CONNECTOR environment variable to delete a "file". If + the call to H5Fdelete() fails, the tool will attempt to use + the POSIX remove(3) call to remove the file. - Note that the HDF5 library does currently have support for - H5Fdelete() in the native VOL connector. + Note that the HDF5 library does currently have support for + H5Fdelete() in the native VOL connector. - (DER - 2020/12/16) + (DER - 2020/12/16) - h5repack added options to control how external links are handled. - Currently h5repack preserves external links and cannot copy and merge - data from the external files. Two options, merge and prune, were added to - control how to merge data from an external link into the resulting file. - --merge Follow external soft link recursively and merge data. - --prune Do not follow external soft links and remove link. - --merge --prune Follow external link, merge data and remove dangling link. + Currently h5repack preserves external links and cannot copy and merge + data from the external files. Two options, merge and prune, were added to + control how to merge data from an external link into the resulting file. + --merge Follow external soft link recursively and merge data. + --prune Do not follow external soft links and remove link. + --merge --prune Follow external link, merge data and remove dangling link. - (ADB - 2020/08/05, HDFFV-9984) + (ADB - 2020/08/05, HDFFV-9984) - h5repack was fixed to repack the reference attributes properly. - The code line that checks if the update of reference inside a compound - datatype is misplaced outside the code block loop that carries out the - check. In consequence, the next attribute that is not the reference - type was repacked again as the reference type and caused the failure of - repacking. The fix is to move the corresponding code line to the correct - code block. + The code line that checks if the update of reference inside a compound + datatype is misplaced outside the code block loop that carries out the + check. In consequence, the next attribute that is not the reference + type was repacked again as the reference type and caused the failure of + repacking. The fix is to move the corresponding code line to the correct + code block. - (KY -2020/02/07, HDFFV-11014) + (KY -2020/02/07, HDFFV-11014) - h5diff was updated to use the new reference APIs. - h5diff uses the new reference APIs to compare references. - Attribute references can also be compared. + h5diff uses the new reference APIs to compare references. + Attribute references can also be compared. - (ADB - 2019/12/19, HDFFV-10980) + (ADB - 2019/12/19, HDFFV-10980) - h5dump and h5ls were updated to use the new reference APIs. - The tools library now use the new reference APIs to inspect a - file. Also the DDL spec was updated to reflect the format - changes produced with the new APIs. The export API and support - functions in the JNI were updated to match. + The tools library now use the new reference APIs to inspect a + file. Also the DDL spec was updated to reflect the format + changes produced with the new APIs. The export API and support + functions in the JNI were updated to match. - (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) + (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) High-Level APIs: ---------------- - added set/get for unsigned long long attributes - the attribute writing high-level API has been expanded to include - public set/get functions for ULL attributes, analogously to the - existing set/get for other types. + The attribute writing high-level API has been expanded to include + public set/get functions for ULL attributes, analogously to the + existing set/get for other types. - (AF - 2021/09/08) + (AF - 2021/09/08) C Packet Table API: ------------------- @@ -943,210 +978,262 @@ Bug Fixes since HDF5-1.12.0 release =================================== Library ------- + - Fixed an H5Pget_filter_by_id1/2() assert w/ out of range filter IDs + + Both H5Pget_filter_by_id1 and 2 did not range check the filter ID, which + could trip as assert in debug versions of the library. The library now + returns a normal HDF5 error when the filter ID is out of range. + + (DER - 2021/11/23, HDFFV-11286) + + - Fixed an issue with collective metadata reads being permanently disabled + after a dataset chunk lookup operation. This would usually cause a + mismatched MPI_Bcast and MPI_ERR_TRUNCATE issue in the library for + simple cases of H5Dcreate() -> H5Dwrite() -> H5Dcreate(). + + (JTH - 2021/11/08, HDFFV-11090) + + - Fixed cross platform incompatibility of references within variable length + types + + Reference types within variable length types previously could not be + read on a platform with different endianness from where they were + written. Fixed so cross platform portability is restored. + + (NAF - 2021/09/30) + - Detection of simple data transform function "x" - In the case of the simple data transform function "x" the (parallel) - library recognizes this is the same as not applying this data transform - function. This improves the I/O performance. In the case of the parallel - library, it also avoids breaking to independent I/O, which makes it - possible to apply a filter when writing or reading data to or from - teh HDF5 file. + In the case of the simple data transform function "x" the (parallel) + library recognizes this is the same as not applying this data transform + function. This improves the I/O performance. In the case of the parallel + library, it also avoids breaking to independent I/O, which makes it + possible to apply a filter when writing or reading data to or from + the HDF5 file. - (JWSB - 2021/09/13) + (JWSB - 2021/09/13) - Fixed an invalid read and memory leak when parsing corrupt file space info messages - When the corrupt file from CVE-2020-10810 was parsed by the library, - the code that imports the version 0 file space info object header - message to the version 1 struct could read past the buffer read from - the disk, causing an invalid memory read. Not catching this error would - cause downstream errors that eventually resulted in a previously - allocated buffer to be unfreed when the library shut down. In builds - where the free lists are in use, this could result in an infinite loop - and SIGABRT when the library shuts down. + When the corrupt file from CVE-2020-10810 was parsed by the library, + the code that imports the version 0 file space info object header + message to the version 1 struct could read past the buffer read from + the disk, causing an invalid memory read. Not catching this error would + cause downstream errors that eventually resulted in a previously + allocated buffer to be unfreed when the library shut down. In builds + where the free lists are in use, this could result in an infinite loop + and SIGABRT when the library shuts down. - We now track the buffer size and raise an error on attempts to read - past the end of it. + We now track the buffer size and raise an error on attempts to read + past the end of it. - (DER - 2021/08/12, HDFFV-11053) + (DER - 2021/08/12, HDFFV-11053) - Fixed CVE-2018-14460 - The tool h5repack produced a segfault when the rank in dataspace - message was corrupted, causing invalid read while decoding the - dimension sizes. + The tool h5repack produced a segfault when the rank in dataspace + message was corrupted, causing invalid read while decoding the + dimension sizes. - The problem was fixed by ensuring that decoding the dimension sizes - and max values will not go beyong the end of the buffer. + The problem was fixed by ensuring that decoding the dimension sizes + and max values will not go beyong the end of the buffer. - (BMR - 2021/05/12, HDFFV-11223) + (BMR - 2021/05/12, HDFFV-11223) - Fixed CVE-2018-11206 - The tool h5dump produced a segfault when the size of a fill value - message was corrupted and caused a buffer overflow. + The tool h5dump produced a segfault when the size of a fill value + message was corrupted and caused a buffer overflow. - The problem was fixed by verifying the fill value's size - against the buffer size before attempting to access the buffer. + The problem was fixed by verifying the fill value's size + against the buffer size before attempting to access the buffer. - (BMR - 2021/03/15, HDFFV-10480) + (BMR - 2021/03/15, HDFFV-10480) - Fixed CVE-2018-14033 (same issue as CVE-2020-10811) - The tool h5dump produced a segfault when the storage size message - was corrupted and caused a buffer overflow. + The tool h5dump produced a segfault when the storage size message + was corrupted and caused a buffer overflow. - The problem was fixed by verifying the storage size against the - buffer size before attempting to access the buffer. + The problem was fixed by verifying the storage size against the + buffer size before attempting to access the buffer. - (BMR - 2021/03/15, HDFFV-11159/HDFFV-11049) + (BMR - 2021/03/15, HDFFV-11159/HDFFV-11049) - Remove underscores on header file guards - Header file guards used a variety of underscores at the beginning of the define. + Header file guards used a variety of underscores at the beginning of the define. + + Removed all leading (some trailing) underscores from header file guards. + + (ADB - 2021/03/03, #361) - Removed all leading (some trailing) underscores from header file guards. + - Fixed a segmentation fault - (ADB - 2021/03/03, #361) + A segmentation fault occurred with a Mathworks corrupted file. + + A detection of accessing a null pointer was added to prevent the problem. + + (BMR - 2021/02/19, HDFFV-11150) - Fixed issue with MPI communicator and info object not being copied into new FAPL retrieved from H5F_get_access_plist - Added logic to copy the MPI communicator and info object into - the output FAPL. MPI communicator is retrieved from the VFD, while - the MPI info object is retrieved from the file's original FAPL. + Added logic to copy the MPI communicator and info object into + the output FAPL. MPI communicator is retrieved from the VFD, while + the MPI info object is retrieved from the file's original FAPL. - (JTH - 2021/02/15, HDFFV-11109) + (JTH - 2021/02/15, HDFFV-11109) - Fixed problems with vlens and refs inside compound using H5VLget_file_type() - Modified library to properly ref count H5VL_object_t structs and only - consider file vlen and reference types to be equal if their files are - the same. + Modified library to properly ref count H5VL_object_t structs and only + consider file vlen and reference types to be equal if their files are + the same. + + (NAF - 2021/01/22) + + - Fixed CVE-2018-17432 - (NAF - 2021/01/22) + The tool h5repack produced a segfault on a corrupted file which had + invalid rank for scalar or NULL datatype. + + The problem was fixed by modifying the dataspace encode and decode + functions to detect and report invalid rank. h5repack now fails + with an error message for the corrupted file. + + (BMR - 2020/10/26, HDFFV-10590) - Creation of dataset with optional filter - When the combination of type, space, etc doesn't work for filter - and the filter is optional, it was supposed to be skipped but it was - not skipped and the creation failed. + When the combination of type, space, etc doesn't work for filter + and the filter is optional, it was supposed to be skipped but it was + not skipped and the creation failed. - Allowed the creation of the dataset in such a situation. + Allowed the creation of the dataset in such a situation. - (BMR - 2020/08/13, HDFFV-10933) + (BMR - 2020/08/13, HDFFV-10933) - Explicitly declared dlopen to use RTLD_LOCAL - dlopen documentation states that if neither RTLD_GLOBAL nor - RTLD_LOCAL are specified, then the default behavior is unspecified. - The default on linux is usually RTLD_LOCAL while macos will default - to RTLD_GLOBAL. + dlopen documentation states that if neither RTLD_GLOBAL nor + RTLD_LOCAL are specified, then the default behavior is unspecified. + The default on linux is usually RTLD_LOCAL while macos will default + to RTLD_GLOBAL. - (ADB - 2020/08/12, HDFFV-11127) + (ADB - 2020/08/12, HDFFV-11127) - H5Sset_extent_none() sets the dataspace class to H5S_NO_CLASS which causes asserts/errors when passed to other dataspace API calls. - H5S_NO_CLASS is an internal class value that should not have been - exposed via a public API call. + H5S_NO_CLASS is an internal class value that should not have been + exposed via a public API call. - In debug builds of the library, this can cause assert() function to - trip. In non-debug builds, it will produce normal library errors. + In debug builds of the library, this can cause assert() function to + trip. In non-debug builds, it will produce normal library errors. - The new library behavior is for H5Sset_extent_none() to convert - the dataspace into one of type H5S_NULL, which is better handled - by the library and easier for developers to reason about. + The new library behavior is for H5Sset_extent_none() to convert + the dataspace into one of type H5S_NULL, which is better handled + by the library and easier for developers to reason about. - (DER - 2020/07/27, HDFFV-11027) + (DER - 2020/07/27, HDFFV-11027) - Fixed issues CVE-2018-13870 and CVE-2018-13869 - When a buffer overflow occurred because a name length was corrupted - and became very large, h5dump crashed on memory access violation. + When a buffer overflow occurred because a name length was corrupted + and became very large, h5dump crashed on memory access violation. - A check for reading pass the end of the buffer was added to multiple - locations to prevent the crashes and h5dump now simply fails with an - error message when this error condition occurs. + A check for reading pass the end of the buffer was added to multiple + locations to prevent the crashes and h5dump now simply fails with an + error message when this error condition occurs. - (BMR - 2020/07/22, HDFFV-11120 and HDFFV-11121) + (BMR - 2020/07/22, HDFFV-11120 and HDFFV-11121) - Fixed the segmentation fault when reading attributes with multiple threads - It was reported that the reading of attributes with variable length string - datatype will crash with segmentation fault particularly when the number of - threads is high (>16 threads). The problem was due to the file pointer that - was set in the variable length string datatype for the attribute. That file - pointer was already closed when the attribute was accessed. + It was reported that the reading of attributes with variable length string + datatype will crash with segmentation fault particularly when the number of + threads is high (>16 threads). The problem was due to the file pointer that + was set in the variable length string datatype for the attribute. That file + pointer was already closed when the attribute was accessed. - The problem was fixed by setting the file pointer to the current opened file pointer - when the attribute was accessed. Similar patch up was done before when reading - dataset with variable length string datatype. + The problem was fixed by setting the file pointer to the current opened file pointer + when the attribute was accessed. Similar patch up was done before when reading + dataset with variable length string datatype. - (VC - 2020/07/13, HDFFV-11080) + (VC - 2020/07/13, HDFFV-11080) - Fixed CVE-2020-10810 - The tool h5clear produced a segfault during an error recovery in - the superblock decoding. An internal pointer was reset to prevent - further accessing when it is not assigned with a value. + The tool h5clear produced a segfault during an error recovery in + the superblock decoding. An internal pointer was reset to prevent + further accessing when it is not assigned with a value. - (BMR - 2020/06/29, HDFFV-11053) + (BMR - 2020/06/29, HDFFV-11053) - Fixed CVE-2018-17435 - The tool h52gif produced a segfault when the size of an attribute - message was corrupted and caused a buffer overflow. + The tool h52gif produced a segfault when the size of an attribute + message was corrupted and caused a buffer overflow. - The problem was fixed by verifying the attribute message's size - against the buffer size before accessing the buffer. h52gif was - also fixed to display the failure instead of silently exiting - after the segfault was eliminated. + The problem was fixed by verifying the attribute message's size + against the buffer size before accessing the buffer. h52gif was + also fixed to display the failure instead of silently exiting + after the segfault was eliminated. - (BMR - 2020/06/19, HDFFV-10591) + (BMR - 2020/06/19, HDFFV-10591) Java Library ------------ - JNI utility function does not handle new references. - The JNI utility function for converting reference data to string did - not use the new APIs. In addition to fixing that function, added new - java tests for using the new APIs. + The JNI utility function for converting reference data to string did + not use the new APIs. In addition to fixing that function, added new + java tests for using the new APIs. - (ADB - 2021/02/16, HDFFV-11212) + (ADB - 2021/02/16, HDFFV-11212) - The H5FArray.java class, in which virtually the entire execution time - is spent using the HDFNativeData method that converts from an array - of bytes to an array of the destination Java type. + is spent using the HDFNativeData method that converts from an array + of bytes to an array of the destination Java type. 1. Convert the entire byte array into a 1-d array of the desired type, rather than performing 1 conversion per row; 2. Use the Java Arrays method copyOfRange to grab the section of the array from (1) that is desired to be inserted into the destination array. - (PGT,ADB - 2020/12/13, HDFFV-10865) + (PGT,ADB - 2020/12/13, HDFFV-10865) - Added ability to test java library with VOLs. - Created a new CMake script that combines the java and vol test scripts. + Created a new CMake script that combines the java and vol test scripts. - (ADB - 2020/02/03, HDFFV-10996) + (ADB - 2020/02/03, HDFFV-10996) - Tests fail for non-English locales. - In the JUnit tests with a non-English locale, only the part before - the decimal comma is replaced by XXXX and this leads to a comparison - error. Changed the regex for the Time substitution. + In the JUnit tests with a non-English locale, only the part before + the decimal comma is replaced by XXXX and this leads to a comparison + error. Changed the regex for the Time substitution. - (ADB - 2020/01/09, HDFFV-10995) + (ADB - 2020/01/09, HDFFV-10995) Configuration ------------- + - Corrected path searched by CMake find_package command + + The install path for cmake find_package files had been changed to use + "share/cmake" + for all platforms. However the trailing "hdf5" directory was not removed. + This "hdf5" additional directory has been removed. + + (ADB - 2021/09/27) + - Corrected pkg-config compile script It was discovered that the position of the "$@" argument for the command @@ -1251,88 +1338,88 @@ Bug Fixes since HDF5-1.12.0 release ----- - Changed how h5dump and h5ls identify long double. - Long double support is not consistent across platforms. Tools will always - identify long double as 128-bit [little/big]-endian float nn-bit precision. - New test file created for datasets with attributes for float, double and - long double. In addition any unknown integer or float datatype will now - also show the number of bits for precision. - These files are also used in the java tests. + Long double support is not consistent across platforms. Tools will always + identify long double as 128-bit [little/big]-endian float nn-bit precision. + New test file created for datasets with attributes for float, double and + long double. In addition any unknown integer or float datatype will now + also show the number of bits for precision. + These files are also used in the java tests. - (ADB - 2021/03/24, HDFFV-11229,HDFFV-11113) + (ADB - 2021/03/24, HDFFV-11229,HDFFV-11113) - Fixed tools argument parsing. - Tools parsing used the length of the option from the long array to match - the option from the command line. This incorrectly matched a shorter long - name option that happened to be a subset of another long option. - Changed to match whole names. + Tools parsing used the length of the option from the long array to match + the option from the command line. This incorrectly matched a shorter long + name option that happened to be a subset of another long option. + Changed to match whole names. - (ADB - 2021/01/19, HDFFV-11106) + (ADB - 2021/01/19, HDFFV-11106) - The tools library was updated by standardizing the error stack process. - General sequence is: - h5tools_setprogname(PROGRAMNAME); - h5tools_setstatus(EXIT_SUCCESS); - h5tools_init(); - ... process the command-line (check for error-stack enable) ... - h5tools_error_report(); - ... (do work) ... - h5diff_exit(ret); + General sequence is: + h5tools_setprogname(PROGRAMNAME); + h5tools_setstatus(EXIT_SUCCESS); + h5tools_init(); + ... process the command-line (check for error-stack enable) ... + h5tools_error_report(); + ... (do work) ... + h5diff_exit(ret); - (ADB - 2020/07/20, HDFFV-11066) + (ADB - 2020/07/20, HDFFV-11066) - h5diff fixed a command line parsing error. - h5diff would ignore the argument to -d (delta) if it is smaller than DBL_EPSILON. - The macro H5_DBL_ABS_EQUAL was removed and a direct value comparision was used. + h5diff would ignore the argument to -d (delta) if it is smaller than DBL_EPSILON. + The macro H5_DBL_ABS_EQUAL was removed and a direct value comparision was used. - (ADB - 2020/07/20, HDFFV-10897) + (ADB - 2020/07/20, HDFFV-10897) - h5diff added a command line option to ignore attributes. - h5diff would ignore all objects with a supplied path if the exclude-path argument is used. - Adding the exclude-attribute argument will only exclude attributes, with the supplied path, - from comparison. + h5diff would ignore all objects with a supplied path if the exclude-path argument is used. + Adding the exclude-attribute argument will only exclude attributes, with the supplied path, + from comparison. - (ADB - 2020/07/20, HDFFV-5935) + (ADB - 2020/07/20, HDFFV-5935) - h5diff added another level to the verbose argument to print filenames. - Added verbose level 3 that is level 2 plus the filenames. The levels are: - 0 : Identical to '-v' or '--verbose' - 1 : All level 0 information plus one-line attribute status summary - 2 : All level 1 information plus extended attribute status report - 3 : All level 2 information plus file names + Added verbose level 3 that is level 2 plus the filenames. The levels are: + 0 : Identical to '-v' or '--verbose' + 1 : All level 0 information plus one-line attribute status summary + 2 : All level 1 information plus extended attribute status report + 3 : All level 2 information plus file names - (ADB - 2020/07/20, HDFFV-1005) + (ADB - 2020/07/20, HDFFV-1005) - h5repack was fixed to repack the reference attributes properly. - The code line that checks if the update of reference inside a compound - datatype is misplaced outside the code block loop that carries out the - check. In consequence, the next attribute that is not the reference - type was repacked again as the reference type and caused the failure of - repacking. The fix is to move the corresponding code line to the correct - code block. + The code line that checks if the update of reference inside a compound + datatype is misplaced outside the code block loop that carries out the + check. In consequence, the next attribute that is not the reference + type was repacked again as the reference type and caused the failure of + repacking. The fix is to move the corresponding code line to the correct + code block. - (KY -2020/02/10, HDFFV-11014) + (KY -2020/02/10, HDFFV-11014) - h5diff was updated to use the new reference APIs. - h5diff uses the new reference APIs to compare references. - Attribute references can also be compared. + h5diff uses the new reference APIs to compare references. + Attribute references can also be compared. - (ADB - 2019/12/19, HDFFV-10980) + (ADB - 2019/12/19, HDFFV-10980) - h5dump and h5ls were updated to use the new reference APIs. - The tools library now use the new reference APIs to inspect a - file. Also the DDL spec was updated to reflect the format - changes produced with the new APIs. The export API and support - functions in the JNI were updated to match. + The tools library now use the new reference APIs to inspect a + file. Also the DDL spec was updated to reflect the format + changes produced with the new APIs. The export API and support + functions in the JNI were updated to match. - (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) + (ADB - 2019/12/06, HDFFV-10876 and HDFFV-10877) Performance @@ -1344,12 +1431,16 @@ Bug Fixes since HDF5-1.12.0 release ----------- - Corrected INTERFACE INTENT(IN) to INTENT(OUT) for buf_size in h5fget_file_image_f. - (MSB - 2020/02/18, HDFFV-11029) + (MSB - 2020/02/18, HDFFV-11029) High-Level Library ------------------ - - + - Fixed HL_test_packet, test for packet table vlen of vlen. + + Incorrect length assignment. + + (ADB - 2021/10/14) Fortran High-Level APIs @@ -1371,172 +1462,128 @@ Bug Fixes since HDF5-1.12.0 release -------- - Added DataSet::operator= - Some compilers complain if the copy constructor is given explicitly - but the assignment operator is implicitly set to default. + Some compilers complain if the copy constructor is given explicitly + but the assignment operator is implicitly set to default. - (2021/05/19) + (2021/05/19) Testing ------- - Stopped java/test/junit.sh.in installing libs for testing under ${prefix} - Lib files needed are now copied to a subdirectory in the java/test - directory, and on Macs the loader path for libhdf5.xxxs.so is changed - in the temporary copy of libhdf5_java.dylib. + Lib files needed are now copied to a subdirectory in the java/test + directory, and on Macs the loader path for libhdf5.xxxs.so is changed + in the temporary copy of libhdf5_java.dylib. - (LRK, 2020/07/02, HDFFV-11063) + (LRK, 2020/07/02, HDFFV-11063) -Supported Platforms +Platforms Tested =================== - Linux 2.6.32-696.16.1.el6.ppc64 gcc (GCC) 4.4.7 20120313 (Red Hat 4.4.7-18) - #1 SMP ppc64 GNU/Linux g++ (GCC) 4.4.7 20120313 (Red Hat 4.4.7-18) - (ostrich) GNU Fortran (GCC) 4.4.7 20120313 (Red Hat 4.4.7-18) - IBM XL C/C++ V13.1 - IBM XL Fortran V15.1 - - Linux 3.10.0-327.10.1.el7 GNU C (gcc), Fortran (gfortran), C++ (g++) - #1 SMP x86_64 GNU/Linux compilers: - (kituo/moohan) Version 4.8.5 20150623 (Red Hat 4.8.5-4) - Version 4.9.3, Version 5.2.0, - Intel(R) C (icc), C++ (icpc), Fortran (icc) - compilers: - Version 17.0.0.098 Build 20160721 - MPICH 3.1.4 compiled with GCC 4.9.3 - - SunOS 5.11 32- and 64-bit Sun C 5.12 SunOS_sparc - (emu) Sun Fortran 95 8.6 SunOS_sparc - Sun C++ 5.12 SunOS_sparc - - Windows 10 x64 Visual Studio 2015 w/ Intel Fortran 18 (cmake) - Visual Studio 2017 w/ Intel Fortran 19 (cmake) - Visual Studio 2019 w/ Intel Fortran 19 (cmake) - Visual Studio 2019 w/ MSMPI 10.1 (cmake) - - Mac OS X Yosemite 10.10.5 Apple clang/clang++ version 6.1 from Xcode 7.0 - 64-bit gfortran GNU Fortran (GCC) 4.9.2 - (osx1010dev/osx1010test) Intel icc/icpc/ifort version 15.0.3 - - Mac OS X El Capitan 10.11.6 Apple clang/clang++ version 7.3.0 from Xcode 7.3 - 64-bit gfortran GNU Fortran (GCC) 5.2.0 - (osx1011dev/osx1011test) Intel icc/icpc/ifort version 16.0.2 - - Mac OS Sierra 10.12.6 Apple LLVM version 8.1.0 (clang/clang++-802.0.42) - 64-bit gfortran GNU Fortran (GCC) 7.1.0 - (swallow/kite) Intel icc/icpc/ifort version 17.0.2 - - -Tested Configuration Features Summary -===================================== - - In the tables below - y = tested - n = not tested in this release - C = Cluster - W = Workstation - x = not working in this release - dna = does not apply - ( ) = footnote appears below second table - <blank> = testing incomplete on this feature or platform - -Platform C F90/ F90 C++ zlib SZIP - parallel F2003 parallel -Solaris2.11 32-bit n y/y n y y y -Solaris2.11 64-bit n y/n n y y y -Windows 10 y y/y n y y y -Windows 10 x64 y y/y n y y y -Mac OS X Mountain Lion 10.8.5 64-bit n y/y n y y y -Mac OS X Mavericks 10.9.5 64-bit n y/y n y y ? -Mac OS X Yosemite 10.10.5 64-bit n y/y n y y ? -Mac OS X El Capitan 10.11.6 64-bit n y/y n y y ? -CentOS 6.7 Linux 2.6.18 x86_64 GNU n y/y n y y y -CentOS 6.7 Linux 2.6.18 x86_64 Intel n y/y n y y y -CentOS 6.7 Linux 2.6.32 x86_64 PGI n y/y n y y y -CentOS 7.2 Linux 2.6.32 x86_64 GNU y y/y y y y y -CentOS 7.2 Linux 2.6.32 x86_64 Intel n y/y n y y y -Linux 2.6.32-573.18.1.el6.ppc64 n y/n n y y y - - -Platform Shared Shared Shared Thread- - C libs F90 libs C++ libs safe -Solaris2.11 32-bit y y y y -Solaris2.11 64-bit y y y y -Windows 10 y y y y -Windows 10 x64 y y y y -Mac OS X Mountain Lion 10.8.5 64-bit y n y y -Mac OS X Mavericks 10.9.5 64-bit y n y y -Mac OS X Yosemite 10.10.5 64-bit y n y y -Mac OS X El Capitan 10.11.6 64-bit y n y y -CentOS 6.7 Linux 2.6.18 x86_64 GNU y y y y -CentOS 6.7 Linux 2.6.18 x86_64 Intel y y y n -CentOS 6.7 Linux 2.6.32 x86_64 PGI y y y n -CentOS 7.2 Linux 2.6.32 x86_64 GNU y y y n -CentOS 7.2 Linux 2.6.32 x86_64 Intel y y y n -Linux 2.6.32-573.18.1.el6.ppc64 y y y n - -Compiler versions for each platform are listed in the preceding -"Supported Platforms" table. - - -More Tested Platforms -===================== -The following platforms are not supported but have been tested for this release. - - Linux 2.6.32-573.22.1.el6 GNU C (gcc), Fortran (gfortran), C++ (g++) - #1 SMP x86_64 GNU/Linux compilers: - (mayll/platypus) Version 4.4.7 20120313 - Version 4.9.3, 5.3.0, 6.2.0 - PGI C, Fortran, C++ for 64-bit target on - x86-64; - Version 17.10-0 - Intel(R) C (icc), C++ (icpc), Fortran (icc) - compilers: - Version 17.0.4.196 Build 20170411 - MPICH 3.1.4 compiled with GCC 4.9.3 - - Linux 3.10.0-327.18.2.el7 GNU C (gcc) and C++ (g++) compilers - #1 SMP x86_64 GNU/Linux Version 4.8.5 20150623 (Red Hat 4.8.5-4) - (jelly) with NAG Fortran Compiler Release 6.1(Tozai) - GCC Version 7.1.0 - OpenMPI 3.0.0-GCC-7.2.0-2.29 - Intel(R) C (icc) and C++ (icpc) compilers - Version 17.0.0.098 Build 20160721 - with NAG Fortran Compiler Release 6.1(Tozai) - - Linux 3.10.0-327.10.1.el7 MPICH 3.2 compiled with GCC 5.3.0 - #1 SMP x86_64 GNU/Linux - (moohan) - - Linux 2.6.32-573.18.1.el6.ppc64 MPICH mpich 3.1.4 compiled with - #1 SMP ppc64 GNU/Linux IBM XL C/C++ for Linux, V13.1 - (ostrich) and IBM XL Fortran for Linux, V15.1 - - Fedora33 5.11.18-200.fc33.x86_64 - #1 SMP x86_64 GNU/Linux GNU gcc (GCC) 10.3.1 20210422 (Red Hat 10.3.1-1) - GNU Fortran (GCC) 10.3.1 20210422 (Red Hat 10.3.1-1) - clang version 11.0.0 (Fedora 11.0.0-2.fc33) + Linux 5.13.14-200.fc34 GNU gcc (GCC) 11.2.1 2021078 (Red Hat 11.2.1-1) + #1 SMP x86_64 GNU/Linux GNU Fortran (GCC) 11.2.1 2021078 (Red Hat 11.2.1-1) + Fedora34 clang version 12.0.1 (Fedora 12.0.1-1.fc34) (cmake and autotools) - Ubuntu20.04 5.8.0-53-generic-x86_64 - #60~20.04-Ubuntu SMP x86_64 GNU/Linux GNU gcc (GCC) 9.3.0-17ubuntu1 - GNU Fortran (GCC) 9.3.0-17ubuntu1 - clang version 10.0.0-4ubuntu1 + Linux 5.11.0-34-generic GNU gcc (GCC) 9.3.0-17ubuntu1 + #36-Ubuntu SMP x86_64 GNU/Linux GNU Fortran (GCC) 9.3.0-17ubuntu1 + Ubuntu 20.04 Ubuntu clang version 10.0.0-4 (cmake and autotools) - Ubuntu20.10 5.8.0-53-generic-x86_64 - #60-Ubuntu SMP x86_64 GNU/Linux GNU gcc (GCC) 10.2.0-13ubuntu1 - GNU Fortran (GCC) 10.2.0-13ubuntu1 - Ubuntu clang version 11.0.0-2 + Linux 5.8.0-63-generic GNU gcc (GCC) 10.3.0-1ubuntu1 + #71-Ubuntu SMP x86_64 GNU/Linux GNU Fortran (GCC) 10.3.0-1ubuntu1 + Ubuntu20.10 Ubuntu clang version 11.0.0-2 (cmake and autotools) - SUSE15sp2 5.3.18-22-default - #1 SMP x86_64 GNU/Linux GNU gcc (SUSE Linux) 7.5.0 - GNU Fortran (SUSE Linux) 7.5.0 - clang version 7.0.1 (tags/RELEASE_701/final 349238) + Linux 5.3.18-22-default GNU gcc (SUSE Linux) 7.5.0 + #1 SMP x86_64 GNU/Linux GNU Fortran (SUSE Linux) 7.5.0 + SUSE15sp2 clang version 7.0.1 (tags/RELEASE_701/final 349238) (cmake and autotools) + Linux-4.14.0-115.21.2 spectrum-mpi/rolling-release + #1 SMP ppc64le GNU/Linux clang 8.0.1, 11.0.1 + (lassen) GCC 7.3.1 + XL 16.1.1.2 + (cmake) + + Linux-4.12.14-150.75-default cray-mpich/7.7.10 + #1 SMP x86_64 GNU/Linux GCC 7.3.0, 8.2.0 + (cori) Intel (R) Version 19.0.3.199 + (cmake) + + Linux-4.12.14-197.86-default cray-mpich/7.7.6 + # 1SMP x86_64 GNU/Linux GCC 7.3.0, 9.3.0, 10.2.0 + (mutrino) Intel (R) Version 17.0.4, 18.0.5, 19.1.3 + (cmake) + + Linux 3.10.0-1160.36.2.el7.ppc64 gcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + #1 SMP ppc64be GNU/Linux g++ (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + Power8 (echidna) GNU Fortran (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + + Linux 3.10.0-1160.24.1.el7 GNU C (gcc), Fortran (gfortran), C++ (g++) + #1 SMP x86_64 GNU/Linux compilers: + Centos7 Version 4.8.5 20150623 (Red Hat 4.8.5-4) + (jelly/kituo/moohan) Version 4.9.3, Version 5.3.0, Version 6.3.0, + Version 7.2.0, Version 8.3.0, Version 9.1.0 + Intel(R) C (icc), C++ (icpc), Fortran (icc) + compilers: + Version 17.0.0.098 Build 20160721 + GNU C (gcc) and C++ (g++) 4.8.5 compilers + with NAG Fortran Compiler Release 6.1(Tozai) + Intel(R) C (icc) and C++ (icpc) 17.0.0.098 compilers + with NAG Fortran Compiler Release 6.1(Tozai) + MPICH 3.1.4 compiled with GCC 4.9.3 + MPICH 3.3 compiled with GCC 7.2.0 + OpenMPI 2.1.6 compiled with icc 18.0.1 + OpenMPI 3.1.3 and 4.0.0 compiled with GCC 7.2.0 + PGI C, Fortran, C++ for 64-bit target on + x86_64; + Version 19.10-0 + + Linux-3.10.0-1127.0.0.1chaos openmpi-4.0.0 + #1 SMP x86_64 GNU/Linux clang 6.0.0, 11.0.1 + (quartz) GCC 7.3.0, 8.1.0 + Intel 16.0.4, 18.0.2, 19.0.4 + + macOS Apple M1 11.6 Apple clang version 12.0.5 (clang-1205.0.22.11) + Darwin 20.6.0 arm64 gfortran GNU Fortran (Homebrew GCC 11.2.0) 11.1.0 + (macmini-m1) Intel icc/icpc/ifort version 2021.3.0 202106092021.3.0 20210609 + + macOS Big Sur 11.3.1 Apple clang version 12.0.5 (clang-1205.0.22.9) + Darwin 20.4.0 x86_64 gfortran GNU Fortran (Homebrew GCC 10.2.0_3) 10.2.0 + (bigsur-1) Intel icc/icpc/ifort version 2021.2.0 20210228 + + macOS High Sierra 10.13.6 Apple LLVM version 10.0.0 (clang-1000.10.44.4) + 64-bit gfortran GNU Fortran (GCC) 6.3.0 + (bear) Intel icc/icpc/ifort version 19.0.4.233 20190416 + + macOS Sierra 10.12.6 Apple LLVM version 9.0.0 (clang-900.39.2) + 64-bit gfortran GNU Fortran (GCC) 7.4.0 + (kite) Intel icc/icpc/ifort version 17.0.2 + + Mac OS X El Capitan 10.11.6 Apple clang version 7.3.0 from Xcode 7.3 + 64-bit gfortran GNU Fortran (GCC) 5.2.0 + (osx1011test) Intel icc/icpc/ifort version 16.0.2 + + + Linux 2.6.32-573.22.1.el6 GNU C (gcc), Fortran (gfortran), C++ (g++) + #1 SMP x86_64 GNU/Linux compilers: + Centos6 Version 4.4.7 20120313 + (platypus) Version 4.9.3, 5.3.0, 6.2.0 + MPICH 3.1.4 compiled with GCC 4.9.3 + PGI C, Fortran, C++ for 64-bit target on + x86_64; + Version 19.10-0 + + Windows 10 x64 Visual Studio 2015 w/ Intel C/C++/Fortran 18 (cmake) + Visual Studio 2017 w/ Intel C/C++/Fortran 19 (cmake) + Visual Studio 2019 w/ clang 12.0.0 + with MSVC-like command-line (C/C++ only - cmake) + Visual Studio 2019 w/ Intel Fortran 19 (cmake) + Visual Studio 2019 w/ MSMPI 10.1 (C only - cmake) + Known Problems ============== @@ -1599,3 +1646,11 @@ The share folder will have the most differences because CMake builds include a number of CMake specific files for support of CMake's find_package and support for the HDF5 Examples CMake project. +The issues with the gif tool are: + HDFFV-10592 CVE-2018-17433 + HDFFV-10593 CVE-2018-17436 + HDFFV-11048 CVE-2020-10809 +These CVE issues have not yet been addressed and can be avoided by not building +the gif tool. Disable building the High-Level tools with these options: + autotools: --disable-hltools + cmake: HDF5_BUILD_HL_TOOLS=OFF diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 955a394..33d31fb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1418,36 +1418,6 @@ endif () # Option to build documentation #----------------------------------------------------------------------------- if (DOXYGEN_FOUND) - set (DOXYGEN_PACKAGE ${HDF5_PACKAGE_NAME}) - set (DOXYGEN_VERSION_STRING ${HDF5_PACKAGE_VERSION_STRING}) - set (DOXYGEN_INCLUDE_ALIASES_PATH ${HDF5_DOXYGEN_DIR}) - set (DOXYGEN_INCLUDE_ALIASES aliases) - set (DOXYGEN_VERBATIM_VARS DOXYGEN_INCLUDE_ALIASES) - set (DOXYGEN_PROJECT_LOGO ${HDF5_DOXYGEN_DIR}/img/HDFG-logo.png) - set (DOXYGEN_PROJECT_BRIEF "C-API Reference") - set (DOXYGEN_INPUT_DIRECTORY "${HDF5_SRC_DIR} ${HDF5_DOXYGEN_DIR}/dox ${HDF5_GENERATED_SOURCE_DIR}") - set (DOXYGEN_OPTIMIZE_OUTPUT_FOR_C YES) - set (DOXYGEN_MACRO_EXPANSION YES) - set (DOXYGEN_OUTPUT_DIRECTORY ${HDF5_BINARY_DIR}/hdf5lib_docs) - set (DOXYGEN_EXAMPLES_DIRECTORY "${HDF5_DOXYGEN_DIR}/examples ${HDF5_SRC_DIR} ${HDF5_SOURCE_DIR}/examples ${HDF5_TEST_SRC_DIR}") - set (DOXYGEN_LAYOUT_FILE ${HDF5_DOXYGEN_DIR}/hdf5doxy_layout.xml) - set (DOXYGEN_HTML_HEADER ${HDF5_DOXYGEN_DIR}/hdf5_header.html) - set (DOXYGEN_HTML_FOOTER ${HDF5_DOXYGEN_DIR}/hdf5_footer.html) - set (DOXYGEN_HTML_EXTRA_STYLESHEET ${HDF5_DOXYGEN_DIR}/hdf5doxy.css) - set (DOXYGEN_HTML_EXTRA_FILES "${HDF5_DOXYGEN_DIR}/hdf5_navtree_hacks.js ${HDF5_DOXYGEN_DIR}/img/ftv2node.png ${HDF5_DOXYGEN_DIR}/img/ftv2pnode.png") - set (DOXYGEN_SERVER_BASED_SEARCH NO) - set (DOXYGEN_EXTERNAL_SEARCH NO) - set (DOXYGEN_SEARCHENGINE_URL) - -# This configure and custom target work together - # Replace variables inside @@ with the current values - configure_file (${HDF5_DOXYGEN_DIR}/Doxyfile.in ${HDF5_BINARY_DIR}/Doxyfile @ONLY) - add_custom_target (hdf5lib_doc ALL - COMMAND ${DOXYGEN_EXECUTABLE} ${HDF5_BINARY_DIR}/Doxyfile - DEPENDS ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c ${HDF5_SRC_BINARY_DIR}/H5lib_settings.c - WORKING_DIRECTORY ${HDF5_SRC_DIR} - COMMENT "Generating HDF5 library Source API documentation with Doxygen" - VERBATIM ) # This cmake function requires that the non-default doxyfile settings are provided with set (DOXYGEN_xxx) commands # In addition the doxyfile aliases @INCLUDE option is not supported and would need to be provided in a set (DOXYGEN_ALIASES) command. # doxygen_add_docs (hdf5lib_doc @@ -1457,11 +1427,15 @@ if (DOXYGEN_FOUND) # WORKING_DIRECTORY ${HDF5_SRC_DIR} # COMMENT "Generating HDF5 library Source Documentation" # ) - install ( - DIRECTORY ${HDF5_BINARY_DIR}/hdf5lib_docs/html - DESTINATION ${HDF5_INSTALL_DATA_DIR} - COMPONENT Documents - ) + +# This custom target and doxygen/configure work together + # Replace variables inside @@ with the current values + add_custom_target (hdf5lib_doc ALL + COMMAND ${DOXYGEN_EXECUTABLE} ${HDF5_BINARY_DIR}/Doxyfile + DEPENDS ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c ${HDF5_SRC_BINARY_DIR}/H5lib_settings.c + WORKING_DIRECTORY ${HDF5_SRC_DIR} + COMMENT "Generating HDF5 library Source API documentation with Doxygen" + VERBATIM ) if (NOT TARGET doxygen) add_custom_target (doxygen) diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index a4e6d60..f7bed0c 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -234,27 +234,27 @@ typedef struct H5AC_proxy_entry_t { /* hbool_t evictions_enabled = */ TRUE, \ /* hbool_t set_initial_size = */ TRUE, \ /* size_t initial_size = */ ( 2 * 1024 * 1024), \ - /* double min_clean_fraction = */ 0.3f, \ + /* double min_clean_fraction = */ 0.3, \ /* size_t max_size = */ (32 * 1024 * 1024), \ /* size_t min_size = */ (1 * 1024 * 1024), \ /* long int epoch_length = */ 50000, \ /* enum H5C_cache_incr_mode incr_mode = */ H5C_incr__threshold, \ - /* double lower_hr_threshold = */ 0.9f, \ - /* double increment = */ 2.0f, \ + /* double lower_hr_threshold = */ 0.9, \ + /* double increment = */ 2.0, \ /* hbool_t apply_max_increment = */ TRUE, \ /* size_t max_increment = */ (4 * 1024 * 1024), \ /* enum H5C_cache_flash_incr_mode */ \ /* flash_incr_mode = */ H5C_flash_incr__add_space, \ - /* double flash_multiple = */ 1.0f, \ - /* double flash_threshold = */ 0.25f, \ + /* double flash_multiple = */ 1.0, \ + /* double flash_threshold = */ 0.25, \ /* enum H5C_cache_decr_mode decr_mode = */ H5C_decr__age_out_with_threshold, \ - /* double upper_hr_threshold = */ 0.999f, \ - /* double decrement = */ 0.9f, \ + /* double upper_hr_threshold = */ 0.999, \ + /* double decrement = */ 0.9, \ /* hbool_t apply_max_decrement = */ TRUE, \ /* size_t max_decrement = */ (1 * 1024 * 1024), \ /* int epochs_before_eviction = */ 3, \ /* hbool_t apply_empty_reserve = */ TRUE, \ - /* double empty_reserve = */ 0.1f, \ + /* double empty_reserve = */ 0.1, \ /* size_t dirty_bytes_threshold = */ (256 * 1024), \ /* int metadata_write_strategy = */ \ H5AC__DEFAULT_METADATA_WRITE_STRATEGY \ @@ -270,7 +270,7 @@ typedef struct H5AC_proxy_entry_t { /* hbool_t evictions_enabled = */ TRUE, \ /* hbool_t set_initial_size = */ TRUE, \ /* size_t initial_size = */ ( 2 * 1024 * 1024), \ - /* double min_clean_fraction = */ 0.01f, \ + /* double min_clean_fraction = */ 0.01, \ /* size_t max_size = */ (32 * 1024 * 1024), \ /* size_t min_size = */ ( 1 * 1024 * 1024), \ /* long int epoch_length = */ 50000, \ diff --git a/src/H5B2int.c b/src/H5B2int.c index 610da6c..ab43a5a 100644 --- a/src/H5B2int.c +++ b/src/H5B2int.c @@ -52,9 +52,9 @@ /********************/ /* Local Prototypes */ /********************/ -static herr_t H5B2__update_child_flush_depends(H5B2_hdr_t *hdr, unsigned depth, - const H5B2_node_ptr_t *node_ptrs, unsigned start_idx, - unsigned end_idx, void *old_parent, void *new_parent); +static herr_t H5B2__update_child_flush_depends(H5B2_hdr_t *hdr, unsigned depth, H5B2_node_ptr_t *node_ptrs, + unsigned start_idx, unsigned end_idx, void *old_parent, + void *new_parent); /*********************/ /* Package Variables */ @@ -1617,7 +1617,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5B2__iterate_node(H5B2_hdr_t *hdr, uint16_t depth, const H5B2_node_ptr_t *curr_node, void *parent, +H5B2__iterate_node(H5B2_hdr_t *hdr, uint16_t depth, H5B2_node_ptr_t *curr_node, void *parent, H5B2_operator_t op, void *op_data) { const H5AC_class_t *curr_node_class = NULL; /* Pointer to current node's class info */ @@ -1642,8 +1642,7 @@ H5B2__iterate_node(H5B2_hdr_t *hdr, uint16_t depth, const H5B2_node_ptr_t *curr_ /* Lock the current B-tree node */ if (NULL == - (internal = H5B2__protect_internal(hdr, parent, (H5B2_node_ptr_t *)curr_node, depth, FALSE, - H5AC__READ_ONLY_FLAG))) /* Casting away const OK -QAK */ + (internal = H5B2__protect_internal(hdr, parent, curr_node, depth, FALSE, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") /* Set up information about current node */ @@ -1739,8 +1738,8 @@ done: *------------------------------------------------------------------------- */ herr_t -H5B2__delete_node(H5B2_hdr_t *hdr, uint16_t depth, const H5B2_node_ptr_t *curr_node, void *parent, - H5B2_remove_t op, void *op_data) +H5B2__delete_node(H5B2_hdr_t *hdr, uint16_t depth, H5B2_node_ptr_t *curr_node, void *parent, H5B2_remove_t op, + void *op_data) { const H5AC_class_t *curr_node_class = NULL; /* Pointer to current node's class info */ void * node = NULL; /* Pointers to current node */ @@ -1759,8 +1758,7 @@ H5B2__delete_node(H5B2_hdr_t *hdr, uint16_t depth, const H5B2_node_ptr_t *curr_n /* Lock the current B-tree node */ if (NULL == - (internal = H5B2__protect_internal(hdr, parent, (H5B2_node_ptr_t *)curr_node, depth, FALSE, - H5AC__NO_FLAGS_SET))) /* Casting away const OK -QAK */ + (internal = H5B2__protect_internal(hdr, parent, curr_node, depth, FALSE, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") /* Set up information about current node */ @@ -1824,7 +1822,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5B2__node_size(H5B2_hdr_t *hdr, uint16_t depth, const H5B2_node_ptr_t *curr_node, void *parent, +H5B2__node_size(H5B2_hdr_t *hdr, uint16_t depth, H5B2_node_ptr_t *curr_node, void *parent, hsize_t *btree_size) { H5B2_internal_t *internal = NULL; /* Pointer to internal node */ @@ -1839,8 +1837,8 @@ H5B2__node_size(H5B2_hdr_t *hdr, uint16_t depth, const H5B2_node_ptr_t *curr_nod HDassert(depth > 0); /* Lock the current B-tree node */ - if (NULL == (internal = H5B2__protect_internal(hdr, parent, (H5B2_node_ptr_t *)curr_node, depth, FALSE, - H5AC__READ_ONLY_FLAG))) /* Casting away const OK -QAK */ + if (NULL == + (internal = H5B2__protect_internal(hdr, parent, curr_node, depth, FALSE, H5AC__READ_ONLY_FLAG))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") /* Recursively descend into child nodes, if we are above the "twig" level in the B-tree */ @@ -1910,7 +1908,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5B2__update_flush_depend(H5B2_hdr_t *hdr, unsigned depth, const H5B2_node_ptr_t *node_ptr, void *old_parent, +H5B2__update_flush_depend(H5B2_hdr_t *hdr, unsigned depth, H5B2_node_ptr_t *node_ptr, void *old_parent, void *new_parent) { const H5AC_class_t *child_class; /* Pointer to child node's class info */ @@ -1941,9 +1939,8 @@ H5B2__update_flush_depend(H5B2_hdr_t *hdr, unsigned depth, const H5B2_node_ptr_t H5B2_internal_t *child_int; /* Protect child */ - if (NULL == (child_int = H5B2__protect_internal( - hdr, new_parent, (H5B2_node_ptr_t *)node_ptr, (uint16_t)(depth - 1), FALSE, - H5AC__NO_FLAGS_SET))) /* Casting away const OK -QAK */ + if (NULL == (child_int = H5B2__protect_internal(hdr, new_parent, node_ptr, (uint16_t)(depth - 1), + FALSE, H5AC__NO_FLAGS_SET))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node") child_class = H5AC_BT2_INT; child = child_int; @@ -2010,7 +2007,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__update_child_flush_depends(H5B2_hdr_t *hdr, unsigned depth, const H5B2_node_ptr_t *node_ptrs, +H5B2__update_child_flush_depends(H5B2_hdr_t *hdr, unsigned depth, H5B2_node_ptr_t *node_ptrs, unsigned start_idx, unsigned end_idx, void *old_parent, void *new_parent) { unsigned u; /* Local index variable */ diff --git a/src/H5B2pkg.h b/src/H5B2pkg.h index 8d620cc..66d04fa 100644 --- a/src/H5B2pkg.h +++ b/src/H5B2pkg.h @@ -321,7 +321,7 @@ extern const H5B2_class_t *const H5B2_client_class_g[H5B2_NUM_BTREE_ID]; /* Generic routines */ H5_DLL herr_t H5B2__create_flush_depend(H5AC_info_t *parent_entry, H5AC_info_t *child_entry); -H5_DLL herr_t H5B2__update_flush_depend(H5B2_hdr_t *hdr, unsigned depth, const H5B2_node_ptr_t *node_ptr, +H5_DLL herr_t H5B2__update_flush_depend(H5B2_hdr_t *hdr, unsigned depth, H5B2_node_ptr_t *node_ptr, void *old_parent, void *new_parent); H5_DLL herr_t H5B2__destroy_flush_depend(H5AC_info_t *parent_entry, H5AC_info_t *child_entry); @@ -390,9 +390,9 @@ H5_DLL herr_t H5B2__update_leaf(H5B2_hdr_t *hdr, H5B2_node_ptr_t *curr_node_ptr, void *op_data); /* Routines for iterating over nodes/records */ -H5_DLL herr_t H5B2__iterate_node(H5B2_hdr_t *hdr, uint16_t depth, const H5B2_node_ptr_t *curr_node, - void *parent, H5B2_operator_t op, void *op_data); -H5_DLL herr_t H5B2__node_size(H5B2_hdr_t *hdr, uint16_t depth, const H5B2_node_ptr_t *curr_node, void *parent, +H5_DLL herr_t H5B2__iterate_node(H5B2_hdr_t *hdr, uint16_t depth, H5B2_node_ptr_t *curr_node, void *parent, + H5B2_operator_t op, void *op_data); +H5_DLL herr_t H5B2__node_size(H5B2_hdr_t *hdr, uint16_t depth, H5B2_node_ptr_t *curr_node, void *parent, hsize_t *op_data); /* Routines for locating records */ @@ -423,8 +423,8 @@ H5_DLL herr_t H5B2__remove_leaf_by_idx(H5B2_hdr_t *hdr, H5B2_node_ptr_t *curr_no void *op_data); /* Routines for deleting nodes */ -H5_DLL herr_t H5B2__delete_node(H5B2_hdr_t *hdr, uint16_t depth, const H5B2_node_ptr_t *curr_node, - void *parent, H5B2_remove_t op, void *op_data); +H5_DLL herr_t H5B2__delete_node(H5B2_hdr_t *hdr, uint16_t depth, H5B2_node_ptr_t *curr_node, void *parent, + H5B2_remove_t op, void *op_data); /* Debugging routines for dumping file structures */ H5_DLL herr_t H5B2__hdr_debug(H5F_t *f, haddr_t addr, FILE *stream, int indent, int fwidth, diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 71a15b0..cd54c3a 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -1403,6 +1403,19 @@ H5D__chunk_mem_xfree(void *chk, const void *_pline) } /* H5D__chunk_mem_xfree() */ /*------------------------------------------------------------------------- + * Function: H5D__chunk_mem_free + * + * Purpose: Wrapper with H5MM_free_t-compatible signature that just + * calls H5D__chunk_mem_xfree and discards the return value. + *------------------------------------------------------------------------- + */ +static void +H5D__chunk_mem_free(void *chk, const void *_pline) +{ + (void)H5D__chunk_mem_xfree(chk, _pline); +} + +/*------------------------------------------------------------------------- * Function: H5D__chunk_mem_realloc * * Purpose: Reallocate space for a chunk in memory. This routine allocates @@ -3193,11 +3206,14 @@ H5D__chunk_hash_val(const H5D_shared_t *shared, const hsize_t *scaled) herr_t H5D__chunk_lookup(const H5D_t *dset, const hsize_t *scaled, H5D_chunk_ud_t *udata) { - H5D_rdcc_ent_t * ent = NULL; /* Cache entry */ - H5O_storage_chunk_t *sc = &(dset->shared->layout.storage.u.chunk); - unsigned idx = 0; /* Index of chunk in cache, if present */ - hbool_t found = FALSE; /* In cache? */ - herr_t ret_value = SUCCEED; /* Return value */ + H5D_rdcc_ent_t * ent = NULL; /* Cache entry */ + H5O_storage_chunk_t *sc = &(dset->shared->layout.storage.u.chunk); + unsigned idx = 0; /* Index of chunk in cache, if present */ + hbool_t found = FALSE; /* In cache? */ +#ifdef H5_HAVE_PARALLEL + hbool_t reenable_coll_md_reads = FALSE; +#endif + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -3268,8 +3284,13 @@ H5D__chunk_lookup(const H5D_t *dset, const hsize_t *scaled, H5D_chunk_ud_t *udat * highly unlikely that users would read the same chunks from all * processes. */ - if (H5F_HAS_FEATURE(idx_info.f, H5FD_FEAT_HAS_MPI)) - H5CX_set_coll_metadata_read(FALSE); + if (H5F_HAS_FEATURE(idx_info.f, H5FD_FEAT_HAS_MPI)) { + hbool_t do_coll_md_reads = H5CX_get_coll_metadata_read(); + if (do_coll_md_reads) { + H5CX_set_coll_metadata_read(FALSE); + reenable_coll_md_reads = TRUE; + } + } #endif /* H5_HAVE_PARALLEL */ /* Go get the chunk information */ @@ -3312,6 +3333,12 @@ H5D__chunk_lookup(const H5D_t *dset, const hsize_t *scaled, H5D_chunk_ud_t *udat } /* end else */ done: +#ifdef H5_HAVE_PARALLEL + /* Re-enable collective metadata reads if we disabled them */ + if (reenable_coll_md_reads) + H5CX_set_coll_metadata_read(TRUE); +#endif /* H5_HAVE_PARALLEL */ + FUNC_LEAVE_NOAPI(ret_value) } /* H5D__chunk_lookup() */ @@ -4427,7 +4454,7 @@ H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, const /* (delay allocating fill buffer for VL datatypes until refilling) */ /* (casting away const OK - QAK) */ if (H5D__fill_init(&fb_info, NULL, (H5MM_allocate_t)H5D__chunk_mem_alloc, (void *)pline, - (H5MM_free_t)H5D__chunk_mem_xfree, (void *)pline, &dset->shared->dcpl_cache.fill, + (H5MM_free_t)H5D__chunk_mem_free, (void *)pline, &dset->shared->dcpl_cache.fill, dset->shared->type, dset->shared->type_id, (size_t)0, orig_chunk_size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill buffer info") fb_info_init = TRUE; diff --git a/src/H5ESpublic.h b/src/H5ESpublic.h index 4cf71c5..c8696b3 100644 --- a/src/H5ESpublic.h +++ b/src/H5ESpublic.h @@ -51,13 +51,13 @@ typedef enum H5ES_status_t { /* Information about operations in an event set */ typedef struct H5ES_op_info_t { /* API call info */ - char *api_name; /* Name of HDF5 API routine called */ - char *api_args; /* "Argument string" for arguments to HDF5 API routine called */ + const char *api_name; /* Name of HDF5 API routine called */ + char * api_args; /* "Argument string" for arguments to HDF5 API routine called */ /* Application info */ - char * app_file_name; /* Name of source file where the HDF5 API routine was called */ - char * app_func_name; /* Name of function where the HDF5 API routine was called */ - unsigned app_line_num; /* Line # of source file where the HDF5 API routine was called */ + const char *app_file_name; /* Name of source file where the HDF5 API routine was called */ + const char *app_func_name; /* Name of function where the HDF5 API routine was called */ + unsigned app_line_num; /* Line # of source file where the HDF5 API routine was called */ /* Operation info */ uint64_t op_ins_count; /* Counter of operation's insertion into event set */ diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h index 0a8850f..7c0dd77 100644 --- a/src/H5FDprivate.h +++ b/src/H5FDprivate.h @@ -168,7 +168,7 @@ typedef struct H5FD_vfd_swmr_idx_entry_t { uint64_t hdf5_page_offset; uint64_t md_file_page_offset; uint32_t length; - uint32_t chksum; + uint32_t checksum; void * entry_ptr; uint64_t tick_of_last_change; hbool_t clean; diff --git a/src/H5FDtest.c b/src/H5FDtest.c index a55d34c..521886f 100644 --- a/src/H5FDtest.c +++ b/src/H5FDtest.c @@ -162,7 +162,7 @@ H5FD__vfd_swmr_reader_md_test(H5FD_t *file, unsigned num_entries, H5FD_vfd_swmr_ HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect md_file_page_offset read from metadata file") - if (vfd_index[i].chksum != index[i].chksum) + if (vfd_index[i].checksum != index[i].checksum) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect chksum read from metadata file") } } diff --git a/src/H5FDvfd_swmr.c b/src/H5FDvfd_swmr.c index 568f5d9..73d8df4 100644 --- a/src/H5FDvfd_swmr.c +++ b/src/H5FDvfd_swmr.c @@ -857,7 +857,7 @@ H5FD__vfd_swmr_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id * signature. */ if (file->pb_configured && entry->length == init_size && - H5_checksum_metadata(buf, entry->length, 0) != entry->chksum) { + H5_checksum_metadata(buf, entry->length, 0) != entry->checksum) { H5FD_vfd_swmr_md_header tmp_header; if (H5FD__vfd_swmr_header_deserialize(file, &tmp_header) != TRUE) @@ -1331,7 +1331,7 @@ H5FD__vfd_swmr_index_deserialize(const H5FD_vfd_swmr_t *file, H5FD_vfd_swmr_md_i UINT32DECODE(p, md_index->entries[i].hdf5_page_offset); UINT32DECODE(p, md_index->entries[i].md_file_page_offset); UINT32DECODE(p, md_index->entries[i].length); - UINT32DECODE(p, md_index->entries[i].chksum); + UINT32DECODE(p, md_index->entries[i].checksum); } } else diff --git a/src/H5Fint.c b/src/H5Fint.c index c3fc346..b80fd15 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -1835,25 +1835,28 @@ done: H5F_t * H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) { - H5F_t * file = NULL; /*the success return value */ - H5F_shared_t * shared = NULL; /*shared part of `file' */ - H5FD_t * lf = NULL; /*file driver part of `shared' */ - unsigned tent_flags; /*tentative flags */ - H5FD_class_t * drvr; /*file driver class info */ - H5P_genplist_t * a_plist; /*file access property list */ - H5F_close_degree_t fc_degree; /*file close degree */ - size_t page_buf_size; - unsigned page_buf_min_meta_perc = 0; - unsigned page_buf_min_raw_perc = 0; - hbool_t set_flag = FALSE; /* Set the status_flags in the superblock */ - hbool_t clear = FALSE; /* Clear the status_flags */ - hbool_t evict_on_close; /* Evict on close value from plist */ - hbool_t use_file_locking = TRUE; /* Using file locks? */ - hbool_t ci_load = FALSE; /* Whether MDC ci load requested */ - hbool_t ci_write = FALSE; /* Whether MDC CI write requested */ - hbool_t file_create = FALSE; /* Creating a new file or not */ - H5F_vfd_swmr_config_t *vfd_swmr_config_ptr = NULL; /* Points to VFD SMWR config info */ - H5F_t * ret_value = NULL; /* Actual return value */ + H5F_t * file = NULL; /*the success return value */ + H5F_shared_t * shared = NULL; /*shared part of `file' */ + H5FD_t * lf = NULL; /*file driver part of `shared' */ + unsigned tent_flags; /*tentative flags */ + H5FD_class_t * drvr; /*file driver class info */ + H5P_genplist_t * a_plist; /*file access property list */ + H5F_close_degree_t fc_degree; /*file close degree */ + size_t page_buf_size; + unsigned page_buf_min_meta_perc = 0; + unsigned page_buf_min_raw_perc = 0; + hbool_t set_flag = FALSE; /* Set the status_flags in the superblock */ + hbool_t clear = FALSE; /* Clear the status_flags */ + hbool_t evict_on_close; /* Evict on close value from plist */ + hbool_t use_file_locking = TRUE; /* Using file locks? */ + hbool_t ci_load = FALSE; /* Whether MDC ci load requested */ + hbool_t ci_write = FALSE; /* Whether MDC ci write requested */ + hbool_t file_create = FALSE; /* Creating a new file or not */ + H5F_vfd_swmr_config_t * vfd_swmr_config_ptr = NULL; /* Points to VFD SMWR config info */ + H5F_generate_md_ck_cb_t cb_info = {NULL}; /* For VFD SWMR NFS testing: + initialize the callback to generate + checksums for metadata files */ + H5F_t *ret_value = NULL; /* Actual return value */ FUNC_ENTER_NOAPI(NULL) @@ -1876,6 +1879,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "file access is writer but VFD SWMR config is reader") if ((flags & H5F_ACC_RDWR) == 0 && vfd_swmr_config_ptr->writer) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "file access is reader but VFD SWMR config is writer") + + /* Retrieve the private property for VFD SWMR testing */ + if (H5P_get(a_plist, H5F_ACS_GENERATE_MD_CK_CB_NAME, &cb_info) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get generate_md_ck_cb info") } /* @@ -2111,6 +2118,9 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) /* Initialization for VFD SWMR writer and reader */ if (1 == shared->nrefs) { + /* Private property for VFD SWMR testing: generate checksum for metadata file */ + if (cb_info.func) + shared->generate_md_ck_cb = cb_info.func; if (H5F_vfd_swmr_init(file, file_create) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "file open fail with initialization for VFD SWMR") } @@ -3764,7 +3774,7 @@ herr_t H5F__start_swmr_write(H5F_t *f) { hbool_t ci_load = FALSE; /* whether MDC ci load requested */ - hbool_t ci_write = FALSE; /* whether MDC CI write requested */ + hbool_t ci_write = FALSE; /* whether MDC ci write requested */ size_t grp_dset_count = 0; /* # of open objects: groups & datasets */ size_t nt_attr_count = 0; /* # of opened named datatypes + opened attributes */ hid_t * obj_ids = NULL; /* List of ids */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 706e41d..ca26941 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -455,13 +455,36 @@ struct H5F_shared_t { uint32_t old_mdf_idx_len; uint32_t old_mdf_idx_entries_used; - /* Metadata file for VFD SWMR writer */ - int vfd_swmr_md_fd; /* POSIX: file descriptor for the - * metadata file - */ - haddr_t vfd_swmr_md_eoa; /* POSIX: eoa for the metadata - * file - */ + /* Metadata file and updater file for VFD SWMR writer */ + int vfd_swmr_md_fd; /* POSIX: file descriptor for the + * metadata file or -1 if the metadata file + * is not currently open. + * The vfd_swmr_config.generate_updater_files + * is FALSE. + */ + /* NFS: + * The vfd_swmr_config.generate_updater_files + * is TRUE and: + * --if vfd_swmr_config.writer is FALSE, + * this field is the file descriptor of the local + * copy of the metadata file, or -1 if the local + * copy is not currently open. + * --if vfd_swmr_config.writer is TRUE, this field + * is not used and is set to -1. + */ + H5F_generate_md_ck_t generate_md_ck_cb; + /* For testing only: + * Invoke the user-defined callback if exists to + * generate checksum for the metadata file + */ + + haddr_t vfd_swmr_md_eoa; /* POSIX: eoa for the metadata + * file + */ + uint64_t updater_seq_num; /* Sequence number of the next updater file to be + * genereated. This field must be initialized to zero, + * and incremented after each updater file is generated. + */ /* Free space manager for the metadata file */ H5FS_t * fs_man_md; /* Free-space manager */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index cef77dc..9a4f363 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -647,11 +647,16 @@ uint64_decode(uint8_t **pp) { \ /* int32_t version = */ 0, /* int32_t tick_len = */ 0, \ /* int32_t max_lag = */ 0, /* hbool_t vfd_swmr_writer = */ FALSE, \ + /* hbool_t maintain_metadata_file = */ FALSE, \ + /* hbool_t generate_updater_files = */ FALSE, \ /* hbool_t flush_raw_data = */ FALSE, /* int32_t md_pages_reserved = */ 0, \ /* int32_t pb_expansion_threshold = */ 0, /* char md_file_path[] = */ "", \ - /* char log_file_path[] = */ "" \ + /* char updater_file_path[] = */ "", /* char log_file_path[] = */ "" \ } +/* For VFD SWMR testing only: private property to generate checksum for metadata file via callback */ +#define H5F_ACS_GENERATE_MD_CK_CB_NAME "generate md ck callback" + /* ======================== File Mount properties ====================*/ #define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */ @@ -784,6 +789,72 @@ uint64_decode(uint8_t **pp) #define H5SM_TABLE_MAGIC "SMTB" /* Shared Message Table */ #define H5SM_LIST_MAGIC "SMLI" /* Shared Message List */ +/* + * VFD SWMR + */ + +/* Updater file header */ +#define H5F_UD_VERSION 0 /* Version of the updater file format */ +#define H5F_UD_HEADER_OFF 0 /* Updater file header offset */ +#define H5F_UD_HEADER_MAGIC "VUDH" /* Updater file header magic */ +#define H5F_SIZEOF_CHKSUM 4 /* Size of checksum */ + +/* Flags in the updater file header */ +#define CREATE_METADATA_FILE_ONLY_FLAG 0x0001 +#define FINAL_UPDATE_FLAG 0x0002 + +/* Size of updater file header */ +#define H5F_UD_HEADER_SIZE \ + (H5_SIZEOF_MAGIC /* Signature */ \ + + 2 /* Version number */ \ + + 2 /* Flags */ \ + + 4 /* Page size */ \ + + 8 /* Sequence number */ \ + + 8 /* Tick number */ \ + + 8 /* Change list offset */ \ + + 8 /* Change list length */ \ + + H5F_SIZEOF_CHKSUM /* Updater file header checksum */ \ + ) + +#define H5F_UD_CL_MAGIC "VUCL" /* Updater file change list magic */ + +/* Size of an updater file change list entry */ +#define H5F_UD_CL_ENTRY_SIZE \ + (4 /* Updater file page offset */ \ + + 4 /* Metadata file page offset */ \ + + 4 /* HDF5 file page offset */ \ + + 4 /* Length */ \ + + H5F_SIZEOF_CHKSUM /* Updater file change list entry checksum */ \ + ) + +/* Size of updater file change list */ +#define H5F_UD_CL_SIZE(N) /* N is number of change list entries */ \ + (H5_SIZEOF_MAGIC /* Signature */ \ + + 8 /* Tick num */ \ + + 4 /* Metadata file header updater file page offset */ \ + + 4 /* Metadata file header length */ \ + + 4 /* Metadata file header checksum */ \ + + 4 /* Metadata file index updater file page offset */ \ + + 8 /* Metadata file index metadata file offset */ \ + + 4 /* Metadata file index length */ \ + + 4 /* Metadata file index checksum */ \ + + 4 /* Number of change list entries */ \ + + (N * H5F_UD_CL_ENTRY_SIZE) /* Change list entries */ \ + + H5F_SIZEOF_CHKSUM /* Updater file change list checksum */ \ + ) + +/* + * For VFD SWMR testing only: + */ + +/* Callback routine to generate checksum for metadata file specified by md_path */ +typedef herr_t (*H5F_generate_md_ck_t)(char *md_path, uint64_t updater_seq_num); + +/* Structure for "generate checksum callback" private property */ +typedef struct H5F_generate_md_ck_cb_t { + H5F_generate_md_ck_t func; +} H5F_generate_md_ck_cb_t; + /****************************/ /* Library Private Typedefs */ /****************************/ @@ -854,6 +925,180 @@ typedef enum H5F_prefix_open_t { H5F_PREFIX_EFILE = 2 /* External file prefix */ } H5F_prefix_open_t; +/* + * VFD SWMR + */ + +/*---------------------------------------------------------------------------- + * + * struct H5F_vfd_swmr_updater_cl_entry_t + * + * An array of instances of H5F_vfd_swmr_updater_cl_entry_t of length equal to + * the number of metadata pages and multi-page metadata entries modified in + * the past tick is used to assemble the associated data in preparation for + * writing an updater file. + * + * Each entry in this array pertains to a given modified metdata page or + * multi-page metadata entry, and contains the following fields: + * + * entry_image_ptr: void pointer to a buffer containing the image of the + * target metadata page or multi-page metadata entry as modified in + * this tick, or NULL if undefined. + * + * entry_image_ud_file_page_offset: Page offset of the entry in the + * updater file, or 0 if undefined. + * + * entry_image_md_file_page_offset: Page offset of the entry in the + * metadata file, or 0 if undefined. + * + * entry_image_h5_file_page_offset: Page offset of the entry in the + * HDF5 file. In this case, a page offset of zero is valid, + * so we havd no easy marker for an invalid value. Instead, + * presume that this field is invalid if the entry_image_md_file_page_offset + * is invalid. + * + * entry_image_len: The size of the metadata page or multi-page metadata + * entry in bytes. + * entry_image_checksum: Checksum of the entry image. + * + *---------------------------------------------------------------------------- + */ +typedef struct H5F_vfd_swmr_updater_cl_entry_t { + void * entry_image_ptr; + uint32_t entry_image_ud_file_page_offset; + uint32_t entry_image_md_file_page_offset; + uint32_t entry_image_h5_file_page_offset; + size_t entry_image_len; + uint32_t entry_image_checksum; +} H5F_vfd_swmr_updater_cl_entry_t; + +/*---------------------------------------------------------------------------- + * + * struct H5F_vfd_swmr_updater_t + * + * Instances of this structure are used to assemble the data required to + * write a metadata file updater file. + * + * Updater file header related fields: + * + * version: Version of the updater file format to be used. At present this + * must be zero. + * + * flags: This field contains any flags to be set in the updater file header. + * Currently defined flags are: + * + * 0x0001 CREATE_METADATA_FILE_ONLY_FLAG + * If set, the auxiliary process should create the metadata file, + * but leave it empty. This flag may only be set if sequence_num + * is zero. + * + * 0x0002 FINAL_UPDATE_FLAG + * If set, the VFD SWMR writer is closing the target file, and this + * updater contains the final set of updates to the metadata file. + * On receipt, the auxiliary process should apply the enclosed + * changes to the metadata file, unlink it, and exit. + * + * sequence_num: This field contains the sequence number of this updater file. + * The sequence number of the first updater file must be zero, and + * this sequence number must be increased by one for each new updater + * file. Note that under some circumstances, the sequence number + * will not match the tick_num. + * + * tick_num: Number of the tick for which this updater file is to be generated. + * This value should match that of the index used to fill our this + * structure. + * + * header_image_ptr: void pointer to the buffer in which the + * updater file header is constructed. + * This field is NULL if the buffer is undefined. + * + * header_image_len: This field contains the length of the updater file + * header in bytes. + * + * change_list_image_ptr: void pointer to a buffer containing the on disk image + * of the updater file change list, or NULL if that buffer does not exist. + * + * change_list_offset: This field contains the offset in bytes of the change + * list in the updater file. This will typically be the offset of + * the first byte in the updater file after the header. + * + * change_list_len: This field contains the size in bytes of the on disk image + * of the change list in the updater file. + * + * Updater File Change List Related Fields: + * + * The updater file change list is a section of the updater file that details the + * locations and lengths of all metadata file entries that must be modified for + * this tick. + * + * md_file_header_image_ptr: void pointer to a buffer containing the on disk image + * of the metadata file header as updated for tick_num. + * + * md_file_header_ud_file_page_offset: This field contains the updater file + * page offset of the metadata file header image. Note that we do + * not store the metadata file page offset of the metadata file header, + * as it is always written to offset 0 in the metadata file. + * + * md_file_header_len: This field contains the size of the metadata file header + * image in bytes. + * + * md_file_index_image: void pointer to a buffer containing the on disk image + * of the metadata file index as updated for tick_num. + * + * md_file_index_md_file_offset: This field contains the offset of the + * metadata file index in the metadata file in bytes. + * + * This value will either be the size of the metadata file header + * (if the metadata file header and index are adjacent), or a page + * aligned value. + * + * md_file_index_ud_file_page_offset: This field contains the page offset of the + * metadata file index in the updater file. + * + * md_file_index_len: This field contains the size of the metadata file index in + * bytes. + * + * num_change_list_entries: This field contains the number of entries in the + * array of H5F_vfd_swmr_updater_cl_ entry_t whose base address + * is stored in the change_list field below. This value is also the + * number of metadata pages and multi-page metadata entries that have + * been modified in the past tick. + * + * If this field is zero, there is no change list, and the change_list + * field below is NULL. + * + * change_list: This field contains the base address of a dynamically allocated + * array of H5F_vfd_swmr_updater_cl_entry_t of length num_change_list_entries, + * or NULL if undefined. + * + *---------------------------------------------------------------------------- + */ +typedef struct H5F_vfd_swmr_updater_t { + /* Updater file header related fields */ + uint16_t version; + uint16_t flags; + uint32_t page_size; + uint64_t sequence_number; + uint64_t tick_num; + void * header_image_ptr; + size_t header_image_len; + void * change_list_image_ptr; + uint64_t change_list_offset; + size_t change_list_len; + /* Updater file change list related fields */ + void * md_file_header_image_ptr; + uint32_t md_file_header_image_chksum; + uint32_t md_file_header_ud_file_page_offset; + size_t md_file_header_len; + void * md_file_index_image_ptr; + uint32_t md_file_index_image_chksum; + uint64_t md_file_index_md_file_offset; + uint32_t md_file_index_ud_file_page_offset; + size_t md_file_index_len; + uint32_t num_change_list_entries; + H5F_vfd_swmr_updater_cl_entry_t *change_list; +} H5F_vfd_swmr_updater_t; + /*****************************/ /* Library-private Variables */ /*****************************/ diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index d241816..7120085 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -275,6 +275,31 @@ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata); * A boolean flag indicating whether the file opened with this FAPL entry * will be opened R/W. (i.e. as a VFD SWMR writer) * + * maintain_metadata_file + * A boolean flag indicating whether the writer should create and + * maintain the metadata file. Note that this field is only revelant + * if the above writer flag is TRUE. + * If this flag is TRUE, the writer must create and maintain the + * metadata file in the location specified in the md_file_path. + * Observe that at least one of maintain_metadata_file and + * generate_updater_files fields must be TRUE if writer is TRUE. + * + * generate_updater_files + * A boolean flag indicating whether the writer should generate a + * sequence of updater files describing how the metadata file + * should be updated at the end of each tick. + * If the flag is TRUE, all modifications to the metadata file + * (including its creation) are described in an ordered sequence of + * updater files. These files are read in order by auxiliary processes, + * and used to generate local copies of the metadata file as required. + * This mechanism exists to allow VFD SWMR to operate on storage + * systems that do not support POSIX semantics. + * This field is only used by the VFD SWMR writer. VFD SWMR readers + * ignore this field, as they always look to the specified metadata + * file, regardless of whether it is generated and maintained + * directly by the VFD SWMR writer, or by an auxiliary process that + * polls for new updater files, and applies them as they appear. + * * flush_raw_data: * A boolean flag indicating whether raw data should be flushed * as part of the end of tick processing. If set to TRUE, raw @@ -307,11 +332,16 @@ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata); * of tick is triggered. * * md_file_path: - * POSIX: this field contains the path of the metadata file. - * NFS: it contains the path and base name of the metadata file - * updater files. - * Object store: it contains the base URL for the objects used - * to store metadata file updater objects. + * If both the writer and maintain_metadata_file fields are TRUE, this + * field contains the path of the metadata file. + * If writer is FALSE, this field contains the path of the (possibly + * local copy of the) metadata file. + * + * updater_file_path: + * If generate_updater_files is TRUE, the contents of this field depends + * on whether the writer field is TRUE. If it is, the field contains + * the path and base name of the metadatea file updater files. + * If writer is FALSE, the field is ignored. * * log_file_path: * This field contains the path to the log file. If defined, this path should @@ -324,10 +354,13 @@ typedef struct H5F_vfd_swmr_config_t { uint32_t tick_len; uint32_t max_lag; hbool_t writer; + hbool_t maintain_metadata_file; + hbool_t generate_updater_files; hbool_t flush_raw_data; uint32_t md_pages_reserved; uint32_t pb_expansion_threshold; char md_file_path[H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1]; + char updater_file_path[H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1]; char log_file_path[H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1]; } H5F_vfd_swmr_config_t; diff --git a/src/H5Ftest.c b/src/H5Ftest.c index b76c579..f4a89fd 100644 --- a/src/H5Ftest.c +++ b/src/H5Ftest.c @@ -344,24 +344,10 @@ H5F__vfd_swmr_writer_create_open_flush_test(hid_t file_id, hbool_t file_create) /* Verify the minimum size for the metadata file */ if (HDstat(f->shared->vfd_swmr_config.md_file_path, &stat_buf) < 0) HGOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to stat the metadata file") - if (stat_buf.st_size < - (HDoff_t)((hsize_t)f->shared->vfd_swmr_config.md_pages_reserved * f->shared->fs_page_size)) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect metadata file size") if (file_create) { /* Creating file */ - uint32_t hdr_magic; - - /* Seek to the beginning of the file */ - if (HDlseek(md_fd, (HDoff_t)H5FD_MD_HEADER_OFF, SEEK_SET) < 0) - HGOTO_ERROR(H5E_FILE, H5E_SEEKERROR, FAIL, "error seeking metadata file") - - /* Try to read the magic for header */ - if (HDread(md_fd, &hdr_magic, H5_SIZEOF_MAGIC) < 0) - HGOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "error reading metadata file") - - /* Verify that there is no header magic in the metadata file */ - if (HDmemcmp(&hdr_magic, H5FD_MD_HEADER_MAGIC, (size_t)H5_SIZEOF_MAGIC) == 0) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "error finding header magic in the metadata file") + if (stat_buf.st_size != 0) + HGOTO_ERROR(H5E_FILE, H5E_READERROR, FAIL, "metadata file should be empty for file create") } else { /* Opening or flushing the file */ @@ -494,7 +480,7 @@ H5F__vfd_swmr_decode_md_idx(int md_fd, H5FD_vfd_swmr_md_header *md_hdr, H5FD_vfd UINT32DECODE(p, md_idx->entries[i].hdf5_page_offset); UINT32DECODE(p, md_idx->entries[i].md_file_page_offset); UINT32DECODE(p, md_idx->entries[i].length); - UINT32DECODE(p, md_idx->entries[i].chksum); + UINT32DECODE(p, md_idx->entries[i].checksum); } /* end for */ } /* end if */ @@ -546,7 +532,7 @@ H5F__vfd_swmr_verify_md_hdr_and_idx(H5F_t *f, H5FD_vfd_swmr_md_header *md_hdr, H HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect index_length read from metadata file") /* Verify index_offset read from header in the metadata file is the size of md header */ - if (md_hdr->index_offset != f->shared->fs_page_size) + if (md_hdr->index_offset != H5FD_MD_HEADER_SIZE) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect index_offset read from metadata file") /* Verify num_entries read from index in the metadata file is num_entries */ @@ -573,8 +559,8 @@ H5F__vfd_swmr_verify_md_hdr_and_idx(H5F_t *f, H5FD_vfd_swmr_md_header *md_hdr, H HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect md_file_page_offset read from metadata file") - if (md_idx->entries[i].chksum != index[i].chksum) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect chksum read from metadata file") + if (md_idx->entries[i].checksum != index[i].checksum) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "incorrect checksum read from metadata file") } } diff --git a/src/H5Fvfd_swmr.c b/src/H5Fvfd_swmr.c index 8da238c..2ac5e6c 100644 --- a/src/H5Fvfd_swmr.c +++ b/src/H5Fvfd_swmr.c @@ -25,6 +25,7 @@ /****************/ #include "H5Fmodule.h" /* This source code file is part of the H5F module */ +#define H5FD_FRIEND /*suppress error about including H5FDpkg */ /***********/ /* Headers */ @@ -36,7 +37,7 @@ #include "H5Dprivate.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* File access */ -#include "H5FDprivate.h" /* File drivers */ +#include "H5FDpkg.h" /* File drivers */ #include "H5Gprivate.h" /* Groups */ #include "H5Iprivate.h" /* IDs */ #include "H5Lprivate.h" /* Links */ @@ -97,13 +98,20 @@ const unsigned int log_err_mesg_length = 14; /********************/ static herr_t H5F__vfd_swmr_update_end_of_tick_and_tick_num(H5F_shared_t *, hbool_t); -static herr_t H5F__vfd_swmr_construct_write_md_hdr(H5F_shared_t *, uint32_t); +static herr_t H5F__vfd_swmr_construct_write_md_hdr(H5F_shared_t *, uint32_t, uint8_t *); static herr_t H5F__vfd_swmr_construct_write_md_idx(H5F_shared_t *, uint32_t, - struct H5FD_vfd_swmr_idx_entry_t[]); + struct H5FD_vfd_swmr_idx_entry_t[], uint8_t *); static herr_t H5F__idx_entry_cmp(const void *_entry1, const void *_entry2); static herr_t H5F__vfd_swmr_create_index(H5F_shared_t *); static herr_t H5F__vfd_swmr_writer__wait_a_tick(H5F_t *); +static herr_t H5F__vfd_swmr_construct_ud_hdr(H5F_vfd_swmr_updater_t *updater); +static herr_t H5F__vfd_swmr_construct_ud_cl(H5F_vfd_swmr_updater_t *updater); +static herr_t H5F__generate_updater_file(H5F_t *f, uint32_t num_entries, uint16_t flags, + uint8_t *md_file_hdr_image_ptr, size_t md_file_hdr_image_len, + uint8_t *md_file_index_image_ptr, uint64_t md_file_index_offset, + size_t md_file_index_image_len); + /*********************/ /* Package Variables */ /*********************/ @@ -170,10 +178,12 @@ H5FL_DEFINE(eot_queue_entry_t); herr_t H5F_vfd_swmr_init(H5F_t *f, hbool_t file_create) { - hsize_t md_size; /* Size of the metadata file */ - haddr_t hdr_addr, idx_addr; /* Addresses returned from H5MV_alloc() */ - herr_t ret_value = SUCCEED; /* Return value */ - H5F_shared_t *shared = f->shared; + hsize_t md_size; /* Size of the metadata file */ + haddr_t hdr_addr; /* Address returned from H5MV_alloc() */ + H5F_shared_t *shared = f->shared; + uint8_t md_idx_image[H5FD_MD_INDEX_SIZE(0)]; /* Buffer for metadata file index */ + uint8_t md_hdr_image[H5FD_MD_HEADER_SIZE]; /* Buffer for metadata file header */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -184,56 +194,59 @@ H5F_vfd_swmr_init(H5F_t *f, hbool_t file_create) if (H5F_SHARED_INTENT(shared) & H5F_ACC_RDWR) { HDassert(shared->vfd_swmr_config.writer); + HDassert(shared->vfd_swmr_config.maintain_metadata_file || + shared->vfd_swmr_config.generate_updater_files); SIMPLEQ_INIT(&shared->lower_defrees); shared->vfd_swmr_writer = TRUE; - shared->tick_num = 1; - - if (H5PB_vfd_swmr__set_tick(shared) < 0) - HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "Can't update page buffer current tick") + shared->tick_num = 0; /* Create the metadata file */ - if (((shared->vfd_swmr_md_fd = HDopen(shared->vfd_swmr_config.md_file_path, O_CREAT | O_RDWR, - H5_POSIX_CREATE_MODE_RW))) < 0) + if (((shared->vfd_swmr_md_fd = HDopen(shared->vfd_swmr_config.md_file_path, + O_CREAT | O_RDWR | O_TRUNC, H5_POSIX_CREATE_MODE_RW))) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, FAIL, "unable to create the metadata file") md_size = (hsize_t)shared->vfd_swmr_config.md_pages_reserved * shared->fs_page_size; - HDassert(shared->fs_page_size >= H5FD_MD_HEADER_SIZE); - - /* Allocate an entire page from the shadow file for the header. */ - if ((hdr_addr = H5MV_alloc(f, shared->fs_page_size)) == HADDR_UNDEF) { + if ((hdr_addr = H5MV_alloc(f, md_size)) == HADDR_UNDEF) { HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error allocating shadow-file header"); } HDassert(H5F_addr_eq(hdr_addr, H5FD_MD_HEADER_OFF)); - idx_addr = H5MV_alloc(f, md_size - shared->fs_page_size); - if (idx_addr == HADDR_UNDEF) { - HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error allocating shadow-file index"); - } - - HDassert(H5F_addr_eq(idx_addr, shared->fs_page_size)); - - shared->writer_index_offset = idx_addr; - - /* Set the metadata file size to md_pages_reserved */ - if (-1 == HDftruncate(shared->vfd_swmr_md_fd, (HDoff_t)md_size)) - HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "truncate fail for the metadata file"); - - /* Set eof for metadata file to md_pages_reserved */ - shared->vfd_swmr_md_eoa = (haddr_t)md_size; + shared->writer_index_offset = H5FD_MD_HEADER_SIZE; + shared->vfd_swmr_md_eoa = (haddr_t)md_size; /* When opening an existing HDF5 file, create header and empty * index in the metadata file */ if (!file_create) { - if (H5F__vfd_swmr_construct_write_md_idx(shared, 0, NULL) < 0) + if (H5F__vfd_swmr_construct_write_md_idx(shared, 0, NULL, md_idx_image) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create index in md"); - if (H5F__vfd_swmr_construct_write_md_hdr(shared, 0) < 0) + if (H5F__vfd_swmr_construct_write_md_hdr(shared, 0, md_hdr_image) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create header in md"); } + + /* For VFD SWMR testing: invoke callback if set to generate metadata file checksum */ + if (shared->generate_md_ck_cb) { + if (shared->generate_md_ck_cb(shared->vfd_swmr_config.md_file_path, shared->updater_seq_num) < 0) + HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "error from generate_md_ck_cb()") + } + + /* Generate updater files if configuration indicates so */ + if (shared->vfd_swmr_config.generate_updater_files) { + shared->updater_seq_num = 0; + if (H5F__generate_updater_file(f, 0, file_create ? CREATE_METADATA_FILE_ONLY_FLAG : 0, + md_hdr_image, H5FD_MD_HEADER_SIZE, md_idx_image, + shared->writer_index_offset, H5FD_MD_INDEX_SIZE(0)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "can't generate updater file") + } + + shared->tick_num = 1; + + if (H5PB_vfd_swmr__set_tick(shared) < 0) + HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "Can't update page buffer current tick") } else { /* VFD SWMR reader */ @@ -254,7 +267,6 @@ H5F_vfd_swmr_init(H5F_t *f, hbool_t file_create) &(shared->mdf_idx_entries_used), shared->mdf_idx) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTLOAD, FAIL, "unable to load/decode metadata file"); - HDassert(shared->tick_num != 0); vfd_swmr_reader_did_increase_tick_to(shared->tick_num); #if 0 /* JRM */ @@ -310,6 +322,8 @@ H5F_vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing) { H5F_shared_t * shared = f->shared; shadow_defree_t *curr; + uint8_t md_idx_image[H5FD_MD_INDEX_SIZE(0)]; /* Buffer for metadata file index */ + uint8_t md_hdr_image[H5FD_MD_HEADER_SIZE]; /* Buffer for metadata file header */ herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI(FAIL) @@ -318,22 +332,26 @@ H5F_vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing) HDassert(shared->vfd_swmr_md_fd >= 0); /* Write empty index to the md file */ - if (H5F__vfd_swmr_construct_write_md_idx(shared, 0, NULL) < 0) + if (H5F__vfd_swmr_construct_write_md_idx(shared, 0, NULL, md_idx_image) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create index in md"); /* Write header to the md file */ - if (H5F__vfd_swmr_construct_write_md_hdr(shared, 0) < 0) + if (H5F__vfd_swmr_construct_write_md_hdr(shared, 0, md_hdr_image) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create header in md"); if (closing) { /* For file close */ - ++shared->tick_num; - /* Close the md file */ if (HDclose(shared->vfd_swmr_md_fd) < 0) HSYS_GOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close the metadata file"); shared->vfd_swmr_md_fd = -1; + /* For VFD SWMR testing: invoke callback if set to generate metadata file checksum */ + if (shared->generate_md_ck_cb) { + if (shared->generate_md_ck_cb(shared->vfd_swmr_config.md_file_path, shared->updater_seq_num) < 0) + HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "error from generate_md_ck_cb()") + } + /* Unlink the md file */ if (HDunlink(shared->vfd_swmr_config.md_file_path) < 0) HSYS_GOTO_ERROR(H5E_FILE, H5E_CANTREMOVE, FAIL, "unable to unlink the metadata file"); @@ -350,6 +368,13 @@ H5F_vfd_swmr_close_or_flush(H5F_t *f, hbool_t closing) } HDassert(TAILQ_EMPTY(&shared->shadow_defrees)); + + if (shared->vfd_swmr_config.generate_updater_files) { + if (H5F__generate_updater_file(f, 0, FINAL_UPDATE_FLAG, md_hdr_image, H5FD_MD_HEADER_SIZE, + md_idx_image, shared->writer_index_offset, + H5FD_MD_INDEX_SIZE(0)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "can't generate updater file") + } } else { /* For file flush */ /* Update end_of_tick */ @@ -436,9 +461,11 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t num_entries, H5FD_vfd_swmr_ H5F_shared_t * shared = f->shared; shadow_defree_t *prev; shadow_defree_t *shadow_defree; - haddr_t md_addr; /* Address in the metadata file */ - uint32_t i; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + haddr_t md_addr; /* Address in the metadata file */ + uint32_t i; /* Local index variable */ + uint8_t * md_idx_image = NULL; + uint8_t md_hdr_image[H5FD_MD_HEADER_SIZE]; /* Buffer for metadata file header */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -460,13 +487,15 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t num_entries, H5FD_vfd_swmr_ * --Compute checksum, update the index entry, write entry to * the metadata file * - * --Set entry_ptr to NULL + * --Set entry_ptr to NULL when not generating updater files */ for (i = 0; i < num_entries; i++) { if (index[i].entry_ptr == NULL) continue; + HDassert(index[i].tick_of_last_change == f->shared->tick_num); + /* Prepend previous image of the entry to the delayed list */ if (index[i].md_file_page_offset) { if (shadow_image_defer_free(shared, &index[i]) == -1) { @@ -482,16 +511,16 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t num_entries, H5FD_vfd_swmr_ /* Compute checksum and update the index entry */ index[i].md_file_page_offset = md_addr / shared->fs_page_size; - index[i].chksum = H5_checksum_metadata(index[i].entry_ptr, index[i].length, 0); + index[i].checksum = H5_checksum_metadata(index[i].entry_ptr, index[i].length, 0); #if 0 /* JRM */ HDfprintf(stderr, - "writing index[%d] fo/mdfo/l/chksum/fc/lc = %lld/%lld/%ld/%lx/%lx/%lx\n", + "writing index[%d] fo/mdfo/l/checksum/fc/lc = %lld/%lld/%ld/%lx/%lx/%lx\n", i, index[i].hdf5_page_offset, index[i].md_file_page_offset, index[i].length, - index[i].chksum, + index[i].checksum, (((char*)(index[i].entry_ptr))[0]), (((char*)(index[i].entry_ptr))[4095])); @@ -500,26 +529,34 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t num_entries, H5FD_vfd_swmr_ HDassert(shared->fs_page_size == 4096); #endif /* JRM */ - /* Seek and write the entry to the metadata file */ - if (HDlseek(shared->vfd_swmr_md_fd, (HDoff_t)md_addr, SEEK_SET) < 0) + if (shared->vfd_swmr_config.maintain_metadata_file) { + + /* Seek and write the entry to the metadata file */ + if (HDlseek(shared->vfd_swmr_md_fd, (HDoff_t)md_addr, SEEK_SET) < 0) - HGOTO_ERROR(H5E_FILE, H5E_SEEKERROR, FAIL, "unable to seek in the metadata file") + HGOTO_ERROR(H5E_FILE, H5E_SEEKERROR, FAIL, "unable to seek in the metadata file") - if (HDwrite(shared->vfd_swmr_md_fd, index[i].entry_ptr, index[i].length) != (ssize_t)index[i].length) + if (HDwrite(shared->vfd_swmr_md_fd, index[i].entry_ptr, index[i].length) != + (ssize_t)index[i].length) - HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, - "error in writing the page/multi-page entry to metadata file") + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, + "error in writing the page/multi-page entry to metadata file") + } - index[i].entry_ptr = NULL; + if (!shared->vfd_swmr_config.generate_updater_files) + index[i].entry_ptr = NULL; } + if ((md_idx_image = HDmalloc(H5FD_MD_INDEX_SIZE(num_entries))) == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for md index") + /* Construct and write index to the metadata file */ - if (H5F__vfd_swmr_construct_write_md_idx(shared, num_entries, index) < 0) + if (H5F__vfd_swmr_construct_write_md_idx(shared, num_entries, index, md_idx_image) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to construct & write index to md") /* Construct and write header to the md file */ - if (H5F__vfd_swmr_construct_write_md_hdr(shared, num_entries) < 0) + if (H5F__vfd_swmr_construct_write_md_hdr(shared, num_entries, md_hdr_image) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to construct & write header to md") @@ -534,26 +571,45 @@ H5F_update_vfd_swmr_metadata_file(H5F_t *f, uint32_t num_entries, H5FD_vfd_swmr_ * --remove the associated entries from the list */ - if (shared->tick_num <= shared->vfd_swmr_config.max_lag) - goto done; // It is too early for any reclamations to be due. + /* if (shared->tick_num <= shared->vfd_swmr_config.max_lag), + it is too early for any reclamations to be due. + */ + if (shared->tick_num > shared->vfd_swmr_config.max_lag) { - TAILQ_FOREACH_REVERSE_SAFE(shadow_defree, &shared->shadow_defrees, shadow_defree_queue, link, prev) - { + TAILQ_FOREACH_REVERSE_SAFE(shadow_defree, &shared->shadow_defrees, shadow_defree_queue, link, prev) + { - if (shadow_defree->tick_num + shared->vfd_swmr_config.max_lag > shared->tick_num) { - break; // No more entries are due for reclamation. - } + if (shadow_defree->tick_num + shared->vfd_swmr_config.max_lag > shared->tick_num) { + break; // No more entries are due for reclamation. + } - if (H5MV_free(f, shadow_defree->offset, shadow_defree->length) < 0) { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush clean entry"); - } + if (H5MV_free(f, shadow_defree->offset, shadow_defree->length) < 0) { + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush clean entry"); + } - TAILQ_REMOVE(&shared->shadow_defrees, shadow_defree, link); + TAILQ_REMOVE(&shared->shadow_defrees, shadow_defree, link); - H5FL_FREE(shadow_defree_t, shadow_defree); + H5FL_FREE(shadow_defree_t, shadow_defree); + } + } + + /* For VFD SWMR testing: invoke callback if set to generate metadata file checksum */ + if (shared->generate_md_ck_cb) { + if (shared->generate_md_ck_cb(shared->vfd_swmr_config.md_file_path, shared->updater_seq_num) < 0) + HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "error from generate_md_ck_cb()") } + /* Generate updater files with num_entries */ + if (shared->vfd_swmr_config.generate_updater_files) + if (H5F__generate_updater_file(f, num_entries, 0, md_hdr_image, H5FD_MD_HEADER_SIZE, md_idx_image, + shared->writer_index_offset, H5FD_MD_INDEX_SIZE(num_entries)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_SYSTEM, FAIL, "can't generate updater file") + done: + + if (md_idx_image) + HDfree(md_idx_image); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_update_vfd_swmr_metadata_file() */ @@ -1571,11 +1627,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5F__vfd_swmr_construct_write_md_hdr(H5F_shared_t *shared, uint32_t num_entries) +H5F__vfd_swmr_construct_write_md_hdr(H5F_shared_t *shared, uint32_t num_entries, uint8_t *image) { - uint8_t image[H5FD_MD_HEADER_SIZE]; /* Buffer for header */ - uint8_t *p = NULL; /* Pointer to buffer */ - uint32_t metadata_chksum; /* Computed metadata checksum value */ + uint8_t *p = NULL; /* Pointer to buffer */ + uint32_t metadata_chksum; /* Computed metadata checksum value */ /* Size of header and index */ const size_t hdr_size = H5FD_MD_HEADER_SIZE; ssize_t nwritten; @@ -1607,15 +1662,17 @@ H5F__vfd_swmr_construct_write_md_hdr(H5F_shared_t *shared, uint32_t num_entries) /* Sanity checks on header */ HDassert(p - image == (ptrdiff_t)hdr_size); - /* Set to beginning of the file */ - if (HDlseek(shared->vfd_swmr_md_fd, H5FD_MD_HEADER_OFF, SEEK_SET) < 0) + if (shared->vfd_swmr_config.maintain_metadata_file) { + /* Set to beginning of the file */ + if (HDlseek(shared->vfd_swmr_md_fd, H5FD_MD_HEADER_OFF, SEEK_SET) < 0) - HGOTO_ERROR(H5E_VFL, H5E_SEEKERROR, FAIL, "unable to seek in metadata file") + HGOTO_ERROR(H5E_VFL, H5E_SEEKERROR, FAIL, "unable to seek in metadata file") - nwritten = HDwrite(shared->vfd_swmr_md_fd, image, hdr_size); - /* Write header to the metadata file */ - if (nwritten != (ssize_t)hdr_size) { - HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing header to metadata file") + nwritten = HDwrite(shared->vfd_swmr_md_fd, image, hdr_size); + /* Write header to the metadata file */ + if (nwritten != (ssize_t)hdr_size) { + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing header to metadata file") + } } done: @@ -1648,10 +1705,9 @@ done: */ static herr_t H5F__vfd_swmr_construct_write_md_idx(H5F_shared_t *shared, uint32_t num_entries, - struct H5FD_vfd_swmr_idx_entry_t index[]) + struct H5FD_vfd_swmr_idx_entry_t index[], uint8_t *image) { - uint8_t *image = NULL; /* Pointer to buffer */ - uint8_t *p = NULL; /* Pointer to buffer */ + uint8_t *p = NULL; /* Pointer to buffer */ uint32_t metadata_chksum; /* Computed metadata checksum value */ /* Size of index */ const size_t idx_size = H5FD_MD_INDEX_SIZE(num_entries); @@ -1663,11 +1719,6 @@ H5F__vfd_swmr_construct_write_md_idx(H5F_shared_t *shared, uint32_t num_entries, HDassert(num_entries == 0 || index != NULL); - /* Allocate space for the buffer to hold the index */ - if ((image = HDmalloc(idx_size)) == NULL) - - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for md index") - /* * Encode metadata file index */ @@ -1688,7 +1739,7 @@ H5F__vfd_swmr_construct_write_md_idx(H5F_shared_t *shared, uint32_t num_entries, UINT32ENCODE(p, index[i].hdf5_page_offset); UINT32ENCODE(p, index[i].md_file_page_offset); UINT32ENCODE(p, index[i].length); - UINT32ENCODE(p, index[i].chksum); + UINT32ENCODE(p, index[i].checksum); } /* Calculate checksum for index */ @@ -1703,22 +1754,20 @@ H5F__vfd_swmr_construct_write_md_idx(H5F_shared_t *shared, uint32_t num_entries, /* Verify the md file descriptor exists */ HDassert(shared->vfd_swmr_md_fd >= 0); - if (HDlseek(shared->vfd_swmr_md_fd, (HDoff_t)shared->writer_index_offset, SEEK_SET) < 0) - HGOTO_ERROR(H5E_VFL, H5E_SEEKERROR, FAIL, "unable to seek in metadata file") + if (shared->vfd_swmr_config.maintain_metadata_file) { + + if (HDlseek(shared->vfd_swmr_md_fd, (HDoff_t)shared->writer_index_offset, SEEK_SET) < 0) + HGOTO_ERROR(H5E_VFL, H5E_SEEKERROR, FAIL, "unable to seek in metadata file") - nwritten = HDwrite(shared->vfd_swmr_md_fd, image, idx_size); - /* Write index to the metadata file */ - if (nwritten != (ssize_t)idx_size) { - HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing index to metadata file") + nwritten = HDwrite(shared->vfd_swmr_md_fd, image, idx_size); + /* Write index to the metadata file */ + if (nwritten != (ssize_t)idx_size) { + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "error in writing index to metadata file") + } } done: - if (image) { - - HDfree(image); - } - FUNC_LEAVE_NOAPI(ret_value) } /* H5F__vfd_swmr_construct_write_idx() */ @@ -2010,3 +2059,388 @@ H5F_post_vfd_swmr_log_entry(H5F_t *f, int entry_type_code, char *log_info) } return; } + +/*------------------------------------------------------------------------- + * + * Function: H5F__vfd_swmr_construct_ud_hdr + * + * Purpose: Encode updater header in the buffer updater->header_image_ptr + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Vailin Choi -- 10/2021 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__vfd_swmr_construct_ud_hdr(H5F_vfd_swmr_updater_t *updater) +{ + uint8_t *p = NULL; /* Pointer to buffer */ + uint8_t *image = (uint8_t *)updater->header_image_ptr; + uint32_t metadata_chksum; /* Computed metadata checksum value */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* + * Encode metadata file header + */ + p = image; + + /* Encode magic for header */ + HDmemcpy(p, H5F_UD_HEADER_MAGIC, (size_t)H5_SIZEOF_MAGIC); + p += H5_SIZEOF_MAGIC; + + /* Encode version number, flags, page size, sequence number, tick number, change list offset, change list + * length */ + UINT16ENCODE(p, H5F_UD_VERSION); + UINT16ENCODE(p, updater->flags); + UINT32ENCODE(p, updater->page_size); + UINT64ENCODE(p, updater->sequence_number); + UINT64ENCODE(p, updater->tick_num); + + UINT64ENCODE(p, updater->change_list_offset); + UINT64ENCODE(p, updater->change_list_len); + + /* Calculate checksum for header */ + metadata_chksum = H5_checksum_metadata(image, (size_t)(p - image), 0); + + /* Encode checksum for header */ + UINT32ENCODE(p, metadata_chksum); + + /* Sanity checks on header */ + HDassert(p - image == (ptrdiff_t)updater->header_image_len); + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5F__vfd_swmr_construct_ud_hdr() */ + +/*------------------------------------------------------------------------- + * + * Function: H5F__vfd_swmr_construct_ud_cl + * + * Purpose: Encode updater change list in the buffer + * updater->change_list_image_ptr + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Vailin Choi -- 10/2021 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__vfd_swmr_construct_ud_cl(H5F_vfd_swmr_updater_t *updater) +{ + uint8_t *p = NULL; /* Pointer to buffer */ + uint8_t *image = (uint8_t *)updater->change_list_image_ptr; + uint32_t metadata_chksum; /* Computed metadata checksum value */ + unsigned i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* + * Encode ud cl + */ + p = image; + + /* Encode magic for ud cl */ + HDmemcpy(p, H5F_UD_CL_MAGIC, H5_SIZEOF_MAGIC); + p += H5_SIZEOF_MAGIC; + + /* Encode tick number */ + UINT64ENCODE(p, updater->tick_num); + + /* Encode Metadata File Header Updater File Page Offset*/ + UINT32ENCODE(p, updater->md_file_header_ud_file_page_offset); + + /* Encode Metadata File Header Length */ + UINT32ENCODE(p, updater->md_file_header_len); + + /* Calculate checksum on the image of the metadata file header */ + updater->md_file_header_image_chksum = + H5_checksum_metadata(updater->md_file_header_image_ptr, (size_t)updater->md_file_header_len, 0); + + /* Encode Metadata File Header Checksum */ + UINT32ENCODE(p, updater->md_file_header_image_chksum); + + /* Encode Metadata File Index Updater File Page Offset*/ + UINT32ENCODE(p, updater->md_file_index_ud_file_page_offset); + + /* Encode Metadata File Index Metadata File Offset */ + UINT64ENCODE(p, updater->md_file_index_md_file_offset); + + /* Encode Metadata File Index Length */ + UINT32ENCODE(p, updater->md_file_index_len); + + /* Calculate checksum on the image of the metadata file index */ + updater->md_file_index_image_chksum = + H5_checksum_metadata(updater->md_file_index_image_ptr, (size_t)updater->md_file_index_len, 0); + + /* Encode Metadata File Index Checksum */ + UINT32ENCODE(p, updater->md_file_index_image_chksum); + + UINT32ENCODE(p, updater->num_change_list_entries); + + /* Encode the ud cl entries */ + for (i = 0; i < updater->num_change_list_entries; i++) { + UINT32ENCODE(p, updater->change_list[i].entry_image_ud_file_page_offset); + UINT32ENCODE(p, updater->change_list[i].entry_image_md_file_page_offset); + UINT32ENCODE(p, updater->change_list[i].entry_image_h5_file_page_offset); + UINT32ENCODE(p, updater->change_list[i].entry_image_len); + UINT32ENCODE(p, updater->change_list[i].entry_image_checksum); + } + + /* Calculate checksum for ud cl */ + metadata_chksum = H5_checksum_metadata(image, (size_t)(p - image), 0); + + /* Encode checksum for index */ + UINT32ENCODE(p, metadata_chksum); + + /* Sanity checks on index */ + HDassert(p - image == (ptrdiff_t)updater->change_list_len); + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5F__vfd_swmr_construct_ud_cl() */ + +/*------------------------------------------------------------------------- + * + * Function: H5F_generate_updater_file() + * + * Purpose: Generate updater file: + * --assemble and initialize data in the updater struct + * --determine num_change_list entries + * --allocate buffers + * --construct on disk image (serialize) of the updater header and change list + * --create updater file using a temporay file name: + * --<shared->vfd_swmr_config.updater_file_path>.ud_tmp + * --allocate space and write the following to the updater file + * --updater file header + * --updater file change list + * --metadata entries + * --metadata file index + * --metadata file header + * --close the updater file + * --rename the updater file with the correct name: + * <shared->vfd_swmr_config.updater_file_path>.<shared->updater_seq_num> + * + * --increment shared->updater_seq_num + * --free buffers + * + * Return: SUCCEED/FAIL + * + * Programmer: Vailin Choi 8/24/2021 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__generate_updater_file(H5F_t *f, uint32_t num_entries, uint16_t flags, uint8_t *md_file_hdr_image_ptr, + size_t md_file_hdr_image_len, uint8_t *md_file_index_image_ptr, + uint64_t md_file_index_offset, size_t md_file_index_image_len) +{ + H5F_shared_t * shared = f->shared; /* shared file pointer */ + H5F_vfd_swmr_updater_t updater; /* Updater struct */ + uint32_t next_page_offset; + H5FD_t * ud_file = NULL; /* Low-level file struct */ + char namebuf[H5F__MAX_VFD_SWMR_FILE_NAME_LEN]; + char newname[H5F__MAX_VFD_SWMR_FILE_NAME_LEN]; + unsigned i; + hsize_t alloc_size; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Updater file header fields */ + updater.version = H5F_UD_VERSION; + updater.flags = flags; + updater.page_size = (uint32_t)shared->fs_page_size; + updater.sequence_number = shared->updater_seq_num; + updater.tick_num = shared->tick_num; + updater.header_image_ptr = NULL; + updater.header_image_len = H5F_UD_HEADER_SIZE; + updater.change_list_image_ptr = NULL; + updater.change_list_offset = 0; + updater.change_list_len = 0; + + /* Updater file change list fields */ + + /* md_file_header related fields */ + updater.md_file_header_ud_file_page_offset = 0; + updater.md_file_header_image_ptr = md_file_hdr_image_ptr; /* parameter */ + updater.md_file_header_len = md_file_hdr_image_len; /* parameter */ + + /* md_file_index related fields */ + updater.md_file_index_ud_file_page_offset = 0; + updater.md_file_index_image_ptr = md_file_index_image_ptr; /* parameter */ + updater.md_file_index_md_file_offset = md_file_index_offset; /* parameter */ + updater.md_file_index_len = md_file_index_image_len; /* parameter */ + + updater.num_change_list_entries = 0; + updater.change_list = NULL; + + /* Scan index to determine updater.num_change_list_entries */ + for (i = 0; i < num_entries; i++) { + if (shared->mdf_idx[i].entry_ptr != NULL && + shared->mdf_idx[i].tick_of_last_change == shared->tick_num) + updater.num_change_list_entries += 1; + } + + if (flags == CREATE_METADATA_FILE_ONLY_FLAG) + HDassert(updater.sequence_number == 0); + /* For file creation, just generate a header with this flag set */ + else { + /* Update 2 updater file header fields: change_list_len, change_list_offset */ + updater.change_list_len = H5F_UD_CL_SIZE(updater.num_change_list_entries); + updater.change_list_offset = updater.header_image_len; + } + + /* Create the updater file with a temporary file name */ + HDsprintf(namebuf, "%s.ud_tmp", shared->vfd_swmr_config.updater_file_path); + + if ((ud_file = H5FD_open(namebuf, H5F_ACC_TRUNC | H5F_ACC_RDWR | H5F_ACC_CREAT, H5P_FILE_ACCESS_DEFAULT, + HADDR_UNDEF)) == NULL) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "fail to open updater file"); + + if ((updater.header_image_ptr = HDmalloc(updater.header_image_len)) == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for ud header") + + /* Serialize updater file hdr in updater.header_image_ptr */ + if (H5F__vfd_swmr_construct_ud_hdr(&updater) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create updater file header "); + + /* Allocate space in updater file for updater file header */ + if (H5FD__alloc_real(ud_file, H5FD_MEM_DEFAULT, updater.header_image_len, NULL, NULL) == HADDR_UNDEF) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to allocate file memory") + + /* Write updater file header */ + if (H5FD_write(ud_file, H5FD_MEM_DEFAULT, H5F_UD_HEADER_OFF, updater.header_image_len, + updater.header_image_ptr) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "ud file write failed") + + if (flags != CREATE_METADATA_FILE_ONLY_FLAG) { + + next_page_offset = + ((uint32_t)(updater.header_image_len + updater.change_list_len) / updater.page_size) + 1; + + if (updater.num_change_list_entries) { + + /* Allocate space for change list entries */ + if ((updater.change_list = HDmalloc(sizeof(H5F_vfd_swmr_updater_cl_entry_t) * + updater.num_change_list_entries)) == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for ud cl") + + /* Initialize change list entries */ + for (i = 0; i < num_entries; i++) { + + if (shared->mdf_idx[i].entry_ptr != NULL && + shared->mdf_idx[i].tick_of_last_change == shared->tick_num) { + + updater.change_list[i].entry_image_ptr = shared->mdf_idx[i].entry_ptr; + updater.change_list[i].entry_image_ud_file_page_offset = 0; + updater.change_list[i].entry_image_md_file_page_offset = + (uint32_t)shared->mdf_idx[i].md_file_page_offset; + updater.change_list[i].entry_image_h5_file_page_offset = + (uint32_t)shared->mdf_idx[i].hdf5_page_offset; + updater.change_list[i].entry_image_len = shared->mdf_idx[i].length; + updater.change_list[i].entry_image_checksum = shared->mdf_idx[i].checksum; + + shared->mdf_idx[i].entry_ptr = NULL; + } + } + + /* Set up page aligned space for all metadata pages */ + for (i = 0; i < updater.num_change_list_entries; i++) { + updater.change_list[i].entry_image_ud_file_page_offset = next_page_offset; + next_page_offset += + (((uint32_t)updater.change_list[i].entry_image_len / updater.page_size) + 1); + } + } + + /* Set up page aligned space for the metadata file index */ + updater.md_file_index_ud_file_page_offset = next_page_offset; + + /* Set up page aligned space for the metadata file header */ + next_page_offset += (((uint32_t)updater.md_file_index_len / updater.page_size) + 1); + updater.md_file_header_ud_file_page_offset = next_page_offset; + + if ((updater.change_list_image_ptr = HDmalloc(updater.change_list_len)) == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for ud cl ") + + /* Serialize updater file change list in updater.change_list_image_ptr */ + if (H5F__vfd_swmr_construct_ud_cl(&updater) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, FAIL, "fail to create updater file cl"); + + /* Allocate space in updater file for updater file change list */ + if (H5FD__alloc_real(ud_file, H5FD_MEM_DEFAULT, updater.header_image_len + updater.change_list_len, + NULL, NULL) == HADDR_UNDEF) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to allocate file memory") + + /* Write updater file change list */ + if (H5FD_write(ud_file, H5FD_MEM_DEFAULT, updater.header_image_len, updater.change_list_len, + updater.change_list_image_ptr) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "ud file write failed") + + /* Allocate and write metadata pages */ + for (i = 0; i < updater.num_change_list_entries; i++) { + alloc_size = updater.change_list[i].entry_image_ud_file_page_offset * updater.page_size + + updater.change_list[i].entry_image_len; + + if (H5FD__alloc_real(ud_file, H5FD_MEM_DEFAULT, alloc_size, NULL, NULL) == HADDR_UNDEF) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to allocate file memory") + + if (H5FD_write(ud_file, H5FD_MEM_DEFAULT, + updater.change_list[i].entry_image_ud_file_page_offset * updater.page_size, + updater.change_list[i].entry_image_len, + updater.change_list[i].entry_image_ptr) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "ud file write failed") + } + + /* Allocate and write metadata file index */ + alloc_size = + updater.md_file_index_ud_file_page_offset * updater.page_size + updater.md_file_index_len; + if (H5FD__alloc_real(ud_file, H5FD_MEM_DEFAULT, alloc_size, NULL, NULL) == HADDR_UNDEF) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to allocate file memory") + + if (H5FD_write(ud_file, H5FD_MEM_DEFAULT, + updater.md_file_index_ud_file_page_offset * updater.page_size, + updater.md_file_index_len, updater.md_file_index_image_ptr) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "ud file write failed") + + /* Allocate and write metadata file header */ + alloc_size = + updater.md_file_header_ud_file_page_offset * updater.page_size + updater.md_file_header_len; + if (H5FD__alloc_real(ud_file, H5FD_MEM_DEFAULT, alloc_size, NULL, NULL) == HADDR_UNDEF) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to allocate file memory") + + if (H5FD_write(ud_file, H5FD_MEM_DEFAULT, + updater.md_file_header_ud_file_page_offset * updater.page_size, + updater.md_file_header_len, updater.md_file_header_image_ptr) < 0) + HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "ud file write failed") + } + + /* Close the updater file and rename the file */ + if (H5FD_close(ud_file) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "unable to close updater file") + HDsprintf(newname, "%s.%lu", shared->vfd_swmr_config.updater_file_path, shared->updater_seq_num); + HDrename(namebuf, newname); + + ++shared->updater_seq_num; + +done: + if (updater.header_image_ptr) + HDfree(updater.header_image_ptr); + if (updater.change_list_image_ptr) + HDfree(updater.change_list_image_ptr); + if (updater.change_list) + HDfree(updater.change_list); + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5F__generate_updater_file() */ @@ -117,8 +117,14 @@ H5HF__op_write(const void *obj, size_t obj_len, void *op_data) { FUNC_ENTER_PACKAGE_NOERR - /* Perform "write", using memcpy() */ - H5MM_memcpy((void *)obj, op_data, obj_len); /* Casting away const OK -QAK */ + /* Perform "write", using memcpy() + * + * We cast away const here because no obj pointer that was originally + * const should ever arrive here. + */ + H5_GCC_CLANG_DIAG_OFF("cast-qual") + H5MM_memcpy((void *)obj, op_data, obj_len); + H5_GCC_CLANG_DIAG_ON("cast-qual") FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HF__op_write() */ @@ -350,10 +356,15 @@ H5HF_insert(H5HF_t *fh, size_t size, const void *obj, void *id /*out*/) /* Check for 'huge' object */ if (size > hdr->max_man_size) { - /* Store 'huge' object in heap */ - /* (Casting away const OK - QAK) */ + /* Store 'huge' object in heap + * + * Although not ideal, we can quiet the const warning here becuase no + * obj pointer that was originally const should ever arrive here. + */ + H5_GCC_CLANG_DIAG_OFF("cast-qual") if (H5HF__huge_insert(hdr, size, (void *)obj, id) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTINSERT, FAIL, "can't store 'huge' object in fractal heap") + H5_GCC_CLANG_DIAG_ON("cast-qual") } /* end if */ /* Check for 'tiny' object */ else if (size <= hdr->tiny_max_len) { diff --git a/src/H5HFcache.c b/src/H5HFcache.c index ea5e730..654fa36 100644 --- a/src/H5HFcache.c +++ b/src/H5HFcache.c @@ -1658,9 +1658,15 @@ H5HF__cache_dblock_verify_chksum(const void *_image, size_t len, void *_udata) /* Update info about direct block */ udata->decompressed = TRUE; len = nbytes; - } /* end if */ - else - read_buf = (void *)image; /* Casting away const OK - QAK */ + } + else { + /* If the data are unfiltered, we just point to the image, which we + * never modify. Casting away const is okay here. + */ + H5_GCC_CLANG_DIAG_OFF("cast-qual") + read_buf = (void *)image; + H5_GCC_CLANG_DIAG_OFF("cast-qual") + } /* Decode checksum */ chk_size = (size_t)(H5HF_MAN_ABS_DIRECT_OVERHEAD(hdr) - H5HF_SIZEOF_CHKSUM); diff --git a/src/H5HFman.c b/src/H5HFman.c index 427be00..a362d99 100644 --- a/src/H5HFman.c +++ b/src/H5HFman.c @@ -487,10 +487,16 @@ H5HF__man_write(H5HF_hdr_t *hdr, const uint8_t *id, const void *obj) HDassert(id); HDassert(obj); - /* Call the internal 'op' routine routine */ - /* (Casting away const OK - QAK) */ + /* Call the internal 'op' routine routine + * + * In this case, the callback operation needs to modify the obj buffer that + * was passed in as const. We quiet the warning here because an obj pointer + * that was originally const should *never* arrive here. + */ + H5_GCC_CLANG_DIAG_OFF("cast-qual") if (H5HF__man_op_real(hdr, id, H5HF__op_write, (void *)obj, H5HF_OP_MODIFY) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTOPERATE, FAIL, "unable to operate on heap object") + H5_GCC_CLANG_DIAG_ON("cast-qual") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5HGprivate.h b/src/H5HGprivate.h index d8e6b46..5deae88 100644 --- a/src/H5HGprivate.h +++ b/src/H5HGprivate.h @@ -50,7 +50,7 @@ typedef struct H5HG_heap_t H5HG_heap_t; /* Size of encoded global heap ID */ /* (size of file address + 32-bit integer) */ -#define H5HG_HEAP_ID_SIZE(F) ((size_t)H5F_SIZEOF_ADDR(F) + H5_SIZEOF_UINT32_T) +#define H5HG_HEAP_ID_SIZE(F) ((size_t)H5F_SIZEOF_ADDR(F) + sizeof(uint32_t)) /* Main global heap routines */ H5_DLL herr_t H5HG_insert(H5F_t *f, size_t size, const void *obj, H5HG_t *hobj /*out*/); @@ -533,7 +533,7 @@ H5M__open_api_common(hid_t loc_id, const char *name, hid_t mapl_id, void **token /* Open the map */ if (H5VL_optional(*vol_obj_ptr, &vol_cb_args, H5P_DATASET_XFER_DEFAULT, token_ptr) < 0) HGOTO_ERROR(H5E_MAP, H5E_CANTOPENOBJ, H5I_INVALID_HID, "unable to open map") - map = map_args.create.map; + map = map_args.open.map; /* Register an ID for the map */ if ((ret_value = H5VL_register(H5I_MAP, map, (*vol_obj_ptr)->connector, TRUE)) < 0) @@ -1360,6 +1360,7 @@ H5Miterate(hid_t map_id, hsize_t *idx, hid_t key_mem_type_id, H5M_iterate_t op, map_args.specific.args.iterate.loc_params.type = H5VL_OBJECT_BY_SELF; map_args.specific.args.iterate.loc_params.obj_type = H5I_get_type(map_id); map_args.specific.args.iterate.idx = (idx ? *idx : 0); + map_args.specific.args.iterate.key_mem_type_id = key_mem_type_id; map_args.specific.args.iterate.op = op; map_args.specific.args.iterate.op_data = op_data; vol_cb_args.op_type = H5VL_MAP_SPECIFIC; @@ -1450,6 +1451,7 @@ H5Miterate_by_name(hid_t loc_id, const char *map_name, hsize_t *idx, hid_t key_m map_args.specific.args.iterate.loc_params.loc_data.loc_by_name.name = map_name; map_args.specific.args.iterate.loc_params.loc_data.loc_by_name.lapl_id = lapl_id; map_args.specific.args.iterate.idx = (idx ? *idx : 0); + map_args.specific.args.iterate.key_mem_type_id = key_mem_type_id; map_args.specific.args.iterate.op = op; map_args.specific.args.iterate.op_data = op_data; vol_cb_args.op_type = H5VL_MAP_SPECIFIC; @@ -2121,7 +2121,7 @@ H5PB_vfd_swmr__update_index(H5F_t *f, uint32_t *idx_ent_added_ptr, uint32_t *idx /* partial initialization of new entry -- rest done later */ ie_ptr->hdf5_page_offset = target_page; ie_ptr->md_file_page_offset = 0; /* undefined at this point */ - ie_ptr->chksum = 0; /* undefined at this point */ + ie_ptr->checksum = 0; /* undefined at this point */ /* ie_ptr->entry_ptr initialized below */ /* ie_ptr->tick_of_last_change initialized below */ /* ie_ptr->clean initialized below */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 3b39f12..ff9ff8a 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -74,7 +74,7 @@ #define H5F_ACS_DATA_CACHE_BYTE_SIZE_DEC H5P__decode_size_t /* Definition for preemption read chunks first */ #define H5F_ACS_PREEMPT_READ_CHUNKS_SIZE sizeof(double) -#define H5F_ACS_PREEMPT_READ_CHUNKS_DEF 0.75f +#define H5F_ACS_PREEMPT_READ_CHUNKS_DEF 0.75 #define H5F_ACS_PREEMPT_READ_CHUNKS_ENC H5P__encode_double #define H5F_ACS_PREEMPT_READ_CHUNKS_DEC H5P__decode_double /* Definition for threshold for alignment */ @@ -318,6 +318,16 @@ #define H5F_ACS_VFD_SWMR_CONFIG_ENC H5P__facc_vfd_swmr_config_enc #define H5F_ACS_VFD_SWMR_CONFIG_DEC H5P__facc_vfd_swmr_config_dec +/* Private property for VFD SWMR testing: + * Callback function to generate checksum for metadata file + */ +#define H5F_ACS_GENERATE_MD_CK_CB_SIZE sizeof(H5F_generate_md_ck_cb_t) + +#define H5F_ACS_GENERATE_MD_CK_CB_DEF \ + { \ + NULL \ + } + /******************/ /* Local Typedefs */ /******************/ @@ -515,6 +525,8 @@ static const hbool_t H5F_def_ignore_disabled_file_locks_g = H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF; /* Default ignore disabled file locks flag */ static const H5F_vfd_swmr_config_t H5F_def_vfd_swmr_config_g = H5F_ACS_VFD_SWMR_CONFIG_DEF; /* Default vfd swmr configuration */ +/* For VFD SWMR testing only: Default to generate checksum for metadata file */ +static const H5F_generate_md_ck_t H5F_def_generate_md_ck_cb_g = H5F_ACS_GENERATE_MD_CK_CB_DEF; /*------------------------------------------------------------------------- * Function: H5P__facc_reg_prop @@ -633,6 +645,14 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the private property of whether to generate checksum for metadata file. + * It's used for VFD SWMR testing only. */ + /* (Note: this property should not have an encode/decode callback -QAK) */ + if (H5P__register_real(pclass, H5F_ACS_GENERATE_MD_CK_CB_NAME, H5F_ACS_GENERATE_MD_CK_CB_SIZE, + &H5F_def_generate_md_ck_cb_g, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the data type of multi driver info */ if (H5P__register_real(pclass, H5F_ACS_MULTI_TYPE_NAME, H5F_ACS_MULTI_TYPE_SIZE, &H5F_def_mem_type_g, NULL, NULL, NULL, H5F_ACS_MULTI_TYPE_ENC, H5F_ACS_MULTI_TYPE_DEC, NULL, NULL, NULL, @@ -4097,11 +4117,16 @@ H5P__facc_vfd_swmr_config_enc(const void *value, void **_pp, size_t *size) INT32ENCODE(*pp, (int32_t)config->tick_len); INT32ENCODE(*pp, (int32_t)config->max_lag); H5_ENCODE_UNSIGNED(*pp, config->writer); + H5_ENCODE_UNSIGNED(*pp, config->maintain_metadata_file); + H5_ENCODE_UNSIGNED(*pp, config->generate_updater_files); H5_ENCODE_UNSIGNED(*pp, config->flush_raw_data); INT32ENCODE(*pp, (int32_t)config->md_pages_reserved); INT32ENCODE(*pp, (int32_t)config->pb_expansion_threshold); HDmemcpy(*pp, (const uint8_t *)(config->md_file_path), (size_t)(H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1)); *pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1; + HDmemcpy(*pp, (const uint8_t *)(config->updater_file_path), + (size_t)(H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1)); + *pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1; HDmemcpy(*pp, (const uint8_t *)(config->log_file_path), (size_t)(H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1)); *pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1; @@ -4109,7 +4134,7 @@ H5P__facc_vfd_swmr_config_enc(const void *value, void **_pp, size_t *size) } /* end if */ /* Compute encoded size */ - *size += ((5 * sizeof(int32_t)) + (2 * sizeof(unsigned)) + (2 * (H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1))); + *size += ((5 * sizeof(int32_t)) + (4 * sizeof(unsigned)) + (3 * (H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1))); FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5P__facc_vfd_swmr_config_enc() */ @@ -4150,6 +4175,8 @@ H5P__facc_vfd_swmr_config_dec(const void **_pp, void *_value) UINT32DECODE(*pp, config->max_lag); H5_DECODE_UNSIGNED(*pp, config->writer); + H5_DECODE_UNSIGNED(*pp, config->maintain_metadata_file); + H5_DECODE_UNSIGNED(*pp, config->generate_updater_files); H5_DECODE_UNSIGNED(*pp, config->flush_raw_data); /* int */ @@ -4159,6 +4186,9 @@ H5P__facc_vfd_swmr_config_dec(const void **_pp, void *_value) HDstrcpy(config->md_file_path, (const char *)(*pp)); *pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1; + HDstrcpy(config->updater_file_path, (const char *)(*pp)); + *pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1; + HDstrcpy(config->log_file_path, (const char *)(*pp)); *pp += H5F__MAX_VFD_SWMR_FILE_NAME_LEN + 1; @@ -4917,8 +4947,7 @@ H5Pset_all_coll_metadata_ops(hid_t plist_id, hbool_t is_collective) /* (Dataset, group, attribute, and named datype access property lists * are sub-classes of link access property lists -QAK) */ - if (TRUE != H5P_isa_class(plist_id, H5P_LINK_ACCESS) && - TRUE != H5P_isa_class(plist_id, H5P_FILE_ACCESS) && TRUE != H5P_isa_class(plist_id, H5P_DATASET_XFER)) + if (TRUE != H5P_isa_class(plist_id, H5P_LINK_ACCESS) && TRUE != H5P_isa_class(plist_id, H5P_FILE_ACCESS)) HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not an access plist") /* set property to either TRUE if > 0, or FALSE otherwise */ @@ -4969,8 +4998,7 @@ H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collective /*out*/) /* (Dataset, group, attribute, and named datype access property lists * are sub-classes of link access property lists -QAK) */ - if (TRUE != H5P_isa_class(plist_id, H5P_LINK_ACCESS) && - TRUE != H5P_isa_class(plist_id, H5P_FILE_ACCESS) && TRUE != H5P_isa_class(plist_id, H5P_DATASET_XFER)) + if (TRUE != H5P_isa_class(plist_id, H5P_LINK_ACCESS) && TRUE != H5P_isa_class(plist_id, H5P_FILE_ACCESS)) HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not an access plist") /* Get value */ @@ -5703,12 +5731,30 @@ H5Pset_vfd_swmr_config(hid_t plist_id, H5F_vfd_swmr_config_t *config_ptr) if (config_ptr->pb_expansion_threshold > H5F__MAX_PB_EXPANSION_THRESHOLD) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "pb_expansion_threshold out of range") - /* Must provide the path for the metadata file */ - name_len = HDstrlen(config_ptr->md_file_path); - if (name_len == 0) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "md_file_path is empty") - else if (name_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN) - HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "md_file_path is too long") + /* If writer is TRUE, at least one of maintain_metadata_file and generate_updater_files must be TRUE */ + if (config_ptr->writer) { + if (!config_ptr->maintain_metadata_file && !config_ptr->generate_updater_files) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, + "either maintain_metadata_file or generate_updater_files must be TRUE") + } + + if ((config_ptr->writer && config_ptr->maintain_metadata_file) || !config_ptr->writer) { + /* Must provide the path and base name of the metadata file */ + name_len = HDstrlen(config_ptr->md_file_path); + if (name_len == 0) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "md_file_path is empty") + else if (name_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "md_file_path is too long") + } + + if (config_ptr->writer && config_ptr->generate_updater_files) { + /* Must provide the path and base name of the metadata updater files */ + name_len = HDstrlen(config_ptr->updater_file_path); + if (name_len == 0) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "updater_file_path is empty") + else if (name_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "updater_file_path is too long") + } name_len = HDstrlen(config_ptr->log_file_path); if (name_len > H5F__MAX_VFD_SWMR_FILE_NAME_LEN) diff --git a/src/H5Pocpl.c b/src/H5Pocpl.c index edb0cca..e442030 100644 --- a/src/H5Pocpl.c +++ b/src/H5Pocpl.c @@ -940,6 +940,8 @@ H5Pget_filter_by_id2(hid_t plist_id, H5Z_filter_t id, unsigned int *flags /*out* H5TRACE8("e", "iZfx*zxzxx", plist_id, id, flags, cd_nelmts, cd_values, namelen, name, filter_config); /* Check args */ + if (id < 0 || id > H5Z_FILTER_MAX) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "filter ID value out of range") if (cd_nelmts || cd_values) { /* * It's likely that users forget to initialize this on input, so @@ -1838,6 +1840,8 @@ H5Pget_filter_by_id1(hid_t plist_id, H5Z_filter_t id, unsigned int *flags /*out* H5TRACE7("e", "iZfx*zxzx", plist_id, id, flags, cd_nelmts, cd_values, namelen, name); /* Check args */ + if (id < 0 || id > H5Z_FILTER_MAX) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "filter ID value out of range") if (cd_nelmts || cd_values) { /* * It's likely that users forget to initialize this on input, so diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index cba7e03..ed61646 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -6402,10 +6402,10 @@ H5_DLL herr_t H5Pset_layout(hid_t plist_id, H5D_layout_t layout); * <td>byte 0</td> * </tr> * <tr> - * <td>????????</td> - * <td>????SPPP</td> - * <td>PPPPPPPP</td> - * <td>PPPP????</td> + * <td> ???????? </td> + * <td> ????SPPP </td> + * <td> PPPPPPPP </td> + * <td> PPPP???? </td> * </tr> * </table> * Note: S - sign bit, P - significant bit, ? - padding bit; For @@ -316,8 +316,16 @@ H5RS_wrap(const char *s) if (NULL == (ret_value = H5FL_MALLOC(H5RS_str_t))) HGOTO_ERROR(H5E_RS, H5E_CANTALLOC, NULL, "memory allocation failed") - /* Set the internal fields */ - ret_value->s = (char *)s; + /* Set the internal fields + * + * We ignore warnings about storing a const char pointer in the struct + * since we never modify or free the string when the wrapped struct + * field is set to TRUE. + */ + H5_GCC_CLANG_DIAG_OFF("cast-qual") + ret_value->s = (char *)s; + H5_GCC_CLANG_DIAG_ON("cast-qual") + ret_value->len = HDstrlen(s); ret_value->end = ret_value->s + ret_value->len; diff --git a/src/H5Rint.c b/src/H5Rint.c index e1a5dcd..7ee4ecc 100644 --- a/src/H5Rint.c +++ b/src/H5Rint.c @@ -1150,7 +1150,7 @@ H5R__encode_obj_token(const H5O_token_t *obj_token, size_t token_size, unsigned /* Encode token */ H5MM_memcpy(p, obj_token, token_size); } - *nalloc = token_size + H5_SIZEOF_UINT8_T; + *nalloc = token_size + sizeof(uint8_t); FUNC_LEAVE_NOAPI(ret_value) } /* end H5R__encode_obj_token() */ @@ -1178,7 +1178,7 @@ H5R__decode_obj_token(const unsigned char *buf, size_t *nbytes, H5O_token_t *obj HDassert(token_size); /* Don't decode if buffer size isn't big enough */ - if (*nbytes < H5_SIZEOF_UINT8_T) + if (*nbytes < sizeof(uint8_t)) HGOTO_ERROR(H5E_REFERENCE, H5E_CANTDECODE, FAIL, "Buffer size is too small") /* Get token size */ @@ -1192,7 +1192,7 @@ H5R__decode_obj_token(const unsigned char *buf, size_t *nbytes, H5O_token_t *obj /* Decode token */ H5MM_memcpy(obj_token, p, *token_size); - *nbytes = (size_t)(*token_size + H5_SIZEOF_UINT8_T); + *nbytes = (size_t)(*token_size + sizeof(uint8_t)); done: FUNC_LEAVE_NOAPI(ret_value) @@ -1225,7 +1225,7 @@ H5R__encode_region(H5S_t *space, unsigned char *buf, size_t *nalloc) "Cannot determine amount of space needed for serializing selection") /* Don't encode if buffer size isn't big enough or buffer is empty */ - if (buf && *nalloc >= ((size_t)buf_size + 2 * H5_SIZEOF_UINT32_T)) { + if (buf && *nalloc >= ((size_t)buf_size + 2 * sizeof(uint32_t))) { int rank; p = (uint8_t *)buf; @@ -1241,7 +1241,7 @@ H5R__encode_region(H5S_t *space, unsigned char *buf, size_t *nalloc) if (H5S_SELECT_SERIALIZE(space, (unsigned char **)&p) < 0) HGOTO_ERROR(H5E_REFERENCE, H5E_CANTENCODE, FAIL, "can't serialize selection") } /* end if */ - *nalloc = (size_t)buf_size + 2 * H5_SIZEOF_UINT32_T; + *nalloc = (size_t)buf_size + 2 * sizeof(uint32_t); done: FUNC_LEAVE_NOAPI(ret_value) @@ -1272,16 +1272,16 @@ H5R__decode_region(const unsigned char *buf, size_t *nbytes, H5S_t **space_ptr) HDassert(space_ptr); /* Don't decode if buffer size isn't big enough */ - if (*nbytes < (2 * H5_SIZEOF_UINT32_T)) + if (*nbytes < (2 * sizeof(uint32_t))) HGOTO_ERROR(H5E_REFERENCE, H5E_CANTDECODE, FAIL, "Buffer size is too small") /* Decode the selection size */ UINT32DECODE(p, buf_size); - buf_size += H5_SIZEOF_UINT32_T; + buf_size += sizeof(uint32_t); /* Decode the extent rank */ UINT32DECODE(p, rank); - buf_size += H5_SIZEOF_UINT32_T; + buf_size += sizeof(uint32_t); /* Don't decode if buffer size isn't big enough */ if (*nbytes < buf_size) diff --git a/src/H5Rpkg.h b/src/H5Rpkg.h index ee5fb71..09144f9 100644 --- a/src/H5Rpkg.h +++ b/src/H5Rpkg.h @@ -43,7 +43,7 @@ #define H5R_REF_ATTRNAME(x) ((x)->info.attr.name) /* Header size */ -#define H5R_ENCODE_HEADER_SIZE (2 * H5_SIZEOF_UINT8_T) +#define H5R_ENCODE_HEADER_SIZE (2 * sizeof(uint8_t)) /****************************/ /* Package Private Typedefs */ @@ -36,13 +36,6 @@ * skip list. The implementation in that document hurts * performance, at least for integer keys. -NAF) * - * (Also, this implementation has a couple of home-grown - * optimizations, including setting the "update" vector to the - * actual 'forward' pointer to update, instead of the node - * containing the forward pointer -QAK - * -No longer uses update vector, as insertions/deletions are now - * always at level 0. -NAF) - * * (Note: This implementation does not have the information for * implementing the "Linear List Operations" (like insert/delete/ * search by position) in section 3.4 of "A Skip List Cookbook", @@ -71,25 +64,14 @@ /* Define the code template for searches for the "OP" in the H5SL_LOCATE macro */ #define H5SL_LOCATE_SEARCH_FOUND(SLIST, X, I) \ { \ - HDassert(!X->removed); \ - HGOTO_DONE(X->item); \ - } /* end block */ - -/* Define the code template for deferred removals for the "OP" in the - * H5SL_LOCATE macro */ -#define H5SL_LOCATE_SEARCH_DEFER_REMOVE_FOUND(SLIST, X, I) \ - { \ - HDassert(!X->removed); \ - X->removed = TRUE; \ HGOTO_DONE(X->item); \ - } /* end block */ + } /* Define the code template for finds for the "OP" in the H5SL_LOCATE macro */ #define H5SL_LOCATE_FIND_FOUND(SLIST, X, I) \ { \ - HDassert(!X->removed); \ HGOTO_DONE(X); \ - } /* end block */ + } /* Define a code template for comparing scalar keys for the "CMP" in the H5SL_LOCATE macro */ #define H5SL_LOCATE_SCALAR_CMP(SLIST, TYPE, PNODE, PKEY, HASHVAL) (*(TYPE *)((PNODE)->key) < *(TYPE *)PKEY) @@ -155,51 +137,19 @@ H5_GLUE3(H5SL_LOCATE_, CMP, _CMP)(SLIST, TYPE, X->forward[_i], KEY, HASHVAL)) { \ X = X->forward[_i]; \ _count++; \ - } /* end while */ \ - } /* end for */ \ + } \ + } \ X = X->forward[0]; \ if (X != NULL && H5_GLUE3(H5SL_LOCATE_, CMP, _EQ)(SLIST, TYPE, X, KEY, HASHVAL)) { \ /* What to do when a node is found */ \ H5_GLUE3(H5SL_LOCATE_, OP, _FOUND)(SLIST, X, _i) \ - } /* end if */ \ - } - -/* Macro used to find node for operation, if there may be "removed" nodes in the - * list (whose keys cannot be read) */ -#define H5SL_LOCATE_SAFE(OP, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ - { \ - int _i; /* Local index variable */ \ - H5SL_node_t *_low = X; \ - H5SL_node_t *_high = NULL; \ - \ - H5_GLUE3(H5SL_LOCATE_, CMP, _HASHINIT) \ - (KEY, HASHVAL) for (_i = (int)SLIST->curr_level; _i >= 0; _i--) \ - { \ - X = _low->forward[_i]; \ - while (X != _high) { \ - if (!X->removed) { \ - if (H5_GLUE3(H5SL_LOCATE_, CMP, _CMP)(SLIST, TYPE, X, KEY, HASHVAL)) \ - _low = X; \ - else \ - break; \ - } /* end if */ \ - X = X->forward[_i]; \ - } /* end while */ \ - _high = X; \ - if (X != NULL && H5_GLUE3(H5SL_LOCATE_, CMP, _EQ)(SLIST, TYPE, X, KEY, HASHVAL)) { \ - /* What to do when a node is found */ \ - H5_GLUE3(H5SL_LOCATE_, OP, _FOUND)(SLIST, X, _i) break; \ - } /* end if */ \ - } /* end for */ \ + } \ } /* Macro used to find node for operation */ #define H5SL_LOCATE(OP, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ { \ - if ((SLIST)->safe_iterating) \ - H5SL_LOCATE_SAFE(OP, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ - else \ - H5SL_LOCATE_OPT(OP, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ + H5SL_LOCATE_OPT(OP, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ } /* Macro used to grow a node by 1. Does not update pointers. LVL is the current @@ -225,13 +175,13 @@ if (NULL == (H5SL_fac_g = (H5FL_fac_head_t **)H5MM_realloc( \ (void *)H5SL_fac_g, H5SL_fac_nalloc_g * sizeof(H5FL_fac_head_t *)))) \ HGOTO_ERROR(H5E_SLIST, H5E_CANTALLOC, ERR, "memory allocation failed") \ - } /* end if */ \ + } \ \ /* Create the new factory */ \ H5SL_fac_g[H5SL_fac_nused_g] = \ H5FL_fac_init((1u << H5SL_fac_nused_g) * sizeof(H5SL_node_t *)); \ H5SL_fac_nused_g++; \ - } /* end if */ \ + } \ \ /* Allocate space for new forward pointers */ \ if (NULL == (_tmp = (H5SL_node_t **)H5FL_FAC_MALLOC(H5SL_fac_g[X->log_nalloc]))) \ @@ -239,7 +189,7 @@ H5MM_memcpy((void *)_tmp, (const void *)X->forward, (LVL + 1) * sizeof(H5SL_node_t *)); \ X->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[X->log_nalloc - 1], (void *)X->forward); \ X->forward = _tmp; \ - } /* end if */ \ + } \ \ X->level++; \ } @@ -260,7 +210,7 @@ H5MM_memcpy((void *)_tmp, (const void *)X->forward, (LVL) * sizeof(H5SL_node_t *)); \ X->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[X->log_nalloc + 1], (void *)X->forward); \ X->forward = _tmp; \ - } /* end if */ \ + } \ \ X->level--; \ } @@ -284,7 +234,7 @@ else { \ HDassert(_lvl < (size_t)SLIST->curr_level); \ X->forward[_lvl + 1] = PREV->forward[_lvl + 1]; \ - } /* end else */ \ + } \ PREV->forward[_lvl + 1] = X; \ } @@ -322,7 +272,7 @@ if (!_drop) \ _drop = X; \ break; \ - } /* end if */ \ + } \ \ /* Check if this node is the start of the next gap */ \ if (!_drop && !H5_GLUE3(H5SL_LOCATE_, CMP, _CMP)(SLIST, TYPE, X->forward[_i], KEY, HASHVAL)) \ @@ -337,7 +287,7 @@ break; \ } \ X = X->forward[_i]; \ - } /* end for */ \ + } \ HDassert(!_drop->forward[_i] || \ !H5_GLUE3(H5SL_LOCATE_, CMP, _CMP)(SLIST, TYPE, _drop->forward[_i], KEY, HASHVAL)); \ \ @@ -350,7 +300,7 @@ /* Prepare to drop down */ \ X = _last = _drop; \ _next = _drop->forward[_i]; \ - } /* end for */ \ + } \ \ if (_next && H5_GLUE3(H5SL_LOCATE_, CMP, _EQ)(SLIST, TYPE, _next, KEY, HASHVAL)) \ HGOTO_ERROR(H5E_SLIST, H5E_CANTINSERT, NULL, "can't insert duplicate key") \ @@ -359,172 +309,167 @@ /* Macro used to remove node */ #define H5SL_REMOVE(CMP, SLIST, X, TYPE, KEY, HASHVAL) \ { \ - /* Check for deferred removal */ \ - if (SLIST->safe_iterating) \ - H5SL_LOCATE(SEARCH_DEFER_REMOVE, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ - else { \ - H5SL_node_t *_last = X; /* Lowest node in the current gap */ \ - H5SL_node_t *_llast = X; /* Lowest node in the previous gap */ \ - H5SL_node_t *_next = NULL; /* Highest node in the currect gap */ \ - H5SL_node_t *_drop = NULL; /* Low node of the gap to drop into */ \ - H5SL_node_t *_ldrop = NULL; /* Low node of gap before the one to drop into */ \ - H5SL_node_t *_head = SLIST->header; /* Head of the skip list */ \ - int _count; /* Number of nodes in the current gap */ \ - int _i = (int)SLIST->curr_level; \ + H5SL_node_t *_last = X; /* Lowest node in the current gap */ \ + H5SL_node_t *_llast = X; /* Lowest node in the previous gap */ \ + H5SL_node_t *_next = NULL; /* Highest node in the currect gap */ \ + H5SL_node_t *_drop = NULL; /* Low node of the gap to drop into */ \ + H5SL_node_t *_ldrop = NULL; /* Low node of gap before the one to drop into */ \ + H5SL_node_t *_head = SLIST->header; /* Head of the skip list */ \ + int _count; /* Number of nodes in the current gap */ \ + int _i = (int)SLIST->curr_level; \ \ - if (_i < 0) \ - HGOTO_DONE(NULL); \ + if (_i < 0) \ + HGOTO_DONE(NULL); \ \ - H5_GLUE3(H5SL_LOCATE_, CMP, _HASHINIT) \ - (KEY, HASHVAL) \ + H5_GLUE3(H5SL_LOCATE_, CMP, _HASHINIT) \ + (KEY, HASHVAL) \ \ - /* Find the gap to drop in to at the highest level */ \ - while (X && (!X->key || H5_GLUE3(H5SL_LOCATE_, CMP, _CMP)(SLIST, TYPE, X, KEY, HASHVAL))) \ - { \ - _llast = _last; \ - _last = X; \ - X = X->forward[_i]; \ - } \ - _next = X; \ + /* Find the gap to drop in to at the highest level */ \ + while (X && (!X->key || H5_GLUE3(H5SL_LOCATE_, CMP, _CMP)(SLIST, TYPE, X, KEY, HASHVAL))) \ + { \ + _llast = _last; \ + _last = X; \ + X = X->forward[_i]; \ + } \ + _next = X; \ \ - /* Main loop */ \ - for (_i--; _i >= 0; _i--) { \ - /* Search for the node to drop into, also count the number of */ \ - /* nodes of height _i in this gap and keep track of of the node */ \ - /* before the one to drop into (_ldrop will become _llast, */ \ - /* _drop will become _last). */ \ - X = _ldrop = _last; \ - _drop = NULL; \ - for (_count = 0;; _count++) { \ - /* Terminate if this is the last node in the gap */ \ - if (X->forward[_i] == _next) { \ - if (!_drop) \ - _drop = X; \ - break; \ - } /* end if */ \ + /* Main loop */ \ + for (_i--; _i >= 0; _i--) { \ + /* Search for the node to drop into, also count the number of */ \ + /* nodes of height _i in this gap and keep track of of the node */ \ + /* before the one to drop into (_ldrop will become _llast, */ \ + /* _drop will become _last). */ \ + X = _ldrop = _last; \ + _drop = NULL; \ + for (_count = 0;; _count++) { \ + /* Terminate if this is the last node in the gap */ \ + if (X->forward[_i] == _next) { \ + if (!_drop) \ + _drop = X; \ + break; \ + } \ \ - /* If we have already found the node to drop into and there */ \ - /* is more than one node in this gap, we can stop searching */ \ - if (_drop) { \ - HDassert(_count >= 1); \ - _count = 2; \ - break; \ + /* If we have already found the node to drop into and there */ \ + /* is more than one node in this gap, we can stop searching */ \ + if (_drop) { \ + HDassert(_count >= 1); \ + _count = 2; \ + break; \ + } \ + else { /* !_drop */ \ + /* Check if this node is the start of the next gap */ \ + if (!H5_GLUE3(H5SL_LOCATE_, CMP, _CMP)(SLIST, TYPE, X->forward[_i], KEY, HASHVAL)) { \ + _drop = X; \ + /* Again check if we can stop searching */ \ + if (_count) { \ + _count = 2; \ + break; \ + } \ } \ - else { /* !_drop */ \ - /* Check if this node is the start of the next gap */ \ - if (!H5_GLUE3(H5SL_LOCATE_, CMP, _CMP)(SLIST, TYPE, X->forward[_i], KEY, HASHVAL)) { \ - _drop = X; \ - /* Again check if we can stop searching */ \ - if (_count) { \ - _count = 2; \ - break; \ - } /* end if */ \ - } /* end if */ \ - else \ - _ldrop = X; \ - } /* end else */ \ + else \ + _ldrop = X; \ + } \ \ - /* No need to check the last node in the gap if there are */ \ - /* 3, as there cannot be a fourth */ \ - if (_count == 2) { \ - if (!_drop) \ - _drop = X->forward[_i]; \ - break; \ - } /* end if */ \ - X = X->forward[_i]; \ - } /* end for */ \ - HDassert(_count >= 1 && _count <= 3); \ - HDassert(!_drop->forward[_i] || \ - !H5_GLUE3(H5SL_LOCATE_, CMP, _CMP)(SLIST, TYPE, _drop->forward[_i], KEY, HASHVAL)); \ + /* No need to check the last node in the gap if there are */ \ + /* 3, as there cannot be a fourth */ \ + if (_count == 2) { \ + if (!_drop) \ + _drop = X->forward[_i]; \ + break; \ + } \ + X = X->forward[_i]; \ + } \ + HDassert(_count >= 1 && _count <= 3); \ + HDassert(!_drop->forward[_i] || \ + !H5_GLUE3(H5SL_LOCATE_, CMP, _CMP)(SLIST, TYPE, _drop->forward[_i], KEY, HASHVAL)); \ \ - /* Check if we need to adjust node heights */ \ - if (_count == 1) { \ - /* Check if we are in the first gap */ \ - if (_llast == _last) { \ - /* We are in the first gap, count the number of nodes */ \ - /* of height _i in the next gap. We need only check */ \ - /* onenode to see if we should promote the first node */ \ - /* in the next gap */ \ - _llast = _next->forward[_i + 1]; \ + /* Check if we need to adjust node heights */ \ + if (_count == 1) { \ + /* Check if we are in the first gap */ \ + if (_llast == _last) { \ + /* We are in the first gap, count the number of nodes */ \ + /* of height _i in the next gap. We need only check */ \ + /* onenode to see if we should promote the first node */ \ + /* in the next gap */ \ + _llast = _next->forward[_i + 1]; \ \ - /* Demote the separator node */ \ - H5SL_DEMOTE(_next, _last) \ + /* Demote the separator node */ \ + H5SL_DEMOTE(_next, _last) \ \ - /* If there are 2 or more nodes, promote the first */ \ - if (_next->forward[_i]->forward[_i] != _llast) { \ - X = _next->forward[_i]; \ - H5SL_PROMOTE(SLIST, X, _last, NULL) \ - } \ - else if (!_head->forward[_i + 1]) { \ - /* shrink the header */ \ - HDassert(_i == SLIST->curr_level - 1); \ - HDassert((size_t)SLIST->curr_level == _head->level); \ + /* If there are 2 or more nodes, promote the first */ \ + if (_next->forward[_i]->forward[_i] != _llast) { \ + X = _next->forward[_i]; \ + H5SL_PROMOTE(SLIST, X, _last, NULL) \ + } \ + else if (!_head->forward[_i + 1]) { \ + /* shrink the header */ \ + HDassert(_i == SLIST->curr_level - 1); \ + HDassert((size_t)SLIST->curr_level == _head->level); \ \ - H5SL_SHRINK(_head, (size_t)(_i + 1)) \ - SLIST->curr_level--; \ - } /* end else */ \ + H5SL_SHRINK(_head, (size_t)(_i + 1)) \ + SLIST->curr_level--; \ } \ - else { \ - /* We are not in the first gap, count the number of */ \ - /* nodes of height _i in the previous gap. Note we */ \ - /* "look ahead" in this loop so X has the value of the */ \ - /* last node in the previous gap. */ \ - X = _llast->forward[_i]; \ - for (_count = 1; _count < 3 && X->forward[_i] != _last; _count++) \ - X = X->forward[_i]; \ - HDassert(X->forward[_i] == _last); \ + } \ + else { \ + /* We are not in the first gap, count the number of */ \ + /* nodes of height _i in the previous gap. Note we */ \ + /* "look ahead" in this loop so X has the value of the */ \ + /* last node in the previous gap. */ \ + X = _llast->forward[_i]; \ + for (_count = 1; _count < 3 && X->forward[_i] != _last; _count++) \ + X = X->forward[_i]; \ + HDassert(X->forward[_i] == _last); \ \ - /* Demote the separator node */ \ - H5SL_DEMOTE(_last, _llast) \ + /* Demote the separator node */ \ + H5SL_DEMOTE(_last, _llast) \ \ - /* If there are 2 or more nodes, promote the last */ \ - if (_count >= 2) \ - H5SL_PROMOTE(SLIST, X, _llast, NULL) \ - else if (!_head->forward[_i + 1]) { \ - /* shrink the header */ \ - HDassert(_i == SLIST->curr_level - 1); \ - HDassert((size_t)SLIST->curr_level == _head->level); \ + /* If there are 2 or more nodes, promote the last */ \ + if (_count >= 2) \ + H5SL_PROMOTE(SLIST, X, _llast, NULL) \ + else if (!_head->forward[_i + 1]) { \ + /* shrink the header */ \ + HDassert(_i == SLIST->curr_level - 1); \ + HDassert((size_t)SLIST->curr_level == _head->level); \ \ - H5SL_SHRINK(_head, (size_t)(_i + 1)) \ - SLIST->curr_level--; \ - } /* end else */ \ - } /* end else */ \ - } /* end if */ \ + H5SL_SHRINK(_head, (size_t)(_i + 1)) \ + SLIST->curr_level--; \ + } \ + } \ + } \ \ - /* Prepare to drop down */ \ - _llast = _ldrop; \ - _last = _drop; \ - _next = _drop->forward[_i]; \ - } /* end for */ \ + /* Prepare to drop down */ \ + _llast = _ldrop; \ + _last = _drop; \ + _next = _drop->forward[_i]; \ + } \ \ - /* Check if we've found the node */ \ - if (_next && H5_GLUE3(H5SL_LOCATE_, CMP, _EQ)(SLIST, TYPE, _next, KEY, HASHVAL)) { \ - void *tmp = _next->item; \ - X = _next; \ + /* Check if we've found the node */ \ + if (_next && H5_GLUE3(H5SL_LOCATE_, CMP, _EQ)(SLIST, TYPE, _next, KEY, HASHVAL)) { \ + void *tmp = _next->item; \ + X = _next; \ \ - /* If the node has a height > 0, swap it with its (lower) */ \ - /* neighbor */ \ - if (X->level) { \ - X = X->backward; \ - _next->key = X->key; \ - _next->item = X->item; \ - _next->hashval = X->hashval; \ - } /* end if */ \ - HDassert(!X->level); \ + /* If the node has a height > 0, swap it with its (lower) */ \ + /* neighbor */ \ + if (X->level) { \ + X = X->backward; \ + _next->key = X->key; \ + _next->item = X->item; \ + _next->hashval = X->hashval; \ + } \ + HDassert(!X->level); \ \ - /* Remove the node */ \ - X->backward->forward[0] = X->forward[0]; \ - if (SLIST->last == X) \ - SLIST->last = X->backward; \ - else \ - X->forward[0]->backward = X->backward; \ - SLIST->nobjs--; \ - X->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[0], X->forward); \ - X = H5FL_FREE(H5SL_node_t, X); \ + /* Remove the node */ \ + X->backward->forward[0] = X->forward[0]; \ + if (SLIST->last == X) \ + SLIST->last = X->backward; \ + else \ + X->forward[0]->backward = X->backward; \ + SLIST->nobjs--; \ + X->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[0], X->forward); \ + X = H5FL_FREE(H5SL_node_t, X); \ \ - HGOTO_DONE(tmp); \ - } /* end if */ \ - } /* end else */ \ + HGOTO_DONE(tmp); \ + } \ } /* Macro used to search for node */ @@ -542,7 +487,6 @@ struct H5SL_node_t { size_t level; /* The level of this node */ size_t log_nalloc; /* log2(Number of slots allocated in forward) */ uint32_t hashval; /* Hash value for key (only for strings, currently) */ - hbool_t removed; /* Whether the node is "removed" (actual removal deferred) */ struct H5SL_node_t **forward; /* Array of forward pointers from this node */ struct H5SL_node_t * backward; /* Backward pointer from this node */ }; @@ -558,8 +502,6 @@ struct H5SL_t { size_t nobjs; /* Number of active objects in skip list */ H5SL_node_t *header; /* Header for nodes in skip list */ H5SL_node_t *last; /* Pointer to last node in skip list */ - hbool_t safe_iterating; /* Whether a routine is "safely" iterating over the list and removals should be - deferred */ }; /* Static functions */ @@ -651,11 +593,11 @@ H5SL_term_package(void) for (i = 0; i < H5SL_fac_nused_g; i++) { ret = H5FL_fac_term(H5SL_fac_g[i]); HDassert(ret >= 0); - } /* end if */ + } H5SL_fac_nused_g = 0; n++; - } /* end if */ + } /* Free the list of factories */ if (H5SL_fac_g) { @@ -663,12 +605,12 @@ H5SL_term_package(void) H5SL_fac_nalloc_g = 0; n++; - } /* end if */ + } /* Mark the interface as uninitialized */ if (0 == n) H5_PKG_INIT_VAR = FALSE; - } /* end if */ + } FUNC_LEAVE_NOAPI(n) } /* H5SL_term_package() */ @@ -711,11 +653,10 @@ H5SL__new_node(void *item, const void *key, uint32_t hashval) ret_value->item = item; ret_value->level = 0; ret_value->hashval = hashval; - ret_value->removed = FALSE; if (NULL == (ret_value->forward = (H5SL_node_t **)H5FL_FAC_MALLOC(H5SL_fac_g[0]))) { ret_value = H5FL_FREE(H5SL_node_t, ret_value); HGOTO_ERROR(H5E_SLIST, H5E_NOSPACE, NULL, "memory allocation failed") - } /* end if */ + } ret_value->log_nalloc = 0; done: @@ -805,7 +746,7 @@ H5SL__insert_common(H5SL_t *slist, void *item, const void *key) default: HDassert(0 && "Unknown skiplist type!"); - } /* end switch */ + } /* 'key' must not have been found in existing list, if we get here */ @@ -880,15 +821,22 @@ H5SL__release_common(H5SL_t *slist, H5SL_operator_t op, void *op_data) while (node) { next_node = node->forward[0]; - /* Call callback, if one is given */ + /* Call callback, if one is given. + * + * Ignoring const here is fine as we only need the value to be const + * with respect to the list code, which should never modify the + * elements. The library code that is making use of the skip list + * container can do what it likes with the elements. + */ + H5_GCC_CLANG_DIAG_OFF("cast-qual") if (op) - /* Casting away const OK -QAK */ (void)(op)(node->item, (void *)node->key, op_data); + H5_GCC_CLANG_DIAG_ON("cast-qual") node->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[node->log_nalloc], node->forward); node = H5FL_FREE(H5SL_node_t, node); node = next_node; - } /* end while */ + } /* Reset the header pointers */ slist->header->forward = @@ -1001,9 +949,8 @@ H5SL_create(H5SL_type_t type, H5SL_cmp_t cmp) new_slist->cmp = cmp; /* Set the dynamic internal fields */ - new_slist->curr_level = -1; - new_slist->nobjs = 0; - new_slist->safe_iterating = FALSE; + new_slist->curr_level = -1; + new_slist->nobjs = 0; /* Allocate the header node */ if (NULL == (header = H5SL__new_node(NULL, NULL, (uint32_t)ULONG_MAX))) @@ -1027,7 +974,7 @@ done: if (ret_value == NULL) { if (new_slist != NULL) new_slist = H5FL_FREE(H5SL_t, new_slist); - } /* end if */ + } FUNC_LEAVE_NOAPI(ret_value) } /* end H5SL_create() */ @@ -1058,9 +1005,6 @@ H5SL_count(H5SL_t *slist) /* Check args */ HDassert(slist); - /* Not currently supported */ - HDassert(!slist->safe_iterating); - /* Check internal consistency */ /* (Pre-condition) */ @@ -1099,9 +1043,6 @@ H5SL_insert(H5SL_t *slist, void *item, const void *key) HDassert(slist); HDassert(key); - /* Not currently supported */ - HDassert(!slist->safe_iterating); - /* Check internal consistency */ /* (Pre-condition) */ @@ -1148,9 +1089,6 @@ H5SL_add(H5SL_t *slist, void *item, const void *key) HDassert(slist); HDassert(key); - /* Not currently supported */ - HDassert(!slist->safe_iterating); - /* Check internal consistency */ /* (Pre-condition) */ @@ -1242,7 +1180,7 @@ H5SL_remove(H5SL_t *slist, const void *key) default: HDassert(0 && "Unknown skiplist type!"); - } /* end switch */ + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1281,9 +1219,6 @@ H5SL_remove_first(H5SL_t *slist) /* Check args */ HDassert(slist); - /* Not currently supported */ - HDassert(!slist->safe_iterating); - /* Assign level */ H5_CHECK_OVERFLOW(slist->curr_level, int, size_t); level = (size_t)slist->curr_level; @@ -1345,12 +1280,12 @@ H5SL_remove_first(H5SL_t *slist) H5SL_SHRINK(head, level) slist->curr_level--; - } /* end else */ + } } else break; - } /* end for */ - } /* end if */ + } + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1436,7 +1371,7 @@ H5SL_search(H5SL_t *slist, const void *key) default: HDassert(0 && "Unknown skiplist type!"); - } /* end switch */ + } /* 'key' must not have been found in list, if we get here */ ret_value = NULL; @@ -1480,9 +1415,6 @@ H5SL_less(H5SL_t *slist, const void *key) HDassert(slist); HDassert(key); - /* Not currently supported */ - HDassert(!slist->safe_iterating); - /* Check internal consistency */ /* (Pre-condition) */ @@ -1531,7 +1463,7 @@ H5SL_less(H5SL_t *slist, const void *key) default: HDassert(0 && "Unknown skiplist type!"); - } /* end switch */ + } /* An exact match for 'key' must not have been found in list, if we get here */ /* Check for a node with a key that is less than the given 'key' */ @@ -1541,13 +1473,13 @@ H5SL_less(H5SL_t *slist, const void *key) ret_value = slist->last->item; else ret_value = NULL; - } /* end if */ + } else { if (x->backward != slist->header) ret_value = x->backward->item; else ret_value = NULL; - } /* end else */ + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1588,9 +1520,6 @@ H5SL_greater(H5SL_t *slist, const void *key) HDassert(slist); HDassert(key); - /* Not currently supported */ - HDassert(!slist->safe_iterating); - /* Check internal consistency */ /* (Pre-condition) */ @@ -1639,7 +1568,7 @@ H5SL_greater(H5SL_t *slist, const void *key) default: HDassert(0 && "Unknown skiplist type!"); - } /* end switch */ + } /* An exact match for 'key' must not have been found in list, if we get here */ /* ('x' must be the next node with a key greater than the 'key', or NULL) */ @@ -1734,7 +1663,7 @@ H5SL_find(H5SL_t *slist, const void *key) default: HDassert(0 && "Unknown skiplist type!"); - } /* end switch */ + } /* 'key' must not have been found in list, if we get here */ ret_value = NULL; @@ -1826,7 +1755,7 @@ H5SL_below(H5SL_t *slist, const void *key) default: HDassert(0 && "Unknown skiplist type!"); - } /* end switch */ + } /* An exact match for 'key' must not have been found in list, if we get here */ /* Check for a node with a key that is less than the given 'key' */ @@ -1836,13 +1765,13 @@ H5SL_below(H5SL_t *slist, const void *key) ret_value = slist->last; else ret_value = NULL; - } /* end if */ + } else { if (x->backward != slist->header) ret_value = x->backward; else ret_value = NULL; - } /* end else */ + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1931,7 +1860,7 @@ H5SL_above(H5SL_t *slist, const void *key) default: HDassert(0 && "Unknown skiplist type!"); - } /* end switch */ + } /* An exact match for 'key' must not have been found in list, if we get here */ /* ('x' must be the next node with a key greater than the 'key', or NULL) */ @@ -1971,9 +1900,6 @@ H5SL_first(H5SL_t *slist) /* Check args */ HDassert(slist); - /* Not currently supported */ - HDassert(!slist->safe_iterating); - /* Check internal consistency */ /* (Pre-condition) */ @@ -2007,9 +1933,6 @@ H5SL_next(H5SL_node_t *slist_node) /* Check args */ HDassert(slist_node); - /* Not currently supported */ - HDassert(!slist_node->removed); - /* Check internal consistency */ /* (Pre-condition) */ @@ -2043,9 +1966,6 @@ H5SL_prev(H5SL_node_t *slist_node) /* Check args */ HDassert(slist_node); - /* Not currently supported */ - HDassert(!slist_node->removed); - /* Check internal consistency */ /* (Pre-condition) */ @@ -2080,9 +2000,6 @@ H5SL_last(H5SL_t *slist) /* Check args */ HDassert(slist); - /* Not currently supported */ - HDassert(!slist->safe_iterating); - /* Check internal consistency */ /* (Pre-condition) */ @@ -2116,9 +2033,6 @@ H5SL_item(H5SL_node_t *slist_node) /* Check args */ HDassert(slist_node); - /* Not currently supported */ - HDassert(!slist_node->removed); - /* Check internal consistency */ /* (Pre-condition) */ @@ -2179,15 +2093,21 @@ H5SL_iterate(H5SL_t *slist, H5SL_operator_t op, void *op_data) /* Protect against the node being deleted by the callback */ next = node->forward[0]; - /* Call the iterator callback */ - /* Casting away const OK -QAK */ - if (!node->removed) - if ((ret_value = (op)(node->item, (void *)node->key, op_data)) != 0) - break; + /* Call the iterator callback + * + * Ignoring const here is fine as we only need the value to be const + * with respect to the list code, which should never modify the + * elements. The library code that is making use of the skip list + * container can do what it likes with the elements. + */ + H5_GCC_CLANG_DIAG_OFF("cast-qual") + if ((ret_value = (op)(node->item, (void *)node->key, op_data)) != 0) + break; + H5_GCC_CLANG_DIAG_ON("cast-qual") /* Advance to next node */ node = next; - } /* end while */ + } FUNC_LEAVE_NOAPI(ret_value) } /* end H5SL_iterate() */ @@ -2222,9 +2142,6 @@ H5SL_release(H5SL_t *slist) /* Check args */ HDassert(slist); - /* Not currently supported */ - HDassert(!slist->safe_iterating); - /* Check internal consistency */ /* (Pre-condition) */ @@ -2274,9 +2191,6 @@ H5SL_free(H5SL_t *slist, H5SL_operator_t op, void *op_data) /* Check args */ HDassert(slist); - /* Not currently supported */ - HDassert(!slist->safe_iterating); - /* Check internal consistency */ /* (Pre-condition) */ @@ -2290,186 +2204,6 @@ done: /*-------------------------------------------------------------------------- NAME - H5SL_try_free_safe - PURPOSE - Makes the supplied callback on all nodes in the skip list, freeing each - node that the callback returns TRUE for. - USAGE - herr_t PURPOSE(slist,op,opdata) - H5SL_t *slist; IN/OUT: Pointer to skip list to release nodes - H5SL_try_free_op_t op; IN: Callback function to try to free item & key - void *op_data; IN/OUT: Pointer to application data for callback - - RETURNS - Returns non-negative on success, negative on failure. - DESCRIPTION - Makes the supplied callback on all nodes in the skip list, freeing each - node that the callback returns TRUE for. The iteration is performed in - a safe manner, such that the callback can call H5SL_remove(), - H5SL_search(), H5SL_find(), and H5SL_iterate() on nodes in this - skiplist, except H5SL_remove() may not be call on *this* node. - GLOBAL VARIABLES - COMMENTS, BUGS, ASSUMPTIONS - This function is written to be most efficient when most nodes are - removed from the skiplist, as it rebuilds the nodes afterwards. - EXAMPLES - REVISION LOG ---------------------------------------------------------------------------*/ -herr_t -H5SL_try_free_safe(H5SL_t *slist, H5SL_try_free_op_t op, void *op_data) -{ - H5SL_node_t *node, *next_node, *last_node; /* Pointers to skip list nodes */ - htri_t op_ret; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT - - /* Check args */ - HDassert(slist); - HDassert(op); - - /* Not currently supported */ - HDassert(!slist->safe_iterating); - - /* Check internal consistency */ - /* (Pre-condition) */ - - /* Mark skip list as safe iterating, so nodes aren't freed out from under - * us */ - slist->safe_iterating = TRUE; - - /* Iterate over skip list nodes, making the callback for each and marking - * them as removed if requested by the callback */ - node = slist->header->forward[0]; - while (node) { - /* Check if the node was already removed */ - if (!node->removed) { - /* Call callback */ - /* Casting away const OK -NAF */ - if ((op_ret = (op)(node->item, (void *)node->key, op_data)) < 0) - HGOTO_ERROR(H5E_SLIST, H5E_CALLBACK, FAIL, "callback operation failed") - - /* Check if op indicated that the node should be removed */ - if (op_ret) - /* Mark the node as removed */ - node->removed = TRUE; - } /* end if */ - - /* Advance node */ - node = node->forward[0]; - } /* end while */ - - /* Reset safe_iterating */ - slist->safe_iterating = FALSE; - - /* Iterate over nodes, freeing ones marked as removed */ - node = slist->header->forward[0]; - last_node = slist->header; - while (node) { - /* Save next node */ - next_node = node->forward[0]; - - /* Check if the node was marked as removed */ - if (node->removed) { - /* Remove the node */ - node->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[node->log_nalloc], node->forward); - node = H5FL_FREE(H5SL_node_t, node); - slist->nobjs--; - } /* end if */ - else { - /* Update backwards and forwards[0] pointers, and set the level to - * 0. Since the list is flattened we must rebuild the skiplist - * afterwards. */ - /* Set level to 0. Note there is no need to preserve - * node->forward[0] since it was cached above and will always be - * updated later. */ - if (node->level > 0) { - node->forward = - (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[node->log_nalloc], (void *)node->forward); - if (NULL == (node->forward = (H5SL_node_t **)H5FL_FAC_MALLOC(H5SL_fac_g[0]))) - HGOTO_ERROR(H5E_SLIST, H5E_CANTALLOC, FAIL, "memory allocation failed") - node->log_nalloc = 0; - node->level = 0; - } /* end if */ - - /* Update pointers */ - last_node->forward[0] = node; - node->backward = last_node; - last_node = node; - } /* end else */ - - /* Advance node */ - node = next_node; - } /* end while */ - - /* Final pointer update */ - last_node->forward[0] = NULL; - slist->last = last_node; - - /* Demote skip list to level 0 */ - if (slist->curr_level > 0) { - HDassert(slist->header->level == (size_t)slist->curr_level); - - node = slist->header->forward[0]; - slist->header->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[slist->header->log_nalloc], - (void *)slist->header->forward); - if (NULL == (slist->header->forward = (H5SL_node_t **)H5FL_FAC_MALLOC(H5SL_fac_g[0]))) - HGOTO_ERROR(H5E_SLIST, H5E_CANTALLOC, FAIL, "memory allocation failed") - slist->header->forward[0] = node; - slist->header->log_nalloc = 0; - slist->header->level = 0; - } /* end if */ - - /* Check if there are any nodes left */ - if (slist->nobjs > 0) { - int i; - - HDassert(slist->header->forward[0]); - - /* Set skiplist level to 0 */ - slist->curr_level = 0; - - /* Rebuild the forward arrays */ - for (i = 0; slist->curr_level >= i; i++) { - HDassert(slist->curr_level == i); - - /* Promote every third node this level until we run out of nodes */ - node = last_node = slist->header; - while (1) { - /* Check second node in gap, if not present, no need to promote - * further this level. */ - HDassert(node->forward[i]); - node = node->forward[i]->forward[i]; - if (!node) - break; - - /* Check third and fourth node in gap, if either is not present, - * no need to promote further this level. */ - node = node->forward[i]; - if (!node || !node->forward[i]) - break; - - /* Promote the third node in the gap */ - H5SL_PROMOTE(slist, node, last_node, FAIL) - last_node = node; - } /* end while */ - } /* end for */ - } /* end if */ - else { - HDassert(!slist->header->forward[0]); - HDassert(slist->last == slist->header); - HDassert(slist->nobjs == 0); - - /* Reset the skiplist level */ - slist->curr_level = -1; - } /* end else */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5SL_try_free_safe() */ - -/*-------------------------------------------------------------------------- - NAME H5SL_destroy PURPOSE Close a skip list, deallocating it and freeing all its nodes. diff --git a/src/H5SLprivate.h b/src/H5SLprivate.h index c9e1147..be6f7b6 100644 --- a/src/H5SLprivate.h +++ b/src/H5SLprivate.h @@ -60,9 +60,6 @@ typedef int (*H5SL_cmp_t)(const void *key1, const void *key2); /* Typedef for iteration operations */ typedef herr_t (*H5SL_operator_t)(void *item, void *key, void *operator_data /*in,out*/); -/* Typedef for H5SL_try_free_safe operation callback */ -typedef htri_t (*H5SL_try_free_op_t)(void *item, void *key, void *operator_data /*in,out*/); - /********************/ /* Private routines */ /********************/ @@ -86,7 +83,6 @@ H5_DLL void * H5SL_item(H5SL_node_t *slist_node); H5_DLL herr_t H5SL_iterate(H5SL_t *slist, H5SL_operator_t op, void *op_data); H5_DLL herr_t H5SL_release(H5SL_t *slist); H5_DLL herr_t H5SL_free(H5SL_t *slist, H5SL_operator_t op, void *op_data); -H5_DLL herr_t H5SL_try_free_safe(H5SL_t *slist, H5SL_try_free_op_t op, void *op_data); H5_DLL herr_t H5SL_close(H5SL_t *slist); H5_DLL herr_t H5SL_destroy(H5SL_t *slist, H5SL_operator_t op, void *op_data); H5_DLL int H5SL_term_interface(void); @@ -423,31 +423,29 @@ hid_t H5T_C_S1_g = FAIL; hid_t H5T_FORTRAN_S1_g = FAIL; -hid_t H5T_NATIVE_SCHAR_g = FAIL; -hid_t H5T_NATIVE_UCHAR_g = FAIL; -hid_t H5T_NATIVE_SHORT_g = FAIL; -hid_t H5T_NATIVE_USHORT_g = FAIL; -hid_t H5T_NATIVE_INT_g = FAIL; -hid_t H5T_NATIVE_UINT_g = FAIL; -hid_t H5T_NATIVE_LONG_g = FAIL; -hid_t H5T_NATIVE_ULONG_g = FAIL; -hid_t H5T_NATIVE_LLONG_g = FAIL; -hid_t H5T_NATIVE_ULLONG_g = FAIL; -hid_t H5T_NATIVE_FLOAT_g = FAIL; -hid_t H5T_NATIVE_DOUBLE_g = FAIL; -#if H5_SIZEOF_LONG_DOUBLE != 0 +hid_t H5T_NATIVE_SCHAR_g = FAIL; +hid_t H5T_NATIVE_UCHAR_g = FAIL; +hid_t H5T_NATIVE_SHORT_g = FAIL; +hid_t H5T_NATIVE_USHORT_g = FAIL; +hid_t H5T_NATIVE_INT_g = FAIL; +hid_t H5T_NATIVE_UINT_g = FAIL; +hid_t H5T_NATIVE_LONG_g = FAIL; +hid_t H5T_NATIVE_ULONG_g = FAIL; +hid_t H5T_NATIVE_LLONG_g = FAIL; +hid_t H5T_NATIVE_ULLONG_g = FAIL; +hid_t H5T_NATIVE_FLOAT_g = FAIL; +hid_t H5T_NATIVE_DOUBLE_g = FAIL; hid_t H5T_NATIVE_LDOUBLE_g = FAIL; -#endif -hid_t H5T_NATIVE_B8_g = FAIL; -hid_t H5T_NATIVE_B16_g = FAIL; -hid_t H5T_NATIVE_B32_g = FAIL; -hid_t H5T_NATIVE_B64_g = FAIL; -hid_t H5T_NATIVE_OPAQUE_g = FAIL; -hid_t H5T_NATIVE_HADDR_g = FAIL; -hid_t H5T_NATIVE_HSIZE_g = FAIL; -hid_t H5T_NATIVE_HSSIZE_g = FAIL; -hid_t H5T_NATIVE_HERR_g = FAIL; -hid_t H5T_NATIVE_HBOOL_g = FAIL; +hid_t H5T_NATIVE_B8_g = FAIL; +hid_t H5T_NATIVE_B16_g = FAIL; +hid_t H5T_NATIVE_B32_g = FAIL; +hid_t H5T_NATIVE_B64_g = FAIL; +hid_t H5T_NATIVE_OPAQUE_g = FAIL; +hid_t H5T_NATIVE_HADDR_g = FAIL; +hid_t H5T_NATIVE_HSIZE_g = FAIL; +hid_t H5T_NATIVE_HSSIZE_g = FAIL; +hid_t H5T_NATIVE_HERR_g = FAIL; +hid_t H5T_NATIVE_HBOOL_g = FAIL; hid_t H5T_NATIVE_INT8_g = FAIL; hid_t H5T_NATIVE_UINT8_g = FAIL; @@ -483,21 +481,19 @@ hid_t H5T_NATIVE_UINT_FAST64_g = FAIL; * datatype or C structures, which are different from the alignments for memory * address below this group of variables. */ -size_t H5T_NATIVE_SCHAR_COMP_ALIGN_g = 0; -size_t H5T_NATIVE_UCHAR_COMP_ALIGN_g = 0; -size_t H5T_NATIVE_SHORT_COMP_ALIGN_g = 0; -size_t H5T_NATIVE_USHORT_COMP_ALIGN_g = 0; -size_t H5T_NATIVE_INT_COMP_ALIGN_g = 0; -size_t H5T_NATIVE_UINT_COMP_ALIGN_g = 0; -size_t H5T_NATIVE_LONG_COMP_ALIGN_g = 0; -size_t H5T_NATIVE_ULONG_COMP_ALIGN_g = 0; -size_t H5T_NATIVE_LLONG_COMP_ALIGN_g = 0; -size_t H5T_NATIVE_ULLONG_COMP_ALIGN_g = 0; -size_t H5T_NATIVE_FLOAT_COMP_ALIGN_g = 0; -size_t H5T_NATIVE_DOUBLE_COMP_ALIGN_g = 0; -#if H5_SIZEOF_LONG_DOUBLE != 0 +size_t H5T_NATIVE_SCHAR_COMP_ALIGN_g = 0; +size_t H5T_NATIVE_UCHAR_COMP_ALIGN_g = 0; +size_t H5T_NATIVE_SHORT_COMP_ALIGN_g = 0; +size_t H5T_NATIVE_USHORT_COMP_ALIGN_g = 0; +size_t H5T_NATIVE_INT_COMP_ALIGN_g = 0; +size_t H5T_NATIVE_UINT_COMP_ALIGN_g = 0; +size_t H5T_NATIVE_LONG_COMP_ALIGN_g = 0; +size_t H5T_NATIVE_ULONG_COMP_ALIGN_g = 0; +size_t H5T_NATIVE_LLONG_COMP_ALIGN_g = 0; +size_t H5T_NATIVE_ULLONG_COMP_ALIGN_g = 0; +size_t H5T_NATIVE_FLOAT_COMP_ALIGN_g = 0; +size_t H5T_NATIVE_DOUBLE_COMP_ALIGN_g = 0; size_t H5T_NATIVE_LDOUBLE_COMP_ALIGN_g = 0; -#endif size_t H5T_POINTER_COMP_ALIGN_g = 0; size_t H5T_HVL_COMP_ALIGN_g = 0; @@ -509,21 +505,19 @@ size_t H5T_REF_COMP_ALIGN_g = 0; * Alignment constraints for native types. These are initialized at run time * in H5Tinit.c */ -size_t H5T_NATIVE_SCHAR_ALIGN_g = 0; -size_t H5T_NATIVE_UCHAR_ALIGN_g = 0; -size_t H5T_NATIVE_SHORT_ALIGN_g = 0; -size_t H5T_NATIVE_USHORT_ALIGN_g = 0; -size_t H5T_NATIVE_INT_ALIGN_g = 0; -size_t H5T_NATIVE_UINT_ALIGN_g = 0; -size_t H5T_NATIVE_LONG_ALIGN_g = 0; -size_t H5T_NATIVE_ULONG_ALIGN_g = 0; -size_t H5T_NATIVE_LLONG_ALIGN_g = 0; -size_t H5T_NATIVE_ULLONG_ALIGN_g = 0; -size_t H5T_NATIVE_FLOAT_ALIGN_g = 0; -size_t H5T_NATIVE_DOUBLE_ALIGN_g = 0; -#if H5_SIZEOF_LONG_DOUBLE != 0 +size_t H5T_NATIVE_SCHAR_ALIGN_g = 0; +size_t H5T_NATIVE_UCHAR_ALIGN_g = 0; +size_t H5T_NATIVE_SHORT_ALIGN_g = 0; +size_t H5T_NATIVE_USHORT_ALIGN_g = 0; +size_t H5T_NATIVE_INT_ALIGN_g = 0; +size_t H5T_NATIVE_UINT_ALIGN_g = 0; +size_t H5T_NATIVE_LONG_ALIGN_g = 0; +size_t H5T_NATIVE_ULONG_ALIGN_g = 0; +size_t H5T_NATIVE_LLONG_ALIGN_g = 0; +size_t H5T_NATIVE_ULLONG_ALIGN_g = 0; +size_t H5T_NATIVE_FLOAT_ALIGN_g = 0; +size_t H5T_NATIVE_DOUBLE_ALIGN_g = 0; size_t H5T_NATIVE_LDOUBLE_ALIGN_g = 0; -#endif /* * Alignment constraints for C9x types. These are initialized at run time in @@ -760,42 +754,40 @@ DESCRIPTION herr_t H5T__init_package(void) { - H5T_t *native_schar = NULL; /* Datatype structure for native signed char */ - H5T_t *native_uchar = NULL; /* Datatype structure for native unsigned char */ - H5T_t *native_short = NULL; /* Datatype structure for native short */ - H5T_t *native_ushort = NULL; /* Datatype structure for native unsigned short */ - H5T_t *native_int = NULL; /* Datatype structure for native int */ - H5T_t *native_uint = NULL; /* Datatype structure for native unsigned int */ - H5T_t *native_long = NULL; /* Datatype structure for native long */ - H5T_t *native_ulong = NULL; /* Datatype structure for native unsigned long */ - H5T_t *native_llong = NULL; /* Datatype structure for native long long */ - H5T_t *native_ullong = NULL; /* Datatype structure for native unsigned long long */ - H5T_t *native_float = NULL; /* Datatype structure for native float */ - H5T_t *native_double = NULL; /* Datatype structure for native double */ -#if H5_SIZEOF_LONG_DOUBLE != 0 - H5T_t *native_ldouble = NULL; /* Datatype structure for native long double */ -#endif - H5T_t * std_u8le = NULL; /* Datatype structure for unsigned 8-bit little-endian integer */ - H5T_t * std_u8be = NULL; /* Datatype structure for unsigned 8-bit big-endian integer */ - H5T_t * std_u16le = NULL; /* Datatype structure for unsigned 16-bit little-endian integer */ - H5T_t * std_u16be = NULL; /* Datatype structure for unsigned 16-bit big-endian integer */ - H5T_t * std_u32le = NULL; /* Datatype structure for unsigned 32-bit little-endian integer */ - H5T_t * std_u32be = NULL; /* Datatype structure for unsigned 32-bit big-endian integer */ - H5T_t * std_u64le = NULL; /* Datatype structure for unsigned 64-bit little-endian integer */ - H5T_t * std_u64be = NULL; /* Datatype structure for unsigned 64-bit big-endian integer */ - H5T_t * dt = NULL; - H5T_t * fixedpt = NULL; /* Datatype structure for native int */ - H5T_t * floatpt = NULL; /* Datatype structure for native float */ - H5T_t * string = NULL; /* Datatype structure for C string */ - H5T_t * bitfield = NULL; /* Datatype structure for bitfield */ - H5T_t * compound = NULL; /* Datatype structure for compound objects */ - H5T_t * enum_type = NULL; /* Datatype structure for enum objects */ - H5T_t * vlen = NULL; /* Datatype structure for vlen objects */ - H5T_t * array = NULL; /* Datatype structure for array objects */ - H5T_t * objref = NULL; /* Datatype structure for deprecated reference objects */ - H5T_t * regref = NULL; /* Datatype structure for deprecated region references */ - H5T_t * ref = NULL; /* Datatype structure for opaque references */ - hsize_t dim[1] = {1}; /* Dimension info for array datatype */ + H5T_t * native_schar = NULL; /* Datatype structure for native signed char */ + H5T_t * native_uchar = NULL; /* Datatype structure for native unsigned char */ + H5T_t * native_short = NULL; /* Datatype structure for native short */ + H5T_t * native_ushort = NULL; /* Datatype structure for native unsigned short */ + H5T_t * native_int = NULL; /* Datatype structure for native int */ + H5T_t * native_uint = NULL; /* Datatype structure for native unsigned int */ + H5T_t * native_long = NULL; /* Datatype structure for native long */ + H5T_t * native_ulong = NULL; /* Datatype structure for native unsigned long */ + H5T_t * native_llong = NULL; /* Datatype structure for native long long */ + H5T_t * native_ullong = NULL; /* Datatype structure for native unsigned long long */ + H5T_t * native_float = NULL; /* Datatype structure for native float */ + H5T_t * native_double = NULL; /* Datatype structure for native double */ + H5T_t * native_ldouble = NULL; /* Datatype structure for native long double */ + H5T_t * std_u8le = NULL; /* Datatype structure for unsigned 8-bit little-endian integer */ + H5T_t * std_u8be = NULL; /* Datatype structure for unsigned 8-bit big-endian integer */ + H5T_t * std_u16le = NULL; /* Datatype structure for unsigned 16-bit little-endian integer */ + H5T_t * std_u16be = NULL; /* Datatype structure for unsigned 16-bit big-endian integer */ + H5T_t * std_u32le = NULL; /* Datatype structure for unsigned 32-bit little-endian integer */ + H5T_t * std_u32be = NULL; /* Datatype structure for unsigned 32-bit big-endian integer */ + H5T_t * std_u64le = NULL; /* Datatype structure for unsigned 64-bit little-endian integer */ + H5T_t * std_u64be = NULL; /* Datatype structure for unsigned 64-bit big-endian integer */ + H5T_t * dt = NULL; + H5T_t * fixedpt = NULL; /* Datatype structure for native int */ + H5T_t * floatpt = NULL; /* Datatype structure for native float */ + H5T_t * string = NULL; /* Datatype structure for C string */ + H5T_t * bitfield = NULL; /* Datatype structure for bitfield */ + H5T_t * compound = NULL; /* Datatype structure for compound objects */ + H5T_t * enum_type = NULL; /* Datatype structure for enum objects */ + H5T_t * vlen = NULL; /* Datatype structure for vlen objects */ + H5T_t * array = NULL; /* Datatype structure for array objects */ + H5T_t * objref = NULL; /* Datatype structure for deprecated reference objects */ + H5T_t * regref = NULL; /* Datatype structure for deprecated region references */ + H5T_t * ref = NULL; /* Datatype structure for opaque references */ + hsize_t dim[1] = {1}; /* Dimension info for array datatype */ herr_t status; hbool_t copied_dtype = TRUE; /* Flag to indicate whether datatype was copied or allocated (for error cleanup) */ @@ -843,10 +835,8 @@ H5T__init_package(void) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype object") if (NULL == (native_double = (H5T_t *)H5I_object(H5T_NATIVE_DOUBLE_g))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype object") -#if H5_SIZEOF_LONG_DOUBLE != 0 if (NULL == (native_ldouble = (H5T_t *)H5I_object(H5T_NATIVE_LDOUBLE_g))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype object") -#endif /*------------------------------------------------------------ * Derived native types @@ -1116,7 +1106,6 @@ H5T__init_package(void) H5T__register_int(H5T_PERS_HARD, "flt_dbl", native_float, native_double, H5T__conv_float_double); status |= H5T__register_int(H5T_PERS_HARD, "dbl_flt", native_double, native_float, H5T__conv_double_float); -#if H5_SIZEOF_LONG_DOUBLE != 0 status |= H5T__register_int(H5T_PERS_HARD, "flt_ldbl", native_float, native_ldouble, H5T__conv_float_ldouble); status |= @@ -1125,7 +1114,6 @@ H5T__init_package(void) H5T__register_int(H5T_PERS_HARD, "ldbl_flt", native_ldouble, native_float, H5T__conv_ldouble_float); status |= H5T__register_int(H5T_PERS_HARD, "ldbl_dbl", native_ldouble, native_double, H5T__conv_ldouble_double); -#endif /* H5_SIZEOF_LONG_DOUBLE != 0 */ /* from long long */ status |= @@ -1663,31 +1651,29 @@ H5T_top_term_package(void) H5T_FORTRAN_S1_g = FAIL; - H5T_NATIVE_SCHAR_g = FAIL; - H5T_NATIVE_UCHAR_g = FAIL; - H5T_NATIVE_SHORT_g = FAIL; - H5T_NATIVE_USHORT_g = FAIL; - H5T_NATIVE_INT_g = FAIL; - H5T_NATIVE_UINT_g = FAIL; - H5T_NATIVE_LONG_g = FAIL; - H5T_NATIVE_ULONG_g = FAIL; - H5T_NATIVE_LLONG_g = FAIL; - H5T_NATIVE_ULLONG_g = FAIL; - H5T_NATIVE_FLOAT_g = FAIL; - H5T_NATIVE_DOUBLE_g = FAIL; -#if H5_SIZEOF_LONG_DOUBLE != 0 + H5T_NATIVE_SCHAR_g = FAIL; + H5T_NATIVE_UCHAR_g = FAIL; + H5T_NATIVE_SHORT_g = FAIL; + H5T_NATIVE_USHORT_g = FAIL; + H5T_NATIVE_INT_g = FAIL; + H5T_NATIVE_UINT_g = FAIL; + H5T_NATIVE_LONG_g = FAIL; + H5T_NATIVE_ULONG_g = FAIL; + H5T_NATIVE_LLONG_g = FAIL; + H5T_NATIVE_ULLONG_g = FAIL; + H5T_NATIVE_FLOAT_g = FAIL; + H5T_NATIVE_DOUBLE_g = FAIL; H5T_NATIVE_LDOUBLE_g = FAIL; -#endif - H5T_NATIVE_B8_g = FAIL; - H5T_NATIVE_B16_g = FAIL; - H5T_NATIVE_B32_g = FAIL; - H5T_NATIVE_B64_g = FAIL; - H5T_NATIVE_OPAQUE_g = FAIL; - H5T_NATIVE_HADDR_g = FAIL; - H5T_NATIVE_HSIZE_g = FAIL; - H5T_NATIVE_HSSIZE_g = FAIL; - H5T_NATIVE_HERR_g = FAIL; - H5T_NATIVE_HBOOL_g = FAIL; + H5T_NATIVE_B8_g = FAIL; + H5T_NATIVE_B16_g = FAIL; + H5T_NATIVE_B32_g = FAIL; + H5T_NATIVE_B64_g = FAIL; + H5T_NATIVE_OPAQUE_g = FAIL; + H5T_NATIVE_HADDR_g = FAIL; + H5T_NATIVE_HSIZE_g = FAIL; + H5T_NATIVE_HSSIZE_g = FAIL; + H5T_NATIVE_HERR_g = FAIL; + H5T_NATIVE_HBOOL_g = FAIL; H5T_NATIVE_INT8_g = FAIL; H5T_NATIVE_UINT8_g = FAIL; diff --git a/src/H5Tconv.c b/src/H5Tconv.c index 18cbcf1..c06c895 100644 --- a/src/H5Tconv.c +++ b/src/H5Tconv.c @@ -7009,14 +7009,12 @@ H5T__conv_float_double(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t ne * *------------------------------------------------------------------------- */ -#if H5_SIZEOF_LONG_DOUBLE != 0 herr_t H5T__conv_float_ldouble(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, void H5_ATTR_UNUSED *bkg) { H5T_CONV_fF(FLOAT, LDOUBLE, float, long double, -, -); } -#endif /* H5_SIZEOF_LONG_DOUBLE != 0 */ /*------------------------------------------------------------------------- * Function: H5T__conv_double_float @@ -7051,14 +7049,12 @@ H5T__conv_double_float(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t ne * *------------------------------------------------------------------------- */ -#if H5_SIZEOF_LONG_DOUBLE != 0 herr_t H5T__conv_double_ldouble(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, void H5_ATTR_UNUSED *bkg) { H5T_CONV_fF(DOUBLE, LDOUBLE, double, long double, -, -); } -#endif /* H5_SIZEOF_LONG_DOUBLE != 0 */ /*------------------------------------------------------------------------- * Function: H5T__conv_ldouble_float @@ -7073,14 +7069,12 @@ H5T__conv_double_ldouble(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t * *------------------------------------------------------------------------- */ -#if H5_SIZEOF_LONG_DOUBLE != 0 herr_t H5T__conv_ldouble_float(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, void H5_ATTR_UNUSED *bkg) { H5T_CONV_Ff(LDOUBLE, FLOAT, long double, float, -FLT_MAX, FLT_MAX); } -#endif /* H5_SIZEOF_LONG_DOUBLE != 0 */ /*------------------------------------------------------------------------- * Function: H5T__conv_ldouble_double @@ -7095,14 +7089,12 @@ H5T__conv_ldouble_float(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t n * *------------------------------------------------------------------------- */ -#if H5_SIZEOF_LONG_DOUBLE != 0 herr_t H5T__conv_ldouble_double(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, size_t buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void *buf, void H5_ATTR_UNUSED *bkg) { H5T_CONV_Ff(LDOUBLE, DOUBLE, long double, double, -DBL_MAX, DBL_MAX); } -#endif /* H5_SIZEOF_LONG_DOUBLE != 0 */ /*------------------------------------------------------------------------- * Function: H5T__conv_schar_float diff --git a/src/H5Tnative.c b/src/H5Tnative.c index 2688b23..c9a2907 100644 --- a/src/H5Tnative.c +++ b/src/H5Tnative.c @@ -718,9 +718,7 @@ H5T__get_native_float(size_t size, H5T_direction_t direction, size_t *struct_ali enum match_type { /* The different kinds of floating point types we can match */ H5T_NATIVE_FLOAT_MATCH_FLOAT, H5T_NATIVE_FLOAT_MATCH_DOUBLE, -#if H5_SIZEOF_LONG_DOUBLE != 0 H5T_NATIVE_FLOAT_MATCH_LDOUBLE, -#endif H5T_NATIVE_FLOAT_MATCH_UNKNOWN } match = H5T_NATIVE_FLOAT_MATCH_UNKNOWN; H5T_t *ret_value = NULL; /* Return value */ @@ -738,24 +736,16 @@ H5T__get_native_float(size_t size, H5T_direction_t direction, size_t *struct_ali match = H5T_NATIVE_FLOAT_MATCH_DOUBLE; native_size = sizeof(double); } -#if H5_SIZEOF_LONG_DOUBLE != 0 else if (size <= sizeof(long double)) { match = H5T_NATIVE_FLOAT_MATCH_LDOUBLE; native_size = sizeof(long double); } -#endif else { /* If not match, return the biggest datatype */ -#if H5_SIZEOF_LONG_DOUBLE != 0 match = H5T_NATIVE_FLOAT_MATCH_LDOUBLE; native_size = sizeof(long double); -#else - match = H5T_NATIVE_FLOAT_MATCH_DOUBLE; - native_size = sizeof(double); -#endif } } else { -#if H5_SIZEOF_LONG_DOUBLE != 0 if (size > sizeof(double)) { match = H5T_NATIVE_FLOAT_MATCH_LDOUBLE; native_size = sizeof(long double); @@ -768,16 +758,6 @@ H5T__get_native_float(size_t size, H5T_direction_t direction, size_t *struct_ali match = H5T_NATIVE_FLOAT_MATCH_FLOAT; native_size = sizeof(float); } -#else - if (size > sizeof(float)) { - match = H5T_NATIVE_FLOAT_MATCH_DOUBLE; - native_size = sizeof(double); - } - else { - match = H5T_NATIVE_FLOAT_MATCH_FLOAT; - native_size = sizeof(float); - } -#endif } /* Set the appropriate native floating point information */ @@ -792,12 +772,11 @@ H5T__get_native_float(size_t size, H5T_direction_t direction, size_t *struct_ali align = H5T_NATIVE_DOUBLE_COMP_ALIGN_g; break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case H5T_NATIVE_FLOAT_MATCH_LDOUBLE: tid = H5T_NATIVE_LDOUBLE; align = H5T_NATIVE_LDOUBLE_COMP_ALIGN_g; break; -#endif + case H5T_NATIVE_FLOAT_MATCH_UNKNOWN: default: HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "Unknown native floating-point match") diff --git a/src/H5Tpkg.h b/src/H5Tpkg.h index 51ecaca..19593dd 100644 --- a/src/H5Tpkg.h +++ b/src/H5Tpkg.h @@ -401,9 +401,7 @@ H5_DLLVAR size_t H5T_NATIVE_LONG_COMP_ALIGN_g; H5_DLLVAR size_t H5T_NATIVE_LLONG_COMP_ALIGN_g; H5_DLLVAR size_t H5T_NATIVE_FLOAT_COMP_ALIGN_g; H5_DLLVAR size_t H5T_NATIVE_DOUBLE_COMP_ALIGN_g; -#if H5_SIZEOF_LONG_DOUBLE != 0 H5_DLLVAR size_t H5T_NATIVE_LDOUBLE_COMP_ALIGN_g; -#endif H5_DLLVAR size_t H5T_POINTER_COMP_ALIGN_g; H5_DLLVAR size_t H5T_HVL_COMP_ALIGN_g; @@ -429,9 +427,7 @@ H5_DLLVAR size_t H5T_NATIVE_LLONG_ALIGN_g; H5_DLLVAR size_t H5T_NATIVE_ULLONG_ALIGN_g; H5_DLLVAR size_t H5T_NATIVE_FLOAT_ALIGN_g; H5_DLLVAR size_t H5T_NATIVE_DOUBLE_ALIGN_g; -#if H5_SIZEOF_LONG_DOUBLE != 0 H5_DLLVAR size_t H5T_NATIVE_LDOUBLE_ALIGN_g; -#endif /* C9x alignment constraints */ H5_DLLVAR size_t H5T_NATIVE_INT8_ALIGN_g; @@ -468,10 +464,8 @@ H5_DLLVAR float H5T_NATIVE_FLOAT_POS_INF_g; H5_DLLVAR float H5T_NATIVE_FLOAT_NEG_INF_g; H5_DLLVAR double H5T_NATIVE_DOUBLE_POS_INF_g; H5_DLLVAR double H5T_NATIVE_DOUBLE_NEG_INF_g; -#if H5_SIZEOF_LONG_DOUBLE != 0 H5_DLLVAR double H5T_NATIVE_LDOUBLE_POS_INF_g; H5_DLLVAR double H5T_NATIVE_LDOUBLE_NEG_INF_g; -#endif /* Declare extern the free lists for H5T_t's and H5T_shared_t's */ H5FL_EXTERN(H5T_t); diff --git a/src/H5Tpublic.h b/src/H5Tpublic.h index bb5b0ef..911efdf 100644 --- a/src/H5Tpublic.h +++ b/src/H5Tpublic.h @@ -803,13 +803,11 @@ H5_DLLVAR hid_t H5T_VAX_F64_g; * C-style \Code{double} */ #define H5T_NATIVE_DOUBLE (H5OPEN H5T_NATIVE_DOUBLE_g) -#if H5_SIZEOF_LONG_DOUBLE != 0 /** * \ingroup PDTNAT * C-style \Code{long double} */ #define H5T_NATIVE_LDOUBLE (H5OPEN H5T_NATIVE_LDOUBLE_g) -#endif /** * \ingroup PDTNAT * HDF5 8-bit bitfield based on native types @@ -872,9 +870,7 @@ H5_DLLVAR hid_t H5T_NATIVE_LLONG_g; H5_DLLVAR hid_t H5T_NATIVE_ULLONG_g; H5_DLLVAR hid_t H5T_NATIVE_FLOAT_g; H5_DLLVAR hid_t H5T_NATIVE_DOUBLE_g; -#if H5_SIZEOF_LONG_DOUBLE != 0 H5_DLLVAR hid_t H5T_NATIVE_LDOUBLE_g; -#endif H5_DLLVAR hid_t H5T_NATIVE_B8_g; H5_DLLVAR hid_t H5T_NATIVE_B16_g; H5_DLLVAR hid_t H5T_NATIVE_B32_g; diff --git a/src/H5Tref.c b/src/H5Tref.c index cac8cf6..511c531 100644 --- a/src/H5Tref.c +++ b/src/H5Tref.c @@ -309,8 +309,8 @@ H5T__ref_set_loc(H5T_t *dt, H5VL_object_t *file, H5T_loc_t loc) HGOTO_ERROR(H5E_REFERENCE, H5E_CANTGET, FAIL, "can't get encode size") /* Size on disk, memory size is different */ - dt->shared->size = MAX(H5_SIZEOF_UINT32_T + H5R_ENCODE_HEADER_SIZE + cont_info.blob_id_size, - ref_encode_size); + dt->shared->size = + MAX(sizeof(uint32_t) + H5R_ENCODE_HEADER_SIZE + cont_info.blob_id_size, ref_encode_size); dt->shared->u.atomic.prec = 8 * dt->shared->size; /* Set up the function pointers to access the information on @@ -778,7 +778,7 @@ H5T__ref_disk_isnull(const H5VL_object_t *src_file, const void *src_buf, hbool_t H5VL_blob_specific_args_t vol_cb_args; /* Arguments to VOL callback */ /* Skip the size / header */ - p = (const uint8_t *)src_buf + H5R_ENCODE_HEADER_SIZE + H5_SIZEOF_UINT32_T; + p = (const uint8_t *)src_buf + H5R_ENCODE_HEADER_SIZE + sizeof(uint32_t); /* Set up VOL callback arguments */ vol_cb_args.op_type = H5VL_BLOB_ISNULL; @@ -819,7 +819,7 @@ H5T__ref_disk_setnull(H5VL_object_t *dst_file, void *dst_buf, void *bg_buf) /* TODO Should get rid of bg stuff */ if (p_bg) { /* Skip the size / header */ - p_bg += (H5_SIZEOF_UINT32_T + H5R_ENCODE_HEADER_SIZE); + p_bg += (sizeof(uint32_t) + H5R_ENCODE_HEADER_SIZE); /* Set up VOL callback arguments */ vol_cb_args.op_type = H5VL_BLOB_DELETE; @@ -929,8 +929,8 @@ H5T__ref_disk_read(H5VL_object_t *src_file, const void *src_buf, size_t H5_ATTR_ blob_size -= H5R_ENCODE_HEADER_SIZE; /* Skip the size */ - p += H5_SIZEOF_UINT32_T; - HDassert(src_size > (H5R_ENCODE_HEADER_SIZE + H5_SIZEOF_UINT32_T)); + p += sizeof(uint32_t); + HDassert(src_size > (H5R_ENCODE_HEADER_SIZE + sizeof(uint32_t))); /* Retrieve blob */ if (H5VL_blob_get(src_file, p, q, blob_size, NULL) < 0) @@ -974,9 +974,9 @@ H5T__ref_disk_write(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_buf, size_t p_buf_size_left = dst_size; /* Skip the size / header */ - p_bg += (H5_SIZEOF_UINT32_T + H5R_ENCODE_HEADER_SIZE); - HDassert(p_buf_size_left > (H5_SIZEOF_UINT32_T + H5R_ENCODE_HEADER_SIZE)); - p_buf_size_left -= (H5_SIZEOF_UINT32_T + H5R_ENCODE_HEADER_SIZE); + p_bg += (sizeof(uint32_t) + H5R_ENCODE_HEADER_SIZE); + HDassert(p_buf_size_left > (sizeof(uint32_t) + H5R_ENCODE_HEADER_SIZE)); + p_buf_size_left -= (sizeof(uint32_t) + H5R_ENCODE_HEADER_SIZE); /* Set up VOL callback arguments */ vol_cb_args.op_type = H5VL_BLOB_DELETE; @@ -991,12 +991,12 @@ H5T__ref_disk_write(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_buf, p += H5R_ENCODE_HEADER_SIZE; q += H5R_ENCODE_HEADER_SIZE; src_size -= H5R_ENCODE_HEADER_SIZE; - buf_size_left -= H5_SIZEOF_UINT32_T; + buf_size_left -= sizeof(uint32_t); /* Set the size */ UINT32ENCODE(q, src_size); - HDassert(buf_size_left > H5_SIZEOF_UINT32_T); - buf_size_left -= H5_SIZEOF_UINT32_T; + HDassert(buf_size_left > sizeof(uint32_t)); + buf_size_left -= sizeof(uint32_t); /* Store blob */ if (H5VL_blob_put(dst_file, p, src_size, q, NULL) < 0) @@ -655,6 +655,39 @@ done: FUNC_LEAVE_API(ret_value) } /* H5VLobject() */ +/*--------------------------------------------------------------------------- + * Function: H5VLobject_is_native + * + * Purpose: Determines whether an object ID represents a native VOL + * connector object. + * + * Return: Non-negative on success/Negative on failure + * + *--------------------------------------------------------------------------- + */ +herr_t +H5VLobject_is_native(hid_t obj_id, hbool_t *is_native) +{ + H5VL_object_t *vol_obj = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*b", obj_id, is_native); + + if (!is_native) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "`is_native` argument is NULL") + + /* Get the location object for the ID */ + if (NULL == (vol_obj = H5VL_vol_object(obj_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid object identifier") + + if (H5VL_object_is_native(vol_obj, is_native) < 0) + HGOTO_ERROR(H5E_VOL, H5E_CANTGET, FAIL, "can't determine if object is a native connector object") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5VLobject_is_native() */ + /*------------------------------------------------------------------------- * Function: H5VLget_file_type * diff --git a/src/H5VLpublic.h b/src/H5VLpublic.h index 78e39e3..543c3c8 100644 --- a/src/H5VLpublic.h +++ b/src/H5VLpublic.h @@ -356,6 +356,19 @@ H5_DLL herr_t H5VLunregister_connector(hid_t connector_id); * \since 1.12.0 */ H5_DLL herr_t H5VLquery_optional(hid_t obj_id, H5VL_subclass_t subcls, int opt_type, uint64_t *flags); +/** + * \ingroup H5VL + * \brief Determines whether an object ID represents a native + * VOL connector object. + * + * \param[in] obj_id Object identifier + * \param[in] is_native Boolean determining whether object is a native + * VOL connector object + * \return \herr_t + * + * \since 1.13.0 + */ +H5_DLL herr_t H5VLobject_is_native(hid_t obj_id, hbool_t *is_native); #ifdef __cplusplus } diff --git a/src/H5Zscaleoffset.c b/src/H5Zscaleoffset.c index a8a63bf..48a29cd 100644 --- a/src/H5Zscaleoffset.c +++ b/src/H5Zscaleoffset.c @@ -364,12 +364,12 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ #define H5Z_scaleoffset_max_min_3(i, d_nelmts, buf, filval, max, min, D_val) \ { \ i = 0; \ - while (i < d_nelmts && HDfabs(buf[i] - filval) < HDpow(10.0, -D_val)) \ + while (i < d_nelmts && HDfabs((double)(buf[i] - filval)) < HDpow(10.0, -D_val)) \ i++; \ if (i < d_nelmts) \ min = max = buf[i]; \ for (; i < d_nelmts; i++) { \ - if (HDfabs(buf[i] - filval) < HDpow(10.0, -D_val)) \ + if (HDfabs((double)(buf[i] - filval)) < HDpow(10.0, -D_val)) \ continue; /* ignore fill value */ \ if (buf[i] > max) \ max = buf[i]; \ @@ -425,22 +425,22 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ #define H5Z_scaleoffset_check_3(i, type, pow_fun, round_fun, max, min, minbits, D_val) \ { \ if (sizeof(type) == sizeof(int)) { \ - if (round_fun(max * pow_fun(10.0F, (type)D_val) - min * pow_fun(10.0F, (type)D_val)) > \ - pow_fun(2.0F, (type)(sizeof(int) * 8 - 1))) { \ + if (round_fun(max * pow_fun((type)10, (type)D_val) - min * pow_fun((type)10, (type)D_val)) > \ + pow_fun((type)2, (type)(sizeof(int) * 8 - 1))) { \ *minbits = sizeof(int) * 8; \ goto done; \ } \ } \ else if (sizeof(type) == sizeof(long)) { \ - if (round_fun(max * pow_fun(10.0F, (type)D_val) - min * pow_fun(10.0F, (type)D_val)) > \ - pow_fun(2.0F, (type)(sizeof(long) * 8 - 1))) { \ + if (round_fun(max * pow_fun((type)10, (type)D_val) - min * pow_fun((type)10, (type)D_val)) > \ + pow_fun((type)2, (type)(sizeof(long) * 8 - 1))) { \ *minbits = sizeof(long) * 8; \ goto done; \ } \ } \ else if (sizeof(type) == sizeof(long long)) { \ - if (round_fun(max * pow_fun(10.0F, (type)D_val) - min * pow_fun(10.0F, (type)D_val)) > \ - pow_fun(2.0F, (type)(sizeof(long long) * 8 - 1))) { \ + if (round_fun(max * pow_fun((type)10, (type)D_val) - min * pow_fun((type)10, (type)D_val)) > \ + pow_fun((type)2, (type)(sizeof(long long) * 8 - 1))) { \ *minbits = sizeof(long long) * 8; \ goto done; \ } \ @@ -530,27 +530,27 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ { \ if (sizeof(type) == sizeof(int)) \ for (i = 0; i < d_nelmts; i++) { \ - if (abs_fun(buf[i] - filval) < pow_fun(10.0F, (type)-D_val)) \ + if (abs_fun(buf[i] - filval) < pow_fun((type)10, (type)-D_val)) \ *(int *)((void *)&buf[i]) = (int)(((unsigned int)1 << *minbits) - 1); \ else \ - *(int *)((void *)&buf[i]) = (int)lround_fun(buf[i] * pow_fun(10.0F, (type)D_val) - \ - min * pow_fun(10.0F, (type)D_val)); \ + *(int *)((void *)&buf[i]) = (int)lround_fun(buf[i] * pow_fun((type)10, (type)D_val) - \ + min * pow_fun((type)10, (type)D_val)); \ } \ else if (sizeof(type) == sizeof(long)) \ for (i = 0; i < d_nelmts; i++) { \ - if (abs_fun(buf[i] - filval) < pow_fun(10.0F, (type)-D_val)) \ + if (abs_fun(buf[i] - filval) < pow_fun((type)10, (type)-D_val)) \ *(long *)((void *)&buf[i]) = (long)(((unsigned long)1 << *minbits) - 1); \ else \ - *(long *)((void *)&buf[i]) = lround_fun(buf[i] * pow_fun(10.0F, (type)D_val) - \ - min * pow_fun(10.0F, (type)D_val)); \ + *(long *)((void *)&buf[i]) = lround_fun(buf[i] * pow_fun((type)10, (type)D_val) - \ + min * pow_fun((type)10, (type)D_val)); \ } \ else if (sizeof(type) == sizeof(long long)) \ for (i = 0; i < d_nelmts; i++) { \ - if (abs_fun(buf[i] - filval) < pow_fun(10.0F, (type)-D_val)) \ + if (abs_fun(buf[i] - filval) < pow_fun((type)10, (type)-D_val)) \ *(long long *)((void *)&buf[i]) = (long long)(((unsigned long long)1 << *minbits) - 1); \ else \ - *(long long *)((void *)&buf[i]) = llround_fun(buf[i] * pow_fun(10.0F, (type)D_val) - \ - min * pow_fun(10.0F, (type)D_val)); \ + *(long long *)((void *)&buf[i]) = llround_fun(buf[i] * pow_fun((type)10, (type)D_val) - \ + min * pow_fun((type)10, (type)D_val)); \ } \ else \ HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \ @@ -561,16 +561,16 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ { \ if (sizeof(type) == sizeof(int)) \ for (i = 0; i < d_nelmts; i++) \ - *(int *)((void *)&buf[i]) = (int)lround_fun(buf[i] * pow_fun(10.0F, (type)D_val) - \ - min * pow_fun(10.0F, (type)D_val)); \ + *(int *)((void *)&buf[i]) = (int)lround_fun(buf[i] * pow_fun((type)10, (type)D_val) - \ + min * pow_fun((type)10, (type)D_val)); \ else if (sizeof(type) == sizeof(long)) \ for (i = 0; i < d_nelmts; i++) \ - *(long *)((void *)&buf[i]) = \ - lround_fun(buf[i] * pow_fun(10.0F, (type)D_val) - min * pow_fun(10.0F, (type)D_val)); \ + *(long *)((void *)&buf[i]) = lround_fun(buf[i] * pow_fun((type)10, (type)D_val) - \ + min * pow_fun((type)10, (type)D_val)); \ else if (sizeof(type) == sizeof(long long)) \ for (i = 0; i < d_nelmts; i++) \ - *(long long *)((void *)&buf[i]) = \ - llround_fun(buf[i] * pow_fun(10.0F, (type)D_val) - min * pow_fun(10.0F, (type)D_val)); \ + *(long long *)((void *)&buf[i]) = llround_fun(buf[i] * pow_fun((type)10, (type)D_val) - \ + min * pow_fun((type)10, (type)D_val)); \ else \ HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \ } @@ -606,8 +606,8 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ H5Z_scaleoffset_get_filval_2(type, cd_values, filval) \ H5Z_scaleoffset_max_min_3(i, d_nelmts, buf, filval, max, min, D_val) \ H5Z_scaleoffset_check_3(i, type, pow_fun, round_fun, max, min, minbits, D_val) span = \ - (unsigned long long)(llround_fun(max * pow_fun(10.0F, (type)D_val) - \ - min * pow_fun(10.0F, (type)D_val)) + \ + (unsigned long long)(llround_fun(max * pow_fun((type)10, (type)D_val) - \ + min * pow_fun((type)10, (type)D_val)) + \ 1); \ *minbits = H5Z__scaleoffset_log2(span + 1); \ if (*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \ @@ -617,8 +617,8 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ else { /* fill value undefined */ \ H5Z_scaleoffset_max_min_2(i, d_nelmts, buf, max, min) \ H5Z_scaleoffset_check_3(i, type, pow_fun, round_fun, max, min, minbits, D_val) span = \ - (unsigned long long)(llround_fun(max * pow_fun(10.0F, (type)D_val) - \ - min * pow_fun(10.0F, (type)D_val)) + \ + (unsigned long long)(llround_fun(max * pow_fun((type)10, (type)D_val) - \ + min * pow_fun((type)10, (type)D_val)) + \ 1); \ *minbits = H5Z__scaleoffset_log2(span); \ if (*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \ @@ -685,19 +685,19 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ buf[i] = \ (type)((*(int *)((void *)&buf[i]) == (int)(((unsigned int)1 << minbits) - 1)) \ ? filval \ - : (type)(*(int *)((void *)&buf[i])) / pow_fun(10.0F, (type)D_val) + min); \ + : (type)(*(int *)((void *)&buf[i])) / pow_fun((type)10, (type)D_val) + min); \ else if (sizeof(type) == sizeof(long)) \ for (i = 0; i < d_nelmts; i++) \ buf[i] = \ (type)((*(long *)((void *)&buf[i]) == (long)(((unsigned long)1 << minbits) - 1)) \ ? filval \ - : (type)(*(long *)((void *)&buf[i])) / pow_fun(10.0F, (type)D_val) + min); \ + : (type)(*(long *)((void *)&buf[i])) / pow_fun((type)10, (type)D_val) + min); \ else if (sizeof(type) == sizeof(long long)) \ for (i = 0; i < d_nelmts; i++) \ buf[i] = (type)( \ (*(long long *)((void *)&buf[i]) == (long long)(((unsigned long long)1 << minbits) - 1)) \ ? filval \ - : (type)(*(long long *)((void *)&buf[i])) / pow_fun(10.0F, (type)D_val) + min); \ + : (type)(*(long long *)((void *)&buf[i])) / pow_fun((type)10, (type)D_val) + min); \ else \ HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \ } @@ -707,13 +707,13 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ { \ if (sizeof(type) == sizeof(int)) \ for (i = 0; i < d_nelmts; i++) \ - buf[i] = ((type)(*(int *)((void *)&buf[i])) / pow_fun(10.0F, (type)D_val) + min); \ + buf[i] = ((type)(*(int *)((void *)&buf[i])) / pow_fun((type)10, (type)D_val) + min); \ else if (sizeof(type) == sizeof(long)) \ for (i = 0; i < d_nelmts; i++) \ - buf[i] = ((type)(*(long *)((void *)&buf[i])) / pow_fun(10.0F, (type)D_val) + min); \ + buf[i] = ((type)(*(long *)((void *)&buf[i])) / pow_fun((type)10, (type)D_val) + min); \ else if (sizeof(type) == sizeof(long long)) \ for (i = 0; i < d_nelmts; i++) \ - buf[i] = ((type)(*(long long *)((void *)&buf[i])) / pow_fun(10.0F, (type)D_val) + min); \ + buf[i] = ((type)(*(long long *)((void *)&buf[i])) / pow_fun((type)10, (type)D_val) + min); \ else \ HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \ } diff --git a/src/H5Ztrans.c b/src/H5Ztrans.c index ab7e9be..ea532cf 100644 --- a/src/H5Ztrans.c +++ b/src/H5Ztrans.c @@ -150,7 +150,6 @@ static void H5Z__xform_reduce_tree(H5Z_node *tree); HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Unexpected type conversion operation") \ } -#if H5_SIZEOF_LONG_DOUBLE != 0 #if CHAR_MIN >= 0 #define H5Z_XFORM_TYPE_OP(RESL, RESR, TYPE, OP, SIZE) \ { \ @@ -212,65 +211,6 @@ static void H5Z__xform_reduce_tree(H5Z_node *tree); H5Z_XFORM_DO_OP1((RESL), (RESR), long double, OP, (SIZE)) \ } #endif /* CHAR_MIN >= 0 */ -#else -#if CHAR_MIN >= 0 -#define H5Z_XFORM_TYPE_OP(RESL, RESR, TYPE, OP, SIZE) \ - { \ - if ((TYPE) == H5T_NATIVE_CHAR) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), char, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_SCHAR) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), signed char, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_SHORT) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), short, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_USHORT) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), unsigned short, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_INT) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), int, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_UINT) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), unsigned int, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_LONG) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), long, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_ULONG) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), unsigned long, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_LLONG) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), long long, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_ULLONG) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), unsigned long long, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_FLOAT) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), float, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_DOUBLE) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), double, OP, (SIZE)) \ - } -#else /* CHAR_MIN >= 0 */ -#define H5Z_XFORM_TYPE_OP(RESL, RESR, TYPE, OP, SIZE) \ - { \ - if ((TYPE) == H5T_NATIVE_CHAR) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), char, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_UCHAR) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), unsigned char, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_SHORT) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), short, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_USHORT) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), unsigned short, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_INT) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), int, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_UINT) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), unsigned int, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_LONG) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), long, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_ULONG) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), unsigned long, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_LLONG) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), long long, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_ULLONG) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), unsigned long long, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_FLOAT) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), float, OP, (SIZE)) \ - else if ((TYPE) == H5T_NATIVE_DOUBLE) \ - H5Z_XFORM_DO_OP1((RESL), (RESR), double, OP, (SIZE)) \ - } -#endif /* CHAR_MIN >= 0 */ -#endif /*H5_SIZEOF_LONG_DOUBLE */ #define H5Z_XFORM_DO_OP3(OP) \ { \ @@ -1056,10 +996,8 @@ H5Z_xform_eval(H5Z_data_xform_t *data_xform_prop, void *array, size_t array_size H5Z_XFORM_DO_OP5(float, array_size) else if (array_type == H5T_NATIVE_DOUBLE) H5Z_XFORM_DO_OP5(double, array_size) -#if H5_SIZEOF_LONG_DOUBLE != 0 else if (array_type == H5T_NATIVE_LDOUBLE) H5Z_XFORM_DO_OP5(long double, array_size) -#endif } /* end if */ /* Otherwise, do the full data transform */ @@ -1267,11 +1205,9 @@ H5Z__xform_find_type(const H5T_t *type) /* Check for DOUBLE type */ else if ((tmp = (H5T_t *)H5I_object(H5T_NATIVE_DOUBLE)) && 0 == H5T_cmp(type, tmp, FALSE)) HGOTO_DONE(H5T_NATIVE_DOUBLE) -#if H5_SIZEOF_LONG_DOUBLE != 0 /* Check for LONGDOUBLE type */ else if ((tmp = (H5T_t *)H5I_object(H5T_NATIVE_LDOUBLE)) && 0 == H5T_cmp(type, tmp, FALSE)) HGOTO_DONE(H5T_NATIVE_LDOUBLE) -#endif else HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "could not find matching type") diff --git a/src/H5detect.c b/src/H5detect.c index e8cded7..022cb55 100644 --- a/src/H5detect.c +++ b/src/H5detect.c @@ -1412,7 +1412,7 @@ detect_C99_floats(void) */ DETECT_F(double, LDOUBLE, d_g[nd_g]); nd_g++; -#elif H5_SIZEOF_LONG_DOUBLE != 0 +#else DETECT_F(long double, LDOUBLE, d_g[nd_g]); nd_g++; #endif diff --git a/src/H5private.h b/src/H5private.h index 7c76483..761ad17 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -259,10 +259,10 @@ #define H5_ATTR_NORETURN __attribute__((noreturn)) #define H5_ATTR_CONST __attribute__((const)) #define H5_ATTR_PURE __attribute__((pure)) -#if defined(__GNUC__) && __GNUC__ >= 7 && !defined(__INTEL_COMPILER) +#if defined(__clang__) || defined(__GNUC__) && __GNUC__ >= 7 && !defined(__INTEL_COMPILER) #define H5_ATTR_FALLTHROUGH __attribute__((fallthrough)); #else -#define H5_ATTR_FALLTHROUGH /*void*/ +#define H5_ATTR_FALLTHROUGH /* FALLTHROUGH */ #endif #else #define H5_ATTR_FORMAT(X, Y, Z) /*void*/ diff --git a/src/H5public.h b/src/H5public.h index 65709c6..163deed 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -83,15 +83,15 @@ /** * For tweaks, bug-fixes, or development */ -#define H5_VERS_RELEASE 0 +#define H5_VERS_RELEASE 1 /** * For pre-releases like \c snap0. Empty string for official releases. */ -#define H5_VERS_SUBRELEASE "" +#define H5_VERS_SUBRELEASE "1" /** * Full version string */ -#define H5_VERS_INFO "HDF5 library version: 1.13.0" +#define H5_VERS_INFO "HDF5 library version: 1.13.1-1" #define H5check() H5check_version(H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE) diff --git a/src/H5trace.c b/src/H5trace.c index 3a5d420..8daa17d 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -1563,10 +1563,8 @@ H5_trace_args(H5RS_str_t *rs, const char *type, va_list ap) H5RS_acat(rs, "H5T_NATIVE_FLOAT"); else if (obj == H5T_NATIVE_DOUBLE_g) H5RS_acat(rs, "H5T_NATIVE_DOUBLE"); -#if H5_SIZEOF_LONG_DOUBLE != 0 else if (obj == H5T_NATIVE_LDOUBLE_g) H5RS_acat(rs, "H5T_NATIVE_LDOUBLE"); -#endif else if (obj == H5T_IEEE_F32BE_g) H5RS_acat(rs, "H5T_IEEE_F32BE"); else if (obj == H5T_IEEE_F32LE_g) diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in index 44c1540..7fe1a36 100644 --- a/src/libhdf5.settings.in +++ b/src/libhdf5.settings.in @@ -67,28 +67,29 @@ Languages: Features: --------- - Parallel HDF5: @PARALLEL@ -Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ - Large Parallel I/O: @LARGE_PARALLEL_IO@ - High-level library: @HDF5_HL@ - Build HDF5 Tests: @HDF5_TESTS@ - Build HDF5 Tools: @HDF5_TOOLS@ - Threadsafety: @THREADSAFE@ - Default API mapping: @DEFAULT_API_VERSION@ - With deprecated public symbols: @DEPRECATED_SYMBOLS@ - I/O filters (external): @EXTERNAL_FILTERS@ - MPE: @MPE@ - Map (H5M) API: @MAP_API@ - Direct VFD: @DIRECT_VFD@ - Mirror VFD: @MIRROR_VFD@ - (Read-Only) S3 VFD: @ROS3_VFD@ - (Read-Only) HDFS VFD: @HAVE_LIBHDFS@ - dmalloc: @HAVE_DMALLOC@ - Packages w/ extra debug output: @INTERNAL_DEBUG_OUTPUT@ - API tracing: @TRACE_API@ - Using memory checker: @USINGMEMCHECKER@ - Memory allocation sanity checks: @MEMORYALLOCSANITYCHECK@ - Function stack tracing: @CODESTACK@ - Use file locking: @DESIRED_FILE_LOCKING@ - Strict file format checks: @STRICT_FORMAT_CHECKS@ - Optimization instrumentation: @INSTRUMENT_LIBRARY@ + Parallel HDF5: @PARALLEL@ + Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ + Large Parallel I/O: @LARGE_PARALLEL_IO@ + High-level library: @HDF5_HL@ +Dimension scales w/ new references: @DIMENSION_SCALES_WITH_NEW_REF@ + Build HDF5 Tests: @HDF5_TESTS@ + Build HDF5 Tools: @HDF5_TOOLS@ + Threadsafety: @THREADSAFE@ + Default API mapping: @DEFAULT_API_VERSION@ + With deprecated public symbols: @DEPRECATED_SYMBOLS@ + I/O filters (external): @EXTERNAL_FILTERS@ + MPE: @MPE@ + Map (H5M) API: @MAP_API@ + Direct VFD: @DIRECT_VFD@ + Mirror VFD: @MIRROR_VFD@ + (Read-Only) S3 VFD: @ROS3_VFD@ + (Read-Only) HDFS VFD: @HAVE_LIBHDFS@ + dmalloc: @HAVE_DMALLOC@ + Packages w/ extra debug output: @INTERNAL_DEBUG_OUTPUT@ + API tracing: @TRACE_API@ + Using memory checker: @USINGMEMCHECKER@ + Memory allocation sanity checks: @MEMORYALLOCSANITYCHECK@ + Function stack tracing: @CODESTACK@ + Use file locking: @DESIRED_FILE_LOCKING@ + Strict file format checks: @STRICT_FORMAT_CHECKS@ + Optimization instrumentation: @INSTRUMENT_LIBRARY@ diff --git a/src/uthash.h b/src/uthash.h index 8bdca55..ea99839 100644 --- a/src/uthash.h +++ b/src/uthash.h @@ -714,25 +714,35 @@ typedef unsigned char uint8_t; hashv += (unsigned)(keylen); \ switch (_hj_k) { \ case 11: \ - hashv += ((unsigned)_hj_key[10] << 24); /* FALLTHROUGH */ \ + hashv += ((unsigned)_hj_key[10] << 24); \ + H5_ATTR_FALLTHROUGH \ case 10: \ - hashv += ((unsigned)_hj_key[9] << 16); /* FALLTHROUGH */ \ + hashv += ((unsigned)_hj_key[9] << 16); \ + H5_ATTR_FALLTHROUGH \ case 9: \ - hashv += ((unsigned)_hj_key[8] << 8); /* FALLTHROUGH */ \ + hashv += ((unsigned)_hj_key[8] << 8); \ + H5_ATTR_FALLTHROUGH \ case 8: \ - _hj_j += ((unsigned)_hj_key[7] << 24); /* FALLTHROUGH */ \ + _hj_j += ((unsigned)_hj_key[7] << 24); \ + H5_ATTR_FALLTHROUGH \ case 7: \ - _hj_j += ((unsigned)_hj_key[6] << 16); /* FALLTHROUGH */ \ + _hj_j += ((unsigned)_hj_key[6] << 16); \ + H5_ATTR_FALLTHROUGH \ case 6: \ - _hj_j += ((unsigned)_hj_key[5] << 8); /* FALLTHROUGH */ \ + _hj_j += ((unsigned)_hj_key[5] << 8); \ + H5_ATTR_FALLTHROUGH \ case 5: \ - _hj_j += _hj_key[4]; /* FALLTHROUGH */ \ + _hj_j += _hj_key[4]; \ + H5_ATTR_FALLTHROUGH \ case 4: \ - _hj_i += ((unsigned)_hj_key[3] << 24); /* FALLTHROUGH */ \ + _hj_i += ((unsigned)_hj_key[3] << 24); \ + H5_ATTR_FALLTHROUGH \ case 3: \ - _hj_i += ((unsigned)_hj_key[2] << 16); /* FALLTHROUGH */ \ + _hj_i += ((unsigned)_hj_key[2] << 16); \ + H5_ATTR_FALLTHROUGH \ case 2: \ - _hj_i += ((unsigned)_hj_key[1] << 8); /* FALLTHROUGH */ \ + _hj_i += ((unsigned)_hj_key[1] << 8); \ + H5_ATTR_FALLTHROUGH \ case 1: \ _hj_i += _hj_key[0]; \ } \ diff --git a/test/cache_api.c b/test/cache_api.c index 7858c65..412888e 100644 --- a/test/cache_api.c +++ b/test/cache_api.c @@ -546,7 +546,7 @@ check_file_mdc_api_calls(unsigned paged, hid_t fcpl_id) /* size_t max_decrement = */ (1 * 1024 * 1024 - 1), /* int epochs_before_eviction = */ 4, /* hbool_t apply_empty_reserve = */ TRUE, - /* double empty_reserve = */ 0.05f, + /* double empty_reserve = */ 0.05, /* int dirty_bytes_threshold = */ (256 * 1024), /* int metadata_write_strategy = */ H5AC__DEFAULT_METADATA_WRITE_STRATEGY}; diff --git a/test/cork.c b/test/cork.c index 80934ad..28c0604 100644 --- a/test/cork.c +++ b/test/cork.c @@ -89,10 +89,11 @@ static unsigned verify_old_dset_cork(void) { /* Variable Declarations */ - hid_t fid = -1; /* File ID */ - hid_t did = -1, did2 = -1, did3 = -1; /* Dataset IDs */ - hid_t dcpl = -1, dcpl2 = -1, dcpl3 = -1; /* Dataset creation property lists */ - hid_t sid = -1, sid2 = -1, sid3 = -1; /* Dataspace IDs */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t did = H5I_INVALID_HID, did2 = H5I_INVALID_HID, did3 = H5I_INVALID_HID; /* Dataset IDs */ + hid_t dcpl = H5I_INVALID_HID, dcpl2 = H5I_INVALID_HID, + dcpl3 = H5I_INVALID_HID; /* Dataset creation property lists */ + hid_t sid = H5I_INVALID_HID, sid2 = H5I_INVALID_HID, sid3 = H5I_INVALID_HID; /* Dataspace IDs */ hsize_t dims[2] = {100, 20}; /* Dataset dimension sizes */ hsize_t max_dims[2] = {100, H5S_UNLIMITED}; /* Dataset maximum dimension sizes */ hsize_t chunk_dims[2] = {2, 5}; /* Dataset chunked dimension sizes */ @@ -295,19 +296,19 @@ static unsigned verify_obj_dset_cork(hbool_t swmr) { /* Variable Declarations */ - hid_t fid = -1; /* File ID */ - hid_t fapl = -1; /* File access property list */ - hid_t aid = -1; /* Attribute ID */ - hid_t sid = -1, sid2 = -1; /* Dataspace IDs */ - hid_t did = -1, did2 = -1; /* Dataset IDs */ - hid_t oid = -1; /* Object ID */ - hid_t dcpl2; /* Dataset creation property list */ - int i = 0; /* Local index variable */ - hsize_t dim[1] = {100}; /* Dataset dimension size */ - hsize_t chunk_dim[1] = {7}; /* Dataset chunk dimension size */ - H5O_info2_t oinfo, oinfo2; /* Object metadata information */ - char attrname[500]; /* Name of attribute */ - unsigned flags; /* File access flags */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + hid_t aid = H5I_INVALID_HID; /* Attribute ID */ + hid_t sid = H5I_INVALID_HID, sid2 = H5I_INVALID_HID; /* Dataspace IDs */ + hid_t did = H5I_INVALID_HID, did2 = H5I_INVALID_HID; /* Dataset IDs */ + hid_t oid = H5I_INVALID_HID; /* Object ID */ + hid_t dcpl2 = H5I_INVALID_HID; /* Dataset creation property list */ + int i = 0; /* Local index variable */ + hsize_t dim[1] = {100}; /* Dataset dimension size */ + hsize_t chunk_dim[1] = {7}; /* Dataset chunk dimension size */ + H5O_info2_t oinfo, oinfo2; /* Object metadata information */ + char attrname[500]; /* Name of attribute */ + unsigned flags; /* File access flags */ if (swmr) { TESTING("cork status for dataset objects with attributes (SWMR)"); @@ -500,11 +501,11 @@ static unsigned verify_dset_cork(hbool_t swmr, hbool_t new_format) { /* Variable Declarations */ - hid_t fid = -1; /* File ID */ - hid_t fapl = -1; /* File access property list */ - hid_t did = -1, did2 = -1, did3 = -1; /* Dataset IDs */ - hid_t dcpl = -1; /* Dataset creation property list */ - hid_t sid = -1, sid2 = -1, sid3 = -1; /* Dataspace IDs */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + hid_t did = H5I_INVALID_HID, did2 = H5I_INVALID_HID, did3 = H5I_INVALID_HID; /* Dataset IDs */ + hid_t dcpl = H5I_INVALID_HID; /* Dataset creation property list */ + hid_t sid = H5I_INVALID_HID, sid2 = H5I_INVALID_HID, sid3 = H5I_INVALID_HID; /* Dataspace IDs */ hsize_t dims[2] = {100, 20}; /* Dataset dimension sizes */ hsize_t max_dims[2] = {100, H5S_UNLIMITED}; /* Dataset maximum dimension sizes */ hsize_t chunk_dims[2] = {2, 5}; /* Dataset chunked dimension sizes */ @@ -758,15 +759,15 @@ static unsigned verify_group_cork(hbool_t swmr) { /* Variable Declarations */ - hid_t fid = -1; /* File ID */ - hid_t fapl = -1; /* File access property list */ - hid_t gid = -1, gid2 = -1, gid3 = -1; /* Group IDs */ - H5O_info2_t oinfo, oinfo2, oinfo3; /* Object metadata information */ - hid_t aid; /* Attribute ID */ - hid_t sid; /* Dataspace ID */ - char attrname[500]; /* Name of attribute */ - unsigned flags; /* File access flags */ - int i = 0; /* Local index variable */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + hid_t gid = H5I_INVALID_HID, gid2 = H5I_INVALID_HID, gid3 = H5I_INVALID_HID; /* Group IDs */ + H5O_info2_t oinfo, oinfo2, oinfo3; /* Object metadata information */ + hid_t aid = H5I_INVALID_HID; /* Attribute ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + char attrname[500]; /* Name of attribute */ + unsigned flags; /* File access flags */ + int i = 0; /* Local index variable */ /* Testing Macro */ if (swmr) { @@ -927,17 +928,17 @@ static unsigned verify_named_cork(hbool_t swmr) { /* Variable Declarations */ - hid_t fid = -1; /* File ID */ - hid_t fapl = -1; /* File access property list */ - hid_t tid = -1, tid2 = -1, tid3 = -1; /* Datatype IDs */ - hid_t gid = -1, gid2 = -1; /* Group IDs */ - H5O_info2_t oinfo, oinfo2, oinfo3, oinfo4; /* Object metadata information */ - hid_t aid = -1; /* Attribute ID */ - hid_t sid; /* Dataspace ID */ - hid_t did; /* Dataset ID */ - char attrname[500]; /* Name of attribute */ - unsigned flags; /* File access flags */ - int i = 0; /* Local index variable */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + hid_t tid = H5I_INVALID_HID, tid2 = H5I_INVALID_HID, tid3 = H5I_INVALID_HID; /* Datatype IDs */ + hid_t gid = H5I_INVALID_HID, gid2 = H5I_INVALID_HID; /* Group IDs */ + H5O_info2_t oinfo, oinfo2, oinfo3, oinfo4; /* Object metadata information */ + hid_t aid = H5I_INVALID_HID; /* Attribute ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t did = H5I_INVALID_HID; /* Dataset ID */ + char attrname[500]; /* Name of attribute */ + unsigned flags; /* File access flags */ + int i = 0; /* Local index variable */ /* Testing Macro */ if (swmr) { @@ -1204,20 +1205,20 @@ static unsigned verify_multiple_cork(hbool_t swmr) { /* Variable Declarations */ - hid_t fid1 = -1, fid2 = -1; /* File ID */ - hid_t fapl = -1; /* File access property list */ - hid_t tid1 = -1, tid2 = -1; /* Datatype IDs */ - hid_t gid1 = -1, gid2 = -1; /* Group IDs */ - hid_t did1 = -1, did2 = -1; /* Dataset ID */ - hid_t aidg1 = -1, aidg2 = -1; /* Attribute ID */ - hid_t aidd1 = -1, aidd2 = -1; /* Attribute ID */ - hid_t aidt1 = -1, aidt2 = -1; /* Attribute ID */ - hid_t sid = -1; /* Dataspace ID */ - H5O_info2_t oinfo1, oinfo2, oinfo3; /* Object metadata information */ - hsize_t dim[1] = {5}; /* Dimension sizes */ - unsigned flags; /* File access flags */ - hbool_t corked; /* Cork status */ - herr_t ret; /* Return value */ + hid_t fid1 = H5I_INVALID_HID, fid2 = H5I_INVALID_HID; /* File ID */ + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + hid_t tid1 = H5I_INVALID_HID, tid2 = H5I_INVALID_HID; /* Datatype IDs */ + hid_t gid1 = H5I_INVALID_HID, gid2 = H5I_INVALID_HID; /* Group IDs */ + hid_t did1 = H5I_INVALID_HID, did2 = H5I_INVALID_HID; /* Dataset ID */ + hid_t aidg1 = H5I_INVALID_HID, aidg2 = H5I_INVALID_HID; /* Attribute ID */ + hid_t aidd1 = H5I_INVALID_HID, aidd2 = H5I_INVALID_HID; /* Attribute ID */ + hid_t aidt1 = H5I_INVALID_HID, aidt2 = H5I_INVALID_HID; /* Attribute ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + H5O_info2_t oinfo1, oinfo2, oinfo3; /* Object metadata information */ + hsize_t dim[1] = {5}; /* Dimension sizes */ + unsigned flags; /* File access flags */ + hbool_t corked; /* Cork status */ + herr_t ret; /* Return value */ /* Testing Macro */ if (swmr) { @@ -1880,18 +1881,18 @@ error: static unsigned test_dset_cork(hbool_t swmr, hbool_t new_format) { - hid_t fid; /* File ID */ - hid_t fapl; /* File access property list */ - hid_t gid; /* Groupd ID */ - hid_t did1, did2; /* Dataset IDs */ - hid_t tid1, tid2; /* Datatype IDs */ - hid_t sid; /* Dataspace ID */ - hid_t dcpl; /* Dataset creation property list */ - hsize_t dims[RANK]; /* Dataset dimensions */ - hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* Maximum dataset dimensions */ - hsize_t cdims[RANK] = {2, 2}; /* Chunk dimensions */ - int fillval = 0; /* Fill value */ - int i, j, k = 0; /* Local index variables */ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t fapl; /* File access property list */ + hid_t gid = H5I_INVALID_HID; /* Groupd ID */ + hid_t did1 = H5I_INVALID_HID, did2 = H5I_INVALID_HID; /* Dataset IDs */ + hid_t tid1 = H5I_INVALID_HID, tid2 = H5I_INVALID_HID; /* Datatype IDs */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dcpl = H5I_INVALID_HID; /* Dataset creation property list */ + hsize_t dims[RANK]; /* Dataset dimensions */ + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* Maximum dataset dimensions */ + hsize_t cdims[RANK] = {2, 2}; /* Chunk dimensions */ + int fillval = 0; /* Fill value */ + int i, j, k = 0; /* Local index variables */ int ** wbuf = NULL; /* Data buffer for writes (pointers to fake 2D array) */ int * wbuf_data = NULL; /* Data buffer for writes (real data) */ int * rbuf_data = NULL; /* Data buffer for reads (real data) */ diff --git a/test/dsets.c b/test/dsets.c index 922f370..e2d281a 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -1399,7 +1399,7 @@ test_conv_buffer(hid_t fid) cf->b[j] = 100.0F * (float)(j + 1) + 0.01F * (float)j; for (j = 0; j < DIM3; j++) - cf->c[j] = 100.0F * (float)(j + 1) + 0.02F * (float)j; + cf->c[j] = 100.0 * (double)(j + 1) + 0.02 * (double)j; /* Create data space */ if ((space = H5Screate(H5S_SCALAR)) < 0) @@ -3442,11 +3442,11 @@ test_nbit_double(hid_t file) /* orig_data[] are initialized to be within the range that can be represented by * dataset datatype (no precision loss during datatype conversion) */ - double orig_data[2][5] = {{(double)1.6081706885101836e+60L, -255.32099170994480f, - (double)1.2677579992621376e-61L, 64568.289448797700f, + double orig_data[2][5] = {{(double)1.6081706885101836e+60L, -255.32099170994480, + (double)1.2677579992621376e-61L, 64568.289448797700, (double)-1.0619721778839084e-75L}, - {(double)2.1499497833454840e+56L, 6.6562295504670740e-3f, -1.5747263393432150f, - 1.0711093225222612f, -9.8971679387636870e-1f}}; + {(double)2.1499497833454840e+56L, 6.6562295504670740e-3, -1.5747263393432150, + 1.0711093225222612, -9.8971679387636870e-1}}; double new_data[2][5]; size_t precision, offset; size_t i, j; @@ -5201,7 +5201,7 @@ test_scaleoffset_float(hid_t file) /* Check that the values read are the same as the values written */ for (i = 0; i < (size_t)size[0]; i++) { for (j = 0; j < (size_t)size[1]; j++) { - if (HDfabs(new_data[i][j] - orig_data[i][j]) > HDpow(10.0, -3.0)) { + if (HDfabs((double)(new_data[i][j] - orig_data[i][j])) > HDpow(10.0, -3.0)) { H5_FAILED(); HDprintf(" Read different values than written.\n"); HDprintf(" At index %lu,%lu\n", (unsigned long)i, (unsigned long)j); @@ -5347,7 +5347,7 @@ test_scaleoffset_float_2(hid_t file) /* Check that the values read are the same as the values written */ for (j = 0; j < (size_t)size[1]; j++) { - if (HDfabs(new_data[0][j] - orig_data[0][j]) > HDpow(10.0, -3.0)) { + if (HDfabs((double)(new_data[0][j] - orig_data[0][j])) > HDpow(10.0, -3.0)) { H5_FAILED(); HDprintf(" Read different values than written.\n"); HDprintf(" At index %lu,%lu\n", (unsigned long)0, (unsigned long)j); @@ -5435,7 +5435,7 @@ test_scaleoffset_double(hid_t file) /* Initialize data */ for (i = 0; i < (size_t)size[0]; i++) for (j = 0; j < (size_t)size[1]; j++) { - orig_data[i][j] = (float)(HDrandom() % 10000000) / 10000000.0F; + orig_data[i][j] = (HDrandom() % 10000000) / 10000000.0; /* even-numbered values are negtive */ if ((i * size[1] + j + 1) % 2 == 0) @@ -5544,7 +5544,7 @@ test_scaleoffset_double_2(hid_t file) goto error; /* Set fill value */ - fillval = 10000.0F; + fillval = 10000.0; if (H5Pset_fill_value(dc, H5T_NATIVE_DOUBLE, &fillval) < 0) goto error; @@ -5581,7 +5581,7 @@ test_scaleoffset_double_2(hid_t file) /* Initialize data of hyperslab */ for (j = 0; j < (size_t)size[1]; j++) { - orig_data[0][j] = (float)(HDrandom() % 10000000) / 10000000.0F; + orig_data[0][j] = (HDrandom() % 10000000) / 10000000.0; /* even-numbered values are negtive */ if ((j + 1) % 2 == 0) @@ -5614,7 +5614,7 @@ test_scaleoffset_double_2(hid_t file) /* Check that the values read are the same as the values written */ for (j = 0; j < (size_t)size[1]; j++) { - if (HDfabs(new_data[0][j] - orig_data[0][j]) > HDpow(10.0, -7.0)) { + if (HDfabs((double)(new_data[0][j] - orig_data[0][j])) > HDpow(10.0, -7.0)) { H5_FAILED(); HDprintf(" Read different values than written.\n"); HDprintf(" At index %lu,%lu\n", (unsigned long)0, (unsigned long)j); @@ -6525,7 +6525,7 @@ test_set_local(hid_t fapl) h5_fixname(FILENAME[5], fapl, filename, sizeof filename); /* Initialize the integer & floating-point dataset */ - n = 1.0F; + n = 1.0; for (i = 0; i < DSET_DIM1; i++) for (j = 0; j < DSET_DIM2; j++) { points[i][j] = (int)n++; @@ -9085,7 +9085,7 @@ test_big_chunks_bypass_cache(hid_t fapl) /* Define cache size to be smaller than chunk size */ rdcc_nelmts = BYPASS_CHUNK_DIM / 5; rdcc_nbytes = sizeof(int) * BYPASS_CHUNK_DIM / 5; - if (H5Pset_cache(fapl_local, 0, rdcc_nelmts, rdcc_nbytes, 0.0F) < 0) + if (H5Pset_cache(fapl_local, 0, rdcc_nelmts, rdcc_nbytes, 0.0) < 0) FAIL_STACK_ERROR /* Create file */ diff --git a/test/dt_arith.c b/test/dt_arith.c index 91e31d5..7e96d29 100644 --- a/test/dt_arith.c +++ b/test/dt_arith.c @@ -68,9 +68,7 @@ typedef enum dtype_t { INT_ULLONG, FLT_FLOAT, FLT_DOUBLE, -#if H5_SIZEOF_LONG_DOUBLE != 0 FLT_LDOUBLE, -#endif OTHER } dtype_t; @@ -457,10 +455,8 @@ reset_hdf5(void) SET_ALIGNMENT(ULLONG, H5_SIZEOF_LONG_LONG); SET_ALIGNMENT(FLOAT, H5_SIZEOF_FLOAT); SET_ALIGNMENT(DOUBLE, H5_SIZEOF_DOUBLE); -#if H5_SIZEOF_LONG_DOUBLE != 0 SET_ALIGNMENT(LDOUBLE, H5_SIZEOF_LONG_DOUBLE); #endif -#endif } /*------------------------------------------------------------------------- @@ -2704,10 +2700,10 @@ my_isnan(dtype_t type, void *val) double x = 0.0; HDmemcpy(&x, val, sizeof(double)); retval = (x != x); -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else if (FLT_LDOUBLE == type) { - long double x = 0.0; + long double x = 0.0L; HDmemcpy(&x, val, sizeof(long double)); retval = (x != x); #endif @@ -2732,10 +2728,10 @@ my_isnan(dtype_t type, void *val) HDmemcpy(&x, val, sizeof(double)); HDsnprintf(s, sizeof(s), "%g", x); -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else if (FLT_LDOUBLE == type) { - long double x = 0.0; + long double x = 0.0L; HDmemcpy(&x, val, sizeof(long double)); HDsnprintf(s, sizeof(s), "%Lg", x); @@ -2898,7 +2894,7 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) else if (H5Tequal(src, H5T_NATIVE_DOUBLE)) { src_type_name = "double"; src_type = FLT_DOUBLE; -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else if (H5Tequal(src, H5T_NATIVE_LDOUBLE)) { src_type_name = "long double"; @@ -2917,7 +2913,7 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) else if (H5Tequal(dst, H5T_NATIVE_DOUBLE)) { dst_type_name = "double"; dst_type = FLT_DOUBLE; -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else if (H5Tequal(dst, H5T_NATIVE_LDOUBLE)) { dst_type_name = "long double"; @@ -3000,7 +2996,7 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) else if (src_type == FLT_DOUBLE) { INIT_FP_NORM(double, DBL_MAX, DBL_MIN, DBL_MAX_10_EXP, DBL_MIN_10_EXP, src_size, dst_size, buf, saved, nelmts); -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else if (src_type == FLT_LDOUBLE) { INIT_FP_NORM(long double, LDBL_MAX, LDBL_MIN, LDBL_MAX_10_EXP, LDBL_MIN_10_EXP, src_size, @@ -3019,7 +3015,7 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) else if (src_type == FLT_DOUBLE) { INIT_FP_DENORM(double, DBL_MANT_DIG, src_size, src_nbits, sendian, dst_size, buf, saved, nelmts); -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else if (src_type == FLT_LDOUBLE) { INIT_FP_DENORM(long double, LDBL_MANT_DIG, src_size, src_nbits, sendian, dst_size, buf, saved, @@ -3037,7 +3033,7 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) } else if (src_type == FLT_DOUBLE) { INIT_FP_SPECIAL(src_size, src_nbits, sendian, DBL_MANT_DIG, dst_size, buf, saved, nelmts); -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else if (src_type == FLT_LDOUBLE) { INIT_FP_SPECIAL(src_size, src_nbits, sendian, LDBL_MANT_DIG, dst_size, buf, saved, nelmts); @@ -3059,7 +3055,7 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) for (j = 0; j < nelmts; j++) { underflow = 0; hw_f = 911.0F; - hw_d = 911.0F; + hw_d = 911.0; #if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE hw_ld = 911.0L; #endif @@ -3108,14 +3104,14 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) if (FLT_FLOAT == dst_type) { hw_f = (float)*((long double *)aligned); hw = (unsigned char *)&hw_f; - underflow = HDfabsl(*((long double *)aligned)) < FLT_MIN; - overflow = HDfabsl(*((long double *)aligned)) > FLT_MAX; + underflow = HDfabsl(*((long double *)aligned)) < (long double)FLT_MIN; + overflow = HDfabsl(*((long double *)aligned)) > (long double)FLT_MAX; } else if (FLT_DOUBLE == dst_type) { hw_d = (double)*((long double *)aligned); hw = (unsigned char *)&hw_d; - underflow = HDfabsl(*((long double *)aligned)) < DBL_MIN; - overflow = HDfabsl(*((long double *)aligned)) > DBL_MAX; + underflow = HDfabsl(*((long double *)aligned)) < (long double)DBL_MIN; + overflow = HDfabsl(*((long double *)aligned)) > (long double)DBL_MAX; } else { hw_ld = *((long double *)aligned); @@ -3133,7 +3129,6 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) * last few unused bytes may have garbage in them. Clean them out with * 0s before compare the values. */ -#if H5_SIZEOF_LONG_DOUBLE != 0 if (sendian == H5T_ORDER_LE && dst_type == FLT_LDOUBLE) { size_t q; @@ -3142,7 +3137,6 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) hw[q] = 0x00; } } -#endif /* Are the two results the same? */ for (k = (dst_size - (dst_nbits / 8)); k < dst_size; k++) @@ -3163,7 +3157,7 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) else if (FLT_DOUBLE == dst_type && my_isnan(dst_type, buf + j * sizeof(double)) && my_isnan(dst_type, hw)) { continue; -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else if (FLT_LDOUBLE == dst_type && my_isnan(dst_type, buf + j * sizeof(long double)) && my_isnan(dst_type, hw)) { @@ -3204,8 +3198,8 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) if (overflow && my_isinf(dendian, buf + j * sizeof(float), dst_size, dst_mpos, dst_msize, dst_epos, dst_esize)) continue; /* all overflowed, no error */ - check_mant[0] = HDfrexpf(x, check_expo + 0); - check_mant[1] = HDfrexpf(hw_f, check_expo + 1); + check_mant[0] = (double)HDfrexpf(x, check_expo + 0); + check_mant[1] = (double)HDfrexpf(hw_f, check_expo + 1); } else if (FLT_DOUBLE == dst_type) { double x = 0.0; @@ -3217,10 +3211,10 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) continue; /* all overflowed, no error */ check_mant[0] = HDfrexp(x, check_expo + 0); check_mant[1] = HDfrexp(hw_d, check_expo + 1); -#if H5_SIZEOF_LONG_DOUBLE != 0 && (H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE) +#if (H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE) } else { - long double x = 0.0; + long double x = 0.0L; HDmemcpy(&x, &buf[j * dst_size], sizeof(long double)); /* dst is largest float, no need to check underflow. */ check_mant[0] = (double)HDfrexpl(x, check_expo + 0); @@ -3278,7 +3272,7 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) #if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else { - long double x = 0.0; + long double x = 0.0L; HDmemcpy(&x, &saved[j * src_size], sizeof(long double)); HDfprintf(stdout, " %29.20Le\n", x); #endif @@ -3300,7 +3294,7 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) #if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else { - long double x = 0.0; + long double x = 0.0L; HDmemcpy(&x, &buf[j * dst_size], sizeof(long double)); HDfprintf(stdout, " %29.20Le\n", x); #endif @@ -3506,7 +3500,7 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) else if (H5Tequal(src, H5T_NATIVE_DOUBLE)) { src_type_name = "double"; src_type = FLT_DOUBLE; -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else if (H5Tequal(src, H5T_NATIVE_LDOUBLE)) { src_type_name = "long double"; @@ -3566,7 +3560,7 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) else if (H5Tequal(dst, H5T_NATIVE_DOUBLE)) { dst_type_name = "double"; dst_type = FLT_DOUBLE; -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else if (H5Tequal(dst, H5T_NATIVE_LDOUBLE)) { dst_type_name = "long double"; @@ -3590,11 +3584,7 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) if ((INT_SCHAR == src_type || INT_UCHAR == src_type || INT_SHORT == src_type || INT_USHORT == src_type || INT_INT == src_type || INT_UINT == src_type || INT_LONG == src_type || INT_ULONG == src_type || INT_LLONG == src_type || INT_ULLONG == src_type) && - (FLT_FLOAT != dst_type && FLT_DOUBLE != dst_type -#if H5_SIZEOF_LONG_DOUBLE != 0 - && FLT_LDOUBLE != dst_type -#endif - )) { + (FLT_FLOAT != dst_type && FLT_DOUBLE != dst_type && FLT_LDOUBLE != dst_type)) { HDsnprintf(str, sizeof(str), "Testing %s %s -> %s conversions", name, src_type_name, dst_type_name); HDprintf("%-70s", str); H5_FAILED(); @@ -3602,11 +3592,7 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) goto error; } - if ((FLT_FLOAT == src_type || FLT_DOUBLE == src_type -#if H5_SIZEOF_LONG_DOUBLE != 0 - || FLT_LDOUBLE == src_type -#endif - ) && + if ((FLT_FLOAT == src_type || FLT_DOUBLE == src_type || FLT_LDOUBLE == src_type) && (INT_SCHAR != dst_type && INT_UCHAR != dst_type && INT_SHORT != dst_type && INT_USHORT != dst_type && INT_INT != dst_type && INT_UINT != dst_type && INT_LONG != dst_type && INT_ULONG != dst_type && INT_LLONG != dst_type && INT_ULLONG != dst_type)) { @@ -3738,7 +3724,7 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) else { INIT_FP_SPECIAL(src_size, src_nbits, sendian, DBL_MANT_DIG, dst_size, buf, saved, nelmts); } -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE } else if (src_type == FLT_LDOUBLE) { if (run_test == TEST_NORMAL) { @@ -3763,11 +3749,7 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) /* Check the results from the library against hardware */ for (j = 0; j < nelmts; j++) { - if (FLT_FLOAT == src_type || FLT_DOUBLE == src_type -#if H5_SIZEOF_LONG_DOUBLE != 0 - || FLT_LDOUBLE == src_type -#endif - ) + if (FLT_FLOAT == src_type || FLT_DOUBLE == src_type || FLT_LDOUBLE == src_type) if (my_isnan(src_type, saved + j * src_size)) continue; @@ -3874,7 +3856,6 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDassert(0 && "Unknown type"); break; } -#if H5_SIZEOF_LONG_DOUBLE != 0 } else if (FLT_LDOUBLE == dst_type) { hw = (unsigned char *)&hw_ldouble; @@ -3927,7 +3908,6 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDassert(0 && "Unknown type"); break; } -#endif } else if (INT_SCHAR == dst_type) { hw = (unsigned char *)&hw_schar; @@ -3940,12 +3920,10 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDmemcpy(aligned, saved + j * sizeof(double), sizeof(double)); hw_schar = (signed char)(*((double *)aligned)); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDmemcpy(aligned, saved + j * sizeof(long double), sizeof(long double)); hw_schar = (signed char)(*((long double *)aligned)); break; -#endif case INT_SCHAR: case INT_UCHAR: case INT_SHORT: @@ -3973,12 +3951,10 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDmemcpy(aligned, saved + j * sizeof(double), sizeof(double)); hw_uchar = (unsigned char)(*((double *)aligned)); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDmemcpy(aligned, saved + j * sizeof(long double), sizeof(long double)); hw_uchar = (unsigned char)(*((long double *)aligned)); break; -#endif case INT_SCHAR: case INT_UCHAR: case INT_SHORT: @@ -4006,12 +3982,10 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDmemcpy(aligned, saved + j * sizeof(double), sizeof(double)); hw_short = (short)(*((double *)aligned)); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDmemcpy(aligned, saved + j * sizeof(long double), sizeof(long double)); hw_short = (short)(*((long double *)aligned)); break; -#endif case INT_SCHAR: case INT_UCHAR: case INT_SHORT: @@ -4039,12 +4013,10 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDmemcpy(aligned, saved + j * sizeof(double), sizeof(double)); hw_ushort = (unsigned short)(*((double *)aligned)); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDmemcpy(aligned, saved + j * sizeof(long double), sizeof(long double)); hw_ushort = (unsigned short)(*((long double *)aligned)); break; -#endif case INT_SCHAR: case INT_UCHAR: case INT_SHORT: @@ -4072,12 +4044,10 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDmemcpy(aligned, saved + j * sizeof(double), sizeof(double)); hw_int = (int)(*((double *)aligned)); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDmemcpy(aligned, saved + j * sizeof(long double), sizeof(long double)); hw_int = (int)(*((long double *)aligned)); break; -#endif case INT_SCHAR: case INT_UCHAR: case INT_SHORT: @@ -4105,12 +4075,10 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDmemcpy(aligned, saved + j * sizeof(double), sizeof(double)); hw_uint = (unsigned int)(*((double *)aligned)); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDmemcpy(aligned, saved + j * sizeof(long double), sizeof(long double)); hw_uint = (unsigned int)(*((long double *)aligned)); break; -#endif case INT_SCHAR: case INT_UCHAR: case INT_SHORT: @@ -4138,12 +4106,10 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDmemcpy(aligned, saved + j * sizeof(double), sizeof(double)); hw_long = (long)(*((double *)aligned)); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDmemcpy(aligned, saved + j * sizeof(long double), sizeof(long double)); hw_long = (long)(*((long double *)aligned)); break; -#endif case INT_SCHAR: case INT_UCHAR: case INT_SHORT: @@ -4171,12 +4137,10 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDmemcpy(aligned, saved + j * sizeof(double), sizeof(double)); hw_ulong = (unsigned long)(*((double *)aligned)); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDmemcpy(aligned, saved + j * sizeof(long double), sizeof(long double)); hw_ulong = (unsigned long)(*((long double *)aligned)); break; -#endif case INT_SCHAR: case INT_UCHAR: case INT_SHORT: @@ -4204,12 +4168,10 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDmemcpy(aligned, saved + j * sizeof(double), sizeof(double)); hw_llong = (long long)(*((double *)aligned)); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDmemcpy(aligned, saved + j * sizeof(long double), sizeof(long double)); hw_llong = (long long)(*((long double *)aligned)); break; -#endif case INT_SCHAR: case INT_UCHAR: case INT_SHORT: @@ -4237,12 +4199,10 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDmemcpy(aligned, saved + j * sizeof(double), sizeof(double)); hw_ullong = (unsigned long long)(*((double *)aligned)); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDmemcpy(aligned, saved + j * sizeof(long double), sizeof(long double)); hw_ullong = (unsigned long long)(*((long double *)aligned)); break; -#endif case INT_SCHAR: case INT_UCHAR: case INT_SHORT: @@ -4269,14 +4229,12 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) * bytes may have garbage in them. Clean them out with 0s before compare * the values. */ -#if H5_SIZEOF_LONG_DOUBLE != 0 if (dendian == H5T_ORDER_LE && dst_type == FLT_LDOUBLE) { size_t q; for (q = dst_nbits / 8; q < dst_size; q++) buf[j * dst_size + q] = 0x00; } -#endif /* Are the two results the same? */ for (k = (dst_size - (dst_nbits / 8)); k < dst_size; k++) @@ -4315,11 +4273,7 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) * Try to follow the except_func callback function to check if the * desired value was set. */ - if ((FLT_FLOAT == src_type || FLT_DOUBLE == src_type -#if H5_SIZEOF_LONG_DOUBLE != 0 - || FLT_LDOUBLE == src_type -#endif - ) && + if ((FLT_FLOAT == src_type || FLT_DOUBLE == src_type || FLT_LDOUBLE == src_type) && (INT_SCHAR == dst_type || INT_SHORT == dst_type || INT_INT == dst_type || INT_LONG == dst_type || INT_LLONG == dst_type)) { if (0 == H5T__bit_get_d(src_bits, src_nbits - 1, (size_t)1) && @@ -4359,11 +4313,7 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) } } - if ((FLT_FLOAT == src_type || FLT_DOUBLE == src_type -#if H5_SIZEOF_LONG_DOUBLE != 0 - || FLT_LDOUBLE == src_type -#endif - ) && + if ((FLT_FLOAT == src_type || FLT_DOUBLE == src_type || FLT_LDOUBLE == src_type) && (INT_UCHAR == dst_type || INT_USHORT == dst_type || INT_UINT == dst_type || INT_ULONG == dst_type || INT_ULLONG == dst_type)) { if (H5T__bit_get_d(src_bits, src_nbits - 1, (size_t)1)) { @@ -4462,12 +4412,10 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDmemcpy(aligned, saved + j * sizeof(double), sizeof(double)); HDprintf(" %29f\n", *((double *)aligned)); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDmemcpy(aligned, saved + j * sizeof(long double), sizeof(long double)); HDprintf(" %29Lf\n", *((long double *)aligned)); break; -#endif case OTHER: default: HDassert(0 && "Unknown type"); @@ -4527,12 +4475,10 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) HDmemcpy(aligned, buf + j * sizeof(double), sizeof(double)); HDprintf(" %29f\n", *((double *)aligned)); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDmemcpy(aligned, buf + j * sizeof(long double), sizeof(long double)); HDprintf(" %29Lf\n", *((long double *)aligned)); break; -#endif case OTHER: default: HDassert(0 && "Unknown type"); @@ -4580,11 +4526,9 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) case FLT_DOUBLE: HDprintf(" %29f\n", *((double *)((void *)hw))); break; -#if H5_SIZEOF_LONG_DOUBLE != 0 case FLT_LDOUBLE: HDprintf(" %29Lf\n", *((long double *)((void *)hw))); break; -#endif case OTHER: default: HDassert(0 && "Unknown type"); @@ -4923,16 +4867,14 @@ run_fp_tests(const char *name) if (!HDstrcmp(name, "noop")) { nerrors += test_conv_flt_1("noop", TEST_NOOP, H5T_NATIVE_FLOAT, H5T_NATIVE_FLOAT); nerrors += test_conv_flt_1("noop", TEST_NOOP, H5T_NATIVE_DOUBLE, H5T_NATIVE_DOUBLE); -#if H5_SIZEOF_LONG_DOUBLE != 0 nerrors += test_conv_flt_1("noop", TEST_NOOP, H5T_NATIVE_LDOUBLE, H5T_NATIVE_LDOUBLE); -#endif goto done; } /*Test normalized values. TEST_NORMAL indicates normalized values.*/ nerrors += test_conv_flt_1(name, TEST_NORMAL, H5T_NATIVE_FLOAT, H5T_NATIVE_DOUBLE); nerrors += test_conv_flt_1(name, TEST_NORMAL, H5T_NATIVE_DOUBLE, H5T_NATIVE_FLOAT); -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE nerrors += test_conv_flt_1(name, TEST_NORMAL, H5T_NATIVE_FLOAT, H5T_NATIVE_LDOUBLE); nerrors += test_conv_flt_1(name, TEST_NORMAL, H5T_NATIVE_DOUBLE, H5T_NATIVE_LDOUBLE); nerrors += test_conv_flt_1(name, TEST_NORMAL, H5T_NATIVE_LDOUBLE, H5T_NATIVE_FLOAT); @@ -4942,7 +4884,7 @@ run_fp_tests(const char *name) /*Test denormalized values. TEST_DENORM indicates denormalized values.*/ nerrors += test_conv_flt_1(name, TEST_DENORM, H5T_NATIVE_FLOAT, H5T_NATIVE_DOUBLE); nerrors += test_conv_flt_1(name, TEST_DENORM, H5T_NATIVE_DOUBLE, H5T_NATIVE_FLOAT); -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE nerrors += test_conv_flt_1(name, TEST_DENORM, H5T_NATIVE_FLOAT, H5T_NATIVE_LDOUBLE); nerrors += test_conv_flt_1(name, TEST_DENORM, H5T_NATIVE_DOUBLE, H5T_NATIVE_LDOUBLE); #ifndef H5_DISABLE_SOME_LDOUBLE_CONV @@ -4955,11 +4897,7 @@ run_fp_tests(const char *name) "float"); HDprintf("%-70s", str); SKIPPED(); -#if H5_SIZEOF_LONG_DOUBLE != 0 HDputs(" Test skipped due to the conversion problem on IBM ppc64le cpu."); -#else - HDputs(" Test skipped due to disabled long double."); -#endif } #endif @@ -4969,7 +4907,7 @@ run_fp_tests(const char *name) /*Test special values, +/-0, +/-infinity, +/-QNaN, +/-SNaN.*/ nerrors += test_conv_flt_1(name, TEST_SPECIAL, H5T_NATIVE_FLOAT, H5T_NATIVE_DOUBLE); nerrors += test_conv_flt_1(name, TEST_SPECIAL, H5T_NATIVE_DOUBLE, H5T_NATIVE_FLOAT); -#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE nerrors += test_conv_flt_1(name, TEST_SPECIAL, H5T_NATIVE_FLOAT, H5T_NATIVE_LDOUBLE); nerrors += test_conv_flt_1(name, TEST_SPECIAL, H5T_NATIVE_DOUBLE, H5T_NATIVE_LDOUBLE); #ifndef H5_DISABLE_SOME_LDOUBLE_CONV @@ -4983,11 +4921,7 @@ run_fp_tests(const char *name) "float or double"); HDprintf("%-70s", str); SKIPPED(); -#if H5_SIZEOF_LONG_DOUBLE != 0 HDputs(" Test skipped due to the conversion problem on IBM ppc64le cpu."); -#else - HDputs(" Test skipped due to disabled long double."); -#endif } #endif #endif @@ -5068,11 +5002,7 @@ run_int_fp_conv(const char *name) "long double"); HDprintf("%-70s", str); SKIPPED(); -#if H5_SIZEOF_LONG_DOUBLE != 0 HDputs(" Test skipped due to the special algorithm of hardware conversion."); -#else - HDputs(" Test skipped due to disabled long double."); -#endif } #endif #endif /* H5_SIZEOF_LONG!=H5_SIZEOF_INT */ @@ -5193,14 +5123,10 @@ run_fp_int_conv(const char *name) "signed and unsigned char, short, int, long"); HDprintf("%-70s", str); SKIPPED(); -#if H5_SIZEOF_LONG_DOUBLE != 0 HDputs(" Test skipped due to the conversion problem on IBM ppc64le cpu."); -#else - HDputs(" Test skipped due to disabled long double."); -#endif #endif } -#if H5_SIZEOF_LONG != H5_SIZEOF_INT && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG != H5_SIZEOF_INT #ifndef H5_LDOUBLE_TO_LONG_SPECIAL if (test_values != TEST_SPECIAL && test_values != TEST_NORMAL) { nerrors += test_conv_int_fp(name, test_values, H5T_NATIVE_LDOUBLE, H5T_NATIVE_LONG); @@ -5220,35 +5146,27 @@ run_fp_int_conv(const char *name) "(unsigned) long"); HDprintf("%-70s", str); SKIPPED(); -#if H5_SIZEOF_LONG_DOUBLE != 0 HDputs(" Test skipped due to the special algorithm of hardware conversion."); -#else - HDputs(" Test skipped due to disabled long double."); -#endif } #endif -#endif /*H5_SIZEOF_LONG!=H5_SIZEOF_INT && H5_SIZEOF_LONG_DOUBLE!=0 */ +#endif /*H5_SIZEOF_LONG!=H5_SIZEOF_INT */ -#if H5_SIZEOF_LONG_LONG != H5_SIZEOF_LONG && H5_SIZEOF_LONG_DOUBLE != 0 +#if H5_SIZEOF_LONG_LONG != H5_SIZEOF_LONG #ifdef H5_LDOUBLE_TO_LLONG_ACCURATE nerrors += test_conv_int_fp(name, test_values, H5T_NATIVE_LDOUBLE, H5T_NATIVE_LLONG); -#else /*H5_LDOUBLE_TO_LLONG_ACCURATE*/ +#else /*H5_LDOUBLE_TO_LLONG_ACCURATE*/ { char str[256]; /*string */ HDsnprintf(str, sizeof(str), "Testing %s %s -> %s conversions", name, "long double", "long long"); HDprintf("%-70s", str); SKIPPED(); -#if H5_SIZEOF_LONG_DOUBLE != 0 HDputs(" Test skipped due to hardware conversion error."); -#else - HDputs(" Test skipped due to disabled long double."); -#endif } #endif /*H5_LDOUBLE_TO_LLONG_ACCURATE*/ #if defined(H5_LDOUBLE_TO_LLONG_ACCURATE) nerrors += test_conv_int_fp(name, test_values, H5T_NATIVE_LDOUBLE, H5T_NATIVE_ULLONG); -#else /*H5_LDOUBLE_TO_LLONG_ACCURATE*/ +#else /*H5_LDOUBLE_TO_LLONG_ACCURATE*/ { char str[256]; /*string */ @@ -5256,11 +5174,7 @@ run_fp_int_conv(const char *name) "unsigned long long"); HDprintf("%-70s", str); SKIPPED(); -#if H5_SIZEOF_LONG_DOUBLE != 0 HDputs(" Test skipped due to hardware conversion error."); -#else - HDputs(" Test skipped due to disabled long double."); -#endif } #endif /*H5_LDOUBLE_TO_LLONG_ACCURATE*/ #endif diff --git a/test/dtransform.c b/test/dtransform.c index 743103f..50c33ee 100644 --- a/test/dtransform.c +++ b/test/dtransform.c @@ -357,9 +357,7 @@ main(void) TEST_TYPE_CONTIG(dxpl_id_utrans_inv, unsigned long long, H5T_NATIVE_ULLONG, "ullong", transformData, 0); TEST_TYPE_CONTIG(dxpl_id_c_to_f, float, H5T_NATIVE_FLOAT, "float", windchillFfloat, 1); TEST_TYPE_CONTIG(dxpl_id_c_to_f, double, H5T_NATIVE_DOUBLE, "double", windchillFfloat, 1); -#if H5_SIZEOF_LONG_DOUBLE != 0 TEST_TYPE_CONTIG(dxpl_id_c_to_f, long double, H5T_NATIVE_LDOUBLE, "ldouble", windchillFfloat, 1); -#endif TEST_TYPE_CHUNK(dxpl_id_utrans_inv, char, H5T_NATIVE_CHAR, "char", transformData, 0); TEST_TYPE_CHUNK(dxpl_id_utrans_inv, unsigned char, H5T_NATIVE_UCHAR, "uchar", transformData, 0); @@ -374,9 +372,7 @@ main(void) TEST_TYPE_CHUNK(dxpl_id_utrans_inv, unsigned long long, H5T_NATIVE_ULLONG, "ullong", transformData, 0); TEST_TYPE_CHUNK(dxpl_id_c_to_f, float, H5T_NATIVE_FLOAT, "float", windchillFfloat, 1); TEST_TYPE_CHUNK(dxpl_id_c_to_f, double, H5T_NATIVE_DOUBLE, "double", windchillFfloat, 1); -#if H5_SIZEOF_LONG_DOUBLE != 0 TEST_TYPE_CHUNK(dxpl_id_c_to_f, long double, H5T_NATIVE_LDOUBLE, "ldouble", windchillFfloat, 1); -#endif if (test_copy(dxpl_id_c_to_f_copy, dxpl_id_polynomial_copy) < 0) TEST_ERROR; diff --git a/test/dtypes.c b/test/dtypes.c index 8b3101c..f9b4e8c 100644 --- a/test/dtypes.c +++ b/test/dtypes.c @@ -160,10 +160,8 @@ reset_hdf5(void) SET_ALIGNMENT(ULLONG, H5_SIZEOF_LONG_LONG); SET_ALIGNMENT(FLOAT, H5_SIZEOF_FLOAT); SET_ALIGNMENT(DOUBLE, H5_SIZEOF_DOUBLE); -#if H5_SIZEOF_LONG_DOUBLE != 0 SET_ALIGNMENT(LDOUBLE, H5_SIZEOF_LONG_DOUBLE); #endif -#endif } /*------------------------------------------------------------------------- diff --git a/test/filter_plugin.c b/test/filter_plugin.c index 276141a..c373b3b 100644 --- a/test/filter_plugin.c +++ b/test/filter_plugin.c @@ -1305,6 +1305,102 @@ error: } /* end test_path_api_calls() */ /*------------------------------------------------------------------------- + * Function: test_filter_numbers + * + * Purpose: Tests the filter numbers are handled correctly + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +test_filter_numbers(void) +{ + hid_t dcpl_id = H5I_INVALID_HID; + H5Z_filter_t id; + herr_t status = SUCCEED; + size_t nelmts = 0; + unsigned int flags; + unsigned int filter_config; + + HDputs("Testing filter number handling"); + + /* Check that out-of-range filter numbers are handled correctly */ + TESTING(" Filter # out of range"); + + /* Create property list */ + if ((dcpl_id = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + nelmts = 0; + + /* Test id > H5Z_FILTER_MAX and < 0, current version */ + + H5E_BEGIN_TRY + { + id = H5Z_FILTER_MAX + 1; + status = H5Pget_filter_by_id2(dcpl_id, id, &flags, &nelmts, NULL, 0, NULL, &filter_config); + } + H5E_END_TRY; + + /* Should fail */ + if (status != FAIL) + TEST_ERROR; + + H5E_BEGIN_TRY + { + id = -1; + status = H5Pget_filter_by_id2(dcpl_id, id, &flags, &nelmts, NULL, 0, NULL, &filter_config); + } + H5E_END_TRY; + + /* Should fail */ + if (status != FAIL) + TEST_ERROR; + + /* Test id > H5Z_FILTER_MAX and < 0, deprecated version */ + +#ifndef H5_NO_DEPRECATED_SYMBOLS + H5E_BEGIN_TRY + { + id = H5Z_FILTER_MAX + 1; + status = H5Pget_filter_by_id1(dcpl_id, id, &flags, &nelmts, NULL, 0, NULL); + } + H5E_END_TRY; + + /* Should fail */ + if (status != FAIL) + TEST_ERROR; + + H5E_BEGIN_TRY + { + id = -1; + status = H5Pget_filter_by_id1(dcpl_id, id, &flags, &nelmts, NULL, 0, NULL); + } + H5E_END_TRY; + + /* Should fail */ + if (status != FAIL) + TEST_ERROR; +#endif + + if (H5Pclose(dcpl_id) < 0) + TEST_ERROR; + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Pclose(dcpl_id); + } + H5E_END_TRY; + return FAIL; +} /* end test_filter_numbers() */ + +/*------------------------------------------------------------------------- * Function: disable_chunk_cache * * Purpose: Turns the chunk cache off @@ -1523,6 +1619,9 @@ main(void) /* Test the APIs for access to the filter plugin path table */ nerrors += (test_path_api_calls() < 0 ? 1 : 0); + /* Test filter numbers */ + nerrors += (test_filter_numbers() < 0 ? 1 : 0); + if (nerrors) TEST_ERROR; diff --git a/test/objcopy.c b/test/objcopy.c index e01083d..9aae621 100644 --- a/test/objcopy.c +++ b/test/objcopy.c @@ -11092,7 +11092,7 @@ test_copy_dataset_contig_cmpd_vl(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, buf[i].b.p = (int *)HDmalloc(buf[i].b.len * sizeof(int)); for (j = 0; j < buf[i].b.len; j++) ((int *)buf[i].b.p)[j] = (int)(i * 10 + j); - buf[i].c = 1.0F / ((float)i + 1.0F); + buf[i].c = 1.0 / ((double)i + 1.0); } /* end for */ /* Initialize the filenames */ @@ -11268,7 +11268,7 @@ test_copy_dataset_chunked_cmpd_vl(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl buf[i].b.p = (int *)HDmalloc(buf[i].b.len * sizeof(int)); for (j = 0; j < buf[i].b.len; j++) ((int *)buf[i].b.p)[j] = (int)(i * 10 + j); - buf[i].c = 1.0F / ((float)i + 1.0F); + buf[i].c = 1.0 / ((double)i + 1.0); } /* end for */ /* Initialize the filenames */ @@ -11453,7 +11453,7 @@ test_copy_dataset_compact_cmpd_vl(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl buf[i].b.p = (int *)HDmalloc(buf[i].b.len * sizeof(int)); for (j = 0; j < buf[i].b.len; j++) ((int *)buf[i].b.p)[j] = (int)(i * 10 + j); - buf[i].c = 1.0F / ((float)i + 1.0F); + buf[i].c = 1.0 / ((double)i + 1.0); } /* end for */ /* Initialize the filenames */ diff --git a/test/page_buffer.c b/test/page_buffer.c index 61e9696..c7bb965 100644 --- a/test/page_buffer.c +++ b/test/page_buffer.c @@ -97,11 +97,13 @@ error: static int swmr_fapl_augment(hid_t fapl, const char *filename, uint32_t max_lag) { - H5F_vfd_swmr_config_t config = {.version = H5F__CURR_VFD_SWMR_CONFIG_VERSION, - .tick_len = 4, - .max_lag = max_lag, - .writer = true, - .md_pages_reserved = 128}; + H5F_vfd_swmr_config_t config = {.version = H5F__CURR_VFD_SWMR_CONFIG_VERSION, + .tick_len = 4, + .max_lag = max_lag, + .writer = true, + .maintain_metadata_file = true, + .generate_updater_files = false, + .md_pages_reserved = 128}; char * bname = NULL; char * dname = NULL; diff --git a/test/swmr.c b/test/swmr.c index 24a0b7c..9091fe2 100644 --- a/test/swmr.c +++ b/test/swmr.c @@ -5220,7 +5220,7 @@ test_file_lock_swmr_concur(hid_t H5_ATTR_UNUSED in_fapl) static int test_file_lock_swmr_concur(hid_t in_fapl) { - hid_t fid; /* File ID */ + hid_t fid = H5I_INVALID_HID; /* File ID */ hid_t fapl; /* File access property list */ char filename[NAME_BUF_SIZE]; /* file name */ pid_t childpid = 0; /* Child process ID */ @@ -6721,7 +6721,7 @@ test_refresh_concur(hid_t H5_ATTR_UNUSED in_fapl, hbool_t new_format) static int test_refresh_concur(hid_t in_fapl, hbool_t new_format) { - hid_t fid; /* File ID */ + hid_t fid = H5I_INVALID_HID; /* File ID */ hid_t fapl; /* File access property list */ pid_t childpid = 0; /* Child process ID */ pid_t tmppid; /* Child process ID returned by waitpid */ diff --git a/test/tconfig.c b/test/tconfig.c index 101de9a..8bd625f 100644 --- a/test/tconfig.c +++ b/test/tconfig.c @@ -114,30 +114,14 @@ test_config_ctypes(void) vrfy_cint_type(long, unsigned long, H5_SIZEOF_LONG); vrfy_ctype(float, H5_SIZEOF_FLOAT); vrfy_ctype(double, H5_SIZEOF_DOUBLE); -#if H5_SIZEOF_LONG_DOUBLE > 0 vrfy_ctype(long double, H5_SIZEOF_LONG_DOUBLE); -#endif /* standard C99 basic types */ -#if H5_SIZEOF_LONG_LONG > 0 vrfy_cint_type(long long, unsigned long long, H5_SIZEOF_LONG_LONG); -#endif - -#if H5_SIZEOF_INT8_T > 0 vrfy_cint_type(int8_t, uint8_t, H5_SIZEOF_INT8_T); -#endif - -#if H5_SIZEOF_INT16_T > 0 vrfy_cint_type(int16_t, uint16_t, H5_SIZEOF_INT16_T); -#endif - -#if H5_SIZEOF_INT32_T > 0 vrfy_cint_type(int32_t, uint32_t, H5_SIZEOF_INT32_T); -#endif - -#if H5_SIZEOF_INT64_T > 0 vrfy_cint_type(int64_t, uint64_t, H5_SIZEOF_INT64_T); -#endif /* Some vendors have different sizes for the signed and unsigned */ /* fast8_t. Need to check them individually. */ diff --git a/test/testmeta.c b/test/testmeta.c index c59c6cb..7cb3000 100644 --- a/test/testmeta.c +++ b/test/testmeta.c @@ -195,7 +195,7 @@ main(void) start[0] = 0; status = H5Sselect_hyperslab(memspace_id, H5S_SELECT_SET, start, stride, count, NULL); - start[0] = (hssize_t)j; + start[0] = (hsize_t)j; status = H5Sselect_hyperslab(dataspace_id, H5S_SELECT_SET, start, stride, count, NULL); status = H5Dwrite(dataset_id, type_id, memspace_id, dataspace_id, H5P_DEFAULT, &floatval); if (status < 0) { diff --git a/test/tskiplist.c b/test/tskiplist.c index 4bf9b11..31b5cff 100644 --- a/test/tskiplist.c +++ b/test/tskiplist.c @@ -1165,213 +1165,6 @@ test_skiplist_free(void) /**************************************************************** ** -** test_skiplist_try_free_safe(): Test H5SL (skip list) code. -** Tests 'try_free_safe' operation in skip lists. -** -****************************************************************/ -/* Macro definitions */ -#define TEST_TFS_MAX_NOBJS 100 -#define TEST_TFS_MIN_NOBJS 5 -#define TEST_TFS_NITER 50 - -/* Structure to hold the list of objects */ -typedef struct { - H5SL_t * slist; /* Skiplist holding the objects */ - struct test_tfs_obj_t *list; /* Linear list of objects */ - int nobjs; /* Number of objects in list */ - int nobjs_rem; /* Number of objects in list that have not been freed */ -} test_tfs_list_t; - -/* Structure for an object */ -typedef struct test_tfs_obj_t { - int idx; /* Index (key) for this object */ - int nfrees; /* Number of times this object has been freed */ -} test_tfs_obj_t; - -/* op_data struct for H5SL_iterate() */ -typedef struct test_tfs_it_ud_t { - test_tfs_list_t *obj_list; /* List of objects */ - int last_idx; /* Index of last object visited in iteration */ - int ncalls; /* Number of times this callback was called */ -} test_tfs_it_ud_t; - -/* iterate callback */ -static herr_t -test_tfs_iter(void *_obj, void *key, void *_udata) -{ - test_tfs_obj_t * obj = (test_tfs_obj_t *)_obj; - test_tfs_it_ud_t *udata = (test_tfs_it_ud_t *)_udata; - - /* Check consistency */ - CHECK_PTR_EQ((void *)&obj->idx, key, "obj->idx"); - CHECK_PTR_EQ(obj, &udata->obj_list->list[obj->idx], "obj_list->list[obj->idx]"); - - /* Increment number of calls */ - udata->ncalls++; - - /* Verify we were given the correct object */ - do { - udata->last_idx++; - } while (udata->obj_list->list[udata->last_idx].nfrees != 0); - VERIFY(udata->last_idx, obj->idx, "H5SL_iterate"); - - return 0; -} /* end test_tfs_iter() */ - -/* try_free_safe callback */ -static htri_t -test_tfs_free(void *_obj, void *key, void *_obj_list) -{ - test_tfs_obj_t * obj = (test_tfs_obj_t *)_obj; - test_tfs_list_t *obj_list = (test_tfs_list_t *)_obj_list; - test_tfs_it_ud_t iter_ud; - int nrems, rem_idx, i, j; - test_tfs_obj_t * obj_ret; - herr_t ret; /* return value */ - htri_t ret_value; - - /* Check consistency */ - CHECK_PTR_EQ((void *)&obj->idx, key, "obj->idx"); - CHECK_PTR_EQ(obj, &obj_list->list[obj->idx], "obj_list->list[obj->idx]"); - - /* Mark this object as freed (to make sure it isn't recursively freed, that - * is not something we support, we will undo this if we decide later not to - * free the object) */ - obj->nfrees++; - obj_list->nobjs_rem--; - - /* Decide how many objects to remove */ - nrems = (int)(HDrandom() % (long)3); - - /* Remove objects */ - for (i = 0; i < nrems; i++) - /* Check nobjs_rem */ - if (obj_list->nobjs_rem > 0) { - /* Remove a random object from the list */ - rem_idx = (int)(HDrandom() % (long)obj_list->nobjs_rem); - - /* Scan the list, finding the rem_idx'th object that has not been - * freed */ - for (j = 0; j < obj_list->nobjs; j++) - if (obj_list->list[j].nfrees == 0) { - if (rem_idx == 0) - break; - else - rem_idx--; - } /* end if */ - if (j == obj_list->nobjs) - ERROR("invalid obj_list"); - else { - /* Remove the object */ - obj_ret = (test_tfs_obj_t *)H5SL_remove(obj_list->slist, &j); - CHECK_PTR(obj_ret, "H5SL_remove"); - obj_ret->nfrees++; - obj_list->nobjs_rem--; - } /* end else */ - } /* end if */ - - /* Mark this object as not freed so we know to expect it in the iterate call - */ - obj->nfrees--; - obj_list->nobjs_rem++; - - /* Iterate over skip list (maybe) */ - if (HDrandom() % (long)5) { - iter_ud.obj_list = obj_list; - iter_ud.last_idx = -1; - iter_ud.ncalls = 0; - ret = H5SL_iterate(obj_list->slist, test_tfs_iter, &iter_ud); - CHECK(ret, FAIL, "H5SL_iterate"); - VERIFY(iter_ud.ncalls, obj_list->nobjs_rem, "H5SL_iterate"); - } /* end if */ - - /* Verify nobjs_rem is non-negative */ - if (obj_list->nobjs_rem < 0) - ERROR("invalid nobjs_rem"); - - /* Decide whether this object should be freed */ - if (HDrandom() % (long)2) { - /* Free the object */ - ret_value = TRUE; - obj->nfrees++; - obj_list->nobjs_rem--; - } /* end if */ - else - /* Do not free the object */ - ret_value = FALSE; - - return ret_value; -} /* end test_tfs_free() */ - -/* Test function */ -static void -test_skiplist_try_free_safe(void) -{ - test_tfs_list_t obj_list; - test_tfs_obj_t list[TEST_TFS_MAX_NOBJS]; - int i, j; - int nobjs_found; - hsize_t count; - herr_t ret; /* Generic return value */ - - /* Output message about test being performed */ - MESSAGE(7, ("Testing Skip List 'Try Free Safe' Operation\n")); - - /* Create a skip list */ - obj_list.slist = H5SL_create(H5SL_TYPE_INT, NULL); - CHECK_PTR(obj_list.slist, "H5SL_create"); - - /* Init obj_list.list */ - obj_list.list = list; - for (j = 0; j < TEST_TFS_MAX_NOBJS; j++) - list[j].idx = j; - - for (i = 0; i < TEST_TFS_NITER; i++) { - /* Build object list */ - obj_list.nobjs = obj_list.nobjs_rem = - (int)(TEST_TFS_MIN_NOBJS + (HDrandom() % (long)(TEST_TFS_MAX_NOBJS - TEST_TFS_MIN_NOBJS + 1))); - for (j = 0; j < obj_list.nobjs; j++) { - list[j].nfrees = 0; - ret = H5SL_insert(obj_list.slist, &list[j], &list[j].idx); - CHECK(ret, FAIL, "H5SL_insert"); - } /* end for */ - - /* Call H5S_try_free_safe() - free most of the items in the skip list in - * a safe manner */ - ret = H5SL_try_free_safe(obj_list.slist, test_tfs_free, &obj_list); - CHECK(ret, FAIL, "H5SL_try_free_safe"); - - /* Verify list */ - nobjs_found = 0; - for (j = 0; j < obj_list.nobjs; j++) - if (list[j].nfrees == 0) - nobjs_found++; - else - VERIFY(list[j].nfrees, (long)1, "list[j].nfrees"); - - /* Verify number of objects */ - VERIFY(obj_list.nobjs_rem, nobjs_found, "obj_list.nobjs_rem"); - count = H5SL_count(obj_list.slist); - VERIFY(count, (size_t)nobjs_found, "H5SL_count"); - - /* Release the skip list, forcibly freeing all nodes (will not make - * callbacks) */ - ret = H5SL_release(obj_list.slist); - CHECK(ret, FAIL, "H5SL_release"); - - /* Verify number of objects is 0 */ - count = H5SL_count(obj_list.slist); - VERIFY(count, (size_t)0, "H5SL_count"); - } /* end for */ - - /* Close the skip list */ - ret = H5SL_close(obj_list.slist); - CHECK(ret, FAIL, "H5SL_close"); - -} /* end test_skiplist_try_free_safe() */ - -/**************************************************************** -** ** test_skiplist_less(): Test H5SL (skip list) code. ** Tests 'less' operation in skip lists. ** @@ -1796,7 +1589,6 @@ test_skiplist(void) test_skiplist_add(); /* Test 'add' operation */ test_skiplist_destroy(); /* Test 'destroy' operation */ test_skiplist_free(); /* Test 'free' operation */ - test_skiplist_try_free_safe(); /* Test 'try_free_safe' operation */ test_skiplist_less(); /* Test 'less' operation */ test_skiplist_greater(); /* Test 'greater' operation */ test_skiplist_below(); /* Test 'below' operation */ diff --git a/test/tunicode.c b/test/tunicode.c index 1b696ac..7d4dba6 100644 --- a/test/tunicode.c +++ b/test/tunicode.c @@ -34,7 +34,7 @@ #define RANK 1 #define COMP_INT_VAL 7 #define COMP_FLOAT_VAL (-42.0F) -#define COMP_DOUBLE_VAL 42.0F +#define COMP_DOUBLE_VAL 42.0 /* Test function prototypes */ void test_fl_string(hid_t fid, const char *string); diff --git a/test/tvlstr.c b/test/tvlstr.c index 7e47c7b..68f6124 100644 --- a/test/tvlstr.c +++ b/test/tvlstr.c @@ -293,7 +293,7 @@ test_vlstrings_special(void) /* Check data read in */ for (i = 0; i < SPACE1_DIM1; i++) if (rdata[i] != NULL) - TestErrPrintf("VL doesn't match!, rdata[%d]=%p\n", (int)i, rdata[i]); + TestErrPrintf("VL doesn't match!, rdata[%d]=%s\n", (int)i, rdata[i]); /* Write dataset to disk */ ret = H5Dwrite(dataset, tid1, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata); @@ -352,7 +352,7 @@ test_vlstrings_special(void) /* Check data read in */ for (i = 0; i < SPACE1_DIM1; i++) if (rdata[i] != NULL) - TestErrPrintf("VL doesn't match!, rdata[%d]=%p\n", (int)i, rdata[i]); + TestErrPrintf("VL doesn't match!, rdata[%d]=%s\n", (int)i, rdata[i]); /* Try to write nil strings to disk. */ ret = H5Dwrite(dataset, tid1, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata2); @@ -365,7 +365,7 @@ test_vlstrings_special(void) /* Check data read in */ for (i = 0; i < SPACE1_DIM1; i++) if (rdata[i] != NULL) - TestErrPrintf("VL doesn't match!, rdata[%d]=%p\n", (int)i, rdata[i]); + TestErrPrintf("VL doesn't match!, rdata[%d]=%s\n", (int)i, rdata[i]); /* Close Dataset */ ret = H5Dclose(dataset); diff --git a/test/vfd_swmr.c b/test/vfd_swmr.c index 3ed73c1..630b25d 100644 --- a/test/vfd_swmr.c +++ b/test/vfd_swmr.c @@ -41,6 +41,7 @@ #define FS_PAGE_SIZE 512 #define FILENAME "vfd_swmr_file.h5" #define MD_FILENAME "vfd_swmr_metadata_file" +#define UD_FILENAME "vfd_swmr_updater_file" #define FILENAME2 "vfd_swmr_file2.h5" #define MD_FILENAME2 "vfd_swmr_metadata_file2" @@ -48,61 +49,51 @@ #define FILENAME3 "vfd_swmr_file3.h5" #define MD_FILENAME3 "vfd_swmr_metadata_file3" +#define FILENAME4 "vfd_swmr_file4.h5" +#define MD_FILE "vfd_swmr_md_file" +#define UD_FILE "vfd_swmr_ud_file" + #define FNAME "non_vfd_swmr_file.h5" -/* test routines for VFD SWMR */ -static unsigned test_fapl(void); -static unsigned test_file_end_tick(void); -static unsigned test_file_fapl(void); -static unsigned test_writer_md(void); +#define FILE_NAME_LEN 1024 -/* helper routines */ -static hid_t init_vfd_swmr_config_fapl(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t max_lag, - hbool_t is_writer, uint32_t md_pages_reserved, - const char *md_file_path, size_t pbuf_size); +/* Defines used by verify_updater_flags() and verify_ud_chk() helper routine */ -/*------------------------------------------------------------------------- - * Function: init_vfd_swmr_config_fapl - * - * Purpose: Helper routine to initialize the fields for VFD SWMR configuration - * - * Return: void - * - *------------------------------------------------------------------------- - */ -static hid_t -init_vfd_swmr_config_fapl(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t max_lag, - hbool_t is_writer, uint32_t md_pages_reserved, const char *md_file_path, - size_t pbuf_size) -{ - hid_t fapl; +/* Offset of "flags" in updater file header */ +#define UD_HD_FLAGS_OFFSET 6 - HDmemset(config, 0, sizeof(*config)); +/* Offset of "sequence number" in updater file header */ +#define UD_HD_SEQ_NUM_OFFSET 12 - config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; - config->tick_len = tick_len; - config->max_lag = max_lag; - config->writer = is_writer; - config->md_pages_reserved = md_pages_reserved; - HDstrcpy(config->md_file_path, md_file_path); +/* Offset of "change list length" in updater file header */ +#define UD_HD_CHANGE_LIST_LEN_OFFSET 36 - /* Create a copy of the file access property list */ - if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) - return H5I_INVALID_HID; +/* Offset of "number of change list entries" in updater file change list header */ +#define UD_CL_NUM_CHANGE_LIST_ENTRIES_OFFSET H5F_UD_HEADER_SIZE + 44 - if (H5Pset_vfd_swmr_config(fapl, config) < 0) { - (void)H5Pclose(fapl); - return H5I_INVALID_HID; - } +/* Size of "sequence number", "tick number" and "change list length" fields in the updater file header */ +#define UD_SIZE_8 8 - /* Enable page buffering */ - if (pbuf_size != 0 && H5Pset_page_buffer_size(fapl, pbuf_size, 0, 0) < 0) { - (void)H5Pclose(fapl); - return H5I_INVALID_HID; - } +/* Size of checksum and "number of change list entries" field in the updater change list header */ +#define UD_SIZE_4 4 + +/* Size of "flags" field in the updater file header */ +#define UD_SIZE_2 2 + +/* test routines for VFD SWMR */ +static unsigned test_fapl(void); +static unsigned test_file_end_tick(void); +static unsigned test_file_fapl(void); +static unsigned test_writer_md(void); - return fapl; -} /* init_vfd_swmr_config_fapl() */ +static unsigned test_updater_flags(void); +static unsigned test_updater_flags_same_file_opens(void); +static herr_t verify_updater_flags(char *ud_name, uint16_t expected_flags); + +static void clean_chk_ud_files(char *md_file_path, char *updater_file_path); +static herr_t verify_ud_chk(char *md_file_path, char *ud_file_path); +static herr_t md_ck_cb(char *md_file_path, uint64_t tick_num); +static unsigned test_updater_generate_md_checksums(hbool_t file_create); /*------------------------------------------------------------------------- * Function: test_fapl() @@ -114,6 +105,12 @@ init_vfd_swmr_config_fapl(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint * --max_lag: should be >= 3 * --md_pages_reserved: should be >= 2 * --md_file_path: should contain the metadata file path (POSIX) + * --at least one of maintain_metadata_file and generate_updater_files + * must be true + * --if both the writer and maintain_metadata_file fields are true, + * then md_file_path field shouldn't be empty + * --if both the writer and generate_updater_files fields are true, + * then updater_file_path field shouldn't be empty * B) Verify that info set in the fapl is retrieved correctly. * * Return: 0 if test is sucessful @@ -195,9 +192,40 @@ test_fapl(void) if (ret >= 0) TEST_ERROR; + my_config->writer = TRUE; + /* Should fail: at least one of maintain_metadata_file and generate_updater_files must be true */ + H5E_BEGIN_TRY + { + ret = H5Pset_vfd_swmr_config(fapl, my_config); + } + H5E_END_TRY; + if (ret >= 0) + TEST_ERROR; + + my_config->writer = TRUE; + my_config->maintain_metadata_file = TRUE; + /* Should fail: empty md_file_path */ + H5E_BEGIN_TRY + { + ret = H5Pset_vfd_swmr_config(fapl, my_config); + } + H5E_END_TRY; + if (ret >= 0) + TEST_ERROR; + + my_config->generate_updater_files = TRUE; + /* Should fail: empty updater_file_path */ + H5E_BEGIN_TRY + { + ret = H5Pset_vfd_swmr_config(fapl, my_config); + } + H5E_END_TRY; + if (ret >= 0) + TEST_ERROR; + /* Set md_file_path */ HDstrcpy(my_config->md_file_path, MD_FILENAME); - my_config->writer = TRUE; + my_config->generate_updater_files = FALSE; /* Should succeed in setting the configuration info */ if (H5Pset_vfd_swmr_config(fapl, my_config) < 0) @@ -215,6 +243,33 @@ test_fapl(void) TEST_ERROR; if (my_config->md_pages_reserved != 2) TEST_ERROR; + if (my_config->generate_updater_files) + TEST_ERROR; + if (HDstrcmp(my_config->md_file_path, MD_FILENAME) != 0) + TEST_ERROR; + + my_config->generate_updater_files = TRUE; + /* Set updater_file_path */ + HDstrcpy(my_config->updater_file_path, UD_FILENAME); + + /* Should succeed in setting the configuration info */ + if (H5Pset_vfd_swmr_config(fapl, my_config) < 0) + TEST_ERROR; + + /* Clear the configuration structure */ + HDmemset(my_config, 0, sizeof(H5F_vfd_swmr_config_t)); + + /* Retrieve the configuration info just set */ + if (H5Pget_vfd_swmr_config(fapl, my_config) < 0) + TEST_ERROR; + + /* Verify the configuration info */ + if (!my_config->generate_updater_files) + TEST_ERROR; + if (HDstrcmp(my_config->updater_file_path, UD_FILENAME) != 0) + TEST_ERROR; + if (!my_config->maintain_metadata_file) + TEST_ERROR; if (HDstrcmp(my_config->md_file_path, MD_FILENAME) != 0) TEST_ERROR; @@ -290,10 +345,18 @@ test_file_fapl(void) if ((file_config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) FAIL_STACK_ERROR; - /* Configured as VFD SWMR reader + no page buffering */ - fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, FALSE, 2, MD_FILENAME, 0); + /* + * Configured as VFD SWMR reader + no page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 4, 7, FALSE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 0, config1); if (fapl1 == H5I_INVALID_HID) - TEST_ERROR + TEST_ERROR; /* Should fail to create: file access is writer but VFD SWMR config is reader */ H5E_BEGIN_TRY @@ -305,12 +368,21 @@ test_file_fapl(void) TEST_ERROR; if (H5Pclose(fapl1) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; + + /* + * Configured as VFD SWMR writer + no page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 4, 7, TRUE, TRUE, TRUE, TRUE, 2, MD_FILENAME, UD_FILENAME); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 0, config1); - /* Configured as VFD SWMR writer + no page buffering */ - fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, TRUE, 2, MD_FILENAME, 0); if (fapl1 == H5I_INVALID_HID) - TEST_ERROR + TEST_ERROR; /* Should fail to create: page buffering and paged aggregation not enabled */ H5E_BEGIN_TRY @@ -321,13 +393,10 @@ test_file_fapl(void) if (fid >= 0) TEST_ERROR; - /* Create a copy of the file creation property list */ - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR - - /* Set file space strategy */ - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); FAIL_STACK_ERROR; + } /* Should fail to create: no page buffering */ H5E_BEGIN_TRY @@ -339,12 +408,21 @@ test_file_fapl(void) TEST_ERROR; if (H5Pclose(fapl1) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; + + /* + * Configured as VFD SWMR writer + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 4, 7, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1); - /* Configured as VFD SWMR writer + page buffering */ - fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, TRUE, 2, MD_FILENAME, 4096); if (fapl1 == H5I_INVALID_HID) - TEST_ERROR + TEST_ERROR; /* Should succeed to create the file: paged aggregation and page buffering enabled */ if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl1)) < 0) @@ -404,10 +482,19 @@ test_file_fapl(void) if (H5Pclose(fapl1) < 0) FAIL_STACK_ERROR; - /* Set up different VFD SWMR configuration + page_buffering */ - fapl2 = init_vfd_swmr_config_fapl(config2, 4, 10, TRUE, 2, MD_FILENAME, 4096); + /* + * Set up different VFD SWMR configuration + page_buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config2, 4, 10, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl2 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config2); + if (fapl2 == H5I_INVALID_HID) - TEST_ERROR + TEST_ERROR; /* Should succeed to open the file as VFD SWMR writer */ if ((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl2)) < 0) @@ -435,8 +522,17 @@ test_file_fapl(void) /* The file previously opened as VDF SWMR writer is still open */ /* with VFD SWMR configuration in config2 */ - /* Set up as VFD SWMR writer in config1 but different from config2 */ - fapl1 = init_vfd_swmr_config_fapl(config1, 3, 8, TRUE, 3, MD_FILENAME, 4096); + /* + * Set up as VFD SWMR writer in config1 but different from config2 + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 3, 8, TRUE, TRUE, FALSE, TRUE, 3, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1); + if (fapl1 == H5I_INVALID_HID) TEST_ERROR; @@ -454,15 +550,22 @@ test_file_fapl(void) if (H5Pclose(fapl1) < 0) FAIL_STACK_ERROR; - /* Set up as VFD SWMR reader in config1 which is same as config2 */ - fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, TRUE, 2, MD_FILENAME, 4096); + /* + * Set up as VFD SWMR reader in config1 which is same as config2 + */ + + init_vfd_swmr_config(config1, 4, 10, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1); + if (fapl1 == H5I_INVALID_HID) TEST_ERROR; /* Re-open the same file as VFD SWMR writer */ /* Should succeed since config1 is same as the setting in config2 */ if ((fid2 = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1)) < 0) - TEST_ERROR + TEST_ERROR; /* Close fapl1 */ if (H5Pclose(fapl1) < 0) @@ -589,28 +692,52 @@ test_file_end_tick(void) if ((config3 = HDmalloc(sizeof(*config3))) == NULL) FAIL_STACK_ERROR; - /* Configured file 1 as VFD SWMR writer + page buffering */ - fapl1 = init_vfd_swmr_config_fapl(config1, 10, 15, TRUE, 2, MD_FILENAME, 4096); + /* + * Configured file 1 as VFD SWMR writer + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 10, 15, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1); + if (fapl1 == H5I_INVALID_HID) - FAIL_STACK_ERROR; + TEST_ERROR; + + /* + * Configured file 2 as VFD SWMR writer + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config2, 5, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME2, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl2 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config2); - /* Configured file 2 as VFD SWMR writer + page buffering */ - fapl2 = init_vfd_swmr_config_fapl(config2, 5, 6, TRUE, 2, MD_FILENAME2, 4096); if (fapl2 == H5I_INVALID_HID) - FAIL_STACK_ERROR; + TEST_ERROR; - /* Configured file 3 as VFD SWMR writer + page buffering */ - fapl3 = init_vfd_swmr_config_fapl(config3, 3, 6, TRUE, 2, MD_FILENAME3, 4096); - if (fapl3 == H5I_INVALID_HID) - FAIL_STACK_ERROR; + /* + * Configured file 3 as VFD SWMR writer + page buffering + */ - /* Create a copy of the file creation property list */ - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config3, 3, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME3, NULL); - /* Set file space strategy */ - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl3 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config3); + + if (fapl3 == H5I_INVALID_HID) + TEST_ERROR; + + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); FAIL_STACK_ERROR; + } /* Create file 1 with VFD SWMR writer */ if ((fid1 = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl1)) < 0) @@ -776,18 +903,23 @@ test_writer_create_open_flush(void) if ((my_config = HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) FAIL_STACK_ERROR; - /* Set up the VFD SWMR configuration + page buffering */ - fapl = init_vfd_swmr_config_fapl(my_config, 1, 3, TRUE, 2, MD_FILENAME, 4096); - if (fapl == H5I_INVALID_HID) - FAIL_STACK_ERROR; + /* + * Set up the VFD SWMR configuration + page buffering + */ - /* Create a copy of the file creation property list */ - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(my_config, 1, 3, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, my_config); + if (fapl == H5I_INVALID_HID) + TEST_ERROR; - /* Set file space strategy */ - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); FAIL_STACK_ERROR; + } /* Create an HDF5 file with VFD SWMR configured */ if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl)) < 0) @@ -797,15 +929,13 @@ test_writer_create_open_flush(void) if (H5F__vfd_swmr_writer_create_open_flush_test(fid, TRUE) < 0) FAIL_STACK_ERROR; -#ifdef LATER /* Will activate the test when flush is implemented */ /* Flush the HDF5 file */ if (H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Verify info in metadata file when flushing the HDF5 file */ if (H5F__vfd_swmr_writer_create_open_flush_test(fid, FALSE) < 0) - FAIL_STACK_ERROR -#endif + FAIL_STACK_ERROR; /* Close the file */ if (H5Fclose(fid) < 0) @@ -891,6 +1021,7 @@ test_writer_md(void) hsize_t chunk_dims[2] = {2, 5}; /* Dataset chunked dimension sizes */ H5FD_vfd_swmr_idx_entry_t *index = NULL; /* Pointer to the index entries */ H5F_vfd_swmr_config_t * my_config = NULL; /* Configuration for VFD SWMR */ + H5F_t * f = NULL; /* Internal file object pointer */ TESTING("Verify the metadata file for VFD SWMR writer"); @@ -898,25 +1029,28 @@ test_writer_md(void) if ((my_config = HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) FAIL_STACK_ERROR; - /* Set up the VFD SWMR configuration + page buffering */ - fapl = init_vfd_swmr_config_fapl(my_config, 1, 3, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE); - if (fapl == H5I_INVALID_HID) - FAIL_STACK_ERROR + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(my_config, 1, 3, TRUE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL); - /* Create a copy of the file creation property list */ - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, my_config); + if (fapl == H5I_INVALID_HID) + TEST_ERROR; - /* Set file space strategy */ - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) - FAIL_STACK_ERROR; - if (H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0) + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, FS_PAGE_SIZE)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); FAIL_STACK_ERROR; + } /* Create an HDF5 file with VFD SWMR configured */ if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl)) < 0) FAIL_STACK_ERROR; + /* Get a pointer to the internal file object */ + if (NULL == (f = (H5F_t *)H5VL_object(fid))) + FAIL_STACK_ERROR; + /* Allocate num_entries for the data buffer */ if ((buf = HDcalloc(num_entries, FS_PAGE_SIZE)) == NULL) FAIL_STACK_ERROR; @@ -932,24 +1066,25 @@ test_writer_md(void) index[i].md_file_page_offset = 1 + (num_entries - i) * 5; index[i].length = (uint32_t)FS_PAGE_SIZE; index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + index[i].tick_of_last_change = f->shared->tick_num; } /* Update with index and verify info in the metadata file */ /* Also verify that 0 entries will be on the delayed list */ if (H5F__vfd_swmr_writer_md_test(fid, num_entries, index, 0) < 0) - TEST_ERROR + TEST_ERROR; /* Create dataset creation property list */ if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Set to use chunked dataset */ if (H5Pset_chunk(dcpl, 2, chunk_dims) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Create dataspace */ if ((sid = H5Screate_simple(2, dims, max_dims)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Perform activities to ensure that max_lag ticks elapse */ for (i = 0; i < my_config->max_lag + 1; i++) { @@ -958,25 +1093,27 @@ test_writer_md(void) /* Create a chunked dataset */ HDsprintf(dname, "dset %d", i); if ((did = H5Dcreate2(fid, dname, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Get dataset object header address */ if (H5Oget_info3(did, &oinfo, H5O_INFO_BASIC) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Close the dataset */ if (H5Dclose(did) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; } /* (B) Update every other entry in the index */ - for (i = 0; i < num_entries; i += 2) - index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + for (i = 0; i < num_entries; i += 2) { + index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + index[i].tick_of_last_change = f->shared->tick_num; + } /* Update with index and verify info in the metadata file */ /* Also verify that 5 entries will be on the delayed list */ if (H5F__vfd_swmr_writer_md_test(fid, num_entries, index, 5) < 0) - TEST_ERROR + TEST_ERROR; /* Allocate memory for the read/write buffer */ if ((rwbuf = HDmalloc(sizeof(*rwbuf) * (50 * 20))) == NULL) @@ -991,29 +1128,31 @@ test_writer_md(void) /* Open the dataset */ HDsprintf(dname, "dset %d", i); if ((did = H5Dopen2(fid, dname, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Write to the dataset */ if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rwbuf) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Get dataset object info */ if (H5Oget_info3(did, &oinfo, H5O_INFO_BASIC) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Close the dataset */ if (H5Dclose(did) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; } /* (C) Update every 3 entry in the index */ - for (i = 0; i < num_entries; i += 3) - index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + for (i = 0; i < num_entries; i += 3) { + index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + index[i].tick_of_last_change = f->shared->tick_num; + } /* Update with index and verify info in the metadata file */ /* Also verify that 4 entries will be on the delayed list */ if (H5F__vfd_swmr_writer_md_test(fid, num_entries, index, 4) < 0) - TEST_ERROR + TEST_ERROR; /* Clear the read/write buffer */ HDmemset(rwbuf, 0, sizeof(sizeof(int) * (50 * 20))); @@ -1025,29 +1164,31 @@ test_writer_md(void) /* Open the dataset */ HDsprintf(dname, "dset %d", i); if ((did = H5Dopen2(fid, dname, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Read from the dataset */ if (H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rwbuf) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Get dataset object info */ if (H5Oget_info3(did, &oinfo, H5O_INFO_BASIC) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Close the dataset */ if (H5Dclose(did) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; } /* (D) Update two entries in the index */ - index[1].entry_ptr = &buf[1 * FS_PAGE_SIZE]; - index[5].entry_ptr = &buf[5 * FS_PAGE_SIZE]; + index[1].entry_ptr = &buf[1 * FS_PAGE_SIZE]; + index[1].tick_of_last_change = f->shared->tick_num; + index[5].entry_ptr = &buf[5 * FS_PAGE_SIZE]; + index[5].tick_of_last_change = f->shared->tick_num; /* Update with index and verify info in the metadata file */ /* Also verify that 2 entries will be on the delayed list */ if (H5F__vfd_swmr_writer_md_test(fid, num_entries, index, 2) < 0) - TEST_ERROR + TEST_ERROR; /* Closing */ if (H5Fclose(fid) < 0) @@ -1202,21 +1343,23 @@ test_reader_md_concur(void) if ((config_writer = HDmalloc(sizeof(*config_writer))) == NULL) FAIL_STACK_ERROR; - /* Set up the VFD SWMR configuration + page buffering */ - fapl_writer = init_vfd_swmr_config_fapl(config_writer, 1, 3, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE); - if (fapl_writer == H5I_INVALID_HID) - FAIL_STACK_ERROR; + /* + * Set up the VFD SWMR configuration + page buffering + */ - /* Create a copy of the file creation property list */ - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config_writer, 1, 3, TRUE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL); - /* Set file space strategy */ - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) - FAIL_STACK_ERROR; + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl_writer = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_writer); + if (fapl_writer == H5I_INVALID_HID) + TEST_ERROR; - if (H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0) + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, FS_PAGE_SIZE)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); FAIL_STACK_ERROR; + } /* Create an HDF5 file with VFD SWMR configured */ if ((fid_writer = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl_writer)) < 0) @@ -1228,14 +1371,14 @@ test_reader_md_concur(void) /* Create 2 pipes */ if (HDpipe(parent_pfd) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; if (HDpipe(child_pfd) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Fork child process */ if ((childpid = HDfork()) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* * Child process as reader @@ -1276,10 +1419,19 @@ test_reader_md_concur(void) if ((config_reader = HDmalloc(sizeof(*config_reader))) == NULL) HDexit(EXIT_FAILURE); - /* Set up the VFD SWMR configuration as reader + page buffering */ - fapl_reader = init_vfd_swmr_config_fapl(config_reader, 1, 3, FALSE, 256, MD_FILENAME, FS_PAGE_SIZE); + /* + * Set up the VFD SWMR configuration as reader + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config_reader, 1, 3, FALSE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl_reader = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_reader); + if (fapl_reader == H5I_INVALID_HID) - HDexit(EXIT_FAILURE); + TEST_ERROR; /* Open the test file as reader */ if ((fid_reader = H5Fopen(FILENAME, H5F_ACC_RDONLY, fapl_reader)) < 0) @@ -1464,11 +1616,11 @@ test_reader_md_concur(void) /* Close unused read end for writer pipe */ if (HDclose(parent_pfd[0]) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Close unused write end for reader pipe */ if (HDclose(child_pfd[1]) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* * Case A: writer @@ -1479,6 +1631,9 @@ test_reader_md_concur(void) if ((fid_writer = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl_writer)) < 0) FAIL_STACK_ERROR; + /* Get the file pointer */ + file_writer = H5VL_object(fid_writer); + /* Send notification 1 to reader to start verfication */ notify = 1; if (HDwrite(parent_pfd[1], ¬ify, sizeof(int)) < 0) @@ -1499,15 +1654,15 @@ test_reader_md_concur(void) /* Create dataset creation property list */ if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Set to use chunked dataset */ if (H5Pset_chunk(dcpl, 2, chunk_dims) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Create dataspace */ if ((sid = H5Screate_simple(2, dims, max_dims)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Perform activities to ensure that ticks elapse */ for (i = 0; i < config_writer->max_lag + 1; i++) { @@ -1516,15 +1671,15 @@ test_reader_md_concur(void) /* Create a chunked dataset */ HDsprintf(dname, "dset %d", i); if ((did = H5Dcreate2(fid_writer, dname, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Get dataset object header address */ if (H5Oget_info3(did, &oinfo, H5O_INFO_BASIC) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Close the dataset */ if (H5Dclose(did) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; } num_entries = 12; @@ -1544,11 +1699,9 @@ test_reader_md_concur(void) index[i].md_file_page_offset = 1 + (num_entries - i) * 5; index[i].length = (uint32_t)FS_PAGE_SIZE; index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + index[i].tick_of_last_change = file_writer->shared->tick_num; } - /* Get the file pointer */ - file_writer = H5VL_object(fid_writer); - /* Update the metadata file with the index */ if (H5F_update_vfd_swmr_metadata_file(file_writer, num_entries, index) < 0) TEST_ERROR; @@ -1592,21 +1745,23 @@ test_reader_md_concur(void) /* Open the dataset */ HDsprintf(dname, "dset %d", i); if ((did = H5Dopen2(fid_writer, dname, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Write to the dataset */ if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rwbuf) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Close the dataset */ if (H5Dclose(did) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; } /* Update 3 entries in the index */ num_entries = 3; - for (i = 0; i < num_entries; i++) - index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + for (i = 0; i < num_entries; i++) { + index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + index[i].tick_of_last_change = file_writer->shared->tick_num; + } /* Update the metadata file with the index */ if (H5F_update_vfd_swmr_metadata_file(file_writer, num_entries, index) < 0) @@ -1645,21 +1800,23 @@ test_reader_md_concur(void) /* Open the dataset */ HDsprintf(dname, "dset %d", i); if ((did = H5Dopen2(fid_writer, dname, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Read from the dataset */ if (H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rwbuf) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Close the dataset */ if (H5Dclose(did) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; } /* Update 5 entries in the index */ num_entries = 5; - for (i = 0; i < num_entries; i++) - index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + for (i = 0; i < num_entries; i++) { + index[i].entry_ptr = &buf[i * FS_PAGE_SIZE]; + index[i].tick_of_last_change = file_writer->shared->tick_num; + } /* Update the metadata file with the index */ if (H5F_update_vfd_swmr_metadata_file(file_writer, num_entries, index) < 0) @@ -1697,15 +1854,15 @@ test_reader_md_concur(void) /* Open the dataset */ HDsprintf(dname, "dset %d", i); if ((did = H5Dopen2(fid_writer, dname, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Write to the dataset */ if (H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rwbuf) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Close the dataset */ if (H5Dclose(did) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; } /* Update the metadata file with 0 entries and NULL index */ @@ -1729,20 +1886,20 @@ test_reader_md_concur(void) /* Wait for child process to complete */ if ((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Check exit status of child process */ if (WIFEXITED(child_status)) { if ((child_exit_val = WEXITSTATUS(child_status)) != 0) - TEST_ERROR + TEST_ERROR; } else { /* child process terminated abnormally */ - TEST_ERROR + TEST_ERROR; } /* Closing */ if (H5Fclose(fid_writer) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; if (H5Pclose(fapl_writer) < 0) FAIL_STACK_ERROR; if (H5Pclose(fcpl) < 0) @@ -1816,16 +1973,10 @@ test_multiple_file_opens_concur(void) TESTING("EOT queue entries when opening files concurrently with VFD SWMR"); - /* Create a copy of the file creation property list */ - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR - - /* Set file space strategy */ - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) - FAIL_STACK_ERROR; - - if (H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0) + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, FS_PAGE_SIZE)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); FAIL_STACK_ERROR; + } /* Create file A */ if ((fid1 = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) @@ -1845,14 +1996,14 @@ test_multiple_file_opens_concur(void) /* Create 2 pipes */ if (HDpipe(parent_pfd) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; if (HDpipe(child_pfd) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Fork child process */ if ((childpid = HDfork()) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* * Child process @@ -1886,7 +2037,14 @@ test_multiple_file_opens_concur(void) HDexit(EXIT_FAILURE); /* Set the VFD SWMR configuration in fapl_writer + page buffering */ - fapl_writer = init_vfd_swmr_config_fapl(config_writer, 1, 3, TRUE, 256, MD_FILENAME2, FS_PAGE_SIZE); + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config_writer, 1, 3, TRUE, TRUE, FALSE, TRUE, 256, MD_FILENAME2, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl_writer = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_writer); + if (fapl_writer == H5I_INVALID_HID) HDexit(EXIT_FAILURE); @@ -1934,11 +2092,11 @@ test_multiple_file_opens_concur(void) /* Close unused read end for writer pipe */ if (HDclose(parent_pfd[0]) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Close unused write end for reader pipe */ if (HDclose(child_pfd[1]) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* * Set up and open file A as VFD SWMR writer @@ -1946,12 +2104,17 @@ test_multiple_file_opens_concur(void) /* Allocate memory for VFD SWMR configuration */ if ((config1 = HDmalloc(sizeof(*config1))) == NULL) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 7, 10, TRUE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config1); - /* Set the VFD SWMR configuration in fapl1 + page buffering */ - fapl1 = init_vfd_swmr_config_fapl(config1, 7, 10, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE); if (fapl1 == H5I_INVALID_HID) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Open file A as VFD SWMR writer */ if ((fid1 = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1)) < 0) @@ -1959,7 +2122,7 @@ test_multiple_file_opens_concur(void) /* Get a pointer to the internal file object */ if (NULL == (f1 = H5VL_object(fid1))) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Head of EOT queue should be a writer */ if ((curr = TAILQ_FIRST(&eot_queue_g)) == NULL || !curr->vfd_swmr_writer) @@ -1984,12 +2147,17 @@ test_multiple_file_opens_concur(void) /* Allocate memory for VFD SWMR configuration */ if ((config2 = HDmalloc(sizeof(*config2))) == NULL) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config2, 1, 3, FALSE, TRUE, FALSE, TRUE, 256, MD_FILENAME2, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl2 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config2); - /* Set the VFD SWMR configuration in fapl2 + page buffering */ - fapl2 = init_vfd_swmr_config_fapl(config2, 1, 3, FALSE, 256, MD_FILENAME2, FS_PAGE_SIZE); if (fapl2 == H5I_INVALID_HID) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Open file B as VFD SWMR reader */ if ((fid2 = H5Fopen(FILENAME2, H5F_ACC_RDONLY, fapl2)) < 0) @@ -1997,7 +2165,7 @@ test_multiple_file_opens_concur(void) /* Get a pointer to the internal file object */ if (NULL == (f2 = H5VL_object(fid2))) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Head of EOT queue should NOT be a writer */ if ((curr = TAILQ_FIRST(&eot_queue_g)) != NULL && curr->vfd_swmr_writer) @@ -2030,22 +2198,22 @@ test_multiple_file_opens_concur(void) /* Wait for child process to complete */ if ((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Check exit status of child process */ if (WIFEXITED(child_status)) { if ((child_exit_val = WEXITSTATUS(child_status)) != 0) - TEST_ERROR + TEST_ERROR; } else { /* child process terminated abnormally */ - TEST_ERROR + TEST_ERROR; } /* Closing */ if (H5Fclose(fid1) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; if (H5Fclose(fid2) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; if (H5Pclose(fapl1) < 0) FAIL_STACK_ERROR; if (H5Pclose(fapl2) < 0) @@ -2120,21 +2288,24 @@ test_disable_enable_eot_concur(void) if ((config_writer = HDmalloc(sizeof(*config_writer))) == NULL) FAIL_STACK_ERROR; - /* Set up the VFD SWMR configuration + page buffering */ - fapl_writer = init_vfd_swmr_config_fapl(config_writer, 1, 3, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE); - if (fapl_writer == H5I_INVALID_HID) - FAIL_STACK_ERROR; + /* + * Set up the VFD SWMR configuration + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config_writer, 1, 3, TRUE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL); - /* Create a copy of the file creation property list */ - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl_writer = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_writer); - /* Set file space strategy */ - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + if (fapl_writer == H5I_INVALID_HID) FAIL_STACK_ERROR; - if (H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0) + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, FS_PAGE_SIZE)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); FAIL_STACK_ERROR; + } /* Create an HDF5 file with VFD SWMR configured */ if ((fid_writer = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl_writer)) < 0) @@ -2146,14 +2317,14 @@ test_disable_enable_eot_concur(void) /* Create 2 pipes */ if (HDpipe(parent_pfd) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; if (HDpipe(child_pfd) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Fork child process */ if ((childpid = HDfork()) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* * Child process as reader @@ -2197,8 +2368,17 @@ test_disable_enable_eot_concur(void) if ((config_reader = HDmalloc(sizeof(*config_reader))) == NULL) HDexit(EXIT_FAILURE); - /* Set up the VFD SWMR configuration as reader + page buffering */ - fapl_reader = init_vfd_swmr_config_fapl(config_reader, 1, 3, FALSE, 256, MD_FILENAME, FS_PAGE_SIZE); + /* + * Set up the VFD SWMR configuration as reader + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config_reader, 1, 3, FALSE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl_reader = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_reader); + if (fapl_reader == H5I_INVALID_HID) FAIL_STACK_ERROR; @@ -2291,11 +2471,11 @@ test_disable_enable_eot_concur(void) /* Close unused read end for writer pipe */ if (HDclose(parent_pfd[0]) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Close unused write end for reader pipe */ if (HDclose(child_pfd[1]) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* * Open the file as VFD SWMR writer @@ -2322,20 +2502,20 @@ test_disable_enable_eot_concur(void) /* Wait for child process to complete */ if ((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Check exit status of child process */ if (WIFEXITED(child_status)) { if ((child_exit_val = WEXITSTATUS(child_status)) != 0) - TEST_ERROR + TEST_ERROR; } else { /* child process terminated abnormally */ - TEST_ERROR + TEST_ERROR; } /* Closing */ if (H5Fclose(fid_writer) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; if (H5Pclose(fapl_writer) < 0) FAIL_STACK_ERROR; if (H5Pclose(fcpl) < 0) @@ -2400,21 +2580,24 @@ test_file_end_tick_concur(void) if ((config_writer = HDmalloc(sizeof(*config_writer))) == NULL) FAIL_STACK_ERROR; - /* Set up the VFD SWMR configuration + page buffering */ - fapl_writer = init_vfd_swmr_config_fapl(config_writer, 1, 3, TRUE, 256, MD_FILENAME, FS_PAGE_SIZE); - if (fapl_writer == H5I_INVALID_HID) - FAIL_STACK_ERROR; + /* + * Set up the VFD SWMR configuration + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config_writer, 1, 3, TRUE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL); - /* Create a copy of the file creation property list */ - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl_writer = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_writer); - /* Set file space strategy */ - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + if (fapl_writer == H5I_INVALID_HID) FAIL_STACK_ERROR; - if (H5Pset_file_space_page_size(fcpl, FS_PAGE_SIZE) < 0) + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, FS_PAGE_SIZE)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); FAIL_STACK_ERROR; + } /* Create an HDF5 file with VFD SWMR configured */ if ((fid_writer = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl_writer)) < 0) @@ -2426,14 +2609,14 @@ test_file_end_tick_concur(void) /* Create 2 pipes */ if (HDpipe(parent_pfd) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; if (HDpipe(child_pfd) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Fork child process */ if ((childpid = HDfork()) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* * Child process as reader @@ -2477,8 +2660,13 @@ test_file_end_tick_concur(void) if ((config_reader = HDmalloc(sizeof(*config_reader))) == NULL) HDexit(EXIT_FAILURE); - /* Set up the VFD SWMR configuration as reader + page buffering */ - fapl_reader = init_vfd_swmr_config_fapl(config_reader, 1, 3, FALSE, 256, MD_FILENAME, FS_PAGE_SIZE); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config_reader, 1, 3, FALSE, TRUE, FALSE, TRUE, 256, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl_reader = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, FS_PAGE_SIZE, config_reader); + if (fapl_reader == H5I_INVALID_HID) FAIL_STACK_ERROR; @@ -2557,11 +2745,11 @@ test_file_end_tick_concur(void) /* Close unused read end for writer pipe */ if (HDclose(parent_pfd[0]) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Close unused write end for reader pipe */ if (HDclose(child_pfd[1]) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* * Open the file as VFD SWMR writer @@ -2588,20 +2776,20 @@ test_file_end_tick_concur(void) /* Wait for child process to complete */ if ((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Check exit status of child process */ if (WIFEXITED(child_status)) { if ((child_exit_val = WEXITSTATUS(child_status)) != 0) - TEST_ERROR + TEST_ERROR; } else { /* child process terminated abnormally */ - TEST_ERROR + TEST_ERROR; } /* Closing */ if (H5Fclose(fid_writer) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; if (H5Pclose(fapl_writer) < 0) FAIL_STACK_ERROR; if (H5Pclose(fcpl) < 0) @@ -2667,23 +2855,33 @@ test_multiple_file_opens(void) if ((config2 = HDmalloc(sizeof(*config2))) == NULL) FAIL_STACK_ERROR; - /* Configured as VFD SWMR writer + page buffering */ - fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, TRUE, 2, MD_FILENAME, 4096); + /* + * Configured as VFD SWMR writer + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 4, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1); if (fapl1 == H5I_INVALID_HID) FAIL_STACK_ERROR; - /* Configured as VFD SWMR writer + page buffering */ - fapl2 = init_vfd_swmr_config_fapl(config2, 4, 6, TRUE, 2, MD_FILENAME2, 4096); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config2, 4, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME2, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl2 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config2); + if (fapl2 == H5I_INVALID_HID) FAIL_STACK_ERROR; - /* Create a copy of the file creation property list */ - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR - - /* Set file space strategy */ - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); FAIL_STACK_ERROR; + } /* Create a file without VFD SWMR */ if ((fid = H5Fcreate(FNAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) @@ -2691,7 +2889,7 @@ test_multiple_file_opens(void) /* Get a pointer to the internal file object */ if (NULL == (f = H5VL_object(fid))) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Verify the global vfd_swmr_writer_g is not set */ if ((curr = TAILQ_FIRST(&eot_queue_g)) != NULL && curr->vfd_swmr_writer) @@ -2706,7 +2904,7 @@ test_multiple_file_opens(void) /* Get a pointer to the internal file object */ if (NULL == (f1 = H5VL_object(fid1))) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Head of EOT queue should be a writer */ if ((curr = TAILQ_FIRST(&eot_queue_g)) == NULL || !curr->vfd_swmr_writer) @@ -2721,7 +2919,7 @@ test_multiple_file_opens(void) /* Get a pointer to the internal file object */ if (NULL == (f2 = H5VL_object(fid2))) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Head of EOT queue should be a writer */ if ((curr = TAILQ_FIRST(&eot_queue_g)) == NULL || !curr->vfd_swmr_writer) @@ -2734,7 +2932,7 @@ test_multiple_file_opens(void) TAILQ_FOREACH(curr, &eot_queue_g, link) { if (curr->vfd_swmr_file == f) - TEST_ERROR + TEST_ERROR; } /* Close the first file with VFD SWMR */ @@ -2854,13 +3052,10 @@ test_same_file_opens(void) if ((config2 = HDmalloc(sizeof(*config2))) == NULL) FAIL_STACK_ERROR; - /* Create a copy of the file creation property list */ - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR - - /* Set file space strategy */ - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); FAIL_STACK_ERROR; + } /* * Tests for first column @@ -2874,8 +3069,17 @@ test_same_file_opens(void) if (H5Fclose(fid) < 0) FAIL_STACK_ERROR; - /* Set the VFD SWMR configuration in fapl1 + page buffering */ - fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, TRUE, 2, MD_FILENAME, 4096); + /* + * Set the VFD SWMR configuration in fapl1 + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 4, 10, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1); + if (fapl1 == H5I_INVALID_HID) FAIL_STACK_ERROR; @@ -2893,8 +3097,17 @@ test_same_file_opens(void) if (H5Fclose(fid2) < 0) FAIL_STACK_ERROR; - /* Set the VFD SWMR configuration in fapl2 + page buffering */ - fapl2 = init_vfd_swmr_config_fapl(config2, 3, 8, FALSE, 3, MD_FILENAME, 4096); + /* + * Set the VFD SWMR configuration in fapl2 + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config2, 3, 8, FALSE, TRUE, FALSE, TRUE, 3, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl2 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config2); + if (fapl2 == H5I_INVALID_HID) FAIL_STACK_ERROR; @@ -2939,8 +3152,17 @@ test_same_file_opens(void) * Tests for second column */ - /* Set up as VFD SWMR reader + page buffering */ - fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, FALSE, 2, MD_FILENAME, 4096); + /* + * Set up as VFD SWMR reader + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 4, 10, FALSE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1); + if (fapl1 == H5I_INVALID_HID) FAIL_STACK_ERROR; @@ -2967,8 +3189,17 @@ test_same_file_opens(void) if ((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) TEST_ERROR; - /* Set up as VFD SWMR writer + page buffering */ - fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, TRUE, 2, MD_FILENAME, 4096); + /* + * Set up as VFD SWMR writer + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 4, 10, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1); + if (fapl1 == H5I_INVALID_HID) FAIL_STACK_ERROR; @@ -3014,8 +3245,17 @@ test_same_file_opens(void) if ((fid = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) TEST_ERROR; - /* Set up as VFD SWMR writer + page buffering */ - fapl1 = init_vfd_swmr_config_fapl(config1, 4, 10, TRUE, 2, MD_FILENAME, 4096); + /* + * Set up as VFD SWMR writer + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 4, 10, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1); + if (fapl1 == H5I_INVALID_HID) FAIL_STACK_ERROR; @@ -3216,28 +3456,52 @@ test_enable_disable_eot(void) if ((config3 = HDmalloc(sizeof(*config3))) == NULL) FAIL_STACK_ERROR; - /* Configured first file as VFD SWMR writer + page buffering */ - fapl1 = init_vfd_swmr_config_fapl(config1, 4, 6, TRUE, 2, MD_FILENAME, 4096); + /* + * Configured first file as VFD SWMR writer + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 4, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1); + if (fapl1 == H5I_INVALID_HID) FAIL_STACK_ERROR; - /* Configured second file as VFD SWMR writer + page buffering */ - fapl2 = init_vfd_swmr_config_fapl(config2, 4, 6, TRUE, 2, MD_FILENAME2, 4096); + /* + * Configured second file as VFD SWMR writer + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config2, 4, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME2, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl2 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config2); + if (fapl2 == H5I_INVALID_HID) FAIL_STACK_ERROR; - /* Configured third file as VFD SWMR writer + page buffering */ - fapl3 = init_vfd_swmr_config_fapl(config3, 4, 6, TRUE, 2, MD_FILENAME3, 4096); + /* + * Configured third file as VFD SWMR writer + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config3, 4, 6, TRUE, TRUE, FALSE, TRUE, 2, MD_FILENAME3, NULL); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl3 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config3); + if (fapl3 == H5I_INVALID_HID) FAIL_STACK_ERROR; - /* Create a copy of the file creation property list */ - if ((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) - FAIL_STACK_ERROR - - /* Set file space strategy */ - if (H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, FALSE, 1) < 0) + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); FAIL_STACK_ERROR; + } /* Create a file without VFD SWMR */ if ((fid = H5Fcreate(FNAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) @@ -3250,10 +3514,10 @@ test_enable_disable_eot(void) } H5E_END_TRY; if (ret >= 0) - TEST_ERROR + TEST_ERROR; if (H5Fclose(fid) < 0) - TEST_ERROR + TEST_ERROR; /* Create file 1 with VFD SWMR writer */ if ((fid1 = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl1)) < 0) @@ -3275,7 +3539,7 @@ test_enable_disable_eot(void) /* Disable EOT for file 1 */ if (H5Fvfd_swmr_disable_end_of_tick(fid1) < 0) - TEST_ERROR + TEST_ERROR; /* Disable file 1 again should fail because the file has just been disabled */ H5E_BEGIN_TRY @@ -3284,18 +3548,18 @@ test_enable_disable_eot(void) } H5E_END_TRY; if (ret >= 0) - TEST_ERROR + TEST_ERROR; /* Should have 2 files on the EOT queue */ count = 0; TAILQ_FOREACH(curr, &eot_queue_g, link) count++; if (count != 2) - TEST_ERROR + TEST_ERROR; /* Get a pointer to the internal file object */ if (NULL == (f1 = H5VL_object(fid1))) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* Should not find file 1 on the EOT queue */ TAILQ_FOREACH(curr, &eot_queue_g, link) @@ -3304,7 +3568,7 @@ test_enable_disable_eot(void) break; } if (curr != NULL && curr->vfd_swmr_file == f1) - TEST_ERROR + TEST_ERROR; /* Enable EOT for file 2 should fail because the file has not been disabled */ H5E_BEGIN_TRY @@ -3313,11 +3577,11 @@ test_enable_disable_eot(void) } H5E_END_TRY; if (ret >= 0) - TEST_ERROR + TEST_ERROR; /* Get a pointer to the internal file object */ if (NULL == (f2 = H5VL_object(fid2))) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* File 2 should be on the EOT queue */ TAILQ_FOREACH(curr, &eot_queue_g, link) @@ -3326,19 +3590,19 @@ test_enable_disable_eot(void) break; } if (curr == NULL || curr->vfd_swmr_file != f2) - TEST_ERROR + TEST_ERROR; /* Close file 3 */ if (H5Fclose(fid3) < 0) - TEST_ERROR + TEST_ERROR; /* Open file 3 again without VFD SWMR writer */ if ((fid3 = H5Fopen(FILENAME3, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) - TEST_ERROR + TEST_ERROR; /* Get a pointer to the internal file object for file 3 */ if (NULL == (f3 = H5VL_object(fid3))) - FAIL_STACK_ERROR + FAIL_STACK_ERROR; /* File 3 should not exist on the EOT queue */ TAILQ_FOREACH(curr, &eot_queue_g, link) @@ -3347,7 +3611,7 @@ test_enable_disable_eot(void) break; } if (curr != NULL && curr->vfd_swmr_file == f3) - TEST_ERROR + TEST_ERROR; /* Should have 2 files on the EOT queue */ count = 0; @@ -3363,7 +3627,7 @@ test_enable_disable_eot(void) } H5E_END_TRY; if (ret >= 0) - TEST_ERROR + TEST_ERROR; /* Should fail to disable file 3 */ H5E_BEGIN_TRY @@ -3372,7 +3636,7 @@ test_enable_disable_eot(void) } H5E_END_TRY; if (ret >= 0) - TEST_ERROR + TEST_ERROR; /* Closing */ if (H5Fclose(fid1) < 0) @@ -3425,6 +3689,704 @@ error: } /* test_enable_disable_eot() */ /*------------------------------------------------------------------------- + * Function: verify_updater_flags() + * + * Purpose: This is the helper routine used to verify whether + * "flags" in the updater file is as expected. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; October 2021 + * + *------------------------------------------------------------------------- + */ +static herr_t +verify_updater_flags(char *ud_name, uint16_t expected_flags) +{ + FILE * ud_fp = NULL; /* Updater file pointer */ + uint16_t flags = 0; /* The "flags" field in the updater file */ + + /* Open the updater file */ + if ((ud_fp = HDfopen(ud_name, "r")) == NULL) + FAIL_STACK_ERROR; + + /* Seek to the position of "flags" in the updater file's header */ + if (HDfseek(ud_fp, (HDoff_t)UD_HD_FLAGS_OFFSET, SEEK_SET) < 0) + FAIL_STACK_ERROR; + + /* Read "flags" from the updater file */ + if (HDfread(&flags, UD_SIZE_2, 1, ud_fp) != (size_t)1) + FAIL_STACK_ERROR; + + if (flags != expected_flags) + TEST_ERROR; + + if (HDfclose(ud_fp) < 0) + FAIL_STACK_ERROR; + + return 0; + +error: + return 1; + +} /* verify_updater_flags() */ + +/*------------------------------------------------------------------------- + * Function: test_updater_flags + * + * Purpose: Verify "flags" in the updater file is as expected for + * file creation. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; October 2021 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_updater_flags(void) +{ + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fapl = -1; /* File access property list ID */ + hid_t file_fapl = -1; /* File access property list ID associated with the file */ + H5F_vfd_swmr_config_t *config = NULL; /* Configuration for VFD SWMR */ + H5F_vfd_swmr_config_t *file_config = NULL; /* Configuration for VFD SWMR */ + uint64_t seq_num = 0; /* Sequence number for updater file */ + uint64_t i = 0; /* Local index variable */ + char namebuf[H5F__MAX_VFD_SWMR_FILE_NAME_LEN]; /* Updater file path */ + h5_stat_t sb; /* Info returned by stat system call */ + + TESTING("VFD SWMR updater file flags"); + + /* Should succeed without VFD SWMR configured */ + if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Close the file */ + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + /* Allocate memory for the configuration structure */ + if ((config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) + FAIL_STACK_ERROR; + if ((file_config = (H5F_vfd_swmr_config_t *)HDmalloc(sizeof(H5F_vfd_swmr_config_t))) == NULL) + FAIL_STACK_ERROR; + + /* + * Configured as VFD SWMR writer + page buffering + generate updater files + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config, 4, 7, TRUE, TRUE, TRUE, TRUE, 2, MD_FILENAME, UD_FILENAME); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config); + + if (fapl == H5I_INVALID_HID) + TEST_ERROR; + + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); + FAIL_STACK_ERROR; + } + + if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + TEST_ERROR; + + /* Get the file's file access property list */ + if ((file_fapl = H5Fget_access_plist(fid)) < 0) + FAIL_STACK_ERROR; + + /* Retrieve the VFD SWMR configuration from file_fapl */ + if (H5Pget_vfd_swmr_config(file_fapl, file_config) < 0) + TEST_ERROR; + + /* Verify the retrieved info is the same as config1 */ + if (HDmemcmp(config, file_config, sizeof(H5F_vfd_swmr_config_t)) != 0) + TEST_ERROR; + + /* Verify the first updater file: "flags" field and file size */ + HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num); + + /* Verify "flags" of the first updater file */ + if (verify_updater_flags(namebuf, CREATE_METADATA_FILE_ONLY_FLAG) < 0) + TEST_ERROR; + + /* Check updater file size */ + if (HDstat(namebuf, &sb) == 0 && sb.st_size != H5F_UD_HEADER_SIZE) + TEST_ERROR; + + /* Closing */ + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + /* Look for the last updater file */ + for (seq_num = 0;; seq_num++) { + HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num); + if (HDaccess(namebuf, F_OK) != 0) + break; + } + HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num - 1); + + /* Verify "flags" of the last updater file */ + if (verify_updater_flags(namebuf, FINAL_UPDATE_FLAG) < 0) + TEST_ERROR; + + if (H5Pclose(file_fapl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + /* Remove updater files */ + for (i = 0; i < seq_num; i++) { + HDsprintf(namebuf, "%s.%lu", UD_FILENAME, i); + HDremove(namebuf); + } + + /* Free buffers */ + if (config) + HDfree(config); + if (file_config) + HDfree(file_config); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY + { + H5Pclose(fapl); + H5Pclose(file_fapl); + H5Pclose(fcpl); + H5Fclose(fid); + } + H5E_END_TRY; + if (config) + HDfree(config); + if (file_config) + HDfree(file_config); + + return 1; +} /* test_updater_flags() */ + +/*------------------------------------------------------------------------- + * Function: test_updater_flags_same_file_opens() + * + * Purpose: Verify "flags" in the updater file is as expected for + * multiple opens of the same file. + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; October 2021 + * + *------------------------------------------------------------------------- + */ +static unsigned +test_updater_flags_same_file_opens(void) +{ + hid_t fid = -1; /* File ID */ + hid_t fid2 = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fapl1 = -1; /* File access property list ID */ + H5F_vfd_swmr_config_t *config1 = NULL; /* Configuration for VFD SWMR */ + uint64_t seq_num = 0; /* Sequence number for updater file */ + uint64_t i = 0; /* Local index variable */ + char namebuf[H5F__MAX_VFD_SWMR_FILE_NAME_LEN]; /* Updater file path */ + + TESTING("VFD SWMR updater file flags for multiple opens of the same file"); + + /* Should succeed without VFD SWMR configured */ + if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Close the file */ + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + /* Allocate memory for the configuration structure */ + if ((config1 = HDmalloc(sizeof(*config1))) == NULL) + FAIL_STACK_ERROR; + + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); + FAIL_STACK_ERROR; + } + + /* Create the test file */ + if ((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Close the file */ + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + /* + * Set the VFD SWMR configuration in fapl1 + page buffering + */ + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config1, 4, 10, TRUE, TRUE, TRUE, TRUE, 2, MD_FILENAME, UD_FILENAME); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl1 = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config1); + + if (fapl1 == H5I_INVALID_HID) + FAIL_STACK_ERROR; + + /* Open the file as VFD SWMR writer */ + /* Keep the file open */ + if ((fid = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1)) < 0) + TEST_ERROR; + + /* Open the same file again as VFD SWMR writer */ + /* Should succeed: 1st open--VFD SWMR writer, 2nd open--VFD SWMR writer */ + if ((fid2 = H5Fopen(FILENAME, H5F_ACC_RDWR, fapl1)) < 0) + TEST_ERROR; + + /* Close the second file open */ + if (H5Fclose(fid2) < 0) + FAIL_STACK_ERROR; + + /* Verify the first updater file for first file open */ + HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num); + + /* Verify "flags" of the first updater file is 0*/ + if (verify_updater_flags(namebuf, 0) < 0) + TEST_ERROR; + + /* Look for the last updater file */ + for (seq_num = 0;; seq_num++) { + HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num); + if (HDaccess(namebuf, F_OK) != 0) + break; + } + HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num - 1); + + /* Verify "flags" of the last updater file is 0 */ + if (verify_updater_flags(namebuf, 0) < 0) + TEST_ERROR; + + /* Close the 1st open file */ + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + /* Look for the last updater file */ + for (seq_num = 0;; seq_num++) { + HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num); + if (HDaccess(namebuf, F_OK) != 0) + break; + } + HDsprintf(namebuf, "%s.%lu", UD_FILENAME, seq_num - 1); + + /* Verify "flags" of the last updater file after closing file */ + if (verify_updater_flags(namebuf, FINAL_UPDATE_FLAG) < 0) + TEST_ERROR; + + /* Clean up updater files */ + for (i = 0; i < seq_num; i++) { + HDsprintf(namebuf, "%s.%lu", UD_FILENAME, i); + HDremove(namebuf); + } + + if (H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + /* Free buffers */ + if (config1) + HDfree(config1); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY + { + H5Pclose(fapl1); + H5Pclose(fcpl); + H5Fclose(fid); + H5Fclose(fid2); + } + H5E_END_TRY; + if (config1) + HDfree(config1); + return 1; +} /* test_updater_flags_same_file_opens() */ + +/*------------------------------------------------------------------------- + * Function: clean_chk_ud_files() + * + * Purpose: This is the helper routine used to clean up + * the checksum file and the updater files. + * + * Return: void + * + * Programmer: Vailin Choi; October 2021 + * + *------------------------------------------------------------------------- + */ +static void +clean_chk_ud_files(char *md_file_path, char *updater_file_path) +{ + char chk_name[FILE_NAME_LEN]; /* Checksum file name */ + char ud_name[FILE_NAME_LEN]; /* Updater file name */ + uint64_t i; + + /* Name of the checksum file: <md_file_path>.chk */ + HDsprintf(chk_name, "%s.chk", md_file_path); + + /* Remove the checksum file if exists. + If not, the callback will just continue appending + checksums to the existing file */ + if (HDaccess(chk_name, F_OK) == 0) { + HDremove(chk_name); + } + + /* Remove all the updater files if exist: <updater_file_path>.<i> */ + for (i = 0;; i++) { + HDsprintf(ud_name, "%s.%lu", updater_file_path, i); + if (HDaccess(ud_name, F_OK) != 0) + break; + HDremove(ud_name); + } + +} /* clean_chk_ud_files() */ + +/*------------------------------------------------------------------------- + * Function: verify_ud_chk() + * + * Purpose: This is the helper routine used by + * test_updater_generate_md_checksums() to verify + * contents of the checksum file and the updater files. + * --verify the sequence number in each updater's file header + * corresponds to the ith sequence number of the updater + * file name. + * --verify the tick number in each updater's file header + * corresponds to the tick number stored in the checksum file + * --verify the change_list_len in each updater's file header + * is consistent with num_change_list_entries in each updater's + * change list header + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; October 2021 + * + *------------------------------------------------------------------------- + */ +static herr_t +verify_ud_chk(char *md_file_path, char *ud_file_path) +{ + char chk_name[FILE_NAME_LEN]; /* Checksum file name */ + char ud_name[FILE_NAME_LEN]; /* Updater file name */ + FILE * chk_fp = NULL; /* Checksum file pointer */ + FILE * ud_fp = NULL; /* Updater file pointer */ + uint64_t ud_seq_num = 0; /* Sequence number in the updater file */ + uint64_t chk_ud_seq_num = 0; /* Updater sequence number in the checksum file */ + uint64_t i; /* Local index variable */ + long size = 0; /* Size of the file */ + size_t change_list_len = 0; /* change_list_len in the updater file header */ + uint32_t num_change_list_entries = 0; /* num_change_list_entries in the updater change list header */ + + /* Open the checksum file */ + HDsprintf(chk_name, "%s.chk", md_file_path); + if ((chk_fp = HDfopen(chk_name, "r")) == NULL) + FAIL_STACK_ERROR; + + for (i = 0;; i++) { + /* Generate updater file name: <ud_file_path>.<i> */ + HDsprintf(ud_name, "%s.%lu", ud_file_path, i); + + /* Open the updater file */ + if ((ud_fp = HDfopen(ud_name, "r")) == NULL) + break; + else { + /* Seek to the position of the sequence number in the updater file's header */ + if (HDfseek(ud_fp, (off_t)UD_HD_SEQ_NUM_OFFSET, SEEK_SET) < 0) + FAIL_STACK_ERROR; + + /* Read the sequence number from the updater file */ + if (HDfread(&ud_seq_num, UD_SIZE_8, 1, ud_fp) != 1) + FAIL_STACK_ERROR; + + /* Compare the sequence number with i */ + if (ud_seq_num != i) + TEST_ERROR; + + /* Read change_list_len from updater file's header */ + if (HDfseek(ud_fp, (off_t)UD_HD_CHANGE_LIST_LEN_OFFSET, SEEK_SET) < 0) + FAIL_STACK_ERROR; + + if (HDfread(&change_list_len, UD_SIZE_8, 1, ud_fp) != 1) + FAIL_STACK_ERROR; + + if (i != 0) { + + /* Read num_change_list_entries from updater file's change list */ + if (HDfseek(ud_fp, (off_t)UD_CL_NUM_CHANGE_LIST_ENTRIES_OFFSET, SEEK_SET) < 0) + FAIL_STACK_ERROR; + + if (HDfread(&num_change_list_entries, UD_SIZE_4, 1, ud_fp) != 1) + FAIL_STACK_ERROR; + + if (num_change_list_entries == 0) { + if (change_list_len != H5F_UD_CL_SIZE(0)) + TEST_ERROR; + } + else { + if (change_list_len != H5F_UD_CL_SIZE(num_change_list_entries)) + TEST_ERROR; + } + } + + /* Close the updater file */ + if (HDfclose(ud_fp) < 0) + FAIL_STACK_ERROR; + + /* Read the updater sequence number from checksum file */ + if (HDfread(&chk_ud_seq_num, UD_SIZE_8, 1, chk_fp) != 1) + FAIL_STACK_ERROR; + + /* Compare sequence number in updater file with sequence number in checksum file */ + if (ud_seq_num != chk_ud_seq_num) + TEST_ERROR; + + /* Advance checksum file to the next sequence number */ + if (HDfseek(chk_fp, (off_t)UD_SIZE_4, SEEK_CUR) < 0) + FAIL_STACK_ERROR; + } + } + + /* Get the size of the chksum file */ + if ((size = HDftell(chk_fp)) < 0) + FAIL_STACK_ERROR; + + /* Size of sequence number and checksum in the checksum file */ + if ((unsigned)size != (i * (UD_SIZE_8 + UD_SIZE_4))) + TEST_ERROR; + + return 0; + +error: + return (-1); + +} /* verify_ud_chk() */ + +/*------------------------------------------------------------------------- + * Function: md_ck_cb() + * + * Purpose: This is the callback function used by + * test_updater_generate_md_checksums() when the + * H5F_ACS_GENERATE_MD_CK_CB_NAME property is set in fapl. + * --Open and read the metadata file into a buffer. + * --Generate checksum for the metadata file + * --Write the tick number and the checksum to the checksum file + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; October 2021 + * + *------------------------------------------------------------------------- + */ +static herr_t +md_ck_cb(char *md_file_path, uint64_t updater_seq_num) +{ + FILE * md_fp = NULL; /* Metadata file pointer */ + FILE * chk_fp = NULL; /* Checksum file pointer */ + long size = 0; /* File size returned from HDftell() */ + void * buf = NULL; /* Buffer for holding the metadata file content */ + uint32_t chksum = 0; /* The checksum generated for the metadata file */ + char chk_name[FILE_NAME_LEN]; /* Buffer for the checksum file name */ + size_t ret; /* Return value */ + + /* Open the metadata file */ + if ((md_fp = HDfopen(md_file_path, "r")) == NULL) + FAIL_STACK_ERROR; + + /* Set file pointer at end of file.*/ + if (HDfseek(md_fp, 0, SEEK_END) < 0) + FAIL_STACK_ERROR; + + /* Get the current position of the file pointer.*/ + if ((size = HDftell(md_fp)) < 0) + FAIL_STACK_ERROR; + + if (size != 0) { + + HDrewind(md_fp); + + if ((buf = HDmalloc((size_t)size)) == NULL) + FAIL_STACK_ERROR; + + /* Read the metadata file to buf */ + if ((ret = HDfread(buf, 1, (size_t)size, md_fp)) != (size_t)size) + FAIL_STACK_ERROR; + + /* Calculate checksum of the metadata file */ + chksum = H5_checksum_metadata(buf, (size_t)size, 0); + } + + /* Close the metadata file */ + if (md_fp && HDfclose(md_fp) < 0) + FAIL_STACK_ERROR; + + /* + * Checksum file + */ + + /* Generate checksum file name: <md_file_path>.chk */ + HDsprintf(chk_name, "%s.chk", md_file_path); + + /* Open checksum file for append */ + if ((chk_fp = HDfopen(chk_name, "a")) == NULL) + FAIL_STACK_ERROR; + + /* Write the updater sequence number to the checksum file */ + if ((ret = HDfwrite(&updater_seq_num, sizeof(uint64_t), 1, chk_fp)) != 1) + FAIL_STACK_ERROR; + + /* Write the checksum to the checksum file */ + if ((ret = HDfwrite(&chksum, sizeof(uint32_t), 1, chk_fp)) != 1) + FAIL_STACK_ERROR; + + /* Close the checksum file */ + if (chk_fp && HDfclose(chk_fp) != 0) + FAIL_STACK_ERROR; + + if (buf) + HDfree(buf); + + return 0; + +error: + if (buf) + HDfree(buf); + if (md_fp) + HDfclose(md_fp); + if (chk_fp) + HDfclose(chk_fp); + + return -1; +} /* md_ck_cb() */ + +/*------------------------------------------------------------------------- + * Function: test_updater_generate_md_checksums() + * + * Purpose: It enables the generation of checksums for the metadata file + * created by the writer end of tick function. + * It also verifies the contents of the checksum file and the + * updater files. + * + * The test is invoked when the file is created via H5Fcreate() + * and via H5Fopen(). + * + * Return: 0 if test is sucessful + * 1 if test fails + * + * Programmer: Vailin Choi; October 2021 + * + * Note: It is important to clean up the checksum file and the updater files. + * + *------------------------------------------------------------------------- + */ +static unsigned +test_updater_generate_md_checksums(hbool_t file_create) +{ + hid_t fid = -1; /* File ID */ + hid_t fcpl = -1; /* File creation property list ID */ + hid_t fapl = -1; /* File access property list ID */ + H5F_vfd_swmr_config_t config; /* Configuration for VFD SWMR */ + H5F_generate_md_ck_cb_t cb_info; /* Callback */ + + if (file_create) { + TESTING("VFD SWMR updater generate checksums for metadata file with H5Fcreate"); + } + else { + TESTING("VFD SWMR updater generate checksums for metadata file with H5Fopen"); + } + + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(&config, 4, 7, TRUE, TRUE, TRUE, TRUE, 2, MD_FILE, UD_FILE); + + /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ + fapl = vfd_swmr_create_fapl(TRUE, TRUE, FALSE, 4096, &config); + + if (fapl == H5I_INVALID_HID) + TEST_ERROR; + + if ((fcpl = vfd_swmr_create_fcpl(H5F_FSPACE_STRATEGY_PAGE, 4096)) < 0) { + HDprintf("vfd_swmr_create_fcpl() failed"); + FAIL_STACK_ERROR; + } + + /* Set up callback to generate checksums for updater's metadata files */ + cb_info.func = md_ck_cb; + + /* Activate private property to generate checksums for updater's metadata file */ + if (H5Pset(fapl, H5F_ACS_GENERATE_MD_CK_CB_NAME, &cb_info) < 0) + FAIL_STACK_ERROR; + + /* Use file creation or file open for testing */ + if (file_create) { + if ((fid = H5Fcreate(FILENAME4, H5F_ACC_TRUNC, fcpl, fapl)) < 0) + FAIL_STACK_ERROR; + } + else { + if ((fid = H5Fcreate(FILENAME4, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR; + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + if ((fid = H5Fopen(FILENAME4, H5F_ACC_RDWR, fapl)) < 0) + FAIL_STACK_ERROR; + } + + /* Close the file */ + if (H5Fclose(fid) < 0) + FAIL_STACK_ERROR; + + if (H5Pclose(fapl) < 0) + FAIL_STACK_ERROR; + if (H5Pclose(fcpl) < 0) + FAIL_STACK_ERROR; + + /* Verify contents of checksum file and updater files */ + if (verify_ud_chk(config.md_file_path, config.updater_file_path) < 0) + TEST_ERROR; + + /* It's important to clean up the checksum and updater files. */ + clean_chk_ud_files(config.md_file_path, config.updater_file_path); + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY + { + H5Pclose(fapl); + H5Pclose(fcpl); + H5Fclose(fid); + } + H5E_END_TRY; + + /* It's important to clean up the chechsum and updater files. */ + clean_chk_ud_files(config.md_file_path, config.updater_file_path); + + return 1; + +} /* test_updater_generate_md_checksums() */ + +/*------------------------------------------------------------------------- * Function: main() * * Purpose: Main function for VFD SWMR tests. @@ -3481,9 +4443,18 @@ main(void) PUTS_ERROR("Can't get VFD-dependent fapl") } + /* Add nfs/updater testing in this routine */ nerrors += test_fapl(); if (use_file_locking) { + nerrors += test_updater_flags(); + nerrors += test_updater_flags_same_file_opens(); +#ifndef H5_HAVE_WIN32_API + /* VFD SWMR: Fails on windows due to error from generate_md_ck_cb(). */ + nerrors += test_updater_generate_md_checksums(TRUE); + nerrors += test_updater_generate_md_checksums(FALSE); +#endif + nerrors += test_shadow_index_lookup(); nerrors += test_file_fapl(); diff --git a/test/vfd_swmr_addrem_writer.c b/test/vfd_swmr_addrem_writer.c index f115d2e..ddf3f9c 100644 --- a/test/vfd_swmr_addrem_writer.c +++ b/test/vfd_swmr_addrem_writer.c @@ -94,8 +94,9 @@ open_skeleton(const char *filename, unsigned verbose) if ((config = HDcalloc(1, sizeof(*config))) == NULL) goto error; - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, 128, "./rw-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(TRUE, TRUE, FALSE, 4096, config)) < 0) diff --git a/test/vfd_swmr_attrdset_writer.c b/test/vfd_swmr_attrdset_writer.c index f445321..a722f44 100644 --- a/test/vfd_swmr_attrdset_writer.c +++ b/test/vfd_swmr_attrdset_writer.c @@ -1979,8 +1979,9 @@ main(int argc, char **argv) TEST_ERROR; } - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, 4, 7, writer, true, 128, "./attrdset-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(&config, 4, 7, writer, TRUE, FALSE, TRUE, 128, "./attrdset-shadow", NULL); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, 4096, &config)) < 0) { diff --git a/test/vfd_swmr_bigset_writer.c b/test/vfd_swmr_bigset_writer.c index 77c7b26..14cbedf 100644 --- a/test/vfd_swmr_bigset_writer.c +++ b/test/vfd_swmr_bigset_writer.c @@ -2470,9 +2470,10 @@ main(int argc, char **argv) continue; } - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, s.flush_raw_data, 128, - "./bigset-shadow-%zu", i); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(&config, s.tick_len, s.max_lag, s.writer, TRUE, FALSE, s.flush_raw_data, 128, + "./bigset-shadow-%zu", NULL, i); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, s.page_buf_size, &config)) < 0) { diff --git a/test/vfd_swmr_common.c b/test/vfd_swmr_common.c index cfe93aa..9b711dd 100644 --- a/test/vfd_swmr_common.c +++ b/test/vfd_swmr_common.c @@ -344,11 +344,14 @@ await_signal(hid_t fid) #endif /* H5_HAVE_WIN32_API */ /* Revised support routines that can be used for all VFD SWMR integration tests + * NOTE: For tests that call this common routine, md_file_path needs to be set + * regardless of whether maintain_metadata_file is true or false. */ -/* Initialize fields in config with the input parameters */ void init_vfd_swmr_config(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t max_lag, hbool_t writer, - hbool_t flush_raw_data, uint32_t md_pages_reserved, const char *md_file_fmtstr, ...) + hbool_t maintain_metadata_file, hbool_t generate_updater_files, hbool_t flush_raw_data, + uint32_t md_pages_reserved, const char *md_file_fmtstr, const char *updater_file_path, + ...) { va_list ap; @@ -357,16 +360,23 @@ init_vfd_swmr_config(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t config->version = H5F__CURR_VFD_SWMR_CONFIG_VERSION; config->pb_expansion_threshold = 0; - config->tick_len = tick_len; - config->max_lag = max_lag; - config->writer = writer; - config->flush_raw_data = flush_raw_data; - config->md_pages_reserved = md_pages_reserved; + config->tick_len = tick_len; + config->max_lag = max_lag; + config->writer = writer; + config->maintain_metadata_file = maintain_metadata_file; + config->generate_updater_files = generate_updater_files; + config->flush_raw_data = flush_raw_data; + config->md_pages_reserved = md_pages_reserved; + + HDva_start(ap, updater_file_path); - HDva_start(ap, md_file_fmtstr); evsnprintf(config->md_file_path, sizeof(config->md_file_path), md_file_fmtstr, ap); + HDva_end(ap); + if (config->generate_updater_files && updater_file_path != NULL) + HDstrcpy(config->updater_file_path, updater_file_path); + } /* init_vfd_swmr_config() */ /* Initialize the log file path in config, this function should be called after init_vfd_swmr_config. */ diff --git a/test/vfd_swmr_common.h b/test/vfd_swmr_common.h index f5981a5..b842fe7 100644 --- a/test/vfd_swmr_common.h +++ b/test/vfd_swmr_common.h @@ -74,8 +74,10 @@ H5TEST_DLL hid_t vfd_swmr_create_fapl(bool use_latest_format, bool use_vfd_swmr, size_t page_buf_size, H5F_vfd_swmr_config_t *config); H5TEST_DLL void init_vfd_swmr_config(H5F_vfd_swmr_config_t *config, uint32_t tick_len, uint32_t max_lag, - hbool_t writer, hbool_t flush_raw_data, uint32_t md_pages_reserved, - const char *md_file_fmtstr, ...) H5_ATTR_FORMAT(printf, 7, 8); + hbool_t writer, hbool_t maintain_metadata_file, + hbool_t generate_updater_files, hbool_t flush_raw_data, + uint32_t md_pages_reserved, const char *md_file_fmtstr, + const char *updater_file_path, ...) H5_ATTR_FORMAT(printf, 9, 11); H5TEST_DLL void init_vfd_swmr_log(H5F_vfd_swmr_config_t *config, const char *log_file_fmtstr, ...) H5_ATTR_FORMAT(printf, 2, 3); diff --git a/test/vfd_swmr_dsetchks_writer.c b/test/vfd_swmr_dsetchks_writer.c index d264ca9..c09c825 100644 --- a/test/vfd_swmr_dsetchks_writer.c +++ b/test/vfd_swmr_dsetchks_writer.c @@ -2302,8 +2302,10 @@ main(int argc, char **argv) TEST_ERROR; } - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, 4, 7, writer, s.flush_raw_data, 128, "./dsetchks-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(&config, 4, 7, writer, TRUE, FALSE, s.flush_raw_data, 128, "./dsetchks-shadow", + NULL); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, 4096, &config)) < 0) { diff --git a/test/vfd_swmr_dsetops_writer.c b/test/vfd_swmr_dsetops_writer.c index 3704c44..d999d64 100644 --- a/test/vfd_swmr_dsetops_writer.c +++ b/test/vfd_swmr_dsetops_writer.c @@ -1888,8 +1888,9 @@ main(int argc, char **argv) TEST_ERROR; } - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, 4, 7, writer, s.flush_raw_data, 128, "./dsetops-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(&config, 4, 7, writer, TRUE, FALSE, s.flush_raw_data, 128, "./dsetops-shadow", NULL); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(true, s.use_vfd_swmr, true, 4096, &config)) < 0) { diff --git a/test/vfd_swmr_generator.c b/test/vfd_swmr_generator.c index 16b1d4b..1c549ca 100644 --- a/test/vfd_swmr_generator.c +++ b/test/vfd_swmr_generator.c @@ -125,8 +125,9 @@ gen_skeleton(const char *filename, hbool_t verbose, hbool_t vfd_swmr_write, int if ((config = HDcalloc(1, sizeof(*config))) == NULL) return -1; - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(config, 4, 10, vfd_swmr_write, TRUE, 128, "generator-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config, 4, 10, vfd_swmr_write, TRUE, FALSE, TRUE, 128, "generator-shadow", NULL); } /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ diff --git a/test/vfd_swmr_gfail_writer.c b/test/vfd_swmr_gfail_writer.c index 430ed89..aef39a0 100644 --- a/test/vfd_swmr_gfail_writer.c +++ b/test/vfd_swmr_gfail_writer.c @@ -544,8 +544,10 @@ main(int argc, char **argv) TEST_ERROR; } - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, TRUE, 128, "./group-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, TRUE, FALSE, TRUE, 128, "./group-shadow", + NULL); /* If old-style option is chosen, use the earliest file format(H5F_LIBVER_EARLIEST) * as the second parameter of H5Pset_libver_bound() that is called by diff --git a/test/vfd_swmr_gperf_writer.c b/test/vfd_swmr_gperf_writer.c index 3342b40..1ca4483 100644 --- a/test/vfd_swmr_gperf_writer.c +++ b/test/vfd_swmr_gperf_writer.c @@ -2786,8 +2786,10 @@ main(int argc, char **argv) return EXIT_SUCCESS; } - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, FALSE, 128, "./group-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, TRUE, FALSE, FALSE, 128, "./group-shadow", + NULL); /* If the log flag is on, create the log file log-test under the current directory. */ if (s.glog == true) diff --git a/test/vfd_swmr_group_writer.c b/test/vfd_swmr_group_writer.c index 5881300..eb7bb0e 100644 --- a/test/vfd_swmr_group_writer.c +++ b/test/vfd_swmr_group_writer.c @@ -5011,8 +5011,10 @@ main(int argc, char **argv) TEST_ERROR; } - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, TRUE, 128, "./group-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(&config, s.tick_len, s.max_lag, writer, TRUE, FALSE, TRUE, 128, "./group-shadow", + NULL); /* If old-style option is chosen, use the earliest file format(H5F_LIBVER_EARLIEST) * as the second parameter of H5Pset_libver_bound() that is called by diff --git a/test/vfd_swmr_reader.c b/test/vfd_swmr_reader.c index 6265cae..d6a87dd 100644 --- a/test/vfd_swmr_reader.c +++ b/test/vfd_swmr_reader.c @@ -319,8 +319,9 @@ read_records(const char *filename, hbool_t verbose, FILE *verbose_file, unsigned goto error; } - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(config, 4, 5, FALSE, TRUE, 128, "./rw-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config, 4, 5, FALSE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config)) < 0) { diff --git a/test/vfd_swmr_remove_reader.c b/test/vfd_swmr_remove_reader.c index 4389b05..bc00756 100644 --- a/test/vfd_swmr_remove_reader.c +++ b/test/vfd_swmr_remove_reader.c @@ -303,8 +303,9 @@ read_records(const char *filename, unsigned verbose, unsigned long nseconds, uns if ((config = HDcalloc(1, sizeof(*config))) == NULL) goto error; - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(config, 4, 5, FALSE, TRUE, 128, "./rw-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config, 4, 5, FALSE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config)) < 0) { diff --git a/test/vfd_swmr_remove_writer.c b/test/vfd_swmr_remove_writer.c index 8df8fa8..6b0c13f 100644 --- a/test/vfd_swmr_remove_writer.c +++ b/test/vfd_swmr_remove_writer.c @@ -86,8 +86,9 @@ open_skeleton(const char *filename, unsigned verbose, unsigned old H5_ATTR_UNUSE if ((config = (H5F_vfd_swmr_config_t *)HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL) goto error; - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, 128, "./rw-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(TRUE, TRUE, FALSE, 4096, config)) < 0) diff --git a/test/vfd_swmr_sparse_reader.c b/test/vfd_swmr_sparse_reader.c index 943c375..6a7ba2f 100644 --- a/test/vfd_swmr_sparse_reader.c +++ b/test/vfd_swmr_sparse_reader.c @@ -207,8 +207,9 @@ read_records(const char *filename, unsigned verbose, unsigned long nrecords, uns if ((config = HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL) goto error; - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(config, 4, 5, FALSE, TRUE, 128, "./rw-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config, 4, 5, FALSE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(FALSE, TRUE, FALSE, 4096, config)) < 0) { diff --git a/test/vfd_swmr_sparse_writer.c b/test/vfd_swmr_sparse_writer.c index 56d19f3..f78bf8b 100644 --- a/test/vfd_swmr_sparse_writer.c +++ b/test/vfd_swmr_sparse_writer.c @@ -86,8 +86,9 @@ open_skeleton(const char *filename, unsigned verbose) if ((config = (H5F_vfd_swmr_config_t *)HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL) goto error; - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, 128, "./rw-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(TRUE, TRUE, FALSE, 4096, config)) < 0) diff --git a/test/vfd_swmr_vlstr_reader.c b/test/vfd_swmr_vlstr_reader.c index fa9d7b4..2554207 100644 --- a/test/vfd_swmr_vlstr_reader.c +++ b/test/vfd_swmr_vlstr_reader.c @@ -115,8 +115,9 @@ main(int argc, char **argv) if (argc > 0) errx(EXIT_FAILURE, "unexpected command-line arguments"); - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, 4, 7, false, TRUE, 128, "./vlstr-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(&config, 4, 7, false, TRUE, FALSE, TRUE, 128, "./vlstr-shadow", NULL); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ fapl = vfd_swmr_create_fapl(true, use_vfd_swmr, sel == TEST_OOB, 4096, &config); diff --git a/test/vfd_swmr_vlstr_writer.c b/test/vfd_swmr_vlstr_writer.c index 4f7376e..e3905e8 100644 --- a/test/vfd_swmr_vlstr_writer.c +++ b/test/vfd_swmr_vlstr_writer.c @@ -184,8 +184,9 @@ main(int argc, char **argv) if (argc > 0) errx(EXIT_FAILURE, "unexpected command-line arguments"); - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&config, 4, 7, true, TRUE, 128, "./vlstr-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(&config, 4, 7, true, TRUE, FALSE, TRUE, 128, "./vlstr-shadow", NULL); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ fapl = vfd_swmr_create_fapl(true, use_vfd_swmr, sel == TEST_OOB, 4096, &config); diff --git a/test/vfd_swmr_writer.c b/test/vfd_swmr_writer.c index 4693554..a630532 100644 --- a/test/vfd_swmr_writer.c +++ b/test/vfd_swmr_writer.c @@ -88,8 +88,9 @@ open_skeleton(const char *filename, hbool_t verbose, FILE *verbose_file, unsigne if ((config = (H5F_vfd_swmr_config_t *)HDcalloc(1, sizeof(H5F_vfd_swmr_config_t))) == NULL) return -1; - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, 128, "./rw-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(config, 4, 5, TRUE, TRUE, FALSE, TRUE, 128, "rw-shadow", NULL); /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ if ((fapl = vfd_swmr_create_fapl(TRUE, TRUE, FALSE, 4096, config)) < 0) diff --git a/test/vfd_swmr_zoo_writer.c b/test/vfd_swmr_zoo_writer.c index 13794d9..d40f2d9 100644 --- a/test/vfd_swmr_zoo_writer.c +++ b/test/vfd_swmr_zoo_writer.c @@ -473,8 +473,9 @@ main(int argc, char **argv) parse_command_line_options(argc, argv); - /* config, tick_len, max_lag, writer, flush_raw_data, md_pages_reserved, md_file_path */ - init_vfd_swmr_config(&vfd_swmr_config, TICK_LEN, 7, writer, TRUE, 128, "./zoo-shadow"); + /* config, tick_len, max_lag, writer, maintain_metadata_file, generate_updater_files, + * flush_raw_data, md_pages_reserved, md_file_path, updater_file_path */ + init_vfd_swmr_config(&vfd_swmr_config, TICK_LEN, 7, writer, TRUE, FALSE, TRUE, 128, "./zoo-shadow", NULL); /* ? turn off use latest format argument via 1st argument? since later on it reset to early format */ /* use_latest_format, use_vfd_swmr, only_meta_page, page_buf_size, config */ diff --git a/tools/lib/h5diff_array.c b/tools/lib/h5diff_array.c index 2d6c66c..1b0d36f 100644 --- a/tools/lib/h5diff_array.c +++ b/tools/lib/h5diff_array.c @@ -24,12 +24,8 @@ *------------------------------------------------------------------------- */ -#define F_FORMAT "%-15g %-15g %-15g\n" - -#if H5_SIZEOF_LONG_DOUBLE != 0 -#define LD_FORMAT "%-15Lg %-15Lg %-15Lg\n" -#endif - +#define F_FORMAT "%-15g %-15g %-15g\n" +#define LD_FORMAT "%-15Lg %-15Lg %-15Lg\n" #define I_FORMAT "%-15d %-15d %-15d\n" #define S_FORMAT "%-16s %-17s\n" #define UI_FORMAT "%-15u %-15u %-15u\n" @@ -39,12 +35,8 @@ #define ULLI_FORMAT "%-15" H5_PRINTF_LL_WIDTH "u %-15" H5_PRINTF_LL_WIDTH "u %-15" H5_PRINTF_LL_WIDTH "u\n" /* with -p option */ -#define F_FORMAT_P "%-15.10g %-15.10g %-15.10g %-14.10g\n" - -#if H5_SIZEOF_LONG_DOUBLE != 0 -#define LD_FORMAT_P "%-15.10Lg %-15.10Lg %-15.10Lg %-14.10Lg\n" -#endif - +#define F_FORMAT_P "%-15.10g %-15.10g %-15.10g %-14.10g\n" +#define LD_FORMAT_P "%-15.10Lg %-15.10Lg %-15.10Lg %-14.10Lg\n" #define I_FORMAT_P "%-15d %-15d %-15d %-14f\n" #define UI_FORMAT_P "%-15u %-15u %-15u %-14f\n" #define LI_FORMAT_P "%-15ld %-15ld %-15ld %-14f\n" @@ -56,12 +48,8 @@ #define SPACES " " /* not comparable */ -#define F_FORMAT_P_NOTCOMP "%-15.10g %-15.10g %-15.10g not comparable\n" - -#if H5_SIZEOF_LONG_DOUBLE != 0 -#define LD_FORMAT_P_NOTCOMP "%-15.10Lg %-15.10Lg %-15.10Lg not comparable\n" -#endif - +#define F_FORMAT_P_NOTCOMP "%-15.10g %-15.10g %-15.10g not comparable\n" +#define LD_FORMAT_P_NOTCOMP "%-15.10Lg %-15.10Lg %-15.10Lg not comparable\n" #define I_FORMAT_P_NOTCOMP "%-15d %-15d %-15d not comparable\n" #define UI_FORMAT_P_NOTCOMP "%-15u %-15u %-15u not comparable\n" #define LI_FORMAT_P_NOTCOMP "%-15ld %-15ld %-15ld not comparable\n" @@ -145,9 +133,7 @@ static hsize_t character_compare_opt(unsigned char *mem1, unsigned char *mem2, h diff_opt_t *opts); static hbool_t equal_float(float value, float expected, diff_opt_t *opts); static hbool_t equal_double(double value, double expected, diff_opt_t *opts); -#if H5_SIZEOF_LONG_DOUBLE != 0 static hbool_t equal_ldouble(long double value, long double expected, diff_opt_t *opts); -#endif static int print_data(diff_opt_t *opts); static void print_pos(diff_opt_t *opts, hsize_t elemtno, size_t u); @@ -162,10 +148,8 @@ static hsize_t diff_float_element(unsigned char *mem1, unsigned char *mem2, hsiz diff_opt_t *opts); static hsize_t diff_double_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, diff_opt_t *opts); -#if H5_SIZEOF_LONG_DOUBLE != 0 static hsize_t diff_ldouble_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, diff_opt_t *opts); -#endif static hsize_t diff_schar_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, diff_opt_t *opts); static hsize_t diff_uchar_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, @@ -191,12 +175,7 @@ static hsize_t diff_ullong_element(unsigned char *mem1, unsigned char *mem2, hsi *------------------------------------------------------------------------- */ -#if H5_SIZEOF_LONG_DOUBLE != 0 typedef enum dtype_t { FLT_FLOAT, FLT_DOUBLE, FLT_LDOUBLE } dtype_t; -#else - -typedef enum dtype_t { FLT_FLOAT, FLT_DOUBLE } dtype_t; -#endif /*------------------------------------------------------------------------- * XCAO, 11/10/2010 @@ -278,7 +257,6 @@ diff_array(void *_mem1, void *_mem2, diff_opt_t *opts, hid_t container1_id, hid_ return nfound; } /* nelmts */ } -#if H5_SIZEOF_LONG_DOUBLE != 0 else if (H5Tequal(opts->m_tid, H5T_NATIVE_LDOUBLE)) { for (i = 0; i < opts->hs_nelmts; i++) { nfound += diff_ldouble_element(mem1, mem2, i, opts); @@ -289,7 +267,6 @@ diff_array(void *_mem1, void *_mem2, diff_opt_t *opts, hid_t container1_id, hid_ return nfound; } /* nelmts */ } -#endif break; case H5T_INTEGER: @@ -1620,6 +1597,9 @@ character_compare_opt(unsigned char *mem1, unsigned char *mem2, hsize_t elemtno, hbool_t both_zero = FALSE; double per; + /* both_zero is set in the PER_UNSIGN macro but not used in this function */ + (void)both_zero; + HDmemcpy(&temp1_uchar, mem1, sizeof(unsigned char)); HDmemcpy(&temp2_uchar, mem2, sizeof(unsigned char)); H5TOOLS_START_DEBUG(" %d=%d", temp1_uchar, temp2_uchar); @@ -2034,7 +2014,6 @@ diff_double_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, * Return: number of differences found *------------------------------------------------------------------------- */ -#if H5_SIZEOF_LONG_DOUBLE != 0 static hsize_t diff_ldouble_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, diff_opt_t *opts) @@ -2070,7 +2049,7 @@ diff_ldouble_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, /* both not NaN, do the comparison */ if (!isnan1 && !isnan2) { - if (ABS(temp1_double - temp2_double) > opts->delta) { + if ((double)ABS(temp1_double - temp2_double) > opts->delta) { opts->print_percentage = 0; print_pos(opts, elem_idx, 0); if (print_data(opts)) { @@ -2163,7 +2142,7 @@ diff_ldouble_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, } nfound++; } - else if (per > opts->percent && ABS(temp1_double - temp2_double) > opts->delta) { + else if (per > opts->percent && (double)ABS(temp1_double - temp2_double) > opts->delta) { opts->print_percentage = 1; print_pos(opts, elem_idx, 0); if (print_data(opts)) { @@ -2200,7 +2179,6 @@ diff_ldouble_element(unsigned char *mem1, unsigned char *mem2, hsize_t elem_idx, return nfound; } -#endif /* H5_SIZEOF_LONG_DOUBLE */ /*------------------------------------------------------------------------- * Function: diff_schar_element @@ -3203,7 +3181,6 @@ equal_double(double value, double expected, diff_opt_t *opts) *------------------------------------------------------------------------- */ -#if H5_SIZEOF_LONG_DOUBLE != 0 static hbool_t equal_ldouble(long double value, long double expected, diff_opt_t *opts) { @@ -3244,8 +3221,6 @@ equal_ldouble(long double value, long double expected, diff_opt_t *opts) return FALSE; } -#endif /* #if H5_SIZEOF_LONG_DOUBLE !=0 */ - /*------------------------------------------------------------------------- * Function: equal_float * diff --git a/tools/lib/h5diff_util.c b/tools/lib/h5diff_util.c index e487a12..c40de9d 100644 --- a/tools/lib/h5diff_util.c +++ b/tools/lib/h5diff_util.c @@ -133,10 +133,8 @@ print_type(hid_t type) parallel_print("H5T_NATIVE_FLOAT"); else if (H5Tequal(type, H5T_NATIVE_DOUBLE)) parallel_print("H5T_NATIVE_DOUBLE"); -#if H5_SIZEOF_LONG_DOUBLE != 0 else if (H5Tequal(type, H5T_NATIVE_LDOUBLE)) parallel_print("H5T_NATIVE_LDOUBLE"); -#endif else parallel_print("undefined float"); break; diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index 5e60941..78e5c48 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -1903,15 +1903,21 @@ render_bin_output(FILE *stream, hid_t container, hid_t tid, void *_mem, hsize_t hid_t region_id = H5I_INVALID_HID; hid_t region_space = H5I_INVALID_HID; H5S_sel_type region_type; + H5R_ref_t tref; + + if (size > sizeof(tref)) + H5TOOLS_THROW((-1), "unexpectedly large ref"); + + HDmemset(&tref, 0, sizeof(tref)); for (block_index = 0; block_index < block_nelmts; block_index++) { mem = ((unsigned char *)_mem) + block_index * size; - if ((region_id = H5Ropen_object((H5R_ref_t *)mem, H5P_DEFAULT, H5P_DEFAULT)) < 0) + HDmemcpy(&tref, mem, size); + if ((region_id = H5Ropen_object(&tref, H5P_DEFAULT, H5P_DEFAULT)) < 0) H5TOOLS_INFO("H5Ropen_object H5T_STD_REF failed"); else { - if ((region_space = H5Ropen_region((H5R_ref_t *)mem, H5P_DEFAULT, H5P_DEFAULT)) >= - 0) { - if (!h5tools_is_zero(mem, H5Tget_size(H5T_STD_REF))) { + if ((region_space = H5Ropen_region(&tref, H5P_DEFAULT, H5P_DEFAULT)) >= 0) { + if (!h5tools_is_zero(&tref, H5Tget_size(H5T_STD_REF))) { region_type = H5Sget_select_type(region_space); if (region_type == H5S_SEL_POINTS) render_bin_output_region_points(region_space, region_id, stream, diff --git a/tools/lib/h5tools_str.c b/tools/lib/h5tools_str.c index abc0058..3cd12bb 100644 --- a/tools/lib/h5tools_str.c +++ b/tools/lib/h5tools_str.c @@ -705,7 +705,6 @@ h5tools_str_sprint(h5tools_str_t *str, const h5tool_format_t *info, hid_t contai HDmemcpy(&tempdouble, vp, sizeof(double)); h5tools_str_append(str, OPT(info->fmt_double, "%g"), tempdouble); -#if H5_SIZEOF_LONG_DOUBLE != 0 } else if (sizeof(long double) == nsize) { /* if (H5Tequal(type, H5T_NATIVE_LDOUBLE)) */ @@ -713,7 +712,6 @@ h5tools_str_sprint(h5tools_str_t *str, const h5tool_format_t *info, hid_t contai HDmemcpy(&templdouble, vp, sizeof(long double)); h5tools_str_append(str, "%Lg", templdouble); -#endif } else { size_t i; diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index f2407bf..ea9812b 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -450,6 +450,7 @@ free_table(table_t *table) HDfree(table->objs[u].objname); HDfree(table->objs); + HDfree(table); } #ifdef H5DUMP_DEBUG diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index 36114ba..2901241 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -453,11 +453,8 @@ table_list_free(void) /* Free each table */ free_table(table_list.tables[u].group_table); - HDfree(table_list.tables[u].group_table); free_table(table_list.tables[u].dset_table); - HDfree(table_list.tables[u].dset_table); free_table(table_list.tables[u].type_table); - HDfree(table_list.tables[u].type_table); } /* Free the table list */ diff --git a/tools/src/h5import/h5import.c b/tools/src/h5import/h5import.c index 6517e43..7272dff 100644 --- a/tools/src/h5import/h5import.c +++ b/tools/src/h5import/h5import.c @@ -467,14 +467,12 @@ readIntegerData(FILE *strm, struct Input *in) H5DT_INT16 temp16; H5DT_INT32 *in32; H5DT_INT32 temp32; -#ifdef H5_SIZEOF_LONG_LONG H5DT_INT64 *in64; H5DT_INT64 temp64; char buffer[256]; -#endif - hsize_t len = 1; - hsize_t i; - int j; + hsize_t len = 1; + hsize_t i; + int j; const char *err1 = "Unable to get integer value from file.\n"; const char *err2 = "Unrecognized input class type.\n"; @@ -589,7 +587,6 @@ readIntegerData(FILE *strm, struct Input *in) } break; -#ifdef H5_SIZEOF_LONG_LONG case 64: in64 = (H5DT_INT64 *)in->data; switch (in->inputClass) { @@ -626,7 +623,6 @@ readIntegerData(FILE *strm, struct Input *in) return (-1); } break; -#endif /* ifdef H5_SIZEOF_LONG_LONG */ default: (void)HDfprintf(stderr, "%s", err3); @@ -643,17 +639,15 @@ readUIntegerData(FILE *strm, struct Input *in) H5DT_UINT16 temp16; H5DT_UINT32 *in32; H5DT_UINT32 temp32; -#ifdef H5_SIZEOF_LONG_LONG H5DT_UINT64 *in64; H5DT_UINT64 temp64; char buffer[256]; -#endif - hsize_t len = 1; - hsize_t i; - int j; - const char *err1 = "Unable to get unsigned integer value from file.\n"; - const char *err2 = "Unrecognized input class type.\n"; - const char *err3 = "Invalid input size.\n"; + hsize_t len = 1; + hsize_t i; + int j; + const char * err1 = "Unable to get unsigned integer value from file.\n"; + const char * err2 = "Unrecognized input class type.\n"; + const char * err3 = "Invalid input size.\n"; for (j = 0; j < in->rank; j++) len *= in->sizeOfDimension[j]; @@ -760,7 +754,6 @@ readUIntegerData(FILE *strm, struct Input *in) } break; -#ifdef H5_SIZEOF_LONG_LONG case 64: in64 = (H5DT_UINT64 *)in->data; switch (in->inputClass) { @@ -797,7 +790,6 @@ readUIntegerData(FILE *strm, struct Input *in) return (-1); } break; -#endif /* ifdef H5_SIZEOF_LONG_LONG */ default: (void)HDfprintf(stderr, "%s", err3); @@ -2457,9 +2449,6 @@ validateConfigurationParameters(struct Input *in) const char *err4a = "OUTPUT-ARCHITECTURE cannot be STD if OUTPUT-CLASS is floating point (FP).\n"; const char *err4b = "OUTPUT-ARCHITECTURE cannot be IEEE if OUTPUT-CLASS is integer (IN).\n"; const char *err5 = "For OUTPUT-CLASS FP, valid values for OUTPUT-SIZE are (32, 64) .\n"; -#ifndef H5_SIZEOF_LONG_LONG - const char *err6 = "No support for reading 64-bit integer (INPUT-CLASS: IN, TEXTIN, UIN, TEXTUIN files\n"; -#endif /* for class STR other parameters are ignored */ if (in->inputClass == 5) /* STR */ @@ -2505,13 +2494,6 @@ validateConfigurationParameters(struct Input *in) return (-1); } -#ifndef H5_SIZEOF_LONG_LONG - if (in->inputSize == 64 && - (in->inputClass == 0 || in->inputClass == 4 || in->inputClass == 6 || in->inputClass == 7)) { - (void)HDfprintf(stderr, "%s", err6); - return -1; - } -#endif return (0); } @@ -3250,7 +3232,6 @@ getInputClassType(struct Input *in, char *buffer) kindex = 3; } -#if H5_SIZEOF_LONG_DOUBLE != 0 else if (!HDstrcmp(buffer, "H5T_NATIVE_LDOUBLE")) { in->inputSize = H5_SIZEOF_LONG_DOUBLE; in->configOptionVector[INPUT_SIZE] = 1; @@ -3263,7 +3244,6 @@ getInputClassType(struct Input *in, char *buffer) kindex = 3; } -#endif else if (!HDstrcmp(buffer, "H5T_TIME: not yet implemented")) { kindex = -1; } diff --git a/tools/test/h5diff/h5diffgentest.c b/tools/test/h5diff/h5diffgentest.c index 118a3a7..7cefeea 100644 --- a/tools/test/h5diff/h5diffgentest.c +++ b/tools/test/h5diff/h5diffgentest.c @@ -442,7 +442,6 @@ test_basic(const char *fname1, const char *fname2, const char *fname3) write_dset(gid1, 2, dims2, "d2", H5T_NATIVE_DOUBLE, data14); } -#if H5_SIZEOF_LONG_DOUBLE != 0 { /*------------------------------------------------------------------------- @@ -454,7 +453,6 @@ test_basic(const char *fname1, const char *fname2, const char *fname3) write_dset(gid1, 2, dims2, "ld", H5T_NATIVE_LDOUBLE, data15); } -#endif /*------------------------------------------------------------------------- * NaNs in H5T_NATIVE_FLOAT diff --git a/tools/test/h5dump/CMakeTests.cmake b/tools/test/h5dump/CMakeTests.cmake index a8984de..c0f279d 100644 --- a/tools/test/h5dump/CMakeTests.cmake +++ b/tools/test/h5dump/CMakeTests.cmake @@ -412,7 +412,7 @@ # -------------------------------------------------------------------- HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/tbin1.ddl" "${PROJECT_BINARY_DIR}/testfiles/std/tbin1LE.ddl" "h5dump_std_files") - if (WIN32) + if (WIN32 AND CMAKE_VS_WINDOWS_TARGET_PLATFORM_VERSION VERSION_LESS 10.0.18362.0) configure_file(${HDF5_TOOLS_DIR}/testfiles/tbinregR.exp ${PROJECT_BINARY_DIR}/testfiles/std/tbinregR.exp NEWLINE_STYLE CRLF) #file (READ ${HDF5_TOOLS_DIR}/testfiles/tbinregR.exp TEST_STREAM) #file (WRITE ${PROJECT_BINARY_DIR}/testfiles/std/tbinregR.exp "${TEST_STREAM}") diff --git a/tools/test/h5import/h5importtest.c b/tools/test/h5import/h5importtest.c index e49125b..7dbf762 100644 --- a/tools/test/h5import/h5importtest.c +++ b/tools/test/h5import/h5importtest.c @@ -46,11 +46,9 @@ main(void) int rowo4i = 11, colo4i = 21, plno4i = 51; int rowi4i = 1, coli4i = 2, plni4i = 5; -#ifdef H5_SIZEOF_LONG_LONG long long row4i64[3], col4i64[4], pln4i64[5]; long long rowo4i64 = (long long)11, colo4i64 = (long long)21, plno4i64 = (long long)51; long long rowi4i64 = (long long)1, coli4i64 = (long long)2, plni4i64 = (long long)5; -#endif short b16i3[5][3][4]; short row4i16[3], col4i16[4], pln4i16[5]; @@ -101,11 +99,9 @@ main(void) col4i[0] = colo4i; pln4i[0] = plno4i; -#ifdef H5_SIZEOF_LONG_LONG row4i64[0] = rowo4i64; col4i64[0] = colo4i64; pln4i64[0] = plno4i64; -#endif row4i16[0] = rowo4i16; col4i16[0] = colo4i16; @@ -116,33 +112,27 @@ main(void) pln4i8[0] = plno4i8; for (i = 1; i < nrow; i++) { - row4[i] = row4[i - 1] + rowi4; - row8[i] = row8[i - 1] + rowi8; - row4i[i] = row4i[i - 1] + rowi4i; -#ifdef H5_SIZEOF_LONG_LONG + row4[i] = row4[i - 1] + rowi4; + row8[i] = row8[i - 1] + rowi8; + row4i[i] = row4i[i - 1] + rowi4i; row4i64[i] = row4i64[i - 1] + rowi4i64; -#endif row4i16[i] = (short)(row4i16[i - 1] + rowi4i16); row4i8[i] = (char)(row4i8[i - 1] + rowi4i8); } for (j = 1; j < ncol; j++) { - col4[j] = col4[j - 1] + coli4; - col8[j] = col8[j - 1] + coli8; - col4i[j] = col4i[j - 1] + coli4i; -#ifdef H5_SIZEOF_LONG_LONG + col4[j] = col4[j - 1] + coli4; + col8[j] = col8[j - 1] + coli8; + col4i[j] = col4i[j - 1] + coli4i; col4i64[j] = col4i64[j - 1] + coli4i64; -#endif col4i16[j] = (short)(col4i16[j - 1] + coli4i16); col4i8[j] = (char)(col4i8[j - 1] + coli4i8); } for (k = 1; k < npln; k++) { - pln4[k] = pln4[k - 1] + plni4; - pln8[k] = pln8[k - 1] + plni8; - pln4i[k] = pln4i[k - 1] + plni4i; -#ifdef H5_SIZEOF_LONG_LONG + pln4[k] = pln4[k - 1] + plni4; + pln8[k] = pln8[k - 1] + plni8; + pln4i[k] = pln4i[k - 1] + plni4i; pln4i64[k] = pln4i64[k - 1] + plni4i64; -#endif pln4i16[k] = (short)(pln4i16[k - 1] + plni4i16); pln4i8[k] = (char)(pln4i8[k - 1] + plni4i8); } diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake index 037287d..397c3ac 100644 --- a/tools/test/h5repack/CMakeTests.cmake +++ b/tools/test/h5repack/CMakeTests.cmake @@ -1544,7 +1544,7 @@ # the references in attribute of compund or vlen datatype ADD_H5_TEST (HDFFV-5932 "TEST" ${FILE_ATTR_REF}) -# Add test for memory leak in attirbute. This test is verified by CTEST. +# Add test for memory leak in attribute. This test is verified by CTEST. # 1. leak from vlen string # 2. leak from compound type without reference member # (HDFFV-7840, ) @@ -1552,12 +1552,12 @@ ADD_H5_TEST (HDFFV-7840 "TEST" h5diff_attr1.h5) # test CVE-2018-17432 fix - set (arg h5repack_CVE-2018-17432.h5 h5repack__CVE-2018-17432_out.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6) + set (arg h5repack_CVE-2018-17432.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6) set (TESTTYPE "TEST") ADD_H5_FILTER_TEST (HDFFV-10590 "" ${TESTTYPE} 1 ${arg}) # test CVE-2018-14460 fix - set (arg h5repack_CVE-2018-14460.h5 h5repack_CVE-2018-14460_out.h5) + set (arg h5repack_CVE-2018-14460.h5) set (TESTTYPE "TEST") ADD_H5_FILTER_TEST (HDFFV-11223 "" ${TESTTYPE} 1 ${arg}) diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in index 3756a95..1e54670 100644 --- a/tools/test/h5repack/h5repack.sh.in +++ b/tools/test/h5repack/h5repack.sh.in @@ -885,13 +885,24 @@ TOOLTEST_FAIL() ( cd $TESTDIR $ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile - ) >$actual + ) >&$actual RET=$? - if [ $RET == 0 ] ; then + + # Normally h5repack of files tested with this function are expected + # to return not 0, but if the command results in "Segmentation fault" + # or "core dumped" it is a failure regardless of the return value. + failure=`grep -e 'Segmentation fault' -e 'core dumped' $actual` + if [ "$failure" != "" ]; then nerrors="`expr $nerrors + 1`" echo " FAILED" + echo " $failure" else - echo " PASSED" + if [ $RET == 0 ] ; then + nerrors="`expr $nerrors + 1`" + echo " FAILED" + else + echo " PASSED" + fi fi rm -f $outfile } diff --git a/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl b/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl index eeb0f2d..15ae813 100644 --- a/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl +++ b/tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl @@ -11,7 +11,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { @@ -33,7 +33,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { @@ -55,7 +55,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { @@ -77,7 +77,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { @@ -99,7 +99,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { @@ -121,7 +121,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { @@ -143,7 +143,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 13 0 } + PARAMS { 9 1 13 1 } } } FILLVALUE { |
