diff options
| author | Dana Robinson <43805+derobins@users.noreply.github.com> | 2020-11-17 13:47:52 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2020-11-17 13:47:52 (GMT) |
| commit | 5d0be1b196570ac2c08ffe0b61ac6a45b6ab7f3f (patch) | |
| tree | dbe8a76bcb211babe9900647eb1f515d8623ad9b | |
| parent | ec48fb8dfdb4f2f59f22a5efb6b950aafcac449d (diff) | |
| download | hdf5-5d0be1b196570ac2c08ffe0b61ac6a45b6ab7f3f.zip hdf5-5d0be1b196570ac2c08ffe0b61ac6a45b6ab7f3f.tar.gz hdf5-5d0be1b196570ac2c08ffe0b61ac6a45b6ab7f3f.tar.bz2 | |
Manual sync with develop (#95)
Brings all features from develop. Note that RELEASE.txt has not been
updated (will be done in a future PR).
175 files changed, 7656 insertions, 1691 deletions
diff --git a/.clang-format b/.clang-format index 4779a35..5b7d76f 100644 --- a/.clang-format +++ b/.clang-format @@ -5,6 +5,19 @@ AlignConsecutiveMacros: true AlignConsecutiveAssignments: true AlignConsecutiveDeclarations: true AlwaysBreakAfterReturnType: AllDefinitions +# Can enable the following section when llvm 12.x is out +#AttributeMacros: +# - H5_ATTR_FORMAT +# - H5_ATTR_UNUSED +# - H5_ATTR_DEPRECATED_USED +# - H5_ATTR_NDEBUG_UNUSED +# - H5_ATTR_DEBUG_API_USED +# - H5_ATTR_PARALLEL_UNUSED +# - H5_ATTR_PARALLEL_USED +# - H5_ATTR_NORETURN +# - H5_ATTR_CONST +# - H5_ATTR_PURE +# - H5_ATTR_FALLTHROUGH BraceWrapping: AfterFunction: true BeforeCatch: true @@ -192,8 +192,26 @@ ./config/site-specific/BlankForm +./doc/code-conventions.md + ./doxygen/aliases ./doxygen/Doxyfile.in +./doxygen/dox/api-compat-macros.dox +./doxygen/dox/H5Acreate.dox +./doxygen/dox/H5Aiterate.dox +./doxygen/dox/H5Fget_info.dox +./doxygen/dox/H5Lget_info_by_idx.dox +./doxygen/dox/H5Lget_info.dox +./doxygen/dox/H5Literate_by_name.dox +./doxygen/dox/H5Literate.dox +./doxygen/dox/H5Lvisit_by_name.dox +./doxygen/dox/H5Lvisit.dox +./doxygen/dox/H5Oget_info_by_idx.dox +./doxygen/dox/H5Oget_info_by_name.dox +./doxygen/dox/H5Oget_info.dox +./doxygen/dox/H5Ovisit_by_name.dox +./doxygen/dox/H5Ovisit.dox +./doxygen/dox/H5Sencode.dox ./doxygen/dox/mainpage.dox ./doxygen/dox/rm-template.dox ./doxygen/examples/H5Fclose.c @@ -2745,6 +2763,7 @@ ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_test.ddl ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_zero.ddl +./tools/test/h5repack/testfiles/h5repack_HDFFV-10590_CVE-2018-17432.h5 ./tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl ./tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl ./tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl @@ -21,9 +21,9 @@ dnl ------------------------------------------------------------------------- dnl _AC_SYS_LARGEFILE_MACRO_VALUE dnl dnl The following macro overrides the autoconf macro of the same name -dnl with this custom definition. This macro performs the same checks as +dnl with this custom definition. This macro performs the same checks as dnl autoconf's native _AC_SYS_LARGEFILE_MACRO_VALUE, but will also set -dnl AM_CPPFLAGS with the appropriate -D defines so additional configure +dnl AM_CPPFLAGS with the appropriate -D defines so additional configure dnl sizeof checks do not fail. dnl # _AC_SYS_LARGEFILE_MACRO_VALUE(C-MACRO, VALUE, diff --git a/bin/checkposix b/bin/checkposix index 233d15c..c29c290 100755 --- a/bin/checkposix +++ b/bin/checkposix @@ -44,7 +44,27 @@ foreach $arg (@ARGV) { # # If a user specifies one file, process it no matter what so people # can inspect files we normally skip (like H5system.c). + + $ignore = 0; + + # Ignored files in src/ if($#ARGV gt 0 and $filename =~ /H5FDmulti|H5FDstdio|H5VLpassthru|H5system|H5detect|H5make_libsettings/) { + $ignore = 1; + } + # Ignored atomic test files in test/ + if($#ARGV gt 0 and $filename =~ /atomic_reader|atomic_writer/) { + $ignore = 1; + } + # Ignored filter plugins in test/ + if($#ARGV gt 0 and $filename =~ /^filter_plugin\d_/) { + $ignore = 1; + } + # Ignored generators in test/ + if($#ARGV gt 0 and $filename =~ /^gen_/) { + $ignore = 1; + } + + if($ignore) { print "$filename is exempt from using Standard library macro wrappers\n"; next; } @@ -95,13 +115,13 @@ foreach $arg (@ARGV) { # These are really HDF5 functions/macros even though they don't # start with `h' or `H'. - next if $name =~ /^FUNC_(ENTER|LEAVE)(_(NO)?API|_PACKAGE|_STATIC)?(_NOFS|_NOCLEAR|_NOINIT)?(_NOFUNC|_TAG)?$/; + next if $name =~ /^FUNC_(ENTER|LEAVE)(_(NO)?API|_PACKAGE|_STATIC)?(_NOFS|_NOCLEAR|_NOINIT|_NOPUSH)?(_NOFUNC|_TAG)?$/; next if $name =~ /^(BEGIN|END)_FUNC$/; next if $name =~ /^U?INT(8|16|32|64)(ENCODE|DECODE)(_VAR)?$/; next if $name =~ /^CI_(PRINT_STATS|INC_SRC|INC_DST)$/; next if $name =~ /^(ABS|ADDR_OVERFLOW|ALL_MEMBERS|BOUND|CONSTR|DETECT_[I|F|M]|DOWN)$/; next if $name =~ /^(MIN3?|MAX3?|NELMTS|POWER_OF_TWO|REGION_OVERFLOW)$/; - next if $name =~ /^(UNIQUE_MEMBERS|S_ISDIR)$/; + next if $name =~ /^(SIZE_OVERFLOW|UNIQUE_MEMBERS|S_ISDIR)$/; next if $name =~ /^addr_defined$/; # These functions/macros are exempt. @@ -140,15 +160,38 @@ foreach $arg (@ARGV) { next if $name =~ /^(pow_fun|round_fun|abs_fun|lround_fun|llround_fun)$/; } + # This is a macro parameter in H5Rint.c. Ignore it in this file. + if($filename =~ /H5Rint/) { + next if $name =~ /^(func)$/; + } + + # Internal calls in the HDFS VFD (H5FDhdfs.c). Ignore it in this file. + if($filename =~ /H5FDhdfs/) { + next if $name =~ /^(hdfs)/; + } + + # Macros, etc. from the mirror VFD (H5FDmirror.c). Ignore in this file. + if($filename =~ /H5FDmirror/) { + next if $name =~ /^(LOG)/; + next if $name =~ /^(BSWAP_64|is_host_little_endian)$/; + } + + # These are things in H5FDs3comms.c and H5FDros3.c. Ignore them in these files. + if($filename =~ /H5FDs3comms|H5FDros3/) { + next if $name =~ /^(curl_|curlwritecallback|gmnow)/; + next if $name =~ /^(ros3_|ROS3_|S3COMMS_)/; + next if $name =~ /^(EVP_sha256|SHA256|ISO8601NOW)$/; + } + # TESTING (not comprehensive - just noise reduction) # Test macros and functions (testhdf5.h) next if $name =~ /^(AddTest|TestErrPrintf|TestSummary|TestCleanup|TestShutdown)$/; next if $name =~ /^(CHECK|CHECK_PTR|CHECK_PTR_NULL|CHECK_PTR_EQ|CHECK_I)$/; - next if $name =~ /^(VERIFY|VERIFY_STR|VERIFY|TYPE|MESSAGE|ERROR)$/; + next if $name =~ /^(VERIFY|VERIFY_STR|VERIFY_TYPE|MESSAGE|ERROR)$/; # Test macros and functions (h5test.h) - next if $name =~ /^(TESTING|PASSED|SKIPPED|FAIL_PUTS_ERROR|FAIL_STACK_ERROR|TEST_ERROR)$/; + next if $name =~ /^(TESTING|PASSED|SKIPPED|PUTS_ERROR|FAIL_PUTS_ERROR|FAIL_STACK_ERROR|TEST_ERROR|AT)$/; next if $name =~ /^(GetTestExpress)$/; # Ignore functions that start with test_ or check_ @@ -158,9 +201,49 @@ foreach $arg (@ARGV) { # Ignore functions that start with h5_ next if $name =~ /^h5_/; + # Ignore process completed status + next if $name =~ /(WIFEXITED|WEXITSTATUS|WIFSIGNALED|WTERMSIG|WCOREDUMP|WIFSTOPPED|WSTOPSIG)/; + # Ignore usage functions next if $name =~ /^usage$/; + # Ignore callbacks + next if $name =~ /(_cb\d?)$/; + + # Specific tests (not even remotely comprehensive) + + # accum test code + if($filename =~ /accum/) { + next if $name =~ /^(accum_)/; + } + + # cache test code + if($filename =~ /cache/) { + next if $name =~ /(_entry|_entries|_cache|_check|_dependency|_status|_op)$/; + next if $name =~ /^(verify_|smoke_check_|row_major_|col_major_)/; + next if $name =~ /^(resize_configs_are_equal|CACHE_ERROR)$/ + } + + # Splitter VFD test code. Ignore in vfd.c. + if($filename =~ /vfd/) { + next if $name =~ /^(SPLITTER_|splitter_)/; + next if $name =~ /(_splitter_)/; + next if $name =~ /^(file_exists)$/; + } + + # S3 VFD test code. Ignore in ros3.c and s3comms.c. + # HDFS VFD test code. Ignore in hdfs.c. + if($filename =~ /ros3|s3comms|hdfs/) { + next if $name =~ /^(JSVERIFY|JSFAILED_|JSERR_|jserr_|FAIL_)/; + next if $name =~ /^(curl_)/; + next if $name =~ /^(S3COMMS_FORMAT_CREDENTIAL|ISO8601NOW|gmnow)$/; + } + + # VDS test code. Ignore in vds.c. + if($filename =~ /vds/) { + next if $name =~ /^(vds_)/; + } + print "$filename:$.: $name\n"; } diff --git a/c++/src/H5FaccProp.cpp b/c++/src/H5FaccProp.cpp index 0f7184c..b60ff8f 100644 --- a/c++/src/H5FaccProp.cpp +++ b/c++/src/H5FaccProp.cpp @@ -691,6 +691,52 @@ FileAccPropList::getGcReferences() const } //-------------------------------------------------------------------------- +// Function: FileAccPropList::setFileLocking +///\brief Sets file locking flags. +///\param use_file_locking - IN: Flag that determines if file locks should +// be used or not. +///\param ignore_when_disabled - IN: Flag that determines if file locks +// should be be used when disabled on the file system or not. +///\exception H5::PropListIException +///\par Description +/// For information, please refer to the H5Pset_file_locking API in +/// the HDF5 C Reference Manual. +// Programmer Dana Robinson - 2020 +//-------------------------------------------------------------------------- +void +FileAccPropList::setFileLocking(hbool_t use_file_locking, hbool_t ignore_when_disabled) const +{ + herr_t ret_value = H5Pset_file_locking(id, use_file_locking, ignore_when_disabled); + + if (ret_value < 0) { + throw PropListIException("FileAccPropList::setFileLocking", "H5Pset_file_locking failed"); + } +} + +//-------------------------------------------------------------------------- +// Function: FileAccPropList::getFileLocking +///\brief Gets file locking flags. +///\param use_file_locking - OUT: Flag that determines if file locks +// should be used or not. +///\param ignore_when_disabled - OUT: Flag that determines if file locks +// should be be used when disabled on the file system or not. +///\exception H5::PropListIException +///\par Description +/// For information, please refer to the H5Pget_file_locking API in +/// the HDF5 C Reference Manual. +// Programmer Dana Robinson - 2020 +//-------------------------------------------------------------------------- +void +FileAccPropList::getFileLocking(hbool_t &use_file_locking, hbool_t &ignore_when_disabled) const +{ + herr_t ret_value = H5Pget_file_locking(id, &use_file_locking, &ignore_when_disabled); + + if (ret_value < 0) { + throw PropListIException("FileAccPropList::getFileLocking", "H5Pget_file_locking failed"); + } +} + +//-------------------------------------------------------------------------- // Function: FileAccPropList::setLibverBounds ///\brief Sets bounds on versions of library format to be used when creating /// or writing objects. diff --git a/c++/src/H5FaccProp.h b/c++/src/H5FaccProp.h index d1a5051..10bf728 100644 --- a/c++/src/H5FaccProp.h +++ b/c++/src/H5FaccProp.h @@ -122,6 +122,12 @@ class H5_DLLCPP FileAccPropList : public PropList { // Returns garbage collecting references setting. unsigned getGcReferences() const; + // Sets file locking parameters. + void setFileLocking(hbool_t use_file_locking, hbool_t ignore_when_disabled) const; + + // Gets file locking parameters. + void getFileLocking(hbool_t &use_file_locking, hbool_t &ignore_when_disabled) const; + // Sets bounds on versions of library format to be used when creating // or writing objects. void setLibverBounds(H5F_libver_t libver_low, H5F_libver_t libver_high) const; diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 2930969..ab121e9 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -61,6 +61,31 @@ if (HDF5_ENABLE_CODESTACK) endif () MARK_AS_ADVANCED (HDF5_ENABLE_CODESTACK) +# ---------------------------------------------------------------------- +# Check if they would like to use file locking by default +#----------------------------------------------------------------------------- +option (HDF5_USE_FILE_LOCKING "Use file locking by default (mainly for SWMR)" ON) +if (HDF5_USE_FILE_LOCKING) + set (${HDF_PREFIX}_USE_FILE_LOCKING 1) +endif () + +# ---------------------------------------------------------------------- +# Check if they would like to ignore file locks when disabled on a file system +#----------------------------------------------------------------------------- +option (HDF5_IGNORE_DISABLED_FILE_LOCKS "Ignore file locks when disabled on file system" ON) +if (HDF5_IGNORE_DISABLED_FILE_LOCKS) + set (${HDF_PREFIX}_IGNORE_DISABLED_FILE_LOCKS 1) +endif () + +# Set the libhdf5.settings file variable +if (HDF5_IGNORE_DISABLED_FILE_LOCKS AND HDF5_USE_FILE_LOCKING) + set (HDF5_FILE_LOCKING_SETTING "best-effort") +elseif (HDF5_IGNORE_DISABLED_FILE_LOCKS) + set (HDF5_FILE_LOCKING_SETTING "yes") +else () + set (HDF5_FILE_LOCKING_SETTING "no") +endif () + #----------------------------------------------------------------------------- # Are we going to use HSIZE_T #----------------------------------------------------------------------------- diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index e80ebc8..1c9fe75 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -421,9 +421,6 @@ /* Define to 1 if you have the <sys/stat.h> header file. */ #cmakedefine H5_HAVE_SYS_STAT_H @H5_HAVE_SYS_STAT_H@ -/* Define to 1 if you have the <sys/timeb.h> header file. */ -#cmakedefine H5_HAVE_SYS_TIMEB_H @H5_HAVE_SYS_TIMEB_H@ - /* Define to 1 if you have the <sys/time.h> header file. */ #cmakedefine H5_HAVE_SYS_TIME_H @H5_HAVE_SYS_TIME_H@ @@ -492,6 +489,9 @@ /* Define if the compiler understands __inline__ */ #cmakedefine H5_HAVE___INLINE__ @H5_HAVE___INLINE__@ +/* Define if the library will ignore file locks when disabled */ +#cmakedefine H5_IGNORE_DISABLED_FILE_LOCKS @H5_IGNORE_DISABLED_FILE_LOCKS@ + /* Define if the high-level library headers should be included in hdf5.h */ #cmakedefine H5_INCLUDE_HL @H5_INCLUDE_HL@ @@ -729,6 +729,9 @@ /* Define using v1.12 public API symbols by default */ #cmakedefine H5_USE_112_API_DEFAULT @H5_USE_112_API_DEFAULT@ +/* Define if the library will use file locking */ +#cmakedefine H5_USE_FILE_LOCKING @H5_USE_FILE_LOCKING@ + /* Define if a memory checking tool will be used on the library, to cause library to be very picky about memory operations and also disable the internal free list manager code. */ diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in index b745765..ebcbd61 100644 --- a/config/cmake/libhdf5.settings.cmake.in +++ b/config/cmake/libhdf5.settings.cmake.in @@ -85,5 +85,6 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ Using memory checker: @HDF5_ENABLE_USING_MEMCHECKER@ Memory allocation sanity checks: @HDF5_MEMORY_ALLOC_SANITY_CHECK@ Function Stack Tracing: @HDF5_ENABLE_CODESTACK@ + Use file locking: @HDF5_FILE_LOCKING_SETTING@ Strict File Format Checks: @HDF5_STRICT_FORMAT_CHECKS@ Optimization Instrumentation: @HDF5_Enable_Instrument@ diff --git a/config/cmake_ext_mod/HDFLibMacros.cmake b/config/cmake_ext_mod/HDFLibMacros.cmake index fe633bf..9704fbd 100644 --- a/config/cmake_ext_mod/HDFLibMacros.cmake +++ b/config/cmake_ext_mod/HDFLibMacros.cmake @@ -84,6 +84,10 @@ endmacro () #------------------------------------------------------------------------------- macro (EXTERNAL_SZIP_LIBRARY compress_type encoding) if (${compress_type} MATCHES "GIT") +# FetchContent_Declare (SZIP +# GIT_REPOSITORY ${SZIP_URL} +# GIT_TAG ${SZIP_BRANCH} +# ) EXTERNALPROJECT_ADD (SZIP GIT_REPOSITORY ${SZIP_URL} GIT_TAG ${SZIP_BRANCH} @@ -104,6 +108,10 @@ macro (EXTERNAL_SZIP_LIBRARY compress_type encoding) -DPACKAGE_NAMESPACE=${HDF_PACKAGE_NAMESPACE} ) elseif (${compress_type} MATCHES "TGZ") +# FetchContent_Declare (SZIP +# URL ${SZIP_URL} +# URL_HASH "" +# ) EXTERNALPROJECT_ADD (SZIP URL ${SZIP_URL} URL_MD5 "" @@ -125,7 +133,12 @@ macro (EXTERNAL_SZIP_LIBRARY compress_type encoding) ) endif () externalproject_get_property (SZIP BINARY_DIR SOURCE_DIR) - +# FetchContent_GetProperties(SZIP) +# if(NOT SZIP_POPULATED) +# FetchContent_Populate(SZIP) +# add_subdirectory(${szip_SOURCE_DIR} ${szip_BINARY_DIR}) +# endif() +# ##include (${BINARY_DIR}/${SZ_PACKAGE_NAME}${HDF_PACKAGE_EXT}-targets.cmake) # Create imported target szip-static if (USE_LIBAEC) diff --git a/config/gnu-warnings/7 b/config/gnu-warnings/7 index 266f5c1..9d5b44d 100644 --- a/config/gnu-warnings/7 +++ b/config/gnu-warnings/7 @@ -3,5 +3,4 @@ -Wduplicated-branches -Wformat-overflow=2 -Wformat-truncation=1 --Wimplicit-fallthrough=5 -Wrestrict diff --git a/configure.ac b/configure.ac index ea3250e..9880223 100644 --- a/configure.ac +++ b/configure.ac @@ -1211,6 +1211,7 @@ AC_CHECK_HEADERS([stddef.h setjmp.h features.h]) AC_CHECK_HEADERS([dirent.h]) AC_CHECK_HEADERS([stdint.h], [C9x=yes]) AC_CHECK_HEADERS([stdbool.h]) +AC_CHECK_HEADERS([netdb.h netinet/in.h arpa/inet.h]) ## Darwin AC_SUBST([H5_IS_DARWIN]) @@ -2379,6 +2380,56 @@ case "X-$OPTIMIZATION" in esac ## ---------------------------------------------------------------------- +## Check if file locking should be used +## +AC_MSG_CHECKING([enable file locking]) +AC_ARG_ENABLE([file-locking], + [AS_HELP_STRING([--enable-file-locking=(yes|no|best-effort)], + [Sets the default for whether or not to use file + locking when opening files. Can be overridden + with the HDF5_USE_FILE_LOCKING environment variable + and the H5Pset_file_locking() API call. + best-effort attempts to use file locking but does + not fail when file locks have been disabled on + the file system (useful with Lustre). + [default=best-effort] + ])], + [DESIRED_FILE_LOCKING=$enableval]) + +## Set defaults +if test "X-$DESIRED_FILE_LOCKING" = X- ; then + DESIRED_FILE_LOCKING=best-effort +fi + +## Allow this variable to be substituted in +## other files (src/libhdf5.settings.in, etc.) +AC_SUBST([DESIRED_FILE_LOCKING]) +AC_SUBST([USE_FILE_LOCKING]) +AC_SUBST([IGNORE_DISABLED_FILE_LOCKS]) + +case "X-$DESIRED_FILE_LOCKING" in + X-best-effort) + AC_MSG_RESULT([best-effort]) + AC_DEFINE([USE_FILE_LOCKING], [1], + [Define if the library will use file locking]) + AC_DEFINE([IGNORE_DISABLED_FILE_LOCKS], [1], + [Define if the library will ignore file locks when disabled]) + ;; + X-yes) + AC_MSG_RESULT([yes]) + AC_DEFINE([USE_FILE_LOCKING], [1], + [Define if the library will use file locking]) + ;; + X-no) + AC_MSG_RESULT([no]) + ;; + *) + AC_MSG_ERROR([Unrecognized value: $USE_FILE_LOCKING]) + ;; +esac + + +## ---------------------------------------------------------------------- ## Enable/disable internal package-level debugging output ## AC_MSG_CHECKING([for internal debug output]) diff --git a/doc/code-conventions.md b/doc/code-conventions.md new file mode 100644 index 0000000..ff3b4cf --- /dev/null +++ b/doc/code-conventions.md @@ -0,0 +1,57 @@ +# HDF5 Library Code Conventions + +This document describes some practices that are new, or newly +documented, starting in 2020. + +## Function / Variable Attributes + +In H5private.h, the library provides platform-independent macros +for qualifying function and variable definitions. + +### Functions that accept `printf(3)` and `scanf(3)` format strings + +Label functions that accept a `printf(3)`-compliant format string with +`H5_ATTR_FORMAT(printf,format_argno,variadic_argno)`, where +the format string is the `format_argno`th argument (counting from 1) +and the variadic arguments start with the `variadic_argno`th. + +Functions that accept a `scanf(3)`-compliant format string should +be labeled `H5_ATTR_FORMAT(scanf,format_argno,variadic_argno)`. + +### Functions that do never return + +The definition of a function that always causes the program to abort and hang +should be labeled `H5_ATTR_NORETURN` to help the compiler see which flows of +control are infeasible. + +### Other attributes + +**TBD** + +### Unused variables and parameters + +Compilers will warn about unused parameters and variables—developers should pay +attention to those warnings and make an effort to prevent them. + +Some function parameters and variables are unused in *all* configurations of +the project. Ordinarily, such parameters and variables should be deleted. +However, sometimes it is possible to foresee a parameter being used, or +removing it would change an API, or a parameter has to be defined to conform a +function to some function pointer type. In those cases, it's permissible to +mark a symbol `H5_ATTR_UNUSED`. + +Other parameters and variables are unused in *some* configurations of the +project, but not all. A symbol may fall into disuse in some configuration in +the future—then the compiler should warn, and the symbol should not be +defined—so developers should try to label a sometimes-unused symbol with an +attribute that's specific to the configurations where the symbol is (or is not) +expected to be used. The library provides the following attributes for that +purpose: + +* `H5_ATTR_DEPRECATED_USED`: used only if deprecated symbols are enabled +* `H5_ATTR_NDEBUG_UNUSED`: used only if `NDEBUG` is *not* \#defined +* `H5_ATTR_DEBUG_API_USED`: used if the debug API is enabled +* `H5_ATTR_PARALLEL_UNUSED`: used only if Parallel HDF5 *is not* configured +* `H5_ATTR_PARALLEL_USED`: used only if Parallel HDF5 *is* configured + +Some attributes may be phased in or phased out in the future. diff --git a/doxygen/dox/H5Acreate.dox b/doxygen/dox/H5Acreate.dox new file mode 100644 index 0000000..18d648f --- /dev/null +++ b/doxygen/dox/H5Acreate.dox @@ -0,0 +1,9 @@ +/** + * \ingroup H5A + * \def H5Acreate() + * H5Acreate() is a macro that is mapped to either H5Acreate1() or + * H5Acreate2(). + * + * + * \todo Standardize the way we describe these macros! + */ diff --git a/doxygen/dox/H5Aiterate.dox b/doxygen/dox/H5Aiterate.dox new file mode 100644 index 0000000..46b9bb4 --- /dev/null +++ b/doxygen/dox/H5Aiterate.dox @@ -0,0 +1,9 @@ +/** + * \ingroup H5A + * \def H5Aiterate() + * H5Aiterate() is a macro that is mapped to either H5Aiterate1() or + * H5Aiterate2(). + * + * + * \todo Standardize the way we describe these macros! + */ diff --git a/doxygen/dox/H5Fget_info.dox b/doxygen/dox/H5Fget_info.dox new file mode 100644 index 0000000..b2eeb6c --- /dev/null +++ b/doxygen/dox/H5Fget_info.dox @@ -0,0 +1,47 @@ +/** + * \ingroup H5F + * \def H5Fget_info() + * H5Fget_info() is a macro that is mapped to either H5Fget_info1() + * or H5Fget_info2(), depending on the needs of the application. + * Similarly, the macro for the \ref H5F_info_t struct is mapped to either + * H5F_info1_t or H5F_info2_t. + * + * Such macros are provided to facilitate application + * compatibility. Their use and mappings are fully described in "API Compatibility + * Macros in HDF5". + * + * When both the HDF5 library and the application are built and installed with + * no specific compatibility flags, H5Fget_info() is mapped to the most recent + * version of the function, currently H5Fget_info2(). If the library and/or + * application is compiled for Release 1.8 emulation, H5Fget_info() will be + * mapped to H5Fget_info1(). Since there was no H5Fget_info() function in + * Release 1.6, if the library and/or application is compiled for Release 1.6 + * emulation, H5Fget_info() will be mapped to the most recent version of the + * function, currently H5Fget_info2(). Function-specific flags are available to + * override these settings on a function-by-function basis when the application + * is compiled. + * + * Specific compile-time compatibility flags and the resulting + * mappings are as follows: + * + * \Bold{Global settings}\n + * \li No compatibility flag: H5Fget_info2() and H5F_info2_t + * \li Enable deprecated symbols: H5Fget_info2() and H5F_info2_t + * \li Disable deprecated symbols: H5Fget_info2() and H5F_info2_t + * \li Emulate Release 1.6 interface: H5Fget_info2() and H5F_info2_t + * \li Emulate Release 1.8 interface: H5Fget_info1() and H5F_info1_t + * + * \Bold{Function- and struct-level macros}\n + * \li \Code{H5Fget_info_vers=2}: H5Fget_info2() + * \li \Code{H5Fget_info_vers=1}: H5Fget_info1() + * \li \Code{H5F_info_t_vers=2}: H5F_info2_t + * \li \Code{H5F_info_t_vers=1}: H5F_info1_t + * + * \todo Fix the reference. + * + * \version 1.10.0 The C function H5Fget_info() and H5F_info_t renamed to + * H5Fget_info1() and H5F_info1_t, respectively, and deprecated + * in this release. The C macro #H5Fget_info, the C function + * H5Fget_info2(), and the struct H5F_info2_t introduced in this + * release. + */ diff --git a/doxygen/dox/H5Lget_info.dox b/doxygen/dox/H5Lget_info.dox new file mode 100644 index 0000000..686dfe4 --- /dev/null +++ b/doxygen/dox/H5Lget_info.dox @@ -0,0 +1,18 @@ + /** + * \ingroup LMGT + * \def H5Lget_info() + * H5Lget_info() is a macro that is mapped to either H5Lget_info1() + * or H5Lget_info2() Such macros are provided to facilitate application + * compatibility. Their use and mappings are fully described in Compatibility + * Macros in HDF5. If the library and/or application is compiled for Release + * 1.12 emulation, H5Lget_info() will be mapped to H5Lget_info2() and + * H5Lget_info1() is deprecated. With earlier versions, H5Lget_info() is mapped to + * H5Lget_info1(). Specific compile-time compatibility flags and the resulting + * mappings are as follows: + * \li No compatibility flag: H5Lget_info2() (using 1.12 source) H5Lget_info1() + * (using 1.10 or 1.8 source) + * \li Emulate Release 1.12: H5Lget_info2() + * \li Emulate Release 1.8 or 1.10 interface: H5Lget_info1() + * + * \todo Fix the reference. + */ diff --git a/doxygen/dox/H5Lget_info_by_idx.dox b/doxygen/dox/H5Lget_info_by_idx.dox new file mode 100644 index 0000000..8f1511e --- /dev/null +++ b/doxygen/dox/H5Lget_info_by_idx.dox @@ -0,0 +1,18 @@ + /** + * \ingroup LMGT + * \def H5Lget_info_by_idx() + * H5Lget_info_by_idx() is a macro that is mapped to either H5Lget_info_by_idx1() + * or H5Lget_info_by_idx2() Such macros are provided to facilitate application + * compatibility. Their use and mappings are fully described in Compatibility + * Macros in HDF5. If the library and/or application is compiled for Release + * 1.12 emulation, H5Lget_info_by_idx() will be mapped to H5Lget_info_by_idx2() and + * H5Lget_info_by_idx1() is deprecated. With earlier versions, H5Lget_infoby_idx() is mapped to + * H5Lget_info_by_idx1(). Specific compile-time compatibility flags and the resulting + * mappings are as follows: + * \li No compatibility flag: H5Lget_info_by_idx2() (using 1.12 source) H5Lget_info_by_idx1() + * (using 1.10 or 1.8 source) + * \li Emulate Release 1.12: H5Lget_info_by_idx2() + * \li Emulate Release 1.8 or 1.10 interface: H5Lget_info_by_idx1() + * + * \todo Fix the reference. + */ diff --git a/doxygen/dox/H5Literate.dox b/doxygen/dox/H5Literate.dox new file mode 100644 index 0000000..4d3a8dc --- /dev/null +++ b/doxygen/dox/H5Literate.dox @@ -0,0 +1,22 @@ +/** + * \ingroup TRAV + * \def H5Literate() + * H5Literate() is a macro that is mapped to either H5Literate1() or + * H5Literate2() Such macros are provided to facilitate application + * compatibility. Their use and mappings are fully described in + * Compatibility Macros in HDF5. If the library and/or application is + * compiled for Release 1.12 emulation, H5Literate() will be mapped to + * H5Literate2() and H5Literate1() is deprecated. With earlier versions, + * H5Literate() is mapped to H5Literate1(). Specific compile-time compatibility + * flags and the resulting mappings are as follows: + * \li No compatibility flag: H5Literate2() (using 1.12 source) H5Literate1() + * (using 1.10 or 1.8 source) + * \li Emulate Release 1.12: H5Literate2() + * \li Emulate Release 1.8 or 1.10 interface: H5Literate1() + * + * \todo Fix the reference. + * + * \version 1.12.0 The function H5Literate() was renamed to H5Literate1() and + * deprecated in this release. The macro H5Literate() and the + * function H5Literate2() were introduced in this release. + */ diff --git a/doxygen/dox/H5Literate_by_name.dox b/doxygen/dox/H5Literate_by_name.dox new file mode 100644 index 0000000..174157a --- /dev/null +++ b/doxygen/dox/H5Literate_by_name.dox @@ -0,0 +1,23 @@ +/** + * \ingroup TRAV + * \def H5Literate_by_name() + * H5Literate_by_name() is a macro that is mapped to either + * H5Literate_by_name1() or H5Literate_by_name2() Such macros are provided to + * facilitate application compatibility. Their use and mappings are fully + * described in Compatibility Macros in HDF5. If the library and/or application is + * compiled for Release 1.12 emulation, H5Literate_by_name() will be mapped to + * H5Literate_by_name2() and H5Literate_by_name1() is deprecated. With earlier + * versions, H5Literate_by_name() is mapped to H5Literate_by_name1(). + * Specific compile-time compatibility flags and the resulting mappings are as + * follows: + * \li No compatibility flag: H5Literate_by_name2() (using 1.12 source) + * H5Literate_by_name1() (using 1.10 or 1.8 source) + * \li Emulate Release 1.12: H5Literate_by_name2() + * \li Emulate Release 1.8 or 1.10 interface: H5Literate_by_name1() + * + * \todo Fix the reference. + * + * \version 1.12.0 The function H5Literate_by_name() was renamed to H5Literate_by_name1() and + * deprecated in this release. The macro H5Literate_by_name() and the + * function H5Literate_by_name2() were introduced in this release. + */ diff --git a/doxygen/dox/H5Lvisit.dox b/doxygen/dox/H5Lvisit.dox new file mode 100644 index 0000000..e356576 --- /dev/null +++ b/doxygen/dox/H5Lvisit.dox @@ -0,0 +1,22 @@ +/** + * \ingroup TRAV + * \def H5Lvisit() + * H5Lvisit() is a macro that is mapped to either H5Lvisit1() or + * H5Lvisit2() Such macros are provided to facilitate application + * compatibility. Their use and mappings are fully described in + * Compatibility Macros in HDF5. If the library and/or application is + * compiled for Release 1.12 emulation, H5Lvisit() will be mapped to + * H5Lvisit2() and H5Lvisit1() is deprecated. With earlier versions, + * H5Lvisit() is mapped to H5Lvisit1(). Specific compile-time compatibility + * flags and the resulting mappings are as follows: + * \li No compatibility flag: H5Lvisit2() (using 1.12 source) H5Lvisit1() + * (using 1.10 or 1.8 source) + * \li Emulate Release 1.12: H5Lvisit2() + * \li Emulate Release 1.8 or 1.10 interface: H5Lvisit1() + * + * \todo Fix the reference. + * + * \version 1.12.0 The function H5Lvisit() was renamed to H5Lvisit1() and + * deprecated in this release. The macro H5Lvisit() and the + * function H5Lvisit2() were introduced in this release. + */ diff --git a/doxygen/dox/H5Lvisit_by_name.dox b/doxygen/dox/H5Lvisit_by_name.dox new file mode 100644 index 0000000..0bb482e --- /dev/null +++ b/doxygen/dox/H5Lvisit_by_name.dox @@ -0,0 +1,22 @@ +/** + * \ingroup TRAV + * \def H5Lvisit_by_name() + * H5Lvisit_by_name() is a macro that is mapped to either H5Lvisit_by_name1() or + * H5Lvisit_by_name2() Such macros are provided to facilitate application + * compatibility. Their use and mappings are fully described in + * Compatibility Macros in HDF5. If the library and/or application is + * compiled for Release 1.12 emulation, H5Lvisit_by_name() will be mapped to + * H5Lvisit_by_name2() and H5Lvisit_by_name1() is deprecated. With earlier versions, + * H5Lvisit_by_name() is mapped to H5Lvisit_by_name1(). Specific compile-time + * compatibility flags and the resulting mappings are as follows: + * \li No compatibility flag: H5Lvisit_by_name2() (using 1.12 source) H5Lvisit_by_name1() + * (using 1.10 or 1.8 source) + * \li Emulate Release 1.12: H5Lvisit_by_name2() + * \li Emulate Release 1.8 or 1.10 interface: H5Lvisit_by_name1() + * + * \todo Fix the reference. + * + * \version 1.12.0 The function H5Lvisit_by_name() was renamed to H5Lvisit_by_name1() and + * deprecated in this release. The macro H5Lvisit_by_name() and the + * function H5Lvisit_by_name2() were introduced in this release. + */ diff --git a/doxygen/dox/H5Oget_info.dox b/doxygen/dox/H5Oget_info.dox new file mode 100644 index 0000000..e8b7afa --- /dev/null +++ b/doxygen/dox/H5Oget_info.dox @@ -0,0 +1,121 @@ +/** + * \ingroup H5O + * \def H5Oget_info() + * + * #H5Oget_info is a macro that is mapped to: + * \li H5Oget_info3() + * \li H5Oget_info1() + * + * \details Such macros are provided to facilitate application + * compatibility. Their use and mappings are fully described in + * API Compatibility Macros in HDF5; we urge you to read that + * document closely. + * + * In HDF5 versions 1.12 and after, #H5Oget_info is mapped to + * \ref H5Oget_info3() and \ref H5Oget_info1() is deprecated. + * In version 1.10 #H5Oget_info is identical to \ref H5Oget_info1(). + * + * Specific compile-time compatibility flags and the resulting + * mappings are as follows: + * \par + * <table> + * <tr> + * <th>Compatibility setting</th> + * <th>H5Oget_info</th> + * </tr> + * <tr> + * <td>No compatibility flag</td> + * <td>\ref H5Oget_info3() (in release 1.12)</td> + * </tr> + * <tr> + <td></td> + * <td>\ref H5Oget_info1() (in 1.8 or 1.10)</td> + * </tr> + * <tr> + * <td>Emulate Release 1.12</td> + * <td>\ref H5Oget_info3()</td> + * </tr> + * <tr> + * <td>Emulate Release 1.10/1.8 interface</td> + * <td>\ref H5Oget_info1()</td> + * </tr> + * </table> + * + * \note If you are iterating through a lot of different objects to + * retrieve information via the H5Oget_info() family of routines, + * you may see memory building up. This can be due to memory + * allocation for metadata such as object headers and messages + * when the iterated objects are put into the metadata cache. + * \note + * If the memory buildup is not desirable, you can configure a + * smaller cache via H5Fset_mdc_config() or set the file access + * property list via H5Pset_mdc_config(). A smaller sized cache + * will force metadata entries to be evicted from the cache, + * thus freeing the memory associated with the entries. + * + * \todo Fix reference to the document + * + * \par Version + * <table> + * <tr> + * <th>Release</th> + * <th>Change</th> + * </tr> + * <tr> + * <td>1.12.0</td> + * <td>The macro #H5Oget_info and the function \ref H5Oget_info3() were added, + * and \ref H5Oget_info1() was deprecated.</td> + * </tr> + * <tr> + * <td>1.10.5</td> + * <td>The macro #H5Oget_info was removed.</td> + * </tr> + * <tr> + * <td></td> + * <td>The functions \ref H5Oget_info1() and + * #H5Oget_info are identical in this release.</td> + * </tr> + * <tr> + * <td></td> + * <td>This change was added to restore the broken API compatibility + * introduced in HDF5-1.10.3.</td> + * </tr> + * <tr> + * <td>1.10.3</td> + * <td>The function \ref H5Oget_info() was renamed \ref H5Oget_info1.</td> + * </tr> + * <tr> + * <td></td> + * <td>The macro #H5Oget_info and the function \ref H5Oget_info2() were + * introduced in this release.</td> + * </tr> + * <tr> + * <td>1.8.15</td> + * <td>Added a note about the valid values for the \c version field in + * the \ref H5O_hdr_info_t structure.</td> + * </tr> + * <tr> + * <td>1.8.11</td> + * <td>Fortran subroutine introduced in this release.</td> + * </tr> + * <tr> + * <td>1.8.10</td> + * <td>Added \ref H5O_type_t structure to the Description section.</td> + * </tr> + * <tr> + * <td></td> + * <td>Separated \ref H5O_hdr_info_t structure from \ref H5O_info_t + * in the Description section.</td> + * </tr> + * <tr> + * <td></td> + * <td>Clarified the definition and implementation of the time fields.</td> + * </tr> + * <tr> + * <td>1.8.0</td> + * <td>Function introduced in this release.</td> + * </tr> + * </table> + * + * + */ diff --git a/doxygen/dox/H5Oget_info_by_idx.dox b/doxygen/dox/H5Oget_info_by_idx.dox new file mode 100644 index 0000000..2598374 --- /dev/null +++ b/doxygen/dox/H5Oget_info_by_idx.dox @@ -0,0 +1,92 @@ +/** + * \ingroup H5O + * \def H5Oget_info_by_idx() + * + * #H5Oget_info_by_idx is a macro that is mapped to: + * \li H5Oget_info_by_idx3() + * \li H5Oget_info_by_idx1() + * + * \details Such macros are provided to facilitate application + * compatibility. Their use and mappings are fully described in + * API Compatibility Macros in HDF5; we urge you to read that + * document closely. + * + * In HDF5 versions 1.12 and after, #H5Oget_info_by_idx is mapped to + * \ref H5Oget_info_by_idx3() and \ref H5Oget_info_by_idx1() is deprecated. + * In version 1.10 #H5Oget_info_by_idx is identical to \ref H5Oget_info_by_idx1(). + * + * Specific compile-time compatibility flags and the resulting + * mappings are as follows: + * + * \par + * <table> + * <tr> + * <th>Compatibility setting</th> + * <th>H5Oget_info_by_idx</th> + * </tr> + * <tr> + * <td>No compatibility flag</td> + * <td>\ref H5Oget_info_by_idx3() for 1.12</td> + * </tr> + * <tr> + <td></td> + * <td>\ref H5Oget_info_by_idx1() for 1.8/1.10</td> + * </tr> + * <tr> + * <td>Emulate Release 1.12</td> + * <td>\ref H5Oget_info_by_idx3()</td> + * </tr> + * <tr> + * <td>Emulate Release 1.10/1.8 interface</td> + * <td>\ref H5Oget_info_by_idx1()</td> + * </tr> + * </table> + * + * \todo Fix reference to the document + * + * \par Version + * <table> + * <tr> + * <th>Release</th> + * <th>Change</th> + * </tr> + * <tr> + * <td>1.12.0</td> + * <td>The macro #H5Oget_info_by_idx and function \ref H5Oget_info_by_idx3() were added, + * and \ref H5Oget_info_by_idx1() was deprecated.</td> + * </tr> + * <tr> + * <td>1.10.5</td> + * <td>The macro #H5Oget_info_by_idx was removed.</td> + * </tr> + * <tr> + * <td></td> + * <td>The functions \ref H5Oget_info_by_idx() and + * H5Oget_info_by_idx1() are identical in this release.</td> + * </tr> + * <tr> + * <td></td> + * <td>This change was added to restore the broken API compatibility + * introduced in HDF5-1.10.3.</td> + * </tr> + * <tr> + * <td>1.10.3</td> + * <td>The function \ref H5Oget_info_by_idx() was renamed \ref H5Oget_info_by_idx1.</td> + * </tr> + * <tr> + * <td></td> + * <td>The macro #H5Oget_info_by_idx and the function \ref H5Oget_info_by_idx2() were + * introduced in this release.</td> + * </tr> + * <tr> + * <td>1.8.11</td> + * <td>Fortran subroutine introduced in this release.</td> + * </tr> + * <tr> + * <td>1.8.0</td> + * <td>Function introduced in this release.</td> + * </tr> + * </table> + * + * + */ diff --git a/doxygen/dox/H5Oget_info_by_name.dox b/doxygen/dox/H5Oget_info_by_name.dox new file mode 100644 index 0000000..b1b9540 --- /dev/null +++ b/doxygen/dox/H5Oget_info_by_name.dox @@ -0,0 +1,99 @@ +/** + * \ingroup H5O + * \def H5Oget_info_by_name() + * + * #H5Oget_info_by_name is a macro that is mapped to: + * \li H5Oget_info_by_name3() + * \li H5Oget_info_by_name1() + * + * \details Such macros are provided to facilitate application + * compatibility. Their use and mappings are fully described in + * API Compatibility Macros in HDF5; we urge you to read that + * document closely. + * + * In HDF5 versions 1.12 and after, #H5Oget_info_by_name is mapped to + * \ref H5Oget_info_by_name3(). In version 1.10 #H5Oget_info_by_name is + * identical to \ref H5Oget_info_by_name1(). + * + * Specific compile-time compatibility flags and the resulting + * mappings are as follows: + * + * \par + * <table> + * <tr> + * <th>Compatibility setting</th> + * <th>H5Oget_info_by_name</th> + * </tr> + * <tr> + * <td>No compatibility flag</td> + * <td>\ref H5Oget_info_by_name3() for 1.12 and above</td> + * </tr> + * <tr> + <td></td> + * <td>\ref H5Oget_info_by_name1() for 1.8 or 1.10</td> + * </tr> + * <tr> + * <td>Emulate Release 1.12</td> + * <td>\ref H5Oget_info_by_name3()</td> + * </tr> + * <tr> + * <td>Emulate Release 1.10 or 1.8 interface</td> + * <td>\ref H5Oget_info_by_name1()</td> + * </tr> + * </table> + * + * \todo Fix reference to the document; exchange 1.8.8 and 1.8.0 + * + * \par Version + * <table> + * <tr> + * <th>Release</th> + * <th>Change</th> + * </tr> + * <tr> + * <td>1.12.0</td> + * <td>The macro #H5Oget_info_by_name and function \ref H5Oget_info_by_name3() were added + * and \ref H5Oget_info_by_name1() was deprecated.</td> + * </tr> + * <tr> + * <td>1.10.5</td> + * <td>The macro #H5Oget_info_by_name was removed.</td> + * </tr> + * <tr> + * <td></td> + * <td>The functions \ref H5Oget_info_by_name() and + * H5Oget_info_by_name1() are identical in this release.</td> + * </tr> + * <tr> + * <td></td> + * <td>This change was added to restore the broken API compatibility + * introduced in HDF5-1.10.3.</td> + * </tr> + * <tr> + * <td>1.10.3</td> + * <td>The function \ref H5Oget_info_by_name() was renamed + * to \ref H5Oget_info_by_name1.</td> + * </tr> + * <tr> + * <td></td> + * <td>The macro #H5Oget_info_by_name was renamed to + * \ref H5Oget_info_by_name1().</td> + * </tr> + * <tr> + * <td></td> + * <td>The macro #H5Oget_info_by_name and the function \ref H5Oget_info_by_name2() + * were introduced in this release.</td> + * </tr> + * <tr> + * <td>1.8.8</td> + * <td>Fortran 2003 subroutine and \ref H5O_info_t derived + * type introduced in this release.</td> + * </tr> + * <tr> + * <td>1.8.0</td> + * <td>C function introduced in this release.</td> + * </tr> + * </table> + * + * + */ diff --git a/doxygen/dox/H5Ovisit.dox b/doxygen/dox/H5Ovisit.dox new file mode 100644 index 0000000..5030a13 --- /dev/null +++ b/doxygen/dox/H5Ovisit.dox @@ -0,0 +1,92 @@ +/** + * \ingroup H5O + * \def H5Ovisit() + * + * #H5Ovisit is a macro that is mapped to one of the following: + * \li H5Ovisit3() + * \li H5Ovisit1() + * + * \details Such macros are provided to facilitate application + * compatibility. Their use and mappings are fully described in + * API Compatibility Macros in HDF5; we urge you to read that + * document closely. + * + * In HDF5 versions 1.12 and after, \ref H5Ovisit() is mapped to + * \ref H5Ovisit3(). In version 1.10, \ref H5Ovisit() is identical + * to \ref H5Ovisit1(). + * + * Specific compile-time compatibility flags and the resulting + * mappings are as follows: + * + * \par + * <table> + * <tr> + * <th>Compatibility settings</th> + * <th>H5Ovisit</th> + * </tr> + * <tr> + * <td>No compatibility flag</td> + * <td>\ref H5Ovisit3() in 1.12 or after</td> + * </tr> + * <tr> + <td></td> + * <td>\ref H5Ovisit1() for 1.8 and 1.10</td> + * </tr> + * <tr> + * <td>Emulate Release 1.12</td> + * <td>\ref H5Ovisit3()</td> + * </tr> + * <tr> + * <td>Emulate Release 1.10 or 1.8 interface</td> + * <td>\ref H5Ovisit1()</td> + * </tr> + * </table> + * + * \todo Fix reference to the document + * + * \par Version + * <table> + * <tr> + * <th>Release</th> + * <th>Change</th> + * </tr> + * <tr> + * <td>1.12.0</td> + * <td>The macro #H5Ovisit and function \ref H5Ovisit3() were added, + * and \ref H5Ovisit1() was deprecated.</td> + * </tr> + * <tr> + * <td>1.10.5</td> + * <td>The macro #H5Ovisit was removed.</td> + * </tr> + * <tr> + * <td></td> + * <td>The functions \ref H5Ovisit() and + * H5Ovisit1() are identical in this release.</td> + * </tr> + * <tr> + * <td></td> + * <td>This change was added to restore the broken API compatibility + * introduced in HDF5-1.10.3.</td> + * </tr> + * <tr> + * <td>1.10.3</td> + * <td>The function \ref H5Ovisit() was renamed to \ref H5Ovisit1.</td> + * </tr> + * <tr> + * <td></td> + * <td>The macro #H5Ovisit and the function \ref H5Ovisit2() were + * introduced in this release.</td> + * </tr> + * <tr> + * <td>1.8.8</td> + * <td>Fortran subroutine and data structure added.</td> + * </tr> + * <tr> + * <td>1.8.0</td> + * <td>C function introduced.</td> + * </tr> + * </table> + * + * + */ diff --git a/doxygen/dox/H5Ovisit_by_name.dox b/doxygen/dox/H5Ovisit_by_name.dox new file mode 100644 index 0000000..5c6e51a --- /dev/null +++ b/doxygen/dox/H5Ovisit_by_name.dox @@ -0,0 +1,92 @@ +/** + * \ingroup H5O + * \def H5Ovisit_by_name() + * + * #H5Ovisit_by_name is a macro that is mapped to one of the following: + * \li H5Ovisit_by_name3() + * \li H5Ovisit_by_name1() + * + * \details Such macros are provided to facilitate application + * compatibility. Their use and mappings are fully described in + * API Compatibility Macros in HDF5; we urge you to read that + * document closely. + * + * In HDF5 versions 1.12 and after, \ref H5Ovisit_by_name() is mapped to + * \ref H5Ovisit_by_name3(). In version 1.10, \ref H5Ovisit_by_name() + * is identical to \ref H5Ovisit_by_name1(). + * + * Specific compile-time compatibility flags and the resulting + * mappings are as follows: + * + * \par + * <table> + * <tr> + * <th>Compatibility settings</th> + * <th>H5Ovisit_by_name</th> + * </tr> + * <tr> + * <td>No compatibility flag</td> + * <td>\ref H5Ovisit_by_name3() for 1.12 and above</td> + * </tr> + * <tr> + <td></td> + * <td>\ref H5Ovisit_by_name1() for 1.10 or 1.8</td> + * </tr> + * <tr> + * <td>Emulate Release 1.12 interface</td> + * <td>\ref H5Ovisit_by_name3()</td> + * </tr> + * <tr> + * <td>Emulate Release 1.10 or 1.8 interface</td> + * <td>\ref H5Ovisit_by_name1()</td> + * </tr> + * </table> + * + * \todo Fix reference to the document + * + * \par Version + * <table> + * <tr> + * <th>Release</th> + * <th>Change</th> + * </tr> + * <tr> + * <td>1.12.0</td> + * <td>The macro #H5Ovisit_by_name and function \ref H5Ovisit_by_name3() were added. + * </tr> + * <tr> + * <td>1.10.5</td> + * <td>The macro #H5Ovisit_by_name was removed.</td> + * </tr> + * <tr> + * <td></td> + * <td>The functions \ref H5Ovisit_by_name() and + * H5Ovisit_by_name1() are identical in this release.</td> + * </tr> + * <tr> + * <td></td> + * <td>This change was added to restore the broken API compatibility + * introduced in HDF5-1.10.3.</td> + * </tr> + * <tr> + * <td>1.10.3</td> + * <td>The function \ref H5Ovisit_by_name() was renamed + * to \ref H5Ovisit_by_name1.</td> + * </tr> + * <tr> + * <td></td> + * <td>The macro #H5Ovisit_by_name and the function \ref H5Ovisit_by_name2() + * were introduced in this release.</td> + * </tr> + * <tr> + * <td>1.8.8</td> + * <td>Fortran subroutine introduced in this release.</td> + * </tr> + * <tr> + * <td>1.8.0</td> + * <td>Function introduced in this release.</td> + * </tr> + * </table> + * + * + */ diff --git a/doxygen/dox/H5Sencode.dox b/doxygen/dox/H5Sencode.dox new file mode 100644 index 0000000..fe0995c --- /dev/null +++ b/doxygen/dox/H5Sencode.dox @@ -0,0 +1,5 @@ +/** + * \ingroup H5S + * \def H5Sencode() + * H5Sencode() is a macro that is mapped to either H5Sencode1() or H5Sencode2(). +*/ diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index 7e06cf3..38e3aac 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -8263,5 +8263,107 @@ END SUBROUTINE h5pget_virtual_dsetname_f END SUBROUTINE h5pget_vol_id_f +!****s* H5P (F03)/h5pget_file_locking_f_F03 +! +! NAME +! h5pget_file_locking_f +! +! PURPOSE +! Gets the file locking properties. File locking is mainly used to help +! enforce SWMR semantics. +! +! INPUTS +! fapl_id - Target file access property list identifier. +! +! OUTPUTS +! use_file_locking - Whether or not to use file locks. +! ignore_disabled_locks - Whether or not to ignore file locks when locking +! is disabled on a file system. +! hdferr - error code: +! 0 on success and -1 on failure +! +! AUTHOR +! Dana Robinson +! Summer 2020 +! +! Fortran2003 Interface: + SUBROUTINE h5pget_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, hdferr) + IMPLICIT NONE + INTEGER(HID_T) , INTENT(IN) :: fapl_id + LOGICAL , INTENT(OUT) :: use_file_locking + LOGICAL , INTENT(OUT) :: ignore_disabled_locks + INTEGER , INTENT(OUT) :: hdferr +!***** + LOGICAL(C_BOOL) :: c_use_flag + LOGICAL(C_BOOL) :: c_ignore_flag + + INTERFACE + INTEGER FUNCTION h5pget_file_locking(fapl_id, use_file_locking, ignore_disabled_locks) BIND(C, NAME='H5Pget_file_locking') + IMPORT :: HID_T, C_BOOL + IMPLICIT NONE + INTEGER(HID_T), INTENT(IN), VALUE :: fapl_id + LOGICAL(C_BOOL), INTENT(OUT) :: use_file_locking + LOGICAL(C_BOOL), INTENT(OUT) :: ignore_disabled_locks + END FUNCTION h5pget_file_locking + END INTERFACE + + hdferr = INT(h5pget_file_locking(fapl_id, c_use_flag, c_ignore_flag)) + + ! Transfer value of C C_BOOL type to Fortran LOGICAL + use_file_locking = c_use_flag + ignore_disabled_locks = c_ignore_flag + + END SUBROUTINE h5pget_file_locking_f + +!****s* H5P (F03)/h5pset_file_locking_f_F03 +! +! NAME +! h5pset_file_locking_f +! +! PURPOSE +! Sets the file locking properties. File locking is mainly used to help +! enforce SWMR semantics. +! +! INPUTS +! fapl_id - Target file access property list identifier. +! use_file_locking - Whether or not to use file locks. +! ignore_disabled_locks - Whether or not to ignore file locks when locking +! is disabled on a file system. +! hdferr - error code: +! 0 on success and -1 on failure +! +! AUTHOR +! Dana Robinson +! Summer 2020 +! +! Fortran2003 Interface: + SUBROUTINE h5pset_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, hdferr) + IMPLICIT NONE + INTEGER(HID_T) , INTENT(IN) :: fapl_id + LOGICAL , INTENT(IN) :: use_file_locking + LOGICAL , INTENT(IN) :: ignore_disabled_locks + INTEGER , INTENT(OUT) :: hdferr +!***** + LOGICAL(C_BOOL) :: c_use_flag + LOGICAL(C_BOOL) :: c_ignore_flag + + INTERFACE + INTEGER FUNCTION h5pset_file_locking(fapl_id, use_file_locking, ignore_disabled_locks) BIND(C, NAME='H5Pset_file_locking') + IMPORT :: HID_T, C_BOOL + IMPLICIT NONE + INTEGER(HID_T), INTENT(IN), VALUE :: fapl_id + LOGICAL(C_BOOL), INTENT(IN), VALUE :: use_file_locking + LOGICAL(C_BOOL), INTENT(IN), VALUE :: ignore_disabled_locks + END FUNCTION h5pset_file_locking + END INTERFACE + + ! Transfer value of Fortran LOGICAL to C C_BOOL type + c_use_flag = use_file_locking + c_ignore_flag = ignore_disabled_locks + + hdferr = INT(h5pset_file_locking(fapl_id, c_use_flag, c_ignore_flag)) + + END SUBROUTINE h5pset_file_locking_f + END MODULE H5P diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in index 9c69e5a..4207239 100644 --- a/fortran/src/hdf5_fortrandll.def.in +++ b/fortran/src/hdf5_fortrandll.def.in @@ -339,6 +339,8 @@ H5P_mp_H5PGET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_VOL_F H5P_mp_H5PGET_VOL_ID_F +H5P_mp_H5PSET_FILE_LOCKING_F +H5P_mp_H5PGET_FILE_LOCKING_F ; Parallel @H5_NOPAREXP@H5P_mp_H5PSET_FAPL_MPIO_F @H5_NOPAREXP@H5P_mp_H5PGET_FAPL_MPIO_F diff --git a/fortran/test/fortranlib_test.F90 b/fortran/test/fortranlib_test.F90 index 01baef5..1fb3e68 100644 --- a/fortran/test/fortranlib_test.F90 +++ b/fortran/test/fortranlib_test.F90 @@ -186,6 +186,10 @@ PROGRAM fortranlibtest CALL test_chunk_cache(cleanup, ret_total_error) CALL write_test_status(ret_total_error, ' Dataset chunk cache configuration', total_error) + ret_total_error = 0 + CALL test_misc_properties(cleanup, ret_total_error) + CALL write_test_status(ret_total_error, ' Miscellaneous properties', total_error) + ! ! '=========================================' ! 'Testing ATTRIBUTE interface ' diff --git a/fortran/test/tH5P.F90 b/fortran/test/tH5P.F90 index 7fe3971..19bee75 100644 --- a/fortran/test/tH5P.F90 +++ b/fortran/test/tH5P.F90 @@ -724,4 +724,77 @@ SUBROUTINE test_chunk_cache(cleanup, total_error) END SUBROUTINE test_chunk_cache +!------------------------------------------------------------------------- +! Function: test_misc_properties +! +! Purpose: Tests setting and getting of miscellaneous properties. Does +! not test the underlying functionality as that is done in +! the C library tests. +! +! Tests APIs: +! H5P_GET/SET_FILE_LOCKING_F +! +! Return: Success: 0 +! Failure: -1 +! +!------------------------------------------------------------------------- +! +SUBROUTINE test_misc_properties(cleanup, total_error) + + IMPLICIT NONE + LOGICAL, INTENT(IN) :: cleanup + INTEGER, INTENT(INOUT) :: total_error + + INTEGER(hid_t) :: fapl_id = -1 ! Local fapl + LOGICAL :: use_file_locking ! (H5Pset/get_file_locking_f) + LOGICAL :: ignore_disabled_locks ! (H5Pset/get_file_locking_f) + INTEGER :: error + + ! Create a default fapl + CALL H5Pcreate_f(H5P_FILE_ACCESS_F, fapl_id, error) + CALL check("H5Pcreate_f", error, total_error) + + ! Test H5Pset/get_file_locking_f + ! true values + use_file_locking = .TRUE. + ignore_disabled_locks = .TRUE. + CALL h5pset_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, error) + CALL check("h5pset_set_file_locking_f", error, total_error) + use_file_locking = .FALSE. + ignore_disabled_locks = .FALSE. + CALL h5pget_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, error) + CALL check("h5pget_set_file_locking_f", error, total_error) + if(use_file_locking .neqv. .TRUE.) then + total_error = total_error + 1 + write(*,*) "Got wrong use_file_locking flag from h5pget_file_locking_f" + endif + if(ignore_disabled_locks .neqv. .TRUE.) then + total_error = total_error + 1 + write(*,*) "Got wrong ignore_disabled_locks flag from h5pget_file_locking_f" + endif + + ! false values + use_file_locking = .FALSE. + ignore_disabled_locks = .FALSE. + CALL h5pset_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, error) + CALL check("h5pset_set_file_locking_f", error, total_error) + use_file_locking = .TRUE. + ignore_disabled_locks = .TRUE. + CALL h5pget_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, error) + CALL check("h5pget_set_file_locking_f", error, total_error) + if(use_file_locking .neqv. .FALSE.) then + total_error = total_error + 1 + write(*,*) "Got wrong use_file_locking flag from h5pget_file_locking_f" + endif + if(ignore_disabled_locks .neqv. .FALSE.) then + total_error = total_error + 1 + write(*,*) "Got wrong ignore_disabled_locks flag from h5pget_file_locking_f" + endif + + ! Close the fapl + CALL H5Pclose_f(fapl_id, error) + CALL check("H5Pclose_f", error, total_error) + +END SUBROUTINE test_misc_properties + END MODULE TH5P diff --git a/hl/src/H5DS.c b/hl/src/H5DS.c index 216cf46..7cd36b9 100644 --- a/hl/src/H5DS.c +++ b/hl/src/H5DS.c @@ -1881,13 +1881,15 @@ out: htri_t H5DSis_scale(hid_t did) { - hid_t tid = -1; /* attribute type ID */ - hid_t aid = -1; /* attribute ID */ - herr_t has_class; /* has the "CLASS" attribute */ - htri_t is_ds; /* boolean return value */ - H5I_type_t it; /* ID type */ - char * buf; /* Name of attribute */ - hsize_t storage_size; /* Size of storage for attribute */ + hid_t tid = -1; /* attribute type ID */ + hid_t aid = -1; /* attribute ID */ + herr_t attr_class; /* has the "CLASS" attribute */ + htri_t is_ds = -1; /* set to "not a dimension scale" */ + H5I_type_t it; /* type of identifier */ + char * buf = NULL; /* buffer to read name of attribute */ + size_t string_size; /* size of storage for the attribute */ + H5T_class_t type_class; + H5T_str_t strpad; /*------------------------------------------------------------------------ * parameter checking @@ -1895,18 +1897,19 @@ H5DSis_scale(hid_t did) */ /* get ID type */ if ((it = H5Iget_type(did)) < 0) - return FAIL; + goto out; if (H5I_DATASET != it) - return FAIL; + goto out; /* try to find the attribute "CLASS" on the dataset */ - if ((has_class = H5LT_find_attribute(did, "CLASS")) < 0) - return FAIL; + if ((attr_class = H5LT_find_attribute(did, "CLASS")) < 0) + goto out; - if (has_class == 0) + if (attr_class == 0) { is_ds = 0; - + goto out; + } else { if ((aid = H5Aopen(did, "CLASS", H5P_DEFAULT)) < 0) goto out; @@ -1914,19 +1917,33 @@ H5DSis_scale(hid_t did) if ((tid = H5Aget_type(aid)) < 0) goto out; - /* check to make sure attribute is a string */ - if (H5T_STRING != H5Tget_class(tid)) + /* check to make sure attribute is a string; + if not, then it is not dimension scale */ + if ((type_class = H5Tget_class(tid)) < 0) goto out; - - /* check to make sure string is null-terminated */ - if (H5T_STR_NULLTERM != H5Tget_strpad(tid)) + if (H5T_STRING != type_class) { + is_ds = 0; goto out; + } + /* check to make sure string is null-terminated; + if not, then it is not dimension scale */ + if ((strpad = H5Tget_strpad(tid)) < 0) + goto out; + if (H5T_STR_NULLTERM != strpad) { + is_ds = 0; + goto out; + } - /* allocate buffer large enough to hold string */ - if ((storage_size = H5Aget_storage_size(aid)) == 0) + /* According to Spec string is ASCII and its size should be 16 to hold + "DIMENSION_SCALE" string */ + if ((string_size = H5Tget_size(tid)) == 0) goto out; + if (string_size != 16) { + is_ds = 0; + goto out; + } - buf = (char *)HDmalloc((size_t)storage_size * sizeof(char) + 1); + buf = (char *)HDmalloc((size_t)string_size * sizeof(char)); if (buf == NULL) goto out; @@ -1937,8 +1954,6 @@ H5DSis_scale(hid_t did) /* compare strings */ if (HDstrncmp(buf, DIMENSION_SCALE_CLASS, MIN(HDstrlen(DIMENSION_SCALE_CLASS), HDstrlen(buf))) == 0) is_ds = 1; - else - is_ds = 0; HDfree(buf); @@ -1948,18 +1963,17 @@ H5DSis_scale(hid_t did) if (H5Aclose(aid) < 0) goto out; } - - return is_ds; - - /* error zone */ out: - H5E_BEGIN_TRY - { - H5Aclose(aid); - H5Tclose(tid); + if (is_ds < 0) { + HDfree(buf); + H5E_BEGIN_TRY + { + H5Aclose(aid); + H5Tclose(tid); + } + H5E_END_TRY; } - H5E_END_TRY; - return FAIL; + return is_ds; } /*------------------------------------------------------------------------- diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 3c5a124..1c37839 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -8518,6 +8518,55 @@ public class H5 implements java.io.Serializable { public synchronized static native void H5Pset_evict_on_close(long fapl_id, boolean evict_on_close) throws HDF5LibraryException; + /** + * H5Pget_use_file_locking retrieves whether we are using file locking. + * + * @param fapl_id + * IN: File access property list identifier + * + * @return indication if file locking is used. + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * + **/ + public synchronized static native boolean H5Pget_use_file_locking(long fapl_id) + throws HDF5LibraryException; + + /** + * H5Pget_use_file_locking retrieves whether we ignore file locks when they are disabled. + * + * @param fapl_id + * IN: File access property list identifier + * + * @return indication if file locking is ignored. + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * + **/ + public synchronized static native boolean H5Pget_ignore_disabled_file_locking(long fapl_id) + throws HDF5LibraryException; + + /** + * H5Pset_file_locking sets parameters related to file locking. + * + * @param fapl_id + * IN: File access property list identifier + * + * @param use_file_locking + * IN: Whether the library will use file locking when opening files (mainly for SWMR semantics). + * + * @param ignore_when_disabled + * IN: Whether file locking will be ignored when disabled on a file system (useful for Lustre). + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * + **/ + public synchronized static native void H5Pset_file_locking(long fapl_id, boolean use_file_locking, boolean ignore_when_disabled) + throws HDF5LibraryException; + // ///// unimplemented ///// // herr_t H5Pset_vol(hid_t plist_id, hid_t new_vol_id, const void *new_vol_info); // herr_t H5Pget_vol_id(hid_t plist_id, hid_t *vol_id); diff --git a/java/src/hdf/hdf5lib/HDF5Constants.java b/java/src/hdf/hdf5lib/HDF5Constants.java index 2241238..a4f17e6 100644 --- a/java/src/hdf/hdf5lib/HDF5Constants.java +++ b/java/src/hdf/hdf5lib/HDF5Constants.java @@ -215,8 +215,8 @@ public class HDF5Constants { public static final long H5E_CANTLOAD = H5E_CANTLOAD(); /** Minor error codes - Resource errors - Unable to lock object */ public static final long H5E_CANTLOCK = H5E_CANTLOCK(); -// /** Minor error codes - File accessibility errors Unable to lock file */ -// public static final long H5E_CANTLOCKFILE = H5E_CANTLOCKFILE(); + /** Minor error codes - File accessibility errors Unable to lock file */ + public static final long H5E_CANTLOCKFILE = H5E_CANTLOCKFILE(); /** Minor error codes - Cache related errors - Unable to mark a pinned entry as clean */ public static final long H5E_CANTMARKCLEAN = H5E_CANTMARKCLEAN(); /** Minor error codes - Cache related errors - Unable to mark a pinned entry as dirty */ @@ -289,8 +289,8 @@ public class HDF5Constants { public static final long H5E_CANTUNDEPEND = H5E_CANTUNDEPEND(); /** Minor error codes - Resource errors - Unable to unlock object */ public static final long H5E_CANTUNLOCK = H5E_CANTUNLOCK(); -// /** Minor error codes - File accessibility errors Unable to unlock file */ -// public static final long H5E_CANTUNLOCKFILE = H5E_CANTUNLOCKFILE(); + /** Minor error codes - File accessibility errors Unable to unlock file */ + public static final long H5E_CANTUNLOCKFILE = H5E_CANTUNLOCKFILE(); /** Minor error codes - Cache related errors - Unable to un-pin cache entry */ public static final long H5E_CANTUNPIN = H5E_CANTUNPIN(); /** Minor error codes - Cache related errors - Unable to unprotect metadata */ @@ -1262,7 +1262,7 @@ public class HDF5Constants { private static native final long H5E_CANTLOCK(); -// private static native final long H5E_CANTLOCKFILE(); + private static native final long H5E_CANTLOCKFILE(); private static native final long H5E_CANTMARKCLEAN(); @@ -1336,7 +1336,7 @@ public class HDF5Constants { private static native final long H5E_CANTUNLOCK(); -// private static native final long H5E_CANTUNLOCKFILE(); + private static native final long H5E_CANTUNLOCKFILE(); private static native final long H5E_CANTUNPIN(); diff --git a/java/src/jni/h5Constants.c b/java/src/jni/h5Constants.c index ade28e8..66ce8cc 100644 --- a/java/src/jni/h5Constants.c +++ b/java/src/jni/h5Constants.c @@ -594,13 +594,11 @@ Java_hdf_hdf5lib_HDF5Constants_H5E_1CANTLOCK(JNIEnv *env, jclass cls) { return H5E_CANTLOCK; } -/* JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_HDF5Constants_H5E_1CANTLOCKFILE(JNIEnv *env, jclass cls) { return H5E_CANTLOCKFILE; } -*/ JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_HDF5Constants_H5E_1CANTMARKCLEAN(JNIEnv *env, jclass cls) { @@ -781,13 +779,11 @@ Java_hdf_hdf5lib_HDF5Constants_H5E_1CANTUNLOCK(JNIEnv *env, jclass cls) { return H5E_CANTUNLOCK; } -/* JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_HDF5Constants_H5E_1CANTUNLOCKFILE(JNIEnv *env, jclass cls) { return H5E_CANTUNLOCKFILE; } -*/ JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_HDF5Constants_H5E_1CANTUNPIN(JNIEnv *env, jclass cls) { diff --git a/java/src/jni/h5pFAPLImp.c b/java/src/jni/h5pFAPLImp.c index bbeddaf..0077df4 100644 --- a/java/src/jni/h5pFAPLImp.c +++ b/java/src/jni/h5pFAPLImp.c @@ -1364,6 +1364,76 @@ done: /* * Class: hdf_hdf5lib_H5 + * Method: H5Pset_file_locking + * Signature: (JZZ)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Pset_1file_1locking(JNIEnv *env, jclass clss, jlong fapl_id, jboolean use_file_locking, + jboolean ignore_when_disabled) +{ + hbool_t use_file_locking_val = TRUE; + hbool_t ignore_when_disabled_val = TRUE; + + UNUSED(clss); + + use_file_locking_val = (use_file_locking == JNI_TRUE) ? TRUE : FALSE; + ignore_when_disabled_val = (ignore_when_disabled == JNI_TRUE) ? TRUE : FALSE; + + if (H5Pset_file_locking((hid_t)fapl_id, use_file_locking_val, ignore_when_disabled_val) < 0) + H5_LIBRARY_ERROR(ENVONLY); + +done: + return; +} /* end Java_hdf_hdf5lib_H5_H5Pset_1file_1locking */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_use_file_locking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1use_1file_1locking(JNIEnv *env, jclass clss, jlong fapl_id) +{ + hbool_t use_file_locking_val = TRUE; + hbool_t unused = TRUE; + jboolean bval = JNI_FALSE; + + UNUSED(clss); + + if (H5Pget_file_locking((hid_t)fapl_id, &use_file_locking_val, &unused) < 0) + H5_LIBRARY_ERROR(ENVONLY); + + bval = (use_file_locking_val == TRUE) ? JNI_TRUE : JNI_FALSE; + +done: + return bval; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1use_1file_1locking */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_ignore_disabled_file_locking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1ignore_1disabled_1file_1locking(JNIEnv *env, jclass clss, jlong fapl_id) +{ + hbool_t ignore_when_disabled_val = TRUE; + hbool_t unused = TRUE; + jboolean bval = JNI_FALSE; + + UNUSED(clss); + + if (H5Pget_file_locking((hid_t)fapl_id, &unused, &ignore_when_disabled_val) < 0) + H5_LIBRARY_ERROR(ENVONLY); + + bval = (ignore_when_disabled_val == TRUE) ? JNI_TRUE : JNI_FALSE; + +done: + return bval; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1ignore_1disabled_1file_1locking */ + +/* + * Class: hdf_hdf5lib_H5 * Method: H5Pset_metadata_read_attempts * Signature: (JJ)V */ diff --git a/java/src/jni/h5pFAPLImp.h b/java/src/jni/h5pFAPLImp.h index 8c9d7ab..780ebc4 100644 --- a/java/src/jni/h5pFAPLImp.h +++ b/java/src/jni/h5pFAPLImp.h @@ -331,6 +331,32 @@ JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1evict_1on_1close(JNIEnv * /* * Class: hdf_hdf5lib_H5 + * Method: H5Pset_file_locking + * Signature: (JZZ)V + */ +JNIEXPORT void JNICALL Java_hdf_hdf5lib_H5_H5Pset_1file_1locking(JNIEnv *env, jclass clss, jlong fapl_id, + jboolean use_file_locking, + jboolean ignore_when_disabled); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_use_file_locking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1use_1file_1locking(JNIEnv *env, jclass clss, + jlong fapl_id); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_ignore_disabled_file_locking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL Java_hdf_hdf5lib_H5_H5Pget_1ignore_1disabled_1file_1locking(JNIEnv *env, + jclass clss, + jlong fapl_id); + +/* + * Class: hdf_hdf5lib_H5 * Method: H5Pset_metadata_read_attempts * Signature: (JJ)V */ diff --git a/java/test/TestH5Pfapl.java b/java/test/TestH5Pfapl.java index 10a79dd..4233580 100644 --- a/java/test/TestH5Pfapl.java +++ b/java/test/TestH5Pfapl.java @@ -1398,4 +1398,36 @@ public class TestH5Pfapl { fail("H5P_evict_on_close: " + err); } } + + @Test + public void testH5P_file_locking() { + boolean use_file_locking = false; + boolean ignore_disabled_file_locking = false; + try { + // false values (usually not the default) + H5.H5Pset_file_locking(fapl_id, false, false); + use_file_locking = H5.H5Pget_use_file_locking(fapl_id); + ignore_disabled_file_locking = H5.H5Pget_ignore_disabled_file_locking(fapl_id); + assertFalse("H5P_file_locking", use_file_locking); + assertFalse("H5P_file_locking", ignore_disabled_file_locking); + + // true values (typically the default) + H5.H5Pset_file_locking(fapl_id, true, true); + use_file_locking = H5.H5Pget_use_file_locking(fapl_id); + ignore_disabled_file_locking = H5.H5Pget_ignore_disabled_file_locking(fapl_id); + assertTrue("H5P_file_locking", use_file_locking); + assertTrue("H5P_file_locking", ignore_disabled_file_locking); + } + catch (HDF5PropertyListInterfaceException err) { + // parallel is not supported + if (err.getMinorErrorNumber() != HDF5Constants.H5E_UNSUPPORTED) { + err.printStackTrace(); + fail("H5P_test_file_locking: " + err); + } + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5P_test_file_locking: " + err); + } + } } diff --git a/java/test/testfiles/JUnit-TestH5Pfapl.txt b/java/test/testfiles/JUnit-TestH5Pfapl.txt index c4f37d0..c34ac1c 100644 --- a/java/test/testfiles/JUnit-TestH5Pfapl.txt +++ b/java/test/testfiles/JUnit-TestH5Pfapl.txt @@ -28,6 +28,7 @@ JUnit version 4.11 .testH5Pset_elink_fapl .testH5P_hyper_vector_size .testH5P_gc_references +.testH5P_file_locking .testH5P_family_offset .testH5P_fapl_core .testH5P_fapl_muti @@ -37,5 +38,5 @@ JUnit version 4.11 Time: XXXX -OK (35 tests) +OK (36 tests) @@ -449,6 +449,14 @@ done: * Programmer: Robb Matzke * Jul 9 1997 * + * Changes: + * + * In the parallel case, added code to setup the MDC slist + * before the call to H5AC__flush_entries() and take it down + * afterwards. + * + * JRM -- 7/29/20 + * *------------------------------------------------------------------------- */ herr_t @@ -477,15 +485,20 @@ H5AC_dest(H5F_t *f) if (H5C_get_logging_status(f->shared->cache, &log_enabled, &curr_logging) < 0) HGOTO_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to get logging status") - if (log_enabled && curr_logging) + if (log_enabled && curr_logging) { + if (H5C_log_write_destroy_cache_msg(f->shared->cache) < 0) HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + } + /* Tear down logging */ - if (log_enabled) + if (log_enabled) { + if (H5C_log_tear_down(f->shared->cache) < 0) HGOTO_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "mdc logging tear-down failed") + } #ifdef H5_HAVE_PARALLEL @@ -495,19 +508,46 @@ H5AC_dest(H5F_t *f) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_clear_coll_entries() failed") aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(f->shared->cache); - if (aux_ptr) + + if (aux_ptr) { + /* Sanity check */ HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - /* If the file was opened R/W, attempt to flush all entries - * from rank 0 & Bcast clean list to other ranks. - * - * Must not flush in the R/O case, as this will trigger the - * free space manager settle routines. - */ - if (H5F_ACC_RDWR & H5F_INTENT(f)) - if (H5AC__flush_entries(f) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush") + /* If the file was opened R/W, attempt to flush all entries + * from rank 0 & Bcast clean list to other ranks. + * + * Must not flush in the R/O case, as this will trigger the + * free space manager settle routines. + * + * Must also enable the skip list before the call to + * H5AC__flush_entries() and disable it afterwards, as the + * skip list will be disabled after the previous flush. + * + * Note that H5C_dest() does slist setup and take down as well. + * Unfortunately, we can't do the setup and take down just once, + * as H5C_dest() is called directly in the test code. + * + * Fortunately, the cache should be clean or close to it at this + * point, so the overhead should be minimal. + */ + if (H5F_ACC_RDWR & H5F_INTENT(f)) { + + /* enable and load the slist */ + if (H5C_set_slist_enabled(f->shared->cache, TRUE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "set slist enabled failed") + + if (H5AC__flush_entries(f) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush") + + /* disable the slist -- should be empty */ + if (H5C_set_slist_enabled(f->shared->cache, FALSE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "disable slist failed") + } + } #endif /* H5_HAVE_PARALLEL */ /* Destroy the cache */ @@ -902,9 +942,10 @@ H5AC_mark_entry_dirty(void *thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_mark_entry_dirty_msg(cache_ptr, entry_ptr, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_mark_entry_dirty_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_mark_entry_dirty() */ @@ -954,9 +995,10 @@ H5AC_mark_entry_clean(void *thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_mark_entry_clean_msg(cache_ptr, entry_ptr, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_mark_entry_clean_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_mark_entry_clean() */ @@ -995,9 +1037,10 @@ H5AC_mark_entry_unserialized(void *thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_mark_unserialized_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_mark_unserialized_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_mark_entry_unserialized() */ @@ -1035,9 +1078,10 @@ H5AC_mark_entry_serialized(void *thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_mark_serialized_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_mark_serialized_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_mark_entry_serialized() */ @@ -1134,9 +1178,10 @@ H5AC_pin_protected_entry(void *thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_pin_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_pin_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_pin_protected_entry() */ @@ -1179,6 +1224,107 @@ done: } /* H5AC_prep_for_file_close() */ /*------------------------------------------------------------------------- + * + * Function: H5AC_prep_for_file_flush + * + * Purpose: This function should be called just prior to the first + * call to H5AC_flush() during a file flush. + * + * Its purpose is to handly any setup required prior to + * metadata cache flush. + * + * Initially, this means setting up the slist prior to the + * flush. We do this in a seperate call because + * H5F__flush_phase2() make repeated calls to H5AC_flush(). + * Handling this detail in separate calls allows us to avoid + * the overhead of setting up and taking down the skip list + * repeatedly. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 5/5/20 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_prep_for_file_flush(H5F_t *f) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + + if (H5C_set_slist_enabled(f->shared->cache, TRUE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "slist enabled failed") + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5AC_prep_for_file_flush() */ + +/*------------------------------------------------------------------------- + * + * Function: H5AC_secure_from_file_flush + * + * Purpose: This function should be called just after the last + * call to H5AC_flush() during a file flush. + * + * Its purpose is to perform any necessary cleanup after the + * metadata cache flush. + * + * The objective of the call is to allow the metadata cache + * to do any necessary necessary cleanup work after a cache + * flush. + * + * Initially, this means taking down the slist after the + * flush. We do this in a seperate call because + * H5F__flush_phase2() make repeated calls to H5AC_flush(). + * Handling this detail in separate calls allows us to avoid + * the overhead of setting up and taking down the skip list + * repeatedly. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 5/5/20 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_secure_from_file_flush(H5F_t *f) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + + if (H5C_set_slist_enabled(f->shared->cache, FALSE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "slist enabled failed") + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5AC_secure_from_file_flush() */ + +/*------------------------------------------------------------------------- + * * Function: H5AC_create_flush_dependency() * * Purpose: Create a flush dependency between two entries in the metadata @@ -1214,10 +1360,11 @@ H5AC_create_flush_dependency(void *parent_thing, void *child_thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_create_fd_msg(cache_ptr, (H5AC_info_t *)parent_thing, (H5AC_info_t *)child_thing, - ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_create_fd_msg(cache_ptr, (H5AC_info_t *)parent_thing, + (H5AC_info_t *)child_thing, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_create_flush_dependency() */ @@ -1345,9 +1492,10 @@ H5AC_resize_entry(void *thing, size_t new_size) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_resize_entry_msg(cache_ptr, entry_ptr, new_size, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_resize_entry_msg(cache_ptr, entry_ptr, new_size, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_resize_entry() */ @@ -1387,9 +1535,10 @@ H5AC_unpin_entry(void *thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_unpin_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_unpin_entry_msg(cache_ptr, entry_ptr, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_unpin_entry() */ @@ -1429,10 +1578,11 @@ H5AC_destroy_flush_dependency(void *parent_thing, void *child_thing) done: /* If currently logging, generate a message */ - if (cache_ptr->log_info->logging) - if (H5C_log_write_destroy_fd_msg(cache_ptr, (H5AC_info_t *)parent_thing, (H5AC_info_t *)child_thing, - ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache_ptr != NULL && cache_ptr->log_info != NULL) + if (cache_ptr->log_info->logging) + if (H5C_log_write_destroy_fd_msg(cache_ptr, (H5AC_info_t *)parent_thing, + (H5AC_info_t *)child_thing, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_destroy_flush_dependency() */ @@ -2581,9 +2731,10 @@ H5AC_remove_entry(void *_entry) done: /* If currently logging, generate a message */ - if (cache->log_info->logging) - if (H5C_log_write_remove_entry_msg(cache, entry, ret_value) < 0) - HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") + if (cache != NULL && cache->log_info != NULL) + if (cache->log_info->logging) + if (H5C_log_write_remove_entry_msg(cache, entry, ret_value) < 0) + HDONE_ERROR(H5E_CACHE, H5E_LOGGING, FAIL, "unable to emit log message") FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_remove_entry() */ diff --git a/src/H5ACmpio.c b/src/H5ACmpio.c index 7ffa0bc..a806b35 100644 --- a/src/H5ACmpio.c +++ b/src/H5ACmpio.c @@ -773,7 +773,7 @@ H5AC__log_dirtied_entry(const H5AC_info_t *entry_ptr) else { aux_ptr->dirty_bytes += entry_ptr->size; #if H5AC_DEBUG_DIRTY_BYTES_CREATION - aux_ptr->unprotect_dirty_bytes += entry_size; + aux_ptr->unprotect_dirty_bytes += entry_ptr->size; aux_ptr->unprotect_dirty_bytes_updates += 1; #endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ } /* end else */ @@ -971,7 +971,7 @@ H5AC__log_inserted_entry(const H5AC_info_t *entry_ptr) aux_ptr->dirty_bytes += entry_ptr->size; #if H5AC_DEBUG_DIRTY_BYTES_CREATION - aux_ptr->insert_dirty_bytes += size; + aux_ptr->insert_dirty_bytes += entry_ptr->size; aux_ptr->insert_dirty_bytes_updates += 1; #endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ @@ -1849,6 +1849,8 @@ done: * Programmer: John Mainzer * April 28, 2010 * + * Changes: None. + * *------------------------------------------------------------------------- */ static herr_t @@ -1877,10 +1879,12 @@ H5AC__rsp__p0_only__flush(H5F_t *f) * However, when flushing from within the close operation from a file, * it's possible to skip this barrier (on the second flush of the cache). */ - if (!H5CX_get_mpi_file_flushing()) + if (!H5CX_get_mpi_file_flushing()) { + if (MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) + } /* Flush data to disk, from rank 0 process */ if (aux_ptr->mpi_rank == 0) { @@ -1904,8 +1908,10 @@ H5AC__rsp__p0_only__flush(H5F_t *f) * enforce POSIX semantics on the server that pretends to be a * file system in our parallel tests. */ - if (aux_ptr->write_done) + if (aux_ptr->write_done) { + (aux_ptr->write_done)(); + } } /* end if */ /* Propagate cleaned entries to other ranks. */ diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 92f3fb3..7d6ec7b 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -385,6 +385,8 @@ H5_DLL herr_t H5AC_insert_entry(H5F_t *f, const H5AC_class_t *type, haddr_t addr unsigned int flags); H5_DLL herr_t H5AC_pin_protected_entry(void *thing); H5_DLL herr_t H5AC_prep_for_file_close(H5F_t *f); +H5_DLL herr_t H5AC_prep_for_file_flush(H5F_t *f); +H5_DLL herr_t H5AC_secure_from_file_flush(H5F_t *f); H5_DLL herr_t H5AC_create_flush_dependency(void *parent_thing, void *child_thing); H5_DLL void * H5AC_protect(H5F_t *f, const H5AC_class_t *type, haddr_t addr, void *udata, unsigned flags); H5_DLL herr_t H5AC_resize_entry(void *thing, size_t new_size); @@ -216,6 +216,82 @@ H5FL_SEQ_DEFINE_STATIC(H5C_cache_entry_ptr_t); * Programmer: John Mainzer * 6/2/04 * + * Modifications: + * + * JRM -- 7/20/04 + * Updated for the addition of the hash table. + * + * JRM -- 10/5/04 + * Added call to H5C_reset_cache_hit_rate_stats(). Also + * added initialization for cache_is_full flag and for + * resize_ctl. + * + * JRM -- 11/12/04 + * Added initialization for the new size_decreased field. + * + * JRM -- 11/17/04 + * Added/updated initialization for the automatic cache + * size control data structures. + * + * JRM -- 6/24/05 + * Added support for the new write_permitted field of + * the H5C_t structure. + * + * JRM -- 7/5/05 + * Added the new log_flush parameter and supporting code. + * + * JRM -- 9/21/05 + * Added the new aux_ptr parameter and supporting code. + * + * JRM -- 1/20/06 + * Added initialization of the new prefix field in H5C_t. + * + * JRM -- 3/16/06 + * Added initialization for the pinned entry related fields. + * + * JRM -- 5/31/06 + * Added initialization for the trace_file_ptr field. + * + * JRM -- 8/19/06 + * Added initialization for the flush_in_progress field. + * + * JRM -- 8/25/06 + * Added initialization for the slist_len_increase and + * slist_size_increase fields. These fields are used + * for sanity checking in the flush process, and are not + * compiled in unless H5C_DO_SANITY_CHECKS is TRUE. + * + * JRM -- 3/28/07 + * Added initialization for the new is_read_only and + * ro_ref_count fields. + * + * JRM -- 7/27/07 + * Added initialization for the new evictions_enabled + * field of H5C_t. + * + * JRM -- 12/31/07 + * Added initialization for the new flash cache size increase + * related fields of H5C_t. + * + * JRM -- 11/5/08 + * Added initialization for the new clean_index_size and + * dirty_index_size fields of H5C_t. + * + * + * Missing entries? + * + * + * JRM -- 4/20/20 + * Added initialization for the slist_enabled field. Recall + * that the slist is used to flush metadata cache entries + * in (roughly) increasing address order. While this is + * needed at flush and close, it is not used elsewhere. + * The slist_enabled field exists to allow us to construct + * the slist when needed, and leave it empty otherwise -- thus + * avoiding the overhead of maintaining it. + * + * JRM -- 4/29/20 + * *------------------------------------------------------------------------- */ H5C_t * @@ -306,6 +382,8 @@ H5C_create(size_t max_cache_size, size_t min_clean_size, int max_type_id, cache_ptr->ignore_tags = FALSE; cache_ptr->num_objs_corked = 0; + /* slist field initializations */ + cache_ptr->slist_enabled = !H5C__SLIST_OPT_ENABLED; cache_ptr->slist_changed = FALSE; cache_ptr->slist_len = 0; cache_ptr->slist_size = (size_t)0; @@ -750,6 +828,20 @@ done: * Programmer: John Mainzer * 6/2/04 * + * Modifications: + * + * JRM -- 5/15/20 + * + * Updated the function to enable the slist prior to the + * call to H5C__flush_invalidate_cache(). + * + * Arguably, it shouldn't be necessary to re-enable the + * slist after the call to H5C__flush_invalidate_cache(), as + * the metadata cache should be discarded. However, in the + * test code, we make multiple calls to H5C_dest(). Thus + * we re-enable the slist on failure if it and the cache + * still exist. + * *------------------------------------------------------------------------- */ herr_t @@ -770,21 +862,32 @@ H5C_dest(H5F_t *f) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't display cache image stats") #endif /* H5AC_DUMP_IMAGE_STATS_ON_CLOSE */ + /* Enable the slist, as it is needed in the flush */ + if (H5C_set_slist_enabled(f->shared->cache, TRUE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "set slist enabled failed") + /* Flush and invalidate all cache entries */ if (H5C__flush_invalidate_cache(f, H5C__NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush cache") /* Generate & write cache image if requested */ - if (cache_ptr->image_ctl.generate_image) + if (cache_ptr->image_ctl.generate_image) { + if (H5C__generate_cache_image(f, cache_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "Can't generate metadata cache image") + } /* Question: Is it possible for cache_ptr->slist be non-null at this * point? If no, shouldn't this if statement be an assert? */ if (cache_ptr->slist_ptr != NULL) { + + HDassert(cache_ptr->slist_len == 0); + HDassert(cache_ptr->slist_size == 0); + H5SL_close(cache_ptr->slist_ptr); cache_ptr->slist_ptr = NULL; @@ -799,14 +902,19 @@ H5C_dest(H5F_t *f) } /* end if */ - if (cache_ptr->log_info != NULL) + if (cache_ptr->log_info != NULL) { + H5MM_xfree(cache_ptr->log_info); + } #ifndef NDEBUG #if H5C_DO_SANITY_CHECKS - if (cache_ptr->get_entry_ptr_from_addr_counter > 0) - HDfprintf(stdout, "*** %ld calls to H5C_get_entry_ptr_from_add(). ***\n", + + if (cache_ptr->get_entry_ptr_from_addr_counter > 0) { + + HDfprintf(stdout, "*** %" PRId64 " calls to H5C_get_entry_ptr_from_add(). ***\n", cache_ptr->get_entry_ptr_from_addr_counter); + } #endif /* H5C_DO_SANITY_CHECKS */ cache_ptr->magic = 0; @@ -815,6 +923,16 @@ H5C_dest(H5F_t *f) cache_ptr = H5FL_FREE(H5C_t, cache_ptr); done: + + if ((ret_value < 0) && (cache_ptr) && (cache_ptr->slist_ptr)) { + + /* need this for test code -- see change note for details */ + + if (H5C_set_slist_enabled(f->shared->cache, FALSE, FALSE) < 0) + + HDONE_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "disable slist on flush dest failure failed") + } + FUNC_LEAVE_NOAPI(ret_value) } /* H5C_dest() */ @@ -829,6 +947,14 @@ done: * Programmer: Vailin Choi * Dec 2013 * + * Modifications: + * + * JRM -- 5/5/20 + * + * Added code to enable the skip list prior to the call + * to H5C__flush_invalidate_cache(), and disable it + * afterwards. + * *------------------------------------------------------------------------- */ herr_t @@ -841,11 +967,21 @@ H5C_evict(H5F_t *f) /* Sanity check */ HDassert(f); + /* Enable the slist, as it is needed in the flush */ + if (H5C_set_slist_enabled(f->shared->cache, TRUE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "set slist enabled failed") + /* Flush and invalidate all cache entries except the pinned entries */ if (H5C__flush_invalidate_cache(f, H5C__EVICT_ALLOW_LAST_PINS_FLAG) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to evict entries in the cache") + /* Disable the slist */ + if (H5C_set_slist_enabled(f->shared->cache, FALSE, TRUE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "set slist disabled failed") + done: FUNC_LEAVE_NOAPI(ret_value) } /* H5C_evict() */ @@ -2765,6 +2901,174 @@ done: } /* H5C_set_evictions_enabled() */ /*------------------------------------------------------------------------- + * + * Function: H5C_set_slist_enabled() + * + * Purpose: Enable or disable the slist as directed. + * + * The slist (skip list) is an address ordered list of + * dirty entries in the metadata cache. However, this + * list is only needed during flush and close, where we + * use it to write entries in more or less increasing + * address order. + * + * This function sets up and enables further operations + * on the slist, or disable the slist. This in turn + * allows us to avoid the overhead of maintaining the + * slist when it is not needed. + * + * + * If the slist_enabled parameter is TRUE, the function + * + * 1) Verifies that the slist is empty. + * + * 2) Scans the index list, and inserts all dirty entries + * into the slist. + * + * 3) Sets cache_ptr->slist_enabled = TRUE. + * + * Note that the clear_slist parameter is ignored if + * the slist_enabed parameter is TRUE. + * + * + * If the slist_enabled_parameter is FALSE, the function + * shuts down the slist. + * + * Normally the slist will be empty at this point, however + * that need not be the case if H5C_flush_cache() has been + * called with the H5C__FLUSH_MARKED_ENTRIES_FLAG. + * + * Thus shutdown proceeds as follows: + * + * 1) Test to see if the slist is empty. If it is, proceed + * to step 3. + * + * 2) Test to see if the clear_slist parameter is TRUE. + * + * If it is, remove all entries from the slist. + * + * If it isn't, throw an error. + * + * 3) set cache_ptr->slist_enabled = FALSE. + * + * Return: SUCCEED on success, and FAIL on failure. + * + * Programmer: John Mainzer + * 5/1/20 + * + * Modifications: + * + * None. + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_set_slist_enabled(H5C_t *cache_ptr, hbool_t slist_enabled, hbool_t clear_slist) +{ + H5C_cache_entry_t *entry_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + if ((cache_ptr == NULL) || (cache_ptr->magic != H5C__H5C_T_MAGIC)) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad cache_ptr on entry") + +#if H5C__SLIST_OPT_ENABLED + + if (slist_enabled) { + + if (cache_ptr->slist_enabled) { + + HDassert(FALSE); + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "slist already enabled?") + } + + if ((cache_ptr->slist_len != 0) || (cache_ptr->slist_size != 0)) { + + HDassert(FALSE); + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "slist not empty (1)?") + } + + /* set cache_ptr->slist_enabled to TRUE so that the slist + * mainenance macros will be enabled. + */ + cache_ptr->slist_enabled = TRUE; + + /* scan the index list and insert all dirty entries in the slist */ + entry_ptr = cache_ptr->il_head; + + while (entry_ptr != NULL) { + + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + + if (entry_ptr->is_dirty) { + + H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL) + } + + entry_ptr = entry_ptr->il_next; + } + + /* we don't maintain a dirty index len, so we can't do a cross + * check against it. Note that there is no point in cross checking + * against the dirty LRU size, as the dirty LRU may not be maintained, + * and in any case, there is no requirement that all dirty entries + * will reside on the dirty LRU. + */ + HDassert(cache_ptr->dirty_index_size == cache_ptr->slist_size); + } + else { /* take down the skip list */ + + if (!cache_ptr->slist_enabled) { + + HDassert(FALSE); + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "slist already disabled?") + } + + if ((cache_ptr->slist_len != 0) || (cache_ptr->slist_size != 0)) { + + if (clear_slist) { + + H5SL_node_t *node_ptr; + + node_ptr = H5SL_first(cache_ptr->slist_ptr); + + while (node_ptr != NULL) { + + entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + + H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, FALSE); + + node_ptr = H5SL_first(cache_ptr->slist_ptr); + } + } + else { + + HDassert(FALSE); + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "slist not empty (2)?") + } + } + + cache_ptr->slist_enabled = FALSE; + + HDassert(0 == cache_ptr->slist_len); + HDassert(0 == cache_ptr->slist_size); + } + +#else /* H5C__SLIST_OPT_ENABLED is FALSE */ + + HDassert(cache_ptr->slist_enabled); + +#endif /* H5C__SLIST_OPT_ENABLED is FALSE */ + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5C_set_slist_enabled() */ + +/*------------------------------------------------------------------------- * Function: H5C_unpin_entry() * * Purpose: Unpin a cache entry. The entry can be either protected or @@ -2838,6 +3142,81 @@ done: * Programmer: John Mainzer * 6/2/04 * + * Modifications: + * + * JRM -- 7/21/04 + * Updated for the addition of the hash table. + * + * JRM -- 10/28/04 + * Added code to set cache_full to TRUE whenever we try to + * make space in the cache. + * + * JRM -- 11/12/04 + * Added code to call to H5C_make_space_in_cache() after the + * call to H5C__auto_adjust_cache_size() if that function + * sets the size_decreased flag is TRUE. + * + * JRM -- 4/25/05 + * The size_decreased flag can also be set to TRUE in + * H5C_set_cache_auto_resize_config() if a new configuration + * forces an immediate reduction in cache size. Modified + * the code to deal with this eventuallity. + * + * JRM -- 6/24/05 + * Added support for the new write_permitted field of H5C_t. + * + * JRM -- 10/22/05 + * Hand optimizations. + * + * JRM -- 5/3/06 + * Added code to set the new dirtied field in + * H5C_cache_entry_t to FALSE prior to return. + * + * JRM -- 6/23/06 + * Modified code to allow dirty entries to be loaded from + * disk. This is necessary as a bug fix in the object + * header code requires us to modify a header as it is read. + * + * JRM -- 3/28/07 + * Added the flags parameter and supporting code. At least + * for now, this parameter is used to allow the entry to + * be protected read only, thus allowing multiple protects. + * + * Also added code to allow multiple read only protects + * of cache entries. + * + * JRM -- 7/27/07 + * Added code supporting the new evictions_enabled field + * in H5C_t. + * + * JRM -- 1/3/08 + * Added to do a flash cache size increase if appropriate + * when a large entry is loaded. + * + * JRM -- 11/13/08 + * Modified function to call H5C_make_space_in_cache() when + * the min_clean_size is violated, not just when there isn't + * enough space for and entry that has just been loaded. + * + * The purpose of this modification is to avoid "metadata + * blizzards" in the write only case. In such instances, + * the cache was allowed to fill with dirty metadata. When + * we finally needed to evict an entry to make space, we had + * to flush out a whole cache full of metadata -- which has + * interesting performance effects. We hope to avoid (or + * perhaps more accurately hide) this effect by maintaining + * the min_clean_size, which should force us to start flushing + * entries long before we actually have to evict something + * to make space. + * + * + * Missing entries? + * + * + * JRM -- 5/8/20 + * Updated for the possibility that the slist will be + * disabled. + * *------------------------------------------------------------------------- */ herr_t @@ -2879,9 +3258,15 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) HDassert(H5F_addr_defined(addr)); HDassert(thing); HDassert(!(pin_entry && unpin_entry)); - HDassert((!free_file_space) || (deleted)); /* deleted flag must accompany free_file_space */ - HDassert((!take_ownership) || (deleted)); /* deleted flag must accompany take_ownership */ - HDassert(!(free_file_space && take_ownership)); /* can't have both free_file_space & take_ownership */ + + /* deleted flag must accompany free_file_space */ + HDassert((!free_file_space) || (deleted)); + + /* deleted flag must accompany take_ownership */ + HDassert((!take_ownership) || (deleted)); + + /* can't have both free_file_space & take_ownership */ + HDassert(!(free_file_space && take_ownership)); entry_ptr = (H5C_cache_entry_t *)thing; @@ -2983,16 +3368,22 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) /* Mark the entry as dirty if appropriate */ entry_ptr->is_dirty = (entry_ptr->is_dirty || dirtied); - if (dirtied) + if (dirtied) { + if (entry_ptr->image_up_to_date) { entry_ptr->image_up_to_date = FALSE; - if (entry_ptr->flush_dep_nparents > 0) + + if (entry_ptr->flush_dep_nparents > 0) { + if (H5C__mark_flush_dep_unserialized(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "Can't propagate serialization status to fd parents") - } /* end if */ + + } /* end if */ + } /* end if */ + } /* end if */ /* Check for newly dirtied entry */ if (was_clean && entry_ptr->is_dirty) { @@ -3004,16 +3395,20 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) * 'entry dirtied' notice now that the entry is fully * integrated into the cache. */ - if (entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_DIRTIED, entry_ptr) < 0) + if ((entry_ptr->type->notify) && + ((entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_DIRTIED, entry_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag set") /* Propagate the flush dep dirty flag up the flush dependency chain - * if appropriate */ - if (entry_ptr->flush_dep_nparents > 0) + * if appropriate + */ + if (entry_ptr->flush_dep_nparents > 0) { + if (H5C__mark_flush_dep_dirty(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep dirty flag") + } } /* end if */ /* Check for newly clean entry */ else if (!was_clean && !entry_ptr->is_dirty) { @@ -3022,17 +3417,21 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) * 'entry cleaned' notice now that the entry is fully * integrated into the cache. */ - if (entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0) + if ((entry_ptr->type->notify) && + ((entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared") /* Propagate the flush dep clean flag up the flush dependency chain - * if appropriate */ - if (entry_ptr->flush_dep_nparents > 0) + * if appropriate + */ + if (entry_ptr->flush_dep_nparents > 0) { + if (H5C__mark_flush_dep_clean(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "Can't propagate flush dep dirty flag") + } } /* end else-if */ /* Pin or unpin the entry as requested. */ @@ -3064,8 +3463,12 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) if (entry_ptr->is_dirty) { entry_ptr->flush_marker |= set_flush_marker; - if (!entry_ptr->in_slist) + + if (!entry_ptr->in_slist) { + + /* this is a no-op if cache_ptr->slist_enabled is FALSE */ H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL) + } } /* end if */ /* this implementation of the "deleted" option is a bit inefficient, as @@ -3094,17 +3497,22 @@ H5C_unprotect(H5F_t *f, haddr_t addr, void *thing, unsigned flags) "hash table contains multiple entries for addr?!?") /* Set the 'free file space' flag for the flush, if needed */ - if (free_file_space) + if (free_file_space) { + flush_flags |= H5C__FREE_FILE_SPACE_FLAG; + } /* Set the "take ownership" flag for the flush, if needed */ - if (take_ownership) + if (take_ownership) { + flush_flags |= H5C__TAKE_OWNERSHIP_FLAG; + } /* Delete the entry from the skip list on destroy */ flush_flags |= H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG; - HDassert(((!was_clean) || dirtied) == entry_ptr->in_slist); + HDassert((!cache_ptr->slist_enabled) || (((!was_clean) || dirtied) == (entry_ptr->in_slist))); + if (H5C__flush_single_entry(f, entry_ptr, flush_flags) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "Can't flush entry") @@ -4868,6 +5276,7 @@ done: } /* H5C__flash_increase_cache_size() */ /*------------------------------------------------------------------------- + * * Function: H5C__flush_invalidate_cache * * Purpose: Flush and destroy the entries contained in the target @@ -4897,6 +5306,51 @@ done: * Programmer: John Mainzer * 3/24/065 * + * Modifications: + * + * To support the fractal heap, the cache must now deal with + * entries being dirtied, resized, and/or renamed inside + * flush callbacks. Updated function to support this. + * + * -- JRM 8/27/06 + * + * Added code to detect and manage the case in which a + * flush callback changes the s-list out from under + * the function. The only way I can think of in which this + * can happen is if a flush function loads an entry + * into the cache that isn't there already. Quincey tells + * me that this will never happen, but I'm not sure I + * believe him. + * + * Note that this is a pretty bad scenario if it ever + * happens. The code I have added should allow us to + * handle the situation under all but the worst conditions, + * but one can argue that we should just scream and die if + * we ever detect the condition. + * + * -- JRM 10/13/07 + * + * Missing entries? + * + * + * Added support for the H5C__EVICT_ALLOW_LAST_PINS_FLAG. + * This flag is used to flush and evict all entries in + * the metadata cache that are not pinned -- typically, + * everything other than the superblock. + * + * ??? -- ??/??/?? + * + * Added sanity checks to verify that the skip list is + * enabled on entry. On the face of it, it would make + * sense to enable the slist on entry, and disable it + * on exit, as this function is not called repeatedly. + * However, since this function can be called from + * H5C_flush_cache(), this would create cases in the test + * code where we would have to check the flags to determine + * whether we must setup and take down the slist. + * + * JRM -- 5/5/20 + * *------------------------------------------------------------------------- */ static herr_t @@ -4914,6 +5368,7 @@ H5C__flush_invalidate_cache(H5F_t *f, unsigned flags) HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); HDassert(cache_ptr->slist_ptr); + HDassert(cache_ptr->slist_enabled); #if H5C_DO_SANITY_CHECKS { @@ -4933,6 +5388,7 @@ H5C__flush_invalidate_cache(H5F_t *f, unsigned flags) HDassert(cache_ptr->slist_ring_size[H5C_RING_UNDEFINED] == (size_t)0); for (i = H5C_RING_USER; i < H5C_RING_NTYPES; i++) { + index_len += cache_ptr->index_ring_len[i]; index_size += cache_ptr->index_ring_size[i]; clean_index_size += cache_ptr->clean_index_ring_size[i]; @@ -4940,6 +5396,7 @@ H5C__flush_invalidate_cache(H5F_t *f, unsigned flags) slist_len += cache_ptr->slist_ring_len[i]; slist_size += cache_ptr->slist_ring_size[i]; + } /* end for */ HDassert(cache_ptr->index_len == index_len); @@ -4952,49 +5409,66 @@ H5C__flush_invalidate_cache(H5F_t *f, unsigned flags) #endif /* H5C_DO_SANITY_CHECKS */ /* remove ageout markers if present */ - if (cache_ptr->epoch_markers_active > 0) + if (cache_ptr->epoch_markers_active > 0) { + if (H5C__autoadjust__ageout__remove_all_markers(cache_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "error removing all epoch markers") + } /* flush invalidate each ring, starting from the outermost ring and * working inward. */ ring = H5C_RING_USER; + while (ring < H5C_RING_NTYPES) { + if (H5C__flush_invalidate_ring(f, ring, flags) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush invalidate ring failed") ring++; + } /* end while */ /* Invariants, after destroying all entries in the hash table */ if (!(flags & H5C__EVICT_ALLOW_LAST_PINS_FLAG)) { + HDassert(cache_ptr->index_size == 0); HDassert(cache_ptr->clean_index_size == 0); HDassert(cache_ptr->pel_len == 0); HDassert(cache_ptr->pel_size == 0); + } /* end if */ else { + H5C_cache_entry_t *entry_ptr; /* Cache entry */ unsigned u; /* Local index variable */ /* All rings except ring 4 should be empty now */ /* (Ring 4 has the superblock) */ for (u = H5C_RING_USER; u < H5C_RING_SB; u++) { + HDassert(cache_ptr->index_ring_len[u] == 0); HDassert(cache_ptr->index_ring_size[u] == 0); HDassert(cache_ptr->clean_index_ring_size[u] == 0); + } /* end for */ /* Check that any remaining pinned entries are in the superblock ring */ + entry_ptr = cache_ptr->pel_head_ptr; + while (entry_ptr) { + /* Check ring */ HDassert(entry_ptr->ring == H5C_RING_SB); /* Advance to next entry in pinned entry list */ entry_ptr = entry_ptr->next; + } /* end while */ } /* end else */ + HDassert(cache_ptr->dirty_index_size == 0); HDassert(cache_ptr->slist_len == 0); HDassert(cache_ptr->slist_size == 0); @@ -5004,42 +5478,58 @@ H5C__flush_invalidate_cache(H5F_t *f, unsigned flags) HDassert(cache_ptr->LRU_list_size == 0); done: + FUNC_LEAVE_NOAPI(ret_value) + } /* H5C__flush_invalidate_cache() */ /*------------------------------------------------------------------------- * Function: H5C__flush_invalidate_ring * - * Purpose: Flush and destroy the entries contained in the target - * cache and ring. + * Purpose: Flush and destroy the entries contained in the target + * cache and ring. * - * If the ring contains protected entries, the function will - * fail, as protected entries cannot be either flushed or - * destroyed. However all unprotected entries should be - * flushed and destroyed before the function returns failure. + * If the ring contains protected entries, the function will + * fail, as protected entries cannot be either flushed or + * destroyed. However all unprotected entries should be + * flushed and destroyed before the function returns failure. * - * While pinned entries can usually be flushed, they cannot - * be destroyed. However, they should be unpinned when all - * the entries that reference them have been destroyed (thus - * reduding the pinned entry's reference count to 0, allowing - * it to be unpinned). + * While pinned entries can usually be flushed, they cannot + * be destroyed. However, they should be unpinned when all + * the entries that reference them have been destroyed (thus + * reduding the pinned entry's reference count to 0, allowing + * it to be unpinned). * - * If pinned entries are present, the function makes repeated - * passes through the cache, flushing all dirty entries - * (including the pinned dirty entries where permitted) and - * destroying all unpinned entries. This process is repeated - * until either the cache is empty, or the number of pinned - * entries stops decreasing on each pass. + * If pinned entries are present, the function makes repeated + * passes through the cache, flushing all dirty entries + * (including the pinned dirty entries where permitted) and + * destroying all unpinned entries. This process is repeated + * until either the cache is empty, or the number of pinned + * entries stops decreasing on each pass. * - * If flush dependencies appear in the target ring, the - * function makes repeated passes through the cache flushing - * entries in flush dependency order. + * If flush dependencies appear in the target ring, the + * function makes repeated passes through the cache flushing + * entries in flush dependency order. * * Return: Non-negative on success/Negative on failure or if there was - * a request to flush all items and something was protected. + * a request to flush all items and something was protected. * * Programmer: John Mainzer - * 9/1/15 + * 9/1/15 + * + * Changes: Added support for the H5C__EVICT_ALLOW_LAST_PINS_FLAG. + * This flag is used to flush and evict all entries in + * the metadata cache that are not pinned -- typically, + * everything other than the superblock. + * + * ??? -- ??/??/?? + * + * A recent optimization turns off the slist unless a flush + * is in progress. This should not effect this function, as + * it is only called during a flush. Added an assertion to + * verify this. + * + * JRM -- 5/6/20 * *------------------------------------------------------------------------- */ @@ -5067,9 +5557,12 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert(f); HDassert(f->shared); + cache_ptr = f->shared->cache; + HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_ptr->slist_enabled); HDassert(cache_ptr->slist_ptr); HDassert(ring > H5C_RING_UNDEFINED); HDassert(ring < H5C_RING_NTYPES); @@ -5111,19 +5604,25 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) */ /* compute the number of pinned entries in this ring */ + entry_ptr = cache_ptr->pel_head_ptr; cur_ring_pel_len = 0; + while (entry_ptr != NULL) { + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(entry_ptr->ring >= ring); if (entry_ptr->ring == ring) cur_ring_pel_len++; entry_ptr = entry_ptr->next; + } /* end while */ old_ring_pel_len = cur_ring_pel_len; + while (cache_ptr->index_ring_len[ring] > 0) { + /* first, try to flush-destroy any dirty entries. Do this by * making a scan through the slist. Note that new dirty entries * may be created by the flush call backs. Thus it is possible @@ -5166,25 +5665,32 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) /* this done, start the scan of the slist */ restart_slist_scan = TRUE; + while (restart_slist_scan || (node_ptr != NULL)) { + if (restart_slist_scan) { + restart_slist_scan = FALSE; /* Start at beginning of skip list */ node_ptr = H5SL_first(cache_ptr->slist_ptr); + if (node_ptr == NULL) /* the slist is empty -- break out of inner loop */ break; /* Get cache entry for this node */ next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + if (NULL == next_entry_ptr) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!") HDassert(next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(next_entry_ptr->is_dirty); HDassert(next_entry_ptr->in_slist); HDassert(next_entry_ptr->ring >= ring); + } /* end if */ entry_ptr = next_entry_ptr; @@ -5210,18 +5716,25 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) * from the slist. */ node_ptr = H5SL_next(node_ptr); + if (node_ptr != NULL) { + next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + if (NULL == next_entry_ptr) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!") + HDassert(next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(next_entry_ptr->is_dirty); HDassert(next_entry_ptr->in_slist); HDassert(next_entry_ptr->ring >= ring); HDassert(entry_ptr != next_entry_ptr); } /* end if */ - else + else { + next_entry_ptr = NULL; + } /* Note that we now remove nodes from the slist as we flush * the associated entries, instead of leaving them there @@ -5236,17 +5749,23 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) if (((!entry_ptr->flush_me_last) || ((entry_ptr->flush_me_last) && (cache_ptr->num_last_entries >= cache_ptr->slist_len))) && (entry_ptr->flush_dep_nchildren == 0) && (entry_ptr->ring == ring)) { + if (entry_ptr->is_protected) { + /* we have major problems -- but lets flush * everything we can before we flag an error. */ protected_entries++; + } /* end if */ else if (entry_ptr->is_pinned) { + if (H5C__flush_single_entry(f, entry_ptr, H5C__DURING_FLUSH_FLAG) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty pinned entry flush failed") if (cache_ptr->slist_changed) { + /* The slist has been modified by something * other than the simple removal of the * of the flushed entry after the flush. @@ -5257,16 +5776,20 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) restart_slist_scan = TRUE; cache_ptr->slist_changed = FALSE; H5C__UPDATE_STATS_FOR_SLIST_SCAN_RESTART(cache_ptr); + } /* end if */ } /* end else-if */ else { + if (H5C__flush_single_entry(f, entry_ptr, (cooked_flags | H5C__DURING_FLUSH_FLAG | H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty entry flush destroy failed") if (cache_ptr->slist_changed) { + /* The slist has been modified by something * other than the simple removal of the * of the flushed entry after the flush. @@ -5293,8 +5816,10 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) */ if (node_ptr == NULL) { + HDassert(cache_ptr->slist_len == (uint32_t)((int32_t)initial_slist_len + cache_ptr->slist_len_increase)); + HDassert(cache_ptr->slist_size == (size_t)((ssize_t)initial_slist_size + cache_ptr->slist_size_increase)); } /* end if */ @@ -5320,7 +5845,9 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) cache_ptr->entries_relocated_counter = 0; next_entry_ptr = cache_ptr->il_head; + while (next_entry_ptr != NULL) { + entry_ptr = next_entry_ptr; HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(entry_ptr->ring >= ring); @@ -5328,19 +5855,25 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) next_entry_ptr = entry_ptr->il_next; HDassert((next_entry_ptr == NULL) || (next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC)); - if ((!entry_ptr->flush_me_last || - (entry_ptr->flush_me_last && cache_ptr->num_last_entries >= cache_ptr->slist_len)) && - entry_ptr->flush_dep_nchildren == 0 && entry_ptr->ring == ring) { + if (((!entry_ptr->flush_me_last) || + (entry_ptr->flush_me_last && (cache_ptr->num_last_entries >= cache_ptr->slist_len))) && + (entry_ptr->flush_dep_nchildren == 0) && (entry_ptr->ring == ring)) { + if (entry_ptr->is_protected) { + /* we have major problems -- but lets flush and * destroy everything we can before we flag an * error. */ protected_entries++; - if (!entry_ptr->in_slist) + + if (!entry_ptr->in_slist) { + HDassert(!(entry_ptr->is_dirty)); + } } /* end if */ else if (!(entry_ptr->is_pinned)) { + /* if *entry_ptr is dirty, it is possible * that one or more other entries may be * either removed from the cache, loaded @@ -5373,6 +5906,7 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) (cooked_flags | H5C__DURING_FLUSH_FLAG | H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Entry flush destroy failed") /* Restart the index list scan if necessary. Must @@ -5383,7 +5917,7 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) * if this results in the size of the pinned entry * failing to decline during the pass. */ - if ((NULL != next_entry_ptr && NULL == cache_ptr->entry_watched_for_removal) || + if (((NULL != next_entry_ptr) && (NULL == cache_ptr->entry_watched_for_removal)) || (cache_ptr->entries_loaded_counter > 0) || (cache_ptr->entries_inserted_counter > 0) || (cache_ptr->entries_relocated_counter > 0)) { @@ -5395,9 +5929,12 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) cache_ptr->entries_relocated_counter = 0; H5C__UPDATE_STATS_FOR_INDEX_SCAN_RESTART(cache_ptr) + } /* end if */ - else + else { + cache_ptr->entry_watched_for_removal = NULL; + } } /* end if */ } /* end if */ } /* end for loop scanning hash table */ @@ -5413,23 +5950,31 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) old_ring_pel_len = cur_ring_pel_len; entry_ptr = cache_ptr->pel_head_ptr; cur_ring_pel_len = 0; + while (entry_ptr != NULL) { + HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(entry_ptr->ring >= ring); - if (entry_ptr->ring == ring) + if (entry_ptr->ring == ring) { + cur_ring_pel_len++; + } entry_ptr = entry_ptr->next; + } /* end while */ /* Check if the number of pinned entries in the ring is positive, and * it is not declining. Scream and die if so. */ - if (cur_ring_pel_len > 0 && cur_ring_pel_len >= old_ring_pel_len) { + if ((cur_ring_pel_len > 0) && (cur_ring_pel_len >= old_ring_pel_len)) { + /* Don't error if allowed to have pinned entries remaining */ - if (evict_flags) + if (evict_flags) { + HGOTO_DONE(TRUE) + } HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Pinned entry count not decreasing, cur_ring_pel_len = %d, old_ring_pel_len = " @@ -5439,14 +5984,17 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert(protected_entries == cache_ptr->pl_len); - if (protected_entries > 0 && protected_entries == cache_ptr->index_len) + if ((protected_entries > 0) && (protected_entries == cache_ptr->index_len)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Only protected entries left in cache, protected_entries = %d", (int)protected_entries) + } /* main while loop */ /* Invariants, after destroying all entries in the ring */ for (i = (int)H5C_RING_UNDEFINED; i <= (int)ring; i++) { + HDassert(cache_ptr->index_ring_len[i] == 0); HDassert(cache_ptr->index_ring_size[i] == (size_t)0); HDassert(cache_ptr->clean_index_ring_size[i] == (size_t)0); @@ -5454,40 +6002,56 @@ H5C__flush_invalidate_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert(cache_ptr->slist_ring_len[i] == 0); HDassert(cache_ptr->slist_ring_size[i] == (size_t)0); + } /* end for */ HDassert(protected_entries <= cache_ptr->pl_len); - if (protected_entries > 0) + if (protected_entries > 0) { + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Cache has protected entries") - else if (cur_ring_pel_len > 0) + } + else if (cur_ring_pel_len > 0) { + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't unpin all pinned entries in ring") + } done: + FUNC_LEAVE_NOAPI(ret_value) + } /* H5C__flush_invalidate_ring() */ /*------------------------------------------------------------------------- + * * Function: H5C__flush_ring * - * Purpose: Flush the entries contained in the specified cache and - * ring. All entries in rings outside the specified ring - * must have been flushed on entry. + * Purpose: Flush the entries contained in the specified cache and + * ring. All entries in rings outside the specified ring + * must have been flushed on entry. * - * If the cache contains protected entries in the specified - * ring, the function will fail, as protected entries cannot - * be flushed. However all unprotected entries in the target - * ring should be flushed before the function returns failure. + * If the cache contains protected entries in the specified + * ring, the function will fail, as protected entries cannot + * be flushed. However all unprotected entries in the target + * ring should be flushed before the function returns failure. * - * If flush dependencies appear in the target ring, the - * function makes repeated passes through the slist flushing - * entries in flush dependency order. + * If flush dependencies appear in the target ring, the + * function makes repeated passes through the slist flushing + * entries in flush dependency order. * * Return: Non-negative on success/Negative on failure or if there was - * a request to flush all items and something was protected. + * a request to flush all items and something was protected. * * Programmer: John Mainzer - * 9/1/15 + * 9/1/15 + * + * Changes: A recent optimization turns off the slist unless a flush + * is in progress. This should not effect this function, as + * it is only called during a flush. Added an assertion to + * verify this. + * + * JRM -- 5/6/20 + * * *------------------------------------------------------------------------- */ @@ -5515,6 +6079,7 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert(cache_ptr); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_ptr->slist_enabled); HDassert(cache_ptr->slist_ptr); HDassert((flags & H5C__FLUSH_INVALIDATE_FLAG) == 0); HDassert(ring > H5C_RING_UNDEFINED); @@ -5523,15 +6088,20 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) #if H5C_DO_EXTREME_SANITY_CHECKS if ((H5C__validate_protected_entry_list(cache_ptr) < 0) || (H5C__validate_pinned_entry_list(cache_ptr) < 0) || (H5C__validate_lru_list(cache_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "an extreme sanity check failed on entry") #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ ignore_protected = ((flags & H5C__FLUSH_IGNORE_PROTECTED_FLAG) != 0); flush_marked_entries = ((flags & H5C__FLUSH_MARKED_ENTRIES_FLAG) != 0); - if (!flush_marked_entries) - for (i = (int)H5C_RING_UNDEFINED; i < (int)ring; i++) + if (!flush_marked_entries) { + + for (i = (int)H5C_RING_UNDEFINED; i < (int)ring; i++) { + HDassert(cache_ptr->slist_ring_len[i] == 0); + } + } HDassert(cache_ptr->flush_in_progress); @@ -5553,6 +6123,7 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) cache_ptr->slist_changed = FALSE; while ((cache_ptr->slist_ring_len[ring] > 0) && (protected_entries == 0) && (flushed_entries_last_pass)) { + flushed_entries_last_pass = FALSE; #if H5C_DO_SANITY_CHECKS @@ -5597,25 +6168,31 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) restart_slist_scan = TRUE; while ((restart_slist_scan) || (node_ptr != NULL)) { + if (restart_slist_scan) { + restart_slist_scan = FALSE; /* Start at beginning of skip list */ node_ptr = H5SL_first(cache_ptr->slist_ptr); - if (node_ptr == NULL) + if (node_ptr == NULL) { + /* the slist is empty -- break out of inner loop */ break; + } /* Get cache entry for this node */ next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); if (NULL == next_entry_ptr) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!") HDassert(next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(next_entry_ptr->is_dirty); HDassert(next_entry_ptr->in_slist); + } /* end if */ entry_ptr = next_entry_ptr; @@ -5640,52 +6217,69 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert(entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(entry_ptr->in_slist); HDassert(entry_ptr->is_dirty); - if (!flush_marked_entries || entry_ptr->flush_marker) + + if ((!flush_marked_entries) || (entry_ptr->flush_marker)) { + HDassert(entry_ptr->ring >= ring); + } /* Advance node pointer now, before we delete its target * from the slist. */ node_ptr = H5SL_next(node_ptr); + if (node_ptr != NULL) { + next_entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + if (NULL == next_entry_ptr) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "next_entry_ptr == NULL ?!?!") HDassert(next_entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(next_entry_ptr->is_dirty); HDassert(next_entry_ptr->in_slist); - if (!flush_marked_entries || next_entry_ptr->flush_marker) + if (!flush_marked_entries || next_entry_ptr->flush_marker) { + HDassert(next_entry_ptr->ring >= ring); + } HDassert(entry_ptr != next_entry_ptr); + } /* end if */ - else + else { + next_entry_ptr = NULL; + } if ((!flush_marked_entries || entry_ptr->flush_marker) && - (!entry_ptr->flush_me_last || - (entry_ptr->flush_me_last && (cache_ptr->num_last_entries >= cache_ptr->slist_len || - (flush_marked_entries && entry_ptr->flush_marker)))) && - (entry_ptr->flush_dep_nchildren == 0 || entry_ptr->flush_dep_ndirty_children == 0) && - entry_ptr->ring == ring) { + ((!entry_ptr->flush_me_last) || + ((entry_ptr->flush_me_last) && ((cache_ptr->num_last_entries >= cache_ptr->slist_len) || + (flush_marked_entries && entry_ptr->flush_marker)))) && + ((entry_ptr->flush_dep_nchildren == 0) || (entry_ptr->flush_dep_ndirty_children == 0)) && + (entry_ptr->ring == ring)) { HDassert(entry_ptr->flush_dep_nunser_children == 0); if (entry_ptr->is_protected) { + /* we probably have major problems -- but lets * flush everything we can before we decide * whether to flag an error. */ tried_to_flush_protected_entry = TRUE; protected_entries++; + } /* end if */ else { + if (H5C__flush_single_entry(f, entry_ptr, (flags | H5C__DURING_FLUSH_FLAG)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry") if (cache_ptr->slist_changed) { + /* The slist has been modified by something * other than the simple removal of the * of the flushed entry after the flush. @@ -5696,9 +6290,11 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) restart_slist_scan = TRUE; cache_ptr->slist_changed = FALSE; H5C__UPDATE_STATS_FOR_SLIST_SCAN_RESTART(cache_ptr) + } /* end if */ flushed_entries_last_pass = TRUE; + } /* end else */ } /* end if */ } /* while ( ( restart_slist_scan ) || ( node_ptr != NULL ) ) */ @@ -5710,22 +6306,28 @@ H5C__flush_ring(H5F_t *f, H5C_ring_t ring, unsigned flags) HDassert((size_t)((ssize_t)initial_slist_size + cache_ptr->slist_size_increase) == cache_ptr->slist_size); #endif /* H5C_DO_SANITY_CHECKS */ - } /* while */ + + } /* while */ HDassert(protected_entries <= cache_ptr->pl_len); if (((cache_ptr->pl_len > 0) && (!ignore_protected)) || (tried_to_flush_protected_entry)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "cache has protected items") #if H5C_DO_SANITY_CHECKS if (!flush_marked_entries) { + HDassert(cache_ptr->slist_ring_len[ring] == 0); HDassert(cache_ptr->slist_ring_size[ring] == 0); + } /* end if */ #endif /* H5C_DO_SANITY_CHECKS */ done: + FUNC_LEAVE_NOAPI(ret_value) + } /* H5C__flush_ring() */ /*------------------------------------------------------------------------- @@ -5733,32 +6335,89 @@ done: * Function: H5C__flush_single_entry * * Purpose: Flush or clear (and evict if requested) the cache entry - * with the specified address and type. If the type is NULL, - * any unprotected entry at the specified address will be - * flushed (and possibly evicted). + * with the specified address and type. If the type is NULL, + * any unprotected entry at the specified address will be + * flushed (and possibly evicted). * - * Attempts to flush a protected entry will result in an - * error. + * Attempts to flush a protected entry will result in an + * error. * - * If the H5C__FLUSH_INVALIDATE_FLAG flag is set, the entry will - * be cleared and not flushed, and the call can't be part of a + * If the H5C__FLUSH_INVALIDATE_FLAG flag is set, the entry will + * be cleared and not flushed, and the call can't be part of a * sequence of flushes. * - * If the caller knows the address of the skip list node at - * which the target entry resides, it can avoid a lookup - * by supplying that address in the tgt_node_ptr parameter. - * If this parameter is NULL, the function will do a skip list - * search for the entry instead. - * - * The function does nothing silently if there is no entry - * at the supplied address, or if the entry found has the - * wrong type. + * The function does nothing silently if there is no entry + * at the supplied address, or if the entry found has the + * wrong type. * * Return: Non-negative on success/Negative on failure or if there was - * an attempt to flush a protected item. + * an attempt to flush a protected item. * * Programmer: John Mainzer, 5/5/04 * + * Modifications: + * + * JRM -- 7/21/04 + * Updated function for the addition of the hash table. + * + * QAK -- 11/26/04 + * Updated function for the switch from TBBTs to skip lists. + * + * JRM -- 1/6/05 + * Updated function to reset the flush_marker field. + * Also replace references to H5F_FLUSH_INVALIDATE and + * H5F_FLUSH_CLEAR_ONLY with references to + * H5C__FLUSH_INVALIDATE_FLAG and H5C__FLUSH_CLEAR_ONLY_FLAG + * respectively. + * + * JRM -- 6/24/05 + * Added code to remove dirty entries from the slist after + * they have been flushed. Also added a sanity check that + * will scream if we attempt a write when writes are + * completely disabled. + * + * JRM -- 7/5/05 + * Added code to call the new log_flush callback whenever + * a dirty entry is written to disk. Note that the callback + * is not called if the H5C__FLUSH_CLEAR_ONLY_FLAG is set, + * as there is no write to file in this case. + * + * JRM -- 8/21/06 + * Added code maintaining the flush_in_progress and + * destroy_in_progress fields in H5C_cache_entry_t. + * + * Also added flush_flags parameter to the call to + * type_ptr->flush() so that the flush routine can report + * whether the entry has been resized or renamed. Added + * code using the flush_flags variable to detect the case + * in which the target entry is resized during flush, and + * update the caches data structures accordingly. + * + * JRM -- 3/29/07 + * Added sanity checks on the new is_read_only and + * ro_ref_count fields. + * + * QAK -- 2/07/08 + * Separated "destroy entry" concept from "remove entry from + * cache" concept, by adding the 'take_ownership' flag and + * the "destroy_entry" variable. + * + * JRM -- 11/5/08 + * Added call to H5C__UPDATE_INDEX_FOR_ENTRY_CLEAN() to + * maintain the new clean_index_size and clean_index_size + * fields of H5C_t. + * + * + * Missing entries?? + * + * + * JRM -- 5/8/20 + * Updated sanity checks for the possibility that the slist + * is disabled. + * + * Also updated main comment to conform more closely with + * the current state of the code. + * *------------------------------------------------------------------------- */ herr_t @@ -5805,18 +6464,26 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) /* Set the flag for destroying the entry, based on the 'take ownership' * and 'destroy' flags */ - if (take_ownership) + if (take_ownership) { + destroy_entry = FALSE; - else + } + else { + destroy_entry = destroy; + } /* we will write the entry to disk if it exists, is dirty, and if the * clear only flag is not set. */ - if (entry_ptr->is_dirty && !clear_only) + if (entry_ptr->is_dirty && !clear_only) { + write_entry = TRUE; - else + } + else { + write_entry = FALSE; + } /* if we have received close warning, and we have been instructed to * generate a metadata cache image, and we have actually constructed @@ -5825,8 +6492,9 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * Set suppress_image_entry_writes to TRUE if indicated by the * image_ctl flags. */ - if (cache_ptr->close_warning_received && cache_ptr->image_ctl.generate_image && - cache_ptr->num_entries_in_image > 0 && cache_ptr->image_entries) { + if ((cache_ptr->close_warning_received) && (cache_ptr->image_ctl.generate_image) && + (cache_ptr->num_entries_in_image > 0) && (cache_ptr->image_entries != NULL)) { + /* Sanity checks */ HDassert(entry_ptr->image_up_to_date || !(entry_ptr->include_in_image)); HDassert(entry_ptr->image_ptr || !(entry_ptr->include_in_image)); @@ -5836,32 +6504,56 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) suppress_image_entry_frees = TRUE; - if (cache_ptr->image_ctl.flags & H5C_CI__SUPRESS_ENTRY_WRITES) + if (cache_ptr->image_ctl.flags & H5C_CI__SUPRESS_ENTRY_WRITES) { + suppress_image_entry_writes = TRUE; - } /* end if */ + + } /* end if */ + } /* end if */ /* run initial sanity checks */ #if H5C_DO_SANITY_CHECKS - if (entry_ptr->in_slist) { - HDassert(entry_ptr->is_dirty); + if (cache_ptr->slist_enabled) { - if ((entry_ptr->flush_marker) && (!entry_ptr->is_dirty)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry in slist failed sanity checks") - } /* end if */ - else { - HDassert(!entry_ptr->is_dirty); - HDassert(!entry_ptr->flush_marker); + if (entry_ptr->in_slist) { + + HDassert(entry_ptr->is_dirty); + + if ((entry_ptr->flush_marker) && (!entry_ptr->is_dirty)) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry in slist failed sanity checks") + } /* end if */ + else { + + HDassert(!entry_ptr->is_dirty); + HDassert(!entry_ptr->flush_marker); + + if ((entry_ptr->is_dirty) || (entry_ptr->flush_marker)) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry failed sanity checks") + + } /* end else */ + } + else { /* slist is disabled */ + + HDassert(!entry_ptr->in_slist); + + if (!entry_ptr->is_dirty) { + + if (entry_ptr->flush_marker) - if ((entry_ptr->is_dirty) || (entry_ptr->flush_marker)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry failed sanity checks") - } /* end else */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "flush marked clean entry?") + } + } #endif /* H5C_DO_SANITY_CHECKS */ if (entry_ptr->is_protected) { + HDassert(!entry_ptr->is_protected); /* Attempt to flush a protected entry -- scream and die. */ HGOTO_ERROR(H5E_CACHE, H5E_PROTECT, FAIL, "Attempt to flush a protected entry") + } /* end if */ /* Set entry_ptr->flush_in_progress = TRUE and set @@ -5881,25 +6573,33 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * entry. */ if (write_entry || generate_image) { + HDassert(entry_ptr->is_dirty); if (NULL == entry_ptr->image_ptr) { + if (NULL == (entry_ptr->image_ptr = H5MM_malloc(entry_ptr->size + H5C_IMAGE_EXTRA_SPACE))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for on disk image buffer") + #if H5C_DO_MEMORY_SANITY_CHECKS H5MM_memcpy(((uint8_t *)entry_ptr->image_ptr) + entry_ptr->size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); -#endif /* H5C_DO_MEMORY_SANITY_CHECKS */ +#endif /* H5C_DO_MEMORY_SANITY_CHECKS */ + } /* end if */ if (!(entry_ptr->image_up_to_date)) { + /* Sanity check */ HDassert(!entry_ptr->prefetched); /* Generate the entry's image */ if (H5C__generate_image(f, cache_ptr, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't generate entry's image") + } /* end if ( ! (entry_ptr->image_up_to_date) ) */ } /* end if */ @@ -5910,10 +6610,12 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * flag should only be used in test code. */ if (write_entry) { + HDassert(entry_ptr->is_dirty); #if H5C_DO_SANITY_CHECKS - if (cache_ptr->check_write_permitted && !(cache_ptr->write_permitted)) + if ((cache_ptr->check_write_permitted) && (!(cache_ptr->write_permitted))) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Write when writes are always forbidden!?!?!") #endif /* H5C_DO_SANITY_CHECKS */ @@ -5924,38 +6626,50 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * H5AC__CLASS_SKIP_WRITES is set in the entry's type. This * flag should only be used in test code */ - if ((!suppress_image_entry_writes || !entry_ptr->include_in_image) && + if (((!suppress_image_entry_writes) || (!entry_ptr->include_in_image)) && (((entry_ptr->type->flags) & H5C__CLASS_SKIP_WRITES) == 0)) { + H5FD_mem_t mem_type = H5FD_MEM_DEFAULT; #ifdef H5_HAVE_PARALLEL if (cache_ptr->coll_write_list) { + if (H5SL_insert(cache_ptr->coll_write_list, entry_ptr, &entry_ptr->addr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "unable to insert skip list item") } /* end if */ else { #endif /* H5_HAVE_PARALLEL */ if (entry_ptr->prefetched) { + HDassert(entry_ptr->type->id == H5AC_PREFETCHED_ENTRY_ID); + mem_type = cache_ptr->class_table_ptr[entry_ptr->prefetch_type_id]->mem_type; } /* end if */ - else + else { + mem_type = entry_ptr->type->mem_type; + } if (H5F_block_write(f, mem_type, entry_ptr->addr, entry_ptr->size, entry_ptr->image_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't write image to file") + #ifdef H5_HAVE_PARALLEL } -#endif /* H5_HAVE_PARALLEL */ +#endif /* H5_HAVE_PARALLEL */ + } /* end if */ /* if the entry has a notify callback, notify it that we have * just flushed the entry. */ - if (entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_FLUSH, entry_ptr) < 0) + if ((entry_ptr->type->notify) && + ((entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_FLUSH, entry_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client of entry flush") + } /* if ( write_entry ) */ /* At this point, all pre-serialize and serialize calls have been @@ -5968,16 +6682,21 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) /* start by updating the statistics */ if (clear_only) { + /* only log a clear if the entry was dirty */ if (was_dirty) { + H5C__UPDATE_STATS_FOR_CLEAR(cache_ptr, entry_ptr) + } /* end if */ } else if (write_entry) { + HDassert(was_dirty); /* only log a flush if we actually wrote to disk */ H5C__UPDATE_STATS_FOR_FLUSH(cache_ptr, entry_ptr) + } /* end else if */ /* Note that the algorithm below is (very) similar to the set of operations @@ -5987,11 +6706,17 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) /* Update the cache internal data structures. */ if (destroy) { + /* Sanity checks */ - if (take_ownership) + if (take_ownership) { + HDassert(!destroy_entry); - else + } + else { + HDassert(destroy_entry); + } + HDassert(!entry_ptr->is_pinned); /* Update stats, while entry is still in the cache */ @@ -6001,8 +6726,9 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * to be removed from the cache, send a 'before eviction' notice while * the entry is still fully integrated in the cache. */ - if (entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_BEFORE_EVICT, entry_ptr) < 0) + if ((entry_ptr->type->notify) && + ((entry_ptr->type->notify)(H5C_NOTIFY_ACTION_BEFORE_EVICT, entry_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry to evict") /* Update the cache internal data structures as appropriate @@ -6023,14 +6749,19 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) */ H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, FAIL) - if (entry_ptr->in_slist && del_from_slist_on_destroy) + if ((entry_ptr->in_slist) && (del_from_slist_on_destroy)) { + H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) + } #ifdef H5_HAVE_PARALLEL /* Check for collective read access flag */ if (entry_ptr->coll_access) { + entry_ptr->coll_access = FALSE; + H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) + } /* end if */ #endif /* H5_HAVE_PARALLEL */ @@ -6038,16 +6769,19 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) /* Remove entry from tag list */ if (H5C__untag_entry(cache_ptr, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTREMOVE, FAIL, "can't remove entry from tag list") /* verify that the entry is no longer part of any flush dependencies */ HDassert(entry_ptr->flush_dep_nparents == 0); HDassert(entry_ptr->flush_dep_nchildren == 0); + } /* end if */ else { + HDassert(clear_only || write_entry); HDassert(entry_ptr->is_dirty); - HDassert(entry_ptr->in_slist); + HDassert((!cache_ptr->slist_enabled) || (entry_ptr->in_slist)); /* We are either doing a flush or a clear. * @@ -6055,7 +6789,7 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * view of the replacement policy and the slist. * Hence no differentiation between them. * - * JRM -- 7/7/07 + * JRM -- 7/7/07 */ H5C__UPDATE_RP_FOR_FLUSH(cache_ptr, entry_ptr, FAIL) @@ -6072,20 +6806,31 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) /* Check for entry changing status and do notifications, etc. */ if (was_dirty) { - /* If the entry's type has a 'notify' callback send a 'entry cleaned' - * notice now that the entry is fully integrated into the cache. + + /* If the entry's type has a 'notify' callback send a + * 'entry cleaned' notice now that the entry is fully + * integrated into the cache. */ - if (entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0) + if ((entry_ptr->type->notify) && + ((entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared") - /* Propagate the clean flag up the flush dependency chain if appropriate */ - if (entry_ptr->flush_dep_ndirty_children != 0) + /* Propagate the clean flag up the flush dependency chain + * if appropriate + */ + if (entry_ptr->flush_dep_ndirty_children != 0) { + HDassert(entry_ptr->flush_dep_ndirty_children == 0); - if (entry_ptr->flush_dep_nparents > 0) + } + + if (entry_ptr->flush_dep_nparents > 0) { + if (H5C__mark_flush_dep_clean(entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKCLEAN, FAIL, "Can't propagate flush dep clean flag") + } } /* end if */ } /* end else */ @@ -6093,7 +6838,8 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) entry_ptr->flush_in_progress = FALSE; /* capture the cache entry address for the log_flush call at the - end before the entry_ptr gets freed */ + * end before the entry_ptr gets freed + */ entry_addr = entry_ptr->addr; /* Internal cache data structures should now be up to date, and @@ -6102,6 +6848,7 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * Now discard the entry if appropriate. */ if (destroy) { + /* Sanity check */ HDassert(0 == entry_ptr->flush_dep_nparents); @@ -6112,10 +6859,14 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * * Otherwise, free the buffer if it exists. */ - if (suppress_image_entry_frees && entry_ptr->include_in_image) + if (suppress_image_entry_frees && entry_ptr->include_in_image) { + entry_ptr->image_ptr = NULL; - else if (entry_ptr->image_ptr != NULL) + } + else if (entry_ptr->image_ptr != NULL) { + entry_ptr->image_ptr = H5MM_xfree(entry_ptr->image_ptr); + } /* If the entry is not a prefetched entry, verify that the flush * dependency parents addresses array has been transferred. @@ -6124,14 +6875,17 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * the flush dependency parents addresses array if necessary. */ if (!entry_ptr->prefetched) { + HDassert(0 == entry_ptr->fd_parent_count); HDassert(NULL == entry_ptr->fd_parent_addrs); + } /* end if */ /* Check whether we should free the space in the file that * the entry occupies */ if (free_file_space) { + hsize_t fsf_size; /* Sanity checks */ @@ -6152,15 +6906,22 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * Otherwise use entry_ptr->size. */ if (entry_ptr->type->fsf_size) { + if ((entry_ptr->type->fsf_size)((void *)entry_ptr, &fsf_size) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "unable to get file space free size") - } /* end if */ - else /* no file space free size callback -- use entry size */ + + } /* end if */ + else { /* no file space free size callback -- use entry size */ + fsf_size = entry_ptr->size; + } /* Release the space on disk */ if (H5MF_xfree(f, entry_ptr->type->mem_type, entry_ptr->addr, fsf_size) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "unable to free file space for cache entry") + } /* end if ( free_file_space ) */ /* Reset the pointer to the cache the entry is within. -QAK */ @@ -6183,23 +6944,31 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) */ cache_ptr->entries_removed_counter++; cache_ptr->last_entry_removed_ptr = entry_ptr; - if (entry_ptr == cache_ptr->entry_watched_for_removal) + + if (entry_ptr == cache_ptr->entry_watched_for_removal) { + cache_ptr->entry_watched_for_removal = NULL; + } /* Check for actually destroying the entry in memory */ /* (As opposed to taking ownership of it) */ if (destroy_entry) { + if (entry_ptr->is_dirty) { + /* Reset dirty flag */ entry_ptr->is_dirty = FALSE; - /* If the entry's type has a 'notify' callback send a 'entry cleaned' - * notice now that the entry is fully integrated into the cache. + /* If the entry's type has a 'notify' callback send a + * 'entry cleaned' notice now that the entry is fully + * integrated into the cache. */ - if (entry_ptr->type->notify && - (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0) + if ((entry_ptr->type->notify) && + ((entry_ptr->type->notify)(H5C_NOTIFY_ACTION_ENTRY_CLEANED, entry_ptr) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry dirty flag cleared") + } /* end if */ /* we are about to discard the in core representation -- @@ -6212,9 +6981,12 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) HDassert(entry_ptr->image_ptr == NULL); if (entry_ptr->type->free_icr((void *)entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "free_icr callback failed") + } /* end if */ else { + HDassert(take_ownership); /* client is taking ownership of the entry. @@ -6222,6 +6994,7 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * unless the entry is re-inserted properly */ entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC; + } /* end else */ } /* if (destroy) */ @@ -6229,25 +7002,36 @@ H5C__flush_single_entry(H5F_t *f, H5C_cache_entry_t *entry_ptr, unsigned flags) * so it doesn't go out of date */ if (update_page_buffer) { + /* Sanity check */ HDassert(!destroy); HDassert(entry_ptr->image_ptr); - if (f->shared->page_buf && f->shared->page_buf->page_size >= entry_ptr->size) + if ((f->shared->page_buf) && (f->shared->page_buf->page_size >= entry_ptr->size)) { + if (H5PB_update_entry(f->shared->page_buf, entry_ptr->addr, entry_ptr->size, entry_ptr->image_ptr) > 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Failed to update PB with metadata cache") - } /* end if */ + } /* end if */ + } /* end if */ + + if (cache_ptr->log_flush) { - if (cache_ptr->log_flush) if ((cache_ptr->log_flush)(cache_ptr, entry_addr, was_dirty, flags) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "log_flush callback failed") + } /* end if */ + done: + HDassert((ret_value != SUCCEED) || (destroy_entry) || (!entry_ptr->flush_in_progress)); + HDassert((ret_value != SUCCEED) || (destroy_entry) || (take_ownership) || (!entry_ptr->is_dirty)); FUNC_LEAVE_NOAPI(ret_value) + } /* H5C__flush_single_entry() */ /*------------------------------------------------------------------------- @@ -7251,8 +8035,18 @@ H5C__entry_in_skip_list(H5C_t *cache_ptr, H5C_cache_entry_t *target_ptr) * Programmer: Mike McGreevy * November 3, 2010 * + * Changes: Modified function to setup the slist before calling + * H%C_flush_cache(), and take it down afterwards. Note + * that the slist need not be empty after the call to + * H5C_flush_cache() since we are only flushing marked + * entries. Thus must set the clear_slist parameter + * of H5C_set_slist_enabled to TRUE. + * + * JRM -- 5/6/20 + * *------------------------------------------------------------------------- */ + herr_t H5C__flush_marked_entries(H5F_t *f) { @@ -7263,12 +8057,28 @@ H5C__flush_marked_entries(H5F_t *f) /* Assertions */ HDassert(f != NULL); + /* Enable the slist, as it is needed in the flush */ + if (H5C_set_slist_enabled(f->shared->cache, TRUE, FALSE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "set slist enabled failed") + /* Flush all marked entries */ if (H5C_flush_cache(f, H5C__FLUSH_MARKED_ENTRIES_FLAG | H5C__FLUSH_IGNORE_PROTECTED_FLAG) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush cache") + /* Disable the slist. Set the clear_slist parameter to TRUE + * since we called H5C_flush_cache() with the + * H5C__FLUSH_MARKED_ENTRIES_FLAG. + */ + if (H5C_set_slist_enabled(f->shared->cache, FALSE, TRUE) < 0) + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "disable slist failed") + done: + FUNC_LEAVE_NOAPI(ret_value) + } /* H5C__flush_marked_entries */ /*------------------------------------------------------------------------- @@ -8135,6 +8945,10 @@ done: * Programmer: Mohamad Chaarawi * 2/10/16 * + * Changes: Updated sanity checks for the possibility that the skip + * list is disabled. + * JRM 5/16/20 + * *------------------------------------------------------------------------- */ static herr_t @@ -8163,15 +8977,18 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr) old_addr = entry_ptr->addr; /* Call client's pre-serialize callback, if there's one */ - if (entry_ptr->type->pre_serialize && - (entry_ptr->type->pre_serialize)(f, (void *)entry_ptr, entry_ptr->addr, entry_ptr->size, &new_addr, - &new_len, &serialize_flags) < 0) + if ((entry_ptr->type->pre_serialize) && + ((entry_ptr->type->pre_serialize)(f, (void *)entry_ptr, entry_ptr->addr, entry_ptr->size, &new_addr, + &new_len, &serialize_flags) < 0)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to pre-serialize entry") /* Check for any flags set in the pre-serialize callback */ if (serialize_flags != H5C__SERIALIZE_NO_FLAGS_SET) { + /* Check for unexpected flags from serialize callback */ if (serialize_flags & ~(H5C__SERIALIZE_RESIZED_FLAG | H5C__SERIALIZE_MOVED_FLAG)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unknown serialize flag(s)") #ifdef H5_HAVE_PARALLEL @@ -8202,20 +9019,25 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr) * tests will be necessary. */ if (cache_ptr->aux_ptr != NULL) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "resize/move in serialize occurred in parallel case") #endif /* If required, resize the buffer and update the entry and the cache - * data structures */ + * data structures + */ if (serialize_flags & H5C__SERIALIZE_RESIZED_FLAG) { + /* Sanity check */ HDassert(new_len > 0); /* Allocate a new image buffer */ if (NULL == (entry_ptr->image_ptr = H5MM_realloc(entry_ptr->image_ptr, new_len + H5C_IMAGE_EXTRA_SPACE))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for on disk image buffer") + #if H5C_DO_MEMORY_SANITY_CHECKS H5MM_memcpy(((uint8_t *)entry_ptr->image_ptr) + new_len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); @@ -8237,25 +9059,31 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr) /* As we haven't updated the cache data structures for * for the flush or flush destroy yet, the entry should - * be in the slist. Thus update it for the size change. + * be in the slist if the slist is enabled. Since + * H5C__UPDATE_SLIST_FOR_SIZE_CHANGE() is a no-op if the + * slist is enabled, call it un-conditionally. */ HDassert(entry_ptr->is_dirty); - HDassert(entry_ptr->in_slist); + HDassert((entry_ptr->in_slist) || (!cache_ptr->slist_enabled)); + H5C__UPDATE_SLIST_FOR_SIZE_CHANGE(cache_ptr, entry_ptr->size, new_len); /* Finally, update the entry for its new size */ entry_ptr->size = new_len; + } /* end if */ /* If required, udate the entry and the cache data structures * for a move */ if (serialize_flags & H5C__SERIALIZE_MOVED_FLAG) { + /* Update stats and entries relocated counter */ H5C__UPDATE_STATS_FOR_MOVE(cache_ptr, entry_ptr) /* We must update cache data structures for the change in address */ if (entry_ptr->addr == old_addr) { + /* Delete the entry from the hash table and the slist */ H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr, FAIL); H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, FALSE); @@ -8266,19 +9094,25 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr) /* And then reinsert in the index and slist */ H5C__INSERT_IN_INDEX(cache_ptr, entry_ptr, FAIL); H5C__INSERT_ENTRY_IN_SLIST(cache_ptr, entry_ptr, FAIL); - } /* end if */ - else /* move is already done for us -- just do sanity checks */ + + } /* end if */ + else { /* move is already done for us -- just do sanity checks */ + HDassert(entry_ptr->addr == new_addr); + } } /* end if */ } /* end if(serialize_flags != H5C__SERIALIZE_NO_FLAGS_SET) */ /* Serialize object into buffer */ if (entry_ptr->type->serialize(f, entry_ptr->image_ptr, entry_ptr->size, (void *)entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to serialize entry") + #if H5C_DO_MEMORY_SANITY_CHECKS HDassert(0 == HDmemcmp(((uint8_t *)entry_ptr->image_ptr) + entry_ptr->size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE)); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ + entry_ptr->image_up_to_date = TRUE; /* Propagate the fact that the entry is serialized up the @@ -8288,9 +9122,13 @@ H5C__generate_image(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_ptr) * for flush dependency parents. */ HDassert(entry_ptr->flush_dep_nunser_children == 0); - if (entry_ptr->flush_dep_nparents > 0) + + if (entry_ptr->flush_dep_nparents > 0) { + if (H5C__mark_flush_dep_serialized(entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "Can't propagate serialization status to fd parents") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Cdbg.c b/src/H5Cdbg.c index 0752673..a8917ea 100644 --- a/src/H5Cdbg.c +++ b/src/H5Cdbg.c @@ -260,6 +260,12 @@ H5C_dump_cache_LRU(H5C_t *cache_ptr, const char *cache_name) * Programmer: John Mainzer * 11/15/14 * + * Changes: Updated function for the slist_enabled field in H5C_t. + * Recall that to minimize slist overhead, the slist is + * empty and not maintained if cache_ptr->slist_enabled is + * false. + * JRM -- 5/6/20 + * *------------------------------------------------------------------------- */ #ifndef NDEBUG @@ -278,6 +284,7 @@ H5C_dump_cache_skip_list(H5C_t *cache_ptr, char *calling_fcn) HDassert(calling_fcn != NULL); HDfprintf(stdout, "\n\nDumping metadata cache skip list from %s.\n", calling_fcn); + HDfprintf(stdout, " slist %s.\n", cache_ptr->slist_enabled ? "enabled" : "disabled"); HDfprintf(stdout, " slist len = %" PRIu32 ".\n", cache_ptr->slist_len); HDfprintf(stdout, " slist size = %zu.\n", cache_ptr->slist_size); diff --git a/src/H5Cimage.c b/src/H5Cimage.c index e876265..b3f6c12 100644 --- a/src/H5Cimage.c +++ b/src/H5Cimage.c @@ -453,6 +453,10 @@ done: * * Programmer: John Mainzer, 8/10/15 * + * Changes: Updated sanity checks for possibility that the slist + * is disabled. + * JRM -- 5/17/20 + * *------------------------------------------------------------------------- */ herr_t @@ -693,7 +697,10 @@ H5C__deserialize_prefetched_entry(H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t pf_entry_ptr->image_ptr = NULL; if (pf_entry_ptr->is_dirty) { - HDassert(pf_entry_ptr->in_slist); + + HDassert(((cache_ptr->slist_enabled) && (pf_entry_ptr->in_slist)) || + ((!cache_ptr->slist_enabled) && (!pf_entry_ptr->in_slist))); + flush_flags |= H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG; } /* end if */ diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c index 6c0c015..0489ad2 100644 --- a/src/H5Cmpio.c +++ b/src/H5Cmpio.c @@ -154,6 +154,10 @@ static herr_t H5C__flush_candidates_in_ring(H5F_t *f, H5C_ring_t ring, unsigned * Programmer: John Mainzer * 3/17/10 * + * Changes: Updated sanity checks to allow for the possibility that + * the slist is disabled. + * JRM -- 8/3/20 + * *------------------------------------------------------------------------- */ herr_t @@ -190,7 +194,7 @@ H5C_apply_candidate_list(H5F_t *f, H5C_t *cache_ptr, unsigned num_candidates, ha HDassert(cache_ptr != NULL); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); HDassert(num_candidates > 0); - HDassert(num_candidates <= cache_ptr->slist_len); + HDassert((!cache_ptr->slist_enabled) || (num_candidates <= cache_ptr->slist_len)); HDassert(candidates_list_ptr != NULL); HDassert(0 <= mpi_rank); HDassert(mpi_rank < mpi_size); @@ -416,6 +420,16 @@ done: * Programmer: John Mainzer * 3/17/10 * + * Changes: With the slist optimization, the slist is not maintained + * unless a flush is in progress. Thus we can not longer use + * cache_ptr->slist_size to determine the total size of + * the entries we must insert in the candidate list. + * + * To address this, we now use cache_ptr->dirty_index_size + * instead. + * + * JRM -- 7/27/20 + * *------------------------------------------------------------------------- */ herr_t @@ -429,18 +443,22 @@ H5C_construct_candidate_list__clean_cache(H5C_t *cache_ptr) HDassert(cache_ptr != NULL); HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - /* As a sanity check, set space needed to the size of the skip list. - * This should be the sum total of the sizes of all the dirty entries - * in the metadata cache. + /* As a sanity check, set space needed to the dirty_index_size. This + * should be the sum total of the sizes of all the dirty entries + * in the metadata cache. Note that if the slist is enabled, + * cache_ptr->slist_size should equal cache_ptr->dirty_index_size. */ - space_needed = cache_ptr->slist_size; + space_needed = cache_ptr->dirty_index_size; + + HDassert((!cache_ptr->slist_enabled) || (space_needed == cache_ptr->slist_size)); /* Recall that while we shouldn't have any protected entries at this * point, it is possible that some dirty entries may reside on the * pinned list at this point. */ - HDassert(cache_ptr->slist_size <= (cache_ptr->dLRU_list_size + cache_ptr->pel_size)); - HDassert(cache_ptr->slist_len <= (cache_ptr->dLRU_list_len + cache_ptr->pel_len)); + HDassert(cache_ptr->dirty_index_size <= (cache_ptr->dLRU_list_size + cache_ptr->pel_size)); + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_len <= (cache_ptr->dLRU_list_len + cache_ptr->pel_len))); if (space_needed > 0) { /* we have work to do */ @@ -449,7 +467,7 @@ H5C_construct_candidate_list__clean_cache(H5C_t *cache_ptr) size_t nominated_entries_size = 0; haddr_t nominated_addr; - HDassert(cache_ptr->slist_len > 0); + HDassert((!cache_ptr->slist_enabled) || (cache_ptr->slist_len > 0)); /* Scan the dirty LRU list from tail forward and nominate sufficient * entries to free up the necessary space. @@ -457,14 +475,14 @@ H5C_construct_candidate_list__clean_cache(H5C_t *cache_ptr) entry_ptr = cache_ptr->dLRU_tail_ptr; while ((nominated_entries_size < space_needed) && - (nominated_entries_count < cache_ptr->slist_len) && + ((!cache_ptr->slist_enabled) || (nominated_entries_count < cache_ptr->slist_len)) && (entry_ptr != NULL)) { HDassert(!(entry_ptr->is_protected)); HDassert(!(entry_ptr->is_read_only)); HDassert(entry_ptr->ro_ref_count == 0); HDassert(entry_ptr->is_dirty); - HDassert(entry_ptr->in_slist); + HDassert((!cache_ptr->slist_enabled) || (entry_ptr->in_slist)); nominated_addr = entry_ptr->addr; @@ -484,7 +502,9 @@ H5C_construct_candidate_list__clean_cache(H5C_t *cache_ptr) * protected entry list as well -- scan it too if necessary */ entry_ptr = cache_ptr->pel_head_ptr; - while ((nominated_entries_size < space_needed) && (nominated_entries_count < cache_ptr->slist_len) && + + while ((nominated_entries_size < space_needed) && + ((!cache_ptr->slist_enabled) || (nominated_entries_count < cache_ptr->slist_len)) && (entry_ptr != NULL)) { if (entry_ptr->is_dirty) { @@ -510,7 +530,7 @@ H5C_construct_candidate_list__clean_cache(H5C_t *cache_ptr) } /* end while */ - HDassert(nominated_entries_count == cache_ptr->slist_len); + HDassert((!cache_ptr->slist_enabled) || (nominated_entries_count == cache_ptr->slist_len)); HDassert(nominated_entries_size == space_needed); } /* end if */ @@ -537,6 +557,12 @@ done: * Programmer: John Mainzer * 3/17/10 * + * Changes: With the slist optimization, the slist is not maintained + * unless a flush is in progress. Updated sanity checks to + * reflect this. + * + * JRM -- 7/27/20 + * *------------------------------------------------------------------------- */ herr_t @@ -556,17 +582,26 @@ H5C_construct_candidate_list__min_clean(H5C_t *cache_ptr) if (cache_ptr->max_cache_size > cache_ptr->index_size) { if (((cache_ptr->max_cache_size - cache_ptr->index_size) + cache_ptr->cLRU_list_size) >= - cache_ptr->min_clean_size) + cache_ptr->min_clean_size) { + space_needed = 0; - else + } + else { + space_needed = cache_ptr->min_clean_size - ((cache_ptr->max_cache_size - cache_ptr->index_size) + cache_ptr->cLRU_list_size); + } } /* end if */ else { - if (cache_ptr->min_clean_size <= cache_ptr->cLRU_list_size) + + if (cache_ptr->min_clean_size <= cache_ptr->cLRU_list_size) { + space_needed = 0; - else + } + else { + space_needed = cache_ptr->min_clean_size - cache_ptr->cLRU_list_size; + } } /* end else */ if (space_needed > 0) { /* we have work to do */ @@ -575,13 +610,15 @@ H5C_construct_candidate_list__min_clean(H5C_t *cache_ptr) unsigned nominated_entries_count = 0; size_t nominated_entries_size = 0; - HDassert(cache_ptr->slist_len > 0); + HDassert((!cache_ptr->slist_enabled) || (cache_ptr->slist_len > 0)); /* Scan the dirty LRU list from tail forward and nominate sufficient * entries to free up the necessary space. */ entry_ptr = cache_ptr->dLRU_tail_ptr; - while ((nominated_entries_size < space_needed) && (nominated_entries_count < cache_ptr->slist_len) && + + while ((nominated_entries_size < space_needed) && + ((!cache_ptr->slist_enabled) || (nominated_entries_count < cache_ptr->slist_len)) && (entry_ptr != NULL) && (!entry_ptr->flush_me_last)) { haddr_t nominated_addr; @@ -590,7 +627,7 @@ H5C_construct_candidate_list__min_clean(H5C_t *cache_ptr) HDassert(!(entry_ptr->is_read_only)); HDassert(entry_ptr->ro_ref_count == 0); HDassert(entry_ptr->is_dirty); - HDassert(entry_ptr->in_slist); + HDassert((!cache_ptr->slist_enabled) || (entry_ptr->in_slist)); nominated_addr = entry_ptr->addr; @@ -603,7 +640,9 @@ H5C_construct_candidate_list__min_clean(H5C_t *cache_ptr) entry_ptr = entry_ptr->aux_prev; } /* end while */ - HDassert(nominated_entries_count <= cache_ptr->slist_len); + + HDassert((!cache_ptr->slist_enabled) || (nominated_entries_count <= cache_ptr->slist_len)); + HDassert(nominated_entries_size <= cache_ptr->dirty_index_size); HDassert(nominated_entries_size >= space_needed); } /* end if */ diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index 409dccc..410fc3e 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -59,6 +59,13 @@ /* Initial allocated size of the "flush_dep_parent" array */ #define H5C_FLUSH_DEP_PARENT_INIT 8 + +/* Set to TRUE to enable the slist optimization. If this field is TRUE, + * the slist is disabled whenever a flush is not in progress. + */ +#define H5C__SLIST_OPT_ENABLED TRUE + + /**************************************************************************** * * We maintain doubly linked lists of instances of H5C_cache_entry_t for a @@ -1573,9 +1580,27 @@ if ( ( (cache_ptr)->index_size != \ * Added code to maintain the cache_ptr->slist_ring_len * and cache_ptr->slist_ring_size arrays. * + * JRM -- 4/29/20 + * Reworked macro to support the slist_enabled field + * of H5C_t. If slist_enabled == TRUE, the macro + * functions as before. Otherwise, the macro is a no-op, + * and the slist must be empty. + * *------------------------------------------------------------------------- */ +/* NOTE: The H5C__INSERT_ENTRY_IN_SLIST() macro is set up so that + * + * H5C_DO_SANITY_CHECKS + * + * and + * + * H5C_DO_SLIST_SANITY_CHECKS + * + * can be selected independantly. This is easy to miss as the + * two #defines are easy to confuse. + */ + #if H5C_DO_SLIST_SANITY_CHECKS #define ENTRY_IN_SLIST(cache_ptr, entry_ptr) \ @@ -1594,33 +1619,43 @@ if ( ( (cache_ptr)->index_size != \ { \ HDassert( (cache_ptr) ); \ HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ - HDassert( (entry_ptr) ); \ - HDassert( (entry_ptr)->size > 0 ); \ - HDassert( H5F_addr_defined((entry_ptr)->addr) ); \ - HDassert( !((entry_ptr)->in_slist) ); \ - HDassert( !ENTRY_IN_SLIST((cache_ptr), (entry_ptr)) ); \ - HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ - HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ - HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_len ); \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_size ); \ \ - if(H5SL_insert((cache_ptr)->slist_ptr, entry_ptr, &(entry_ptr)->addr) < 0) \ - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, (fail_val), "can't insert entry in skip list") \ + if ( (cache_ptr)->slist_enabled ) { \ \ - (entry_ptr)->in_slist = TRUE; \ - (cache_ptr)->slist_changed = TRUE; \ - (cache_ptr)->slist_len++; \ - (cache_ptr)->slist_size += (entry_ptr)->size; \ - ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])++; \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (entry_ptr)->size; \ - (cache_ptr)->slist_len_increase++; \ - (cache_ptr)->slist_size_increase += (int64_t)((entry_ptr)->size); \ + HDassert( (entry_ptr) ); \ + HDassert( (entry_ptr)->size > 0 ); \ + HDassert( H5F_addr_defined((entry_ptr)->addr) ); \ + HDassert( !((entry_ptr)->in_slist) ); \ + HDassert( ! ENTRY_IN_SLIST((cache_ptr), (entry_ptr)) ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ \ - HDassert( (cache_ptr)->slist_len > 0 ); \ - HDassert( (cache_ptr)->slist_size > 0 ); \ + if ( H5SL_insert((cache_ptr)->slist_ptr, entry_ptr, \ + &((entry_ptr)->addr)) < 0) \ + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, (fail_val), \ + "can't insert entry in skip list") \ \ + (entry_ptr)->in_slist = TRUE; \ + (cache_ptr)->slist_changed = TRUE; \ + (cache_ptr)->slist_len++; \ + (cache_ptr)->slist_size += (entry_ptr)->size; \ + ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])++; \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (entry_ptr)->size;\ + (cache_ptr)->slist_len_increase++; \ + (cache_ptr)->slist_size_increase += (int64_t)((entry_ptr)->size); \ + \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + HDassert( (cache_ptr)->slist_size > 0 ); \ + \ + } else { /* slist disabled */ \ + \ + HDassert( (cache_ptr)->slist_len == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ + } \ } /* H5C__INSERT_ENTRY_IN_SLIST */ #else /* H5C_DO_SANITY_CHECKS */ @@ -1629,31 +1664,42 @@ if ( ( (cache_ptr)->index_size != \ { \ HDassert( (cache_ptr) ); \ HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ - HDassert( (entry_ptr) ); \ - HDassert( (entry_ptr)->size > 0 ); \ - HDassert( H5F_addr_defined((entry_ptr)->addr) ); \ - HDassert( !((entry_ptr)->in_slist) ); \ - HDassert( !ENTRY_IN_SLIST((cache_ptr), (entry_ptr)) ); \ - HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ - HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ - HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_len ); \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_size ); \ \ - if(H5SL_insert((cache_ptr)->slist_ptr, entry_ptr, &(entry_ptr)->addr) < 0) \ - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, (fail_val), "can't insert entry in skip list") \ + if ( (cache_ptr)->slist_enabled ) { \ + \ + HDassert( (entry_ptr) ); \ + HDassert( (entry_ptr)->size > 0 ); \ + HDassert( ! ENTRY_IN_SLIST((cache_ptr), (entry_ptr)) ); \ + HDassert( H5F_addr_defined((entry_ptr)->addr) ); \ + HDassert( !((entry_ptr)->in_slist) ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ + HDassert( (cache_ptr)->slist_ptr ); \ \ - (entry_ptr)->in_slist = TRUE; \ - (cache_ptr)->slist_changed = TRUE; \ - (cache_ptr)->slist_len++; \ - (cache_ptr)->slist_size += (entry_ptr)->size; \ - ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])++; \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (entry_ptr)->size; \ + if ( H5SL_insert((cache_ptr)->slist_ptr, entry_ptr, \ + &((entry_ptr)->addr)) < 0) \ + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, (fail_val), \ + "can't insert entry in skip list") \ \ - HDassert( (cache_ptr)->slist_len > 0 ); \ - HDassert( (cache_ptr)->slist_size > 0 ); \ + (entry_ptr)->in_slist = TRUE; \ + (cache_ptr)->slist_changed = TRUE; \ + (cache_ptr)->slist_len++; \ + (cache_ptr)->slist_size += (entry_ptr)->size; \ + ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])++; \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (entry_ptr)->size;\ \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + HDassert( (cache_ptr)->slist_size > 0 ); \ + \ + } else { /* slist disabled */ \ + \ + HDassert( (cache_ptr)->slist_len == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ + } \ } /* H5C__INSERT_ENTRY_IN_SLIST */ #endif /* H5C_DO_SANITY_CHECKS */ @@ -1671,79 +1717,127 @@ if ( ( (cache_ptr)->index_size != \ * * Programmer: John Mainzer, 5/10/04 * + * Modifications: + * + * JRM -- 7/21/04 + * Updated function for the addition of the hash table. + * + * JRM - 7/27/04 + * Converted from the function H5C_remove_entry_from_tree() + * to the macro H5C__REMOVE_ENTRY_FROM_TREE in the hopes of + * wringing a little more performance out of the cache. + * + * QAK -- 11/27/04 + * Switched over to using skip list routines. + * + * JRM -- 3/28/07 + * Updated sanity checks for the new is_read_only and + * ro_ref_count fields in H5C_cache_entry_t. + * + * JRM -- 12/13/14 + * Added code to set cache_ptr->slist_changed to TRUE + * when an entry is removed from the slist. + * + * JRM -- 4/29/20 + * Reworked macro to support the slist_enabled field + * of H5C_t. If slist_enabled == TRUE, the macro + * functions as before. Otherwise, the macro is a no-op, + * and the slist must be empty. + * *------------------------------------------------------------------------- */ #if H5C_DO_SANITY_CHECKS -#define H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) \ -{ \ - HDassert( (cache_ptr) ); \ - HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ - HDassert( (entry_ptr) ); \ - HDassert( !((entry_ptr)->is_read_only) ); \ - HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ - HDassert( (entry_ptr)->size > 0 ); \ - HDassert( (entry_ptr)->in_slist ); \ - HDassert( (cache_ptr)->slist_ptr ); \ - HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ - HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ - HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_len ); \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_size ); \ - \ - if ( H5SL_remove((cache_ptr)->slist_ptr, &(entry_ptr)->addr) \ - != (entry_ptr) ) \ - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "can't delete entry from skip list") \ - \ - HDassert( (cache_ptr)->slist_len > 0 ); \ - if(!(during_flush)) \ - (cache_ptr)->slist_changed = TRUE; \ - (cache_ptr)->slist_len--; \ - HDassert( (cache_ptr)->slist_size >= (entry_ptr)->size ); \ - (cache_ptr)->slist_size -= (entry_ptr)->size; \ - ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])--; \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >= \ - (entry_ptr)->size ); \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (entry_ptr)->size; \ - (cache_ptr)->slist_len_increase--; \ - (cache_ptr)->slist_size_increase -= (int64_t)((entry_ptr)->size); \ - (entry_ptr)->in_slist = FALSE; \ +#define H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) \ +{ \ + HDassert( (cache_ptr) ); \ + HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ + \ + if ( (cache_ptr)->slist_enabled ) { \ + \ + HDassert( (entry_ptr) ); \ + HDassert( !((entry_ptr)->is_read_only) ); \ + HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ + HDassert( (entry_ptr)->size > 0 ); \ + HDassert( (entry_ptr)->in_slist ); \ + HDassert( (cache_ptr)->slist_ptr ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ + HDassert( (cache_ptr)->slist_size >= (entry_ptr)->size ); \ + \ + if ( H5SL_remove((cache_ptr)->slist_ptr, &(entry_ptr)->addr) \ + != (entry_ptr) ) \ + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, \ + "can't delete entry from skip list") \ + \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + if(!(during_flush)) \ + (cache_ptr)->slist_changed = TRUE; \ + (cache_ptr)->slist_len--; \ + HDassert( (cache_ptr)->slist_size >= (entry_ptr)->size ); \ + (cache_ptr)->slist_size -= (entry_ptr)->size; \ + ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])--; \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >= \ + (entry_ptr)->size ); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (entry_ptr)->size;\ + (cache_ptr)->slist_len_increase--; \ + (cache_ptr)->slist_size_increase -= (int64_t)((entry_ptr)->size); \ + (entry_ptr)->in_slist = FALSE; \ + \ + } else { /* slist disabled */ \ + \ + HDassert( (cache_ptr)->slist_len == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ + } \ } /* H5C__REMOVE_ENTRY_FROM_SLIST */ #else /* H5C_DO_SANITY_CHECKS */ -#define H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) \ -{ \ - HDassert( (cache_ptr) ); \ - HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ - HDassert( (entry_ptr) ); \ - HDassert( !((entry_ptr)->is_read_only) ); \ - HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ - HDassert( (entry_ptr)->in_slist ); \ - HDassert( (cache_ptr)->slist_ptr ); \ - HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ - HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ - HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_len ); \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_size ); \ - \ - if ( H5SL_remove((cache_ptr)->slist_ptr, &(entry_ptr)->addr) \ - != (entry_ptr) ) \ - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "can't delete entry from skip list") \ - \ - HDassert( (cache_ptr)->slist_len > 0 ); \ - if(!(during_flush)) \ - (cache_ptr)->slist_changed = TRUE; \ - (cache_ptr)->slist_len--; \ - HDassert( (cache_ptr)->slist_size >= (entry_ptr)->size ); \ - (cache_ptr)->slist_size -= (entry_ptr)->size; \ - ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])--; \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >= \ - (entry_ptr)->size ); \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (entry_ptr)->size; \ - (entry_ptr)->in_slist = FALSE; \ +#define H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, during_flush) \ +{ \ + HDassert( (cache_ptr) ); \ + HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ + \ + if ( (cache_ptr)->slist_enabled ) { \ + \ + HDassert( (entry_ptr) ); \ + HDassert( !((entry_ptr)->is_read_only) ); \ + HDassert( ((entry_ptr)->ro_ref_count) == 0 ); \ + HDassert( (entry_ptr)->in_slist ); \ + HDassert( (cache_ptr)->slist_ptr ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ + \ + if ( H5SL_remove((cache_ptr)->slist_ptr, &(entry_ptr)->addr) \ + != (entry_ptr) ) \ + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, \ + "can't delete entry from skip list") \ + \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + if(!(during_flush)) \ + (cache_ptr)->slist_changed = TRUE; \ + (cache_ptr)->slist_len--; \ + HDassert( (cache_ptr)->slist_size >= (entry_ptr)->size ); \ + (cache_ptr)->slist_size -= (entry_ptr)->size; \ + ((cache_ptr)->slist_ring_len[(entry_ptr)->ring])--; \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >= \ + (entry_ptr)->size ); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (entry_ptr)->size;\ + (entry_ptr)->in_slist = FALSE; \ + \ + } else { /* slist disabled */ \ + \ + HDassert( (cache_ptr)->slist_len == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ + } \ } /* H5C__REMOVE_ENTRY_FROM_SLIST */ #endif /* H5C_DO_SANITY_CHECKS */ @@ -1781,6 +1875,12 @@ if ( ( (cache_ptr)->index_size != \ * Added code to maintain the cache_ptr->slist_ring_len * and cache_ptr->slist_ring_size arrays. * + * JRM -- 4/29/20 + * Reworked macro to support the slist_enabled field + * of H5C_t. If slist_enabled == TRUE, the macro + * functions as before. Otherwise, the macro is a no-op, + * and the slist must be empty. + * *------------------------------------------------------------------------- */ @@ -1790,32 +1890,43 @@ if ( ( (cache_ptr)->index_size != \ { \ HDassert( (cache_ptr) ); \ HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ - HDassert( (old_size) > 0 ); \ - HDassert( (new_size) > 0 ); \ - HDassert( (old_size) <= (cache_ptr)->slist_size ); \ - HDassert( (cache_ptr)->slist_len > 0 ); \ - HDassert( ((cache_ptr)->slist_len > 1) || \ - ( (cache_ptr)->slist_size == (old_size) ) ); \ - HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ - HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ - HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_len ); \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_size ); \ \ - (cache_ptr)->slist_size -= (old_size); \ - (cache_ptr)->slist_size += (new_size); \ + if ( (cache_ptr)->slist_enabled ) { \ + \ + HDassert( (old_size) > 0 ); \ + HDassert( (new_size) > 0 ); \ + HDassert( (old_size) <= (cache_ptr)->slist_size ); \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + HDassert( ((cache_ptr)->slist_len > 1) || \ + ( (cache_ptr)->slist_size == (old_size) ) ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ + \ + (cache_ptr)->slist_size -= (old_size); \ + (cache_ptr)->slist_size += (new_size); \ + \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] \ + >= (old_size) ); \ \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >=(old_size) ); \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (old_size); \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (new_size); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (old_size); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (new_size); \ \ - (cache_ptr)->slist_size_increase -= (int64_t)(old_size); \ - (cache_ptr)->slist_size_increase += (int64_t)(new_size); \ + (cache_ptr)->slist_size_increase -= (int64_t)(old_size); \ + (cache_ptr)->slist_size_increase += (int64_t)(new_size); \ \ - HDassert( (new_size) <= (cache_ptr)->slist_size ); \ - HDassert( ( (cache_ptr)->slist_len > 1 ) || \ - ( (cache_ptr)->slist_size == (new_size) ) ); \ + HDassert( (new_size) <= (cache_ptr)->slist_size ); \ + HDassert( ( (cache_ptr)->slist_len > 1 ) || \ + ( (cache_ptr)->slist_size == (new_size) ) ); \ + \ + } else { /* slist disabled */ \ + \ + HDassert( (cache_ptr)->slist_len == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ + } \ } /* H5C__UPDATE_SLIST_FOR_SIZE_CHANGE */ #else /* H5C_DO_SANITY_CHECKS */ @@ -1824,29 +1935,39 @@ if ( ( (cache_ptr)->index_size != \ { \ HDassert( (cache_ptr) ); \ HDassert( (cache_ptr)->magic == H5C__H5C_T_MAGIC ); \ - HDassert( (old_size) > 0 ); \ - HDassert( (new_size) > 0 ); \ - HDassert( (old_size) <= (cache_ptr)->slist_size ); \ - HDassert( (cache_ptr)->slist_len > 0 ); \ - HDassert( ((cache_ptr)->slist_len > 1) || \ - ( (cache_ptr)->slist_size == (old_size) ) ); \ - HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ - HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ - HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_len ); \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ - (cache_ptr)->slist_size ); \ \ - (cache_ptr)->slist_size -= (old_size); \ - (cache_ptr)->slist_size += (new_size); \ + if ( (cache_ptr)->slist_enabled ) { \ + \ + HDassert( (old_size) > 0 ); \ + HDassert( (new_size) > 0 ); \ + HDassert( (old_size) <= (cache_ptr)->slist_size ); \ + HDassert( (cache_ptr)->slist_len > 0 ); \ + HDassert( ((cache_ptr)->slist_len > 1) || \ + ( (cache_ptr)->slist_size == (old_size) ) ); \ + HDassert( (entry_ptr)->ring > H5C_RING_UNDEFINED ); \ + HDassert( (entry_ptr)->ring < H5C_RING_NTYPES ); \ + HDassert( (cache_ptr)->slist_ring_len[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_len ); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr)->ring] <= \ + (cache_ptr)->slist_size ); \ + \ + (cache_ptr)->slist_size -= (old_size); \ + (cache_ptr)->slist_size += (new_size); \ \ - HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >=(old_size) ); \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (old_size); \ - ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (new_size); \ + HDassert( (cache_ptr)->slist_ring_size[(entry_ptr->ring)] >= \ + (old_size) ); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) -= (old_size); \ + ((cache_ptr)->slist_ring_size[(entry_ptr)->ring]) += (new_size); \ \ - HDassert( (new_size) <= (cache_ptr)->slist_size ); \ - HDassert( ( (cache_ptr)->slist_len > 1 ) || \ - ( (cache_ptr)->slist_size == (new_size) ) ); \ + HDassert( (new_size) <= (cache_ptr)->slist_size ); \ + HDassert( ( (cache_ptr)->slist_len > 1 ) || \ + ( (cache_ptr)->slist_size == (new_size) ) ); \ + \ + } else { /* slist disabled */ \ + \ + HDassert( (cache_ptr)->slist_len == 0 ); \ + HDassert( (cache_ptr)->slist_size == 0 ); \ + } \ } /* H5C__UPDATE_SLIST_FOR_SIZE_CHANGE */ #endif /* H5C_DO_SANITY_CHECKS */ @@ -3752,6 +3873,36 @@ typedef struct H5C_tag_info_t { * are flushed. (this has been changed -- dirty entries are now removed from * the skip list as they are flushed. JRM - 10/25/05) * + * Update 4/21/20: + * + * Profiling indicates that the cost of maintaining the skip list is + * significant. As it is only used on flush and close, maintaining it + * only when needed is an obvious optimization. + * + * To do this, we add a flag to control maintenanace of the skip list. + * This flag is initially set to FALSE, which disables all operations + * on the skip list. + * + * At the beginning of either flush or close, we scan the index list, + * insert all dirtly entries in the skip list, and enable operations + * on skip list by setting above control flag to true. + * + * At the end of a complete flush, we verify that the skip list is empty, + * and set the control flag back to false, so as to avoid skip list + * maintenance overhead until the next flush or close. + * + * In the case of a partial flush (i.e. flush marked entries), we remove + * all remaining entries from the skip list, and then set the control flag + * back to false -- again avoiding skip list maintenance overhead until + * the next flush or close. + * + * slist_enabled: Boolean flag used to control operation of the skip + * list. If this filed is FALSE, operations on the + * slist are no-ops, and the slist must be empty. If + * it is TRUE, operations on the slist proceed as usual, + * and all dirty entries in the metadata cache must be + * listed in the slist. + * * slist_changed: Boolean flag used to indicate whether the contents of * the slist has changed since the last time this flag was * reset. This is used in the cache flush code to detect @@ -4698,6 +4849,7 @@ struct H5C_t { H5C_cache_entry_t * entry_watched_for_removal; /* Fields for maintaining list of in-order entries, for flushing */ + hbool_t slist_enabled; hbool_t slist_changed; uint32_t slist_len; size_t slist_size; diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 80ab5ff..ac597bf 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -2270,6 +2270,7 @@ H5_DLL herr_t H5C_resize_entry(void *thing, size_t new_size); H5_DLL herr_t H5C_set_cache_auto_resize_config(H5C_t *cache_ptr, H5C_auto_size_ctl_t *config_ptr); H5_DLL herr_t H5C_set_cache_image_config(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_image_ctl_t *config_ptr); H5_DLL herr_t H5C_set_evictions_enabled(H5C_t *cache_ptr, hbool_t evictions_enabled); +H5_DLL herr_t H5C_set_slist_enabled(H5C_t *cache_ptr, hbool_t slist_enabled, hbool_t clear_slist); H5_DLL herr_t H5C_set_prefix(H5C_t *cache_ptr, char *prefix); H5_DLL herr_t H5C_stats(H5C_t *cache_ptr, const char *cache_name, hbool_t display_detailed_stats); H5_DLL void H5C_stats__reset(H5C_t *cache_ptr); diff --git a/src/H5Dint.c b/src/H5Dint.c index cfed336..7a2b93b 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -1277,18 +1277,24 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, hid_t /* Check if the dataset has a non-default DCPL & get important values, if so */ if (new_dset->shared->dcpl_id != H5P_DATASET_CREATE_DEFAULT) { - H5O_layout_t *layout; /* Dataset's layout information */ - H5O_pline_t * pline; /* Dataset's I/O pipeline information */ - H5O_fill_t * fill; /* Dataset's fill value info */ - H5O_efl_t * efl; /* Dataset's external file list info */ + H5O_layout_t *layout; /* Dataset's layout information */ + H5O_pline_t * pline; /* Dataset's I/O pipeline information */ + H5O_fill_t * fill; /* Dataset's fill value info */ + H5O_efl_t * efl; /* Dataset's external file list info */ + htri_t ignore_filters = FALSE; /* Ignore optional filters or not */ - /* Check if the filters in the DCPL can be applied to this dataset */ - if (H5Z_can_apply(new_dset->shared->dcpl_id, new_dset->shared->type_id) < 0) - HGOTO_ERROR(H5E_ARGS, H5E_CANTINIT, NULL, "I/O filters can't operate on this dataset") + if ((ignore_filters = H5Z_ignore_filters(new_dset->shared->dcpl_id, dt, space)) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_CANTINIT, NULL, "H5Z_has_optional_filter() failed") - /* Make the "set local" filter callbacks for this dataset */ - if (H5Z_set_local(new_dset->shared->dcpl_id, new_dset->shared->type_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to set local filter parameters") + if (FALSE == ignore_filters) { + /* Check if the filters in the DCPL can be applied to this dataset */ + if (H5Z_can_apply(new_dset->shared->dcpl_id, new_dset->shared->type_id) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_CANTINIT, NULL, "I/O filters can't operate on this dataset") + + /* Make the "set local" filter callbacks for this dataset */ + if (H5Z_set_local(new_dset->shared->dcpl_id, new_dset->shared->type_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to set local filter parameters") + } /* ignore_filters */ /* Get new dataset's property list object */ if (NULL == (dc_plist = (H5P_genplist_t *)H5I_object(new_dset->shared->dcpl_id))) @@ -1312,9 +1318,11 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, hid_t HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "can't retrieve external file list") efl_copied = TRUE; - /* Check that chunked layout is used if filters are enabled */ - if (pline->nused > 0 && H5D_CHUNKED != layout->type) - HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, NULL, "filters can only be used with chunked layout") + if (FALSE == ignore_filters) { + /* Check that chunked layout is used if filters are enabled */ + if (pline->nused > 0 && H5D_CHUNKED != layout->type) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, NULL, "filters can only be used with chunked layout") + } /* Check if the alloc_time is the default and error out */ if (fill->alloc_time == H5D_ALLOC_TIME_DEFAULT) diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 8c2cace..2588d1c 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -302,22 +302,23 @@ done: herr_t H5D_virtual_update_min_dims(H5O_layout_t *layout, size_t idx) { - H5S_sel_type sel_type; - int rank; - hsize_t bounds_start[H5S_MAX_RANK]; - hsize_t bounds_end[H5S_MAX_RANK]; - int i; - herr_t ret_value = SUCCEED; + H5O_storage_virtual_t * virt = &layout->storage.u.virt; + H5O_storage_virtual_ent_t *ent = &virt->list[idx]; + H5S_sel_type sel_type; + int rank; + hsize_t bounds_start[H5S_MAX_RANK]; + hsize_t bounds_end[H5S_MAX_RANK]; + int i; + herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI(FAIL) HDassert(layout); HDassert(layout->type == H5D_VIRTUAL); - HDassert(idx < layout->storage.u.virt.list_nalloc); + HDassert(idx < virt->list_nalloc); /* Get type of selection */ - if (H5S_SEL_ERROR == - (sel_type = H5S_GET_SELECT_TYPE(layout->storage.u.virt.list[idx].source_dset.virtual_select))) + if (H5S_SEL_ERROR == (sel_type = H5S_GET_SELECT_TYPE(ent->source_dset.virtual_select))) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection type") /* Do not update min_dims for "all" or "none" selections */ @@ -325,20 +326,18 @@ H5D_virtual_update_min_dims(H5O_layout_t *layout, size_t idx) HGOTO_DONE(SUCCEED) /* Get rank of vspace */ - if ((rank = H5S_GET_EXTENT_NDIMS(layout->storage.u.virt.list[idx].source_dset.virtual_select)) < 0) + if ((rank = H5S_GET_EXTENT_NDIMS(ent->source_dset.virtual_select)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions") /* Get selection bounds */ - if (H5S_SELECT_BOUNDS(layout->storage.u.virt.list[idx].source_dset.virtual_select, bounds_start, - bounds_end) < 0) + if (H5S_SELECT_BOUNDS(ent->source_dset.virtual_select, bounds_start, bounds_end) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds") /* Update min_dims */ for (i = 0; i < rank; i++) /* Don't check unlimited dimensions in the selection */ - if ((i != layout->storage.u.virt.list[idx].unlim_dim_virtual) && - (bounds_end[i] >= layout->storage.u.virt.min_dims[i])) - layout->storage.u.virt.min_dims[i] = bounds_end[i] + (hsize_t)1; + if ((i != ent->unlim_dim_virtual) && (bounds_end[i] >= virt->min_dims[i])) + virt->min_dims[i] = bounds_end[i] + (hsize_t)1; done: FUNC_LEAVE_NOAPI(ret_value) @@ -410,31 +409,31 @@ done: herr_t H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout) { - uint8_t *heap_block = NULL; /* Block to add to heap */ - size_t * str_size = NULL; /* Array for VDS entry string lengths */ - uint8_t *heap_block_p; /* Pointer into the heap block, while encoding */ - size_t block_size; /* Total size of block needed */ - hsize_t tmp_nentries; /* Temp. variable for # of VDS entries */ - uint32_t chksum; /* Checksum for heap data */ - size_t i; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_storage_virtual_t *virt = &layout->storage.u.virt; + uint8_t * heap_block = NULL; /* Block to add to heap */ + size_t * str_size = NULL; /* Array for VDS entry string lengths */ + uint8_t * heap_block_p; /* Pointer into the heap block, while encoding */ + size_t block_size; /* Total size of block needed */ + hsize_t tmp_nentries; /* Temp. variable for # of VDS entries */ + uint32_t chksum; /* Checksum for heap data */ + size_t i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE /* Sanity checking */ HDassert(f); HDassert(layout); - HDassert(layout->storage.u.virt.serial_list_hobjid.addr == HADDR_UNDEF); + HDassert(virt->serial_list_hobjid.addr == HADDR_UNDEF); /* Create block if # of used entries > 0 */ - if (layout->storage.u.virt.list_nused > 0) { + if (virt->list_nused > 0) { /* Set the low/high bounds according to 'f' for the API context */ H5CX_set_libver_bounds(f); /* Allocate array for caching results of strlen */ - if (NULL == - (str_size = (size_t *)H5MM_malloc(2 * layout->storage.u.virt.list_nused * sizeof(size_t)))) + if (NULL == (str_size = (size_t *)H5MM_malloc(2 * virt->list_nused * sizeof(size_t)))) HGOTO_ERROR(H5E_OHDR, H5E_RESOURCE, FAIL, "unable to allocate string length array") /* @@ -445,31 +444,30 @@ H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout) block_size = (size_t)1 + H5F_SIZEOF_SIZE(f); /* Calculate size of each entry */ - for (i = 0; i < layout->storage.u.virt.list_nused; i++) { - hssize_t select_serial_size; /* Size of serialized selection */ + for (i = 0; i < virt->list_nused; i++) { + H5O_storage_virtual_ent_t *ent = &virt->list[i]; + hssize_t select_serial_size; /* Size of serialized selection */ - HDassert(layout->storage.u.virt.list[i].source_file_name); - HDassert(layout->storage.u.virt.list[i].source_dset_name); - HDassert(layout->storage.u.virt.list[i].source_select); - HDassert(layout->storage.u.virt.list[i].source_dset.virtual_select); + HDassert(ent->source_file_name); + HDassert(ent->source_dset_name); + HDassert(ent->source_select); + HDassert(ent->source_dset.virtual_select); /* Source file name */ - str_size[2 * i] = HDstrlen(layout->storage.u.virt.list[i].source_file_name) + (size_t)1; + str_size[2 * i] = HDstrlen(ent->source_file_name) + (size_t)1; block_size += str_size[2 * i]; /* Source dset name */ - str_size[(2 * i) + 1] = HDstrlen(layout->storage.u.virt.list[i].source_dset_name) + (size_t)1; + str_size[(2 * i) + 1] = HDstrlen(ent->source_dset_name) + (size_t)1; block_size += str_size[(2 * i) + 1]; /* Source selection */ - if ((select_serial_size = H5S_SELECT_SERIAL_SIZE(layout->storage.u.virt.list[i].source_select)) < - 0) + if ((select_serial_size = H5S_SELECT_SERIAL_SIZE(ent->source_select)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to check dataspace selection size") block_size += (size_t)select_serial_size; /* Virtual dataset selection */ - if ((select_serial_size = - H5S_SELECT_SERIAL_SIZE(layout->storage.u.virt.list[i].source_dset.virtual_select)) < 0) + if ((select_serial_size = H5S_SELECT_SERIAL_SIZE(ent->source_dset.virtual_select)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to check dataspace selection size") block_size += (size_t)select_serial_size; } /* end for */ @@ -490,28 +488,26 @@ H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout) *heap_block_p++ = (uint8_t)H5O_LAYOUT_VDS_GH_ENC_VERS; /* Number of entries */ - tmp_nentries = (hsize_t)layout->storage.u.virt.list_nused; + tmp_nentries = (hsize_t)virt->list_nused; H5F_ENCODE_LENGTH(f, heap_block_p, tmp_nentries) /* Encode each entry */ - for (i = 0; i < layout->storage.u.virt.list_nused; i++) { + for (i = 0; i < virt->list_nused; i++) { + H5O_storage_virtual_ent_t *ent = &virt->list[i]; /* Source file name */ - H5MM_memcpy((char *)heap_block_p, layout->storage.u.virt.list[i].source_file_name, - str_size[2 * i]); + H5MM_memcpy((char *)heap_block_p, ent->source_file_name, str_size[2 * i]); heap_block_p += str_size[2 * i]; /* Source dataset name */ - H5MM_memcpy((char *)heap_block_p, layout->storage.u.virt.list[i].source_dset_name, - str_size[(2 * i) + 1]); + H5MM_memcpy((char *)heap_block_p, ent->source_dset_name, str_size[(2 * i) + 1]); heap_block_p += str_size[(2 * i) + 1]; /* Source selection */ - if (H5S_SELECT_SERIALIZE(layout->storage.u.virt.list[i].source_select, &heap_block_p) < 0) + if (H5S_SELECT_SERIALIZE(ent->source_select, &heap_block_p) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to serialize source selection") /* Virtual selection */ - if (H5S_SELECT_SERIALIZE(layout->storage.u.virt.list[i].source_dset.virtual_select, - &heap_block_p) < 0) + if (H5S_SELECT_SERIALIZE(ent->source_dset.virtual_select, &heap_block_p) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to serialize virtual selection") } /* end for */ @@ -520,7 +516,7 @@ H5D__virtual_store_layout(H5F_t *f, H5O_layout_t *layout) UINT32ENCODE(heap_block_p, chksum) /* Insert block into global heap */ - if (H5HG_insert(f, block_size, heap_block, &(layout->storage.u.virt.serial_list_hobjid)) < + if (H5HG_insert(f, block_size, heap_block, &(virt->serial_list_hobjid)) < 0) /* Casting away const OK --NAF */ HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "unable to insert virtual dataset heap block") } /* end if */ @@ -551,6 +547,7 @@ herr_t H5D__virtual_copy_layout(H5O_layout_t *layout) { H5O_storage_virtual_ent_t *orig_list = NULL; + H5O_storage_virtual_t * virt = &layout->storage.u.virt; hid_t orig_source_fapl; hid_t orig_source_dapl; H5P_genplist_t * plist; @@ -564,133 +561,125 @@ H5D__virtual_copy_layout(H5O_layout_t *layout) /* Save original entry list and top-level property lists and reset in layout * so the originals aren't closed on error */ - orig_source_fapl = layout->storage.u.virt.source_fapl; - layout->storage.u.virt.source_fapl = -1; - orig_source_dapl = layout->storage.u.virt.source_dapl; - layout->storage.u.virt.source_dapl = -1; - orig_list = layout->storage.u.virt.list; - layout->storage.u.virt.list = NULL; + orig_source_fapl = virt->source_fapl; + virt->source_fapl = -1; + orig_source_dapl = virt->source_dapl; + virt->source_dapl = -1; + orig_list = virt->list; + virt->list = NULL; /* Copy entry list */ - if (layout->storage.u.virt.list_nused > 0) { + if (virt->list_nused > 0) { HDassert(orig_list); /* Allocate memory for the list */ - if (NULL == (layout->storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_calloc( - layout->storage.u.virt.list_nused * sizeof(H5O_storage_virtual_ent_t)))) + if (NULL == (virt->list = H5MM_calloc(virt->list_nused * sizeof(virt->list[0])))) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate memory for virtual dataset entry list") - layout->storage.u.virt.list_nalloc = layout->storage.u.virt.list_nused; + virt->list_nalloc = virt->list_nused; /* Copy the list entries, though set source_dset.dset and sub_dset to * NULL */ - for (i = 0; i < layout->storage.u.virt.list_nused; i++) { + for (i = 0; i < virt->list_nused; i++) { + H5O_storage_virtual_ent_t *ent = &virt->list[i]; + /* Copy virtual selection */ - if (NULL == (layout->storage.u.virt.list[i].source_dset.virtual_select = + if (NULL == (ent->source_dset.virtual_select = H5S_copy(orig_list[i].source_dset.virtual_select, FALSE, TRUE))) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") /* Copy original source names */ - if (NULL == (layout->storage.u.virt.list[i].source_file_name = - H5MM_strdup(orig_list[i].source_file_name))) + if (NULL == (ent->source_file_name = H5MM_strdup(orig_list[i].source_file_name))) HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source file name") - if (NULL == (layout->storage.u.virt.list[i].source_dset_name = - H5MM_strdup(orig_list[i].source_dset_name))) + if (NULL == (ent->source_dset_name = H5MM_strdup(orig_list[i].source_dset_name))) HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source dataset name") /* Copy source selection */ - if (NULL == (layout->storage.u.virt.list[i].source_select = - H5S_copy(orig_list[i].source_select, FALSE, TRUE))) + if (NULL == (ent->source_select = H5S_copy(orig_list[i].source_select, FALSE, TRUE))) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy source selection") /* Initialize clipped selections */ if (orig_list[i].unlim_dim_virtual < 0) { - layout->storage.u.virt.list[i].source_dset.clipped_source_select = - layout->storage.u.virt.list[i].source_select; - layout->storage.u.virt.list[i].source_dset.clipped_virtual_select = - layout->storage.u.virt.list[i].source_dset.virtual_select; + ent->source_dset.clipped_source_select = ent->source_select; + ent->source_dset.clipped_virtual_select = ent->source_dset.virtual_select; } /* end if */ /* Copy parsed names */ - if (H5D__virtual_copy_parsed_name(&layout->storage.u.virt.list[i].parsed_source_file_name, + if (H5D__virtual_copy_parsed_name(&ent->parsed_source_file_name, orig_list[i].parsed_source_file_name) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy parsed source file name") - layout->storage.u.virt.list[i].psfn_static_strlen = orig_list[i].psfn_static_strlen; - layout->storage.u.virt.list[i].psfn_nsubs = orig_list[i].psfn_nsubs; - if (H5D__virtual_copy_parsed_name(&layout->storage.u.virt.list[i].parsed_source_dset_name, + ent->psfn_static_strlen = orig_list[i].psfn_static_strlen; + ent->psfn_nsubs = orig_list[i].psfn_nsubs; + if (H5D__virtual_copy_parsed_name(&ent->parsed_source_dset_name, orig_list[i].parsed_source_dset_name) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy parsed source dataset name") - layout->storage.u.virt.list[i].psdn_static_strlen = orig_list[i].psdn_static_strlen; - layout->storage.u.virt.list[i].psdn_nsubs = orig_list[i].psdn_nsubs; + ent->psdn_static_strlen = orig_list[i].psdn_static_strlen; + ent->psdn_nsubs = orig_list[i].psdn_nsubs; /* Copy source names in source dset or add reference as appropriate */ if (orig_list[i].source_dset.file_name) { if (orig_list[i].source_dset.file_name == orig_list[i].source_file_name) - layout->storage.u.virt.list[i].source_dset.file_name = - layout->storage.u.virt.list[i].source_file_name; + ent->source_dset.file_name = ent->source_file_name; else if (orig_list[i].parsed_source_file_name && (orig_list[i].source_dset.file_name != orig_list[i].parsed_source_file_name->name_segment)) { - HDassert(layout->storage.u.virt.list[i].parsed_source_file_name); - HDassert(layout->storage.u.virt.list[i].parsed_source_file_name->name_segment); - layout->storage.u.virt.list[i].source_dset.file_name = - layout->storage.u.virt.list[i].parsed_source_file_name->name_segment; + HDassert(ent->parsed_source_file_name); + HDassert(ent->parsed_source_file_name->name_segment); + ent->source_dset.file_name = ent->parsed_source_file_name->name_segment; } /* end if */ - else if (NULL == (layout->storage.u.virt.list[i].source_dset.file_name = - H5MM_strdup(orig_list[i].source_dset.file_name))) + else if (NULL == + (ent->source_dset.file_name = H5MM_strdup(orig_list[i].source_dset.file_name))) HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source file name") } /* end if */ if (orig_list[i].source_dset.dset_name) { if (orig_list[i].source_dset.dset_name == orig_list[i].source_dset_name) - layout->storage.u.virt.list[i].source_dset.dset_name = - layout->storage.u.virt.list[i].source_dset_name; + ent->source_dset.dset_name = ent->source_dset_name; else if (orig_list[i].parsed_source_dset_name && (orig_list[i].source_dset.dset_name != orig_list[i].parsed_source_dset_name->name_segment)) { - HDassert(layout->storage.u.virt.list[i].parsed_source_dset_name); - HDassert(layout->storage.u.virt.list[i].parsed_source_dset_name->name_segment); - layout->storage.u.virt.list[i].source_dset.dset_name = - layout->storage.u.virt.list[i].parsed_source_dset_name->name_segment; + HDassert(ent->parsed_source_dset_name); + HDassert(ent->parsed_source_dset_name->name_segment); + ent->source_dset.dset_name = ent->parsed_source_dset_name->name_segment; } /* end if */ - else if (NULL == (layout->storage.u.virt.list[i].source_dset.dset_name = - H5MM_strdup(orig_list[i].source_dset.dset_name))) + else if (NULL == + (ent->source_dset.dset_name = H5MM_strdup(orig_list[i].source_dset.dset_name))) HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source dataset name") } /* end if */ /* Copy other fields in entry */ - layout->storage.u.virt.list[i].unlim_dim_source = orig_list[i].unlim_dim_source; - layout->storage.u.virt.list[i].unlim_dim_virtual = orig_list[i].unlim_dim_virtual; - layout->storage.u.virt.list[i].unlim_extent_source = orig_list[i].unlim_extent_source; - layout->storage.u.virt.list[i].unlim_extent_virtual = orig_list[i].unlim_extent_virtual; - layout->storage.u.virt.list[i].clip_size_source = orig_list[i].clip_size_source; - layout->storage.u.virt.list[i].clip_size_virtual = orig_list[i].clip_size_virtual; - layout->storage.u.virt.list[i].source_space_status = orig_list[i].source_space_status; - layout->storage.u.virt.list[i].virtual_space_status = orig_list[i].virtual_space_status; + ent->unlim_dim_source = orig_list[i].unlim_dim_source; + ent->unlim_dim_virtual = orig_list[i].unlim_dim_virtual; + ent->unlim_extent_source = orig_list[i].unlim_extent_source; + ent->unlim_extent_virtual = orig_list[i].unlim_extent_virtual; + ent->clip_size_source = orig_list[i].clip_size_source; + ent->clip_size_virtual = orig_list[i].clip_size_virtual; + ent->source_space_status = orig_list[i].source_space_status; + ent->virtual_space_status = orig_list[i].virtual_space_status; } /* end for */ } /* end if */ else { /* Zero out other fields related to list, just to be sure */ - layout->storage.u.virt.list = NULL; - layout->storage.u.virt.list_nalloc = 0; + virt->list = NULL; + virt->list_nalloc = 0; } /* end else */ /* Copy property lists */ if (orig_source_fapl >= 0) { if (NULL == (plist = (H5P_genplist_t *)H5I_object_verify(orig_source_fapl, H5I_GENPROP_LST))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") - if ((layout->storage.u.virt.source_fapl = H5P_copy_plist(plist, FALSE)) < 0) + if ((virt->source_fapl = H5P_copy_plist(plist, FALSE)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy fapl") } /* end if */ if (orig_source_dapl >= 0) { if (NULL == (plist = (H5P_genplist_t *)H5I_object_verify(orig_source_dapl, H5I_GENPROP_LST))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") - if ((layout->storage.u.virt.source_dapl = H5P_copy_plist(plist, FALSE)) < 0) + if ((virt->source_dapl = H5P_copy_plist(plist, FALSE)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy dapl") } /* end if */ /* New layout is not fully initialized */ - layout->storage.u.virt.init = FALSE; + virt->init = FALSE; done: /* Release allocated resources on failure */ @@ -720,8 +709,9 @@ done: herr_t H5D__virtual_reset_layout(H5O_layout_t *layout) { - size_t i, j; - herr_t ret_value = SUCCEED; + size_t i, j; + H5O_storage_virtual_t *virt = &layout->storage.u.virt; + herr_t ret_value = SUCCEED; FUNC_ENTER_PACKAGE @@ -731,56 +721,54 @@ H5D__virtual_reset_layout(H5O_layout_t *layout) /* Free the list entries. Note we always attempt to free everything even in * the case of a failure. Because of this, and because we free the list * afterwards, we do not need to zero out the memory in the list. */ - for (i = 0; i < layout->storage.u.virt.list_nused; i++) { + for (i = 0; i < virt->list_nused; i++) { + H5O_storage_virtual_ent_t *ent = &virt->list[i]; /* Free source_dset */ - if (H5D__virtual_reset_source_dset(&layout->storage.u.virt.list[i], - &layout->storage.u.virt.list[i].source_dset) < 0) + if (H5D__virtual_reset_source_dset(ent, &ent->source_dset) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to reset source dataset") /* Free original source names */ - (void)H5MM_xfree(layout->storage.u.virt.list[i].source_file_name); - (void)H5MM_xfree(layout->storage.u.virt.list[i].source_dset_name); + (void)H5MM_xfree(ent->source_file_name); + (void)H5MM_xfree(ent->source_dset_name); /* Free sub_dset */ - for (j = 0; j < layout->storage.u.virt.list[i].sub_dset_nalloc; j++) - if (H5D__virtual_reset_source_dset(&layout->storage.u.virt.list[i], - &layout->storage.u.virt.list[i].sub_dset[j]) < 0) + for (j = 0; j < ent->sub_dset_nalloc; j++) + if (H5D__virtual_reset_source_dset(ent, &ent->sub_dset[j]) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to reset source dataset") - layout->storage.u.virt.list[i].sub_dset = - (H5O_storage_virtual_srcdset_t *)H5MM_xfree(layout->storage.u.virt.list[i].sub_dset); + ent->sub_dset = H5MM_xfree(ent->sub_dset); /* Free source_select */ - if (layout->storage.u.virt.list[i].source_select) - if (H5S_close(layout->storage.u.virt.list[i].source_select) < 0) + if (ent->source_select) + if (H5S_close(ent->source_select) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release source selection") /* Free parsed_source_file_name */ - H5D_virtual_free_parsed_name(layout->storage.u.virt.list[i].parsed_source_file_name); + H5D_virtual_free_parsed_name(ent->parsed_source_file_name); /* Free parsed_source_dset_name */ - H5D_virtual_free_parsed_name(layout->storage.u.virt.list[i].parsed_source_dset_name); - } /* end for */ + H5D_virtual_free_parsed_name(ent->parsed_source_dset_name); + } /* Free the list */ - layout->storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_xfree(layout->storage.u.virt.list); - layout->storage.u.virt.list_nalloc = (size_t)0; - layout->storage.u.virt.list_nused = (size_t)0; - (void)HDmemset(layout->storage.u.virt.min_dims, 0, sizeof(layout->storage.u.virt.min_dims)); + virt->list = H5MM_xfree(virt->list); + virt->list_nalloc = (size_t)0; + virt->list_nused = (size_t)0; + (void)HDmemset(virt->min_dims, 0, sizeof(virt->min_dims)); /* Close access property lists */ - if (layout->storage.u.virt.source_fapl >= 0) { - if (H5I_dec_ref(layout->storage.u.virt.source_fapl) < 0) + if (virt->source_fapl >= 0) { + if (H5I_dec_ref(virt->source_fapl) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't close source fapl") - layout->storage.u.virt.source_fapl = -1; - } /* end if */ - if (layout->storage.u.virt.source_dapl >= 0) { - if (H5I_dec_ref(layout->storage.u.virt.source_dapl) < 0) + virt->source_fapl = -1; + } + if (virt->source_dapl >= 0) { + if (H5I_dec_ref(virt->source_dapl) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't close source dapl") - layout->storage.u.virt.source_dapl = -1; - } /* end if */ + virt->source_dapl = -1; + } /* The list is no longer initialized */ - layout->storage.u.virt.init = FALSE; + virt->init = FALSE; /* Note the lack of a done: label. This is because there are no HGOTO_ERROR * calls. If one is added, a done: label must also be added */ @@ -2883,8 +2871,9 @@ H5D__virtual_write_one(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, * extent in the unlimited dimension. -NAF */ /* Project intersection of file space and mapping virtual space onto * mapping source space */ - if (H5S_select_project_intersection(source_dset->virtual_select, source_dset->clipped_source_select, - file_space, &projected_src_space, TRUE) < 0) + if (H5S_select_project_intersection(source_dset->clipped_virtual_select, + source_dset->clipped_source_select, file_space, + &projected_src_space, TRUE) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto source space") diff --git a/src/H5Eprivate.h b/src/H5Eprivate.h index 8f89201..f3d3b6f 100644 --- a/src/H5Eprivate.h +++ b/src/H5Eprivate.h @@ -114,12 +114,18 @@ typedef struct H5E_t H5E_t; #define HSYS_DONE_ERROR(majorcode, minorcode, retcode, str) \ { \ int myerrno = errno; \ + /* Other projects may rely on the description format to get the errno and any changes should be \ + * considered as an API change \ + */ \ HDONE_ERROR(majorcode, minorcode, retcode, "%s, errno = %d, error message = '%s'", str, myerrno, \ HDstrerror(myerrno)); \ } #define HSYS_GOTO_ERROR(majorcode, minorcode, retcode, str) \ { \ int myerrno = errno; \ + /* Other projects may rely on the description format to get the errno and any changes should be \ + * considered as an API change \ + */ \ HGOTO_ERROR(majorcode, minorcode, retcode, "%s, errno = %d, error message = '%s'", str, myerrno, \ HDstrerror(myerrno)); \ } @@ -1542,7 +1542,7 @@ done: } /* end H5FDtruncate() */ /*------------------------------------------------------------------------- - * Function: H5FD_truncate + * Function: H5FD_truncate * * Purpose: Private version of H5FDtruncate() * @@ -1594,7 +1594,7 @@ H5FDlock(H5FD_t *file, hbool_t rw) /* Call private function */ if (H5FD_lock(file, rw) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file lock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "file lock request failed") done: FUNC_LEAVE_API(ret_value) @@ -1622,7 +1622,7 @@ H5FD_lock(H5FD_t *file, hbool_t rw) /* Dispatch to driver */ if (file->cls->lock && (file->cls->lock)(file, rw) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "driver lock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "driver lock request failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -1653,7 +1653,7 @@ H5FDunlock(H5FD_t *file) /* Call private function */ if (H5FD_unlock(file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file unlock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "file unlock request failed") done: FUNC_LEAVE_API(ret_value) @@ -1681,7 +1681,7 @@ H5FD_unlock(H5FD_t *file) /* Dispatch to driver */ if (file->cls->unlock && (file->cls->unlock)(file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "driver unlock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "driver unlock request failed") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDcore.c b/src/H5FDcore.c index 791deb2..61f46c7 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -36,6 +36,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_CORE_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* The skip list node type. Represents a region in the file. */ typedef struct H5FD_core_region_t { haddr_t start; /* Start address of the region */ @@ -56,7 +59,8 @@ typedef struct H5FD_core_t { hbool_t backing_store; /* write to file name on flush */ hbool_t write_tracking; /* Whether to track writes */ size_t bstore_page_size; /* backing store page size */ - int fd; /* backing store file descriptor */ + hbool_t ignore_disabled_file_locks; + int fd; /* backing store file descriptor */ /* Information for determining uniqueness of a file with a backing store */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -413,10 +417,20 @@ done: static herr_t H5FD__init_package(void) { - herr_t ret_value = SUCCEED; + char * lock_env_var = NULL; /* Environment variable pointer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if (H5FD_core_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize core VFD") @@ -794,6 +808,16 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr /* Save file image callbacks */ file->fi_callbacks = file_image_info.callbacks; + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + if (fd >= 0) { /* Retrieve information for determining uniqueness of file */ #ifdef H5_HAVE_WIN32_API @@ -1633,10 +1657,12 @@ H5FD__core_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if (HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") } /* end if */ @@ -1669,10 +1695,12 @@ H5FD__core_unlock(H5FD_t *_file) if (file->fd >= 0) if (HDflock(file->fd, LOCK_UN) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") } diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c index 9637946..4c2d251 100644 --- a/src/H5FDdirect.c +++ b/src/H5FDdirect.c @@ -37,6 +37,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_DIRECT_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* File operations */ #define OP_UNKNOWN 0 #define OP_READ 1 @@ -70,6 +73,7 @@ typedef struct H5FD_direct_t { haddr_t pos; /*current file I/O position */ int op; /*last operation */ H5FD_direct_fapl_t fa; /*file access properties */ + hbool_t ignore_disabled_file_locks; #ifndef H5_HAVE_WIN32_API /* * On most systems the combination of device and i-node number uniquely @@ -187,10 +191,20 @@ DESCRIPTION static herr_t H5FD__init_package(void) { - herr_t ret_value = SUCCEED; + char * lock_env_var = NULL; /* Environment variable pointer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if (H5FD_direct_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize direct VFD") @@ -370,14 +384,13 @@ static void * H5FD__direct_fapl_get(H5FD_t *_file) { H5FD_direct_t *file = (H5FD_direct_t *)_file; - void * ret_value; /* Return value */ + void *ret_value = NULL; /* Return value */ - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR /* Set return value */ ret_value = H5FD__direct_fapl_copy(&(file->fa)); -done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD__direct_fapl_get() */ @@ -433,7 +446,7 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad int o_flags; int fd = (-1); H5FD_direct_t * file = NULL; - H5FD_direct_fapl_t *fa; + const H5FD_direct_fapl_t *fa; #ifdef H5_HAVE_WIN32_API HFILE filehandle; struct _BY_HANDLE_FILE_INFORMATION fileinfo; @@ -482,7 +495,7 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad /* Get the driver specific information */ if (NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") - if (NULL == (fa = (H5FD_direct_fapl_t *)H5P_peek_driver_info(plist))) + if (NULL == (fa = (const H5FD_direct_fapl_t *)H5P_peek_driver_info(plist))) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info") file->fd = fd; @@ -502,6 +515,16 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad file->fa.fbsize = fa->fbsize; file->fa.cbsize = fa->cbsize; + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Try to decide if data alignment is required. The reason to check it here * is to handle correctly the case that the file is in a different file system * than the one where the program is running. @@ -522,7 +545,8 @@ H5FD__direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxad } else { file->fa.must_align = FALSE; - HDftruncate(file->fd, (HDoff_t)0); + if (-1 == HDftruncate(file->fd, (HDoff_t)0)) + HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, NULL, "unable to truncate file") } } else { @@ -769,7 +793,7 @@ H5FD__direct_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { const H5FD_direct_t *file = (const H5FD_direct_t *)_file; - FUNC_ENTER_STATIC + FUNC_ENTER_STATIC_NOERR FUNC_LEAVE_NOAPI(file->eof) } @@ -1293,17 +1317,28 @@ done: static herr_t H5FD__direct_lock(H5FD_t *_file, hbool_t rw) { - H5FD_direct_t *file = (H5FD_direct_t *)_file; /* VFD file struct */ - const int lock = rw ? LOCK_EX : LOCK_SH; - herr_t ret_value = SUCCEED; /* Return value */ + H5FD_direct_t *file = (H5FD_direct_t *)_file; /* VFD file struct */ + int lock_flags; /* file locking flags */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC HDassert(file); - /* Place the lock with non-blocking */ - if (HDflock(file->fd, lock | LOCK_NB) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock file") + /* Set exclusive or shared lock based on rw status */ + lock_flags = rw ? LOCK_EX : LOCK_SH; + + /* Place a non-blocking lock on the file */ + if (HDflock(file->fd, lock_flags | LOCK_NB) < 0) { + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } + else + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1330,8 +1365,16 @@ H5FD__direct_unlock(H5FD_t *_file) HDassert(file); - if (HDflock(file->fd, LOCK_UN) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock (unlock) file") + if (HDflock(file->fd, LOCK_UN) < 0) { + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } + else + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDdirect.h b/src/H5FDdirect.h index 2d88a69..ea2b3f2 100644 --- a/src/H5FDdirect.h +++ b/src/H5FDdirect.h @@ -23,7 +23,7 @@ #ifdef H5_HAVE_DIRECT #define H5FD_DIRECT (H5FD_direct_init()) #else -#define H5FD_DIRECT (-1) +#define H5FD_DIRECT (H5I_INVALID_HID) #endif /* H5_HAVE_DIRECT */ #ifdef H5_HAVE_DIRECT diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c index cf89f26..a0639a7 100644 --- a/src/H5FDfamily.c +++ b/src/H5FDfamily.c @@ -1305,9 +1305,9 @@ H5FD__family_lock(H5FD_t *_file, hbool_t rw) for (v = 0; v < u; v++) { if (H5FD_unlock(file->memb[v]) < 0) /* Push error, but keep going */ - HDONE_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + HDONE_ERROR(H5E_IO, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock member files") } /* end for */ - HGOTO_ERROR(H5E_IO, H5E_CANTLOCK, FAIL, "unable to lock member files") + HGOTO_ERROR(H5E_IO, H5E_CANTLOCKFILE, FAIL, "unable to lock member files") } /* end if */ done: @@ -1338,7 +1338,7 @@ H5FD__family_unlock(H5FD_t *_file) for (u = 0; u < file->nmembs; u++) if (file->memb[u]) if (H5FD_unlock(file->memb[u]) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock member files") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c index c3bfd12..286a8e6 100644 --- a/src/H5FDhdfs.c +++ b/src/H5FDhdfs.c @@ -373,7 +373,7 @@ done: hid_t H5FD_hdfs_init(void) { - hid_t ret_value = H5I_INVALID_HID; /* Return value */ + hid_t ret_value = H5I_INVALID_HID; #if HDFS_STATS unsigned int bin_i; #endif @@ -690,7 +690,7 @@ H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") /* Copy the hdfs fapl data out */ - HDmemcpy(fa_out, fa, sizeof(H5FD_hdfs_fapl_t)); + H5MM_memcpy(fa_out, fa, sizeof(H5FD_hdfs_fapl_t)); done: FUNC_LEAVE_API(ret_value) @@ -725,7 +725,7 @@ H5FD__hdfs_fapl_get(H5FD_t *_file) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "memory allocation failed") /* Copy the fields of the structure */ - HDmemcpy(fa, &(file->fa), sizeof(H5FD_hdfs_fapl_t)); + H5MM_memcpy(fa, &(file->fa), sizeof(H5FD_hdfs_fapl_t)); ret_value = fa; @@ -763,7 +763,7 @@ H5FD__hdfs_fapl_copy(const void *_old_fa) if (new_fa == NULL) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "memory allocation failed") - HDmemcpy(new_fa, old_fa, sizeof(H5FD_hdfs_fapl_t)); + H5MM_memcpy(new_fa, old_fa, sizeof(H5FD_hdfs_fapl_t)); ret_value = new_fa; done: @@ -920,7 +920,7 @@ H5FD__hdfs_open(const char *path, unsigned flags, hid_t fapl_id, haddr_t maxaddr if (file == NULL) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate file struct") file->hdfs_handle = handle; - HDmemcpy(&(file->fa), &fa, sizeof(H5FD_hdfs_fapl_t)); + H5MM_memcpy(&(file->fa), &fa, sizeof(H5FD_hdfs_fapl_t)); #if HDFS_STATS if (FAIL == hdfs__reset_stats(file)) diff --git a/src/H5FDhdfs.h b/src/H5FDhdfs.h index 5fbc54b..9e46954 100644 --- a/src/H5FDhdfs.h +++ b/src/H5FDhdfs.h @@ -25,7 +25,7 @@ #ifdef H5_HAVE_LIBHDFS #define H5FD_HDFS (H5FD_hdfs_init()) #else /* H5_HAVE_LIBHDFS */ -#define H5FD_HDFS (-1) +#define H5FD_HDFS (H5I_INVALID_HID) #endif /* H5_HAVE_LIBHDFS */ /**************************************************************************** diff --git a/src/H5FDlog.c b/src/H5FDlog.c index 929cd9c..fc2cb12 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -39,6 +39,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_LOG_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* Driver-specific file access properties */ typedef struct H5FD_log_fapl_t { char * logfile; /* Allocated log file name */ @@ -68,12 +71,13 @@ static const char *flavors[] = { * occurs), and `op' will be set to H5F_OP_UNKNOWN. */ typedef struct H5FD_log_t { - H5FD_t pub; /* public stuff, must be first */ - int fd; /* the unix file */ - haddr_t eoa; /* end of allocated region */ - haddr_t eof; /* end of file; current file size */ - haddr_t pos; /* current file I/O position */ - H5FD_file_op_t op; /* last operation */ + H5FD_t pub; /* public stuff, must be first */ + int fd; /* the unix file */ + haddr_t eoa; /* end of allocated region */ + haddr_t eof; /* end of file; current file size */ + haddr_t pos; /* current file I/O position */ + H5FD_file_op_t op; /* last operation */ + hbool_t ignore_disabled_file_locks; char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -224,10 +228,20 @@ H5FL_DEFINE_STATIC(H5FD_log_t); static herr_t H5FD__init_package(void) { - herr_t ret_value = SUCCEED; + char * lock_env_var = NULL; /* Environment variable pointer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if (H5FD_log_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize log VFD") @@ -611,6 +625,16 @@ H5FD__log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) } /* end if */ } /* end if */ + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Check for non-default FAPL */ if (H5P_FILE_ACCESS_DEFAULT != fapl_id) { /* This step is for h5repart tool only. If user wants to change file driver from @@ -1699,13 +1723,15 @@ H5FD__log_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if (HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1733,13 +1759,15 @@ H5FD__log_unlock(H5FD_t *_file) HDassert(file); if (HDflock(file->fd, LOCK_UN) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDmirror.c b/src/H5FDmirror.c index 3895b42..7ea9609 100644 --- a/src/H5FDmirror.c +++ b/src/H5FDmirror.c @@ -242,7 +242,7 @@ H5FD_mirror_init(void) { hid_t ret_value = H5I_INVALID_HID; - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI(H5I_INVALID_HID) LOG_OP_CALL(FUNC); @@ -298,7 +298,7 @@ H5FD__mirror_xmit_decode_uint16(uint16_t *out, const unsigned char *_buf) HDassert(_buf && out); - HDmemcpy(&n, _buf, sizeof(n)); + H5MM_memcpy(&n, _buf, sizeof(n)); *out = (uint16_t)HDntohs(n); return 2; /* number of bytes eaten */ @@ -326,7 +326,7 @@ H5FD__mirror_xmit_decode_uint32(uint32_t *out, const unsigned char *_buf) HDassert(_buf && out); - HDmemcpy(&n, _buf, sizeof(n)); + H5MM_memcpy(&n, _buf, sizeof(n)); *out = (uint32_t)HDntohl(n); return 4; /* number of bytes eaten */ @@ -385,7 +385,7 @@ H5FD__mirror_xmit_decode_uint64(uint64_t *out, const unsigned char *_buf) HDassert(_buf && out); - HDmemcpy(&n, _buf, sizeof(n)); + H5MM_memcpy(&n, _buf, sizeof(n)); if (TRUE == is_host_little_endian()) *out = BSWAP_64(n); else @@ -412,7 +412,7 @@ H5FD__mirror_xmit_decode_uint8(uint8_t *out, const unsigned char *_buf) HDassert(_buf && out); - HDmemcpy(out, _buf, sizeof(uint8_t)); + H5MM_memcpy(out, _buf, sizeof(uint8_t)); return 1; /* number of bytes eaten */ } /* end H5FD__mirror_xmit_decode_uint8() */ @@ -439,7 +439,7 @@ H5FD__mirror_xmit_encode_uint16(unsigned char *_dest, uint16_t v) HDassert(_dest); n = (uint16_t)HDhtons(v); - HDmemcpy(_dest, &n, sizeof(n)); + H5MM_memcpy(_dest, &n, sizeof(n)); return 2; } /* end H5FD__mirror_xmit_encode_uint16() */ @@ -466,7 +466,7 @@ H5FD__mirror_xmit_encode_uint32(unsigned char *_dest, uint32_t v) HDassert(_dest); n = (uint32_t)HDhtonl(v); - HDmemcpy(_dest, &n, sizeof(n)); + H5MM_memcpy(_dest, &n, sizeof(n)); return 4; } /* end H5FD__mirror_xmit_encode_uint32() */ @@ -494,7 +494,7 @@ H5FD__mirror_xmit_encode_uint64(unsigned char *_dest, uint64_t v) if (TRUE == is_host_little_endian()) n = BSWAP_64(v); - HDmemcpy(_dest, &n, sizeof(n)); + H5MM_memcpy(_dest, &n, sizeof(n)); return 8; } /* H5FD__mirror_xmit_encode_uint64() */ @@ -519,7 +519,7 @@ H5FD__mirror_xmit_encode_uint8(unsigned char *dest, uint8_t v) HDassert(dest); - HDmemcpy(dest, &v, sizeof(v)); + H5MM_memcpy(dest, &v, sizeof(v)); return 1; } /* end H5FD__mirror_xmit_encode_uint8() */ @@ -1188,7 +1188,7 @@ H5FD__mirror_fapl_get(H5FD_t *_file) if (NULL == fa) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "calloc failed"); - HDmemcpy(fa, &(file->fa), sizeof(H5FD_mirror_fapl_t)); + H5MM_memcpy(fa, &(file->fa), sizeof(H5FD_mirror_fapl_t)); ret_value = fa; @@ -1224,7 +1224,7 @@ H5FD__mirror_fapl_copy(const void *_old_fa) if (new_fa == NULL) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "memory allocation failed"); - HDmemcpy(new_fa, old_fa, sizeof(H5FD_mirror_fapl_t)); + H5MM_memcpy(new_fa, old_fa, sizeof(H5FD_mirror_fapl_t)); ret_value = new_fa; done: @@ -1298,7 +1298,7 @@ H5Pget_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa_out) HDassert(fa->magic == H5FD_MIRROR_FAPL_MAGIC); /* sanity check */ - HDmemcpy(fa_out, fa, sizeof(H5FD_mirror_fapl_t)); + H5MM_memcpy(fa_out, fa, sizeof(H5FD_mirror_fapl_t)); done: FUNC_LEAVE_API(ret_value); diff --git a/src/H5FDmpio.c b/src/H5FDmpio.c index 6042776..c16e01e 100644 --- a/src/H5FDmpio.c +++ b/src/H5FDmpio.c @@ -1746,33 +1746,4 @@ H5FD__mpio_communicator(const H5FD_t *_file) FUNC_LEAVE_NOAPI(file->comm) } /* end H5FD__mpio_communicator() */ -/*------------------------------------------------------------------------- - * Function: H5FD__mpio_get_info - * - * Purpose: Returns the file info of MPIO file driver. - * - * Returns: Non-negative if succeed or negative if fails. - * - * Programmer: John Mainzer - * April 4, 2017 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5FD__mpio_get_info(H5FD_t *_file, void **mpi_info) -{ - H5FD_mpio_t *file = (H5FD_mpio_t *)_file; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_STATIC - - if (!mpi_info) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "mpi info not valid") - - *mpi_info = &(file->info); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5FD__mpio_get_info() */ - #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5FDmpio.h b/src/H5FDmpio.h index c9280c7..0773d9e 100644 --- a/src/H5FDmpio.h +++ b/src/H5FDmpio.h @@ -25,7 +25,7 @@ #ifdef H5_HAVE_PARALLEL #define H5FD_MPIO (H5FD_mpio_init()) #else -#define H5FD_MPIO (-1) +#define H5FD_MPIO (H5I_INVALID_HID) #endif /* H5_HAVE_PARALLEL */ #ifdef H5_HAVE_PARALLEL diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index 8f9257d..d60ddad 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -1860,7 +1860,7 @@ H5FD_multi_lock(H5FD_t *_file, hbool_t rw) } /* end if */ if (nerrors) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "error locking member files", -1) return 0; + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTLOCKFILE, "error locking member files", -1) return 0; } /* H5FD_multi_lock() */ @@ -1897,7 +1897,7 @@ H5FD_multi_unlock(H5FD_t *_file) END_MEMBERS; if (nerrors) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "error unlocking member files", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTUNLOCKFILE, "error unlocking member files", -1) return 0; } /* H5FD_multi_unlock() */ diff --git a/src/H5FDros3.c b/src/H5FDros3.c index fa41b6f..a557f1d 100644 --- a/src/H5FDros3.c +++ b/src/H5FDros3.c @@ -320,7 +320,7 @@ H5FD_ros3_init(void) unsigned int bin_i; #endif - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI(H5I_INVALID_HID) #if ROS3_DEBUG HDfprintf(stdout, "H5FD_ros3_init() called.\n"); @@ -340,7 +340,6 @@ H5FD_ros3_init(void) } #endif - /* Set return value */ ret_value = H5FD_ROS3_g; done: @@ -501,7 +500,7 @@ H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") /* Copy the ros3 fapl data out */ - HDmemcpy(fa_out, fa, sizeof(H5FD_ros3_fapl_t)); + H5MM_memcpy(fa_out, fa, sizeof(H5FD_ros3_fapl_t)); done: FUNC_LEAVE_API(ret_value) @@ -536,7 +535,7 @@ H5FD__ros3_fapl_get(H5FD_t *_file) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* Copy the fields of the structure */ - HDmemcpy(fa, &(file->fa), sizeof(H5FD_ros3_fapl_t)); + H5MM_memcpy(fa, &(file->fa), sizeof(H5FD_ros3_fapl_t)); /* Set return value */ ret_value = fa; @@ -576,7 +575,7 @@ H5FD__ros3_fapl_copy(const void *_old_fa) if (new_fa == NULL) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed"); - HDmemcpy(new_fa, old_fa, sizeof(H5FD_ros3_fapl_t)); + H5MM_memcpy(new_fa, old_fa, sizeof(H5FD_ros3_fapl_t)); ret_value = new_fa; done: @@ -769,7 +768,7 @@ H5FD__ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct") file->s3r_handle = handle; - HDmemcpy(&(file->fa), &fa, sizeof(H5FD_ros3_fapl_t)); + H5MM_memcpy(&(file->fa), &fa, sizeof(H5FD_ros3_fapl_t)); #if ROS3_STATS if (FAIL == ros3_reset_stats(file)) diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c index 2ac27d0..ac58f67 100644 --- a/src/H5FDs3comms.c +++ b/src/H5FDs3comms.c @@ -144,7 +144,7 @@ curlwritecallback(char *ptr, size_t size, size_t nmemb, void *userdata) return written; if (size > 0) { - HDmemcpy(&(sds->data[sds->size]), ptr, product); + H5MM_memcpy(&(sds->data[sds->size]), ptr, product); sds->size += product; written = product; } @@ -263,12 +263,12 @@ H5FD_s3comms_hrb_node_set(hrb_node_t **L, const char *name, const char *value) namecpy = (char *)H5MM_malloc(sizeof(char) * (namelen + 1)); if (namecpy == NULL) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for name copy."); - HDmemcpy(namecpy, name, (namelen + 1)); + H5MM_memcpy(namecpy, name, (namelen + 1)); valuecpy = (char *)H5MM_malloc(sizeof(char) * (valuelen + 1)); if (valuecpy == NULL) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "cannot make space for value copy."); - HDmemcpy(valuecpy, value, (valuelen + 1)); + H5MM_memcpy(valuecpy, value, (valuelen + 1)); nvcat = (char *)H5MM_malloc(sizeof(char) * catwrite); if (nvcat == NULL) @@ -701,14 +701,14 @@ H5FD_s3comms_hrb_init_request(const char *_verb, const char *_resource, const ch res = (char *)H5MM_malloc(sizeof(char) * (reslen + 1)); if (res == NULL) HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, "no space for resource string"); - HDmemcpy(res, _resource, (reslen + 1)); + H5MM_memcpy(res, _resource, (reslen + 1)); } else { res = (char *)H5MM_malloc(sizeof(char) * (reslen + 2)); if (res == NULL) HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, "no space for resource string"); *res = '/'; - HDmemcpy((&res[1]), _resource, (reslen + 1)); + H5MM_memcpy((&res[1]), _resource, (reslen + 1)); HDassert((reslen + 1) == HDstrlen(res)); } /* end if (else resource string not starting with '/') */ @@ -910,7 +910,7 @@ H5FD_s3comms_s3r_getsize(s3r_t *handle) handle->httpverb = (char *)H5MM_malloc(sizeof(char) * 16); if (handle->httpverb == NULL) HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, "unable to allocate space for S3 request HTTP verb"); - HDmemcpy(handle->httpverb, "HEAD", 5); + H5MM_memcpy(handle->httpverb, "HEAD", 5); headerresponse = (char *)H5MM_malloc(sizeof(char) * CURL_MAX_HTTP_HEADER); if (headerresponse == NULL) @@ -1078,19 +1078,19 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const handle->region = (char *)H5MM_malloc(sizeof(char) * tmplen); if (handle->region == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle region copy."); - HDmemcpy(handle->region, region, tmplen); + H5MM_memcpy(handle->region, region, tmplen); tmplen = HDstrlen(id) + 1; handle->secret_id = (char *)H5MM_malloc(sizeof(char) * tmplen); if (handle->secret_id == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle ID copy."); - HDmemcpy(handle->secret_id, id, tmplen); + H5MM_memcpy(handle->secret_id, id, tmplen); tmplen = SHA256_DIGEST_LENGTH; handle->signing_key = (unsigned char *)H5MM_malloc(sizeof(unsigned char) * tmplen); if (handle->signing_key == NULL) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "could not malloc space for handle key copy."); - HDmemcpy(handle->signing_key, signing_key, tmplen); + H5MM_memcpy(handle->signing_key, signing_key, tmplen); } /* if authentication information provided */ /************************ @@ -1138,7 +1138,7 @@ H5FD_s3comms_s3r_open(const char *url, const char *region, const char *id, const *********************/ HDassert(handle->httpverb != NULL); - HDmemcpy(handle->httpverb, "GET", 4); + H5MM_memcpy(handle->httpverb, "GET", 4); ret_value = handle; @@ -1951,7 +1951,7 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha for (buffer_i = 0; buffer_i < 128; buffer_i++) buffer[buffer_i] = 0; - line_buffer = fgets(line_buffer, 128, file); + line_buffer = HDfgets(line_buffer, 128, file); if (line_buffer == NULL) /* reached end of file */ goto done; } while (HDstrncmp(line_buffer, profile_line, HDstrlen(profile_line))); @@ -1963,7 +1963,7 @@ H5FD__s3comms_load_aws_creds_from_file(FILE *file, const char *profile_name, cha buffer[buffer_i] = 0; /* collect a line from file */ - line_buffer = fgets(line_buffer, 128, file); + line_buffer = HDfgets(line_buffer, 128, file); if (line_buffer == NULL) goto done; /* end of file */ @@ -2062,9 +2062,9 @@ H5FD_s3comms_load_aws_profile(const char *profile_name, char *key_id_out, char * #endif #ifdef H5_HAVE_WIN32_API - ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("USERPROFILE")); + ret = HDsnprintf(awspath, 117, "%s/.aws/", HDgetenv("USERPROFILE")); #else - ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("HOME")); + ret = HDsnprintf(awspath, 117, "%s/.aws/", HDgetenv("HOME")); #endif if (ret < 0 || (size_t)ret >= 117) HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, "unable to format home-aws path") @@ -2150,7 +2150,7 @@ H5FD_s3comms_nlowercase(char *dest, const char *s, size_t len) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "destination cannot be null."); if (len > 0) { - HDmemcpy(dest, s, len); + H5MM_memcpy(dest, s, len); do { len--; dest[len] = (char)HDtolower((int)dest[len]); @@ -2686,14 +2686,14 @@ H5FD_s3comms_tostringtosign(char *dest, const char *req, const char *now, const if (ret <= 0 || ret >= 127) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "problem adding day and region to string") - HDmemcpy((dest + d), "AWS4-HMAC-SHA256\n", 17); + H5MM_memcpy((dest + d), "AWS4-HMAC-SHA256\n", 17); d = 17; - HDmemcpy((dest + d), now, HDstrlen(now)); + H5MM_memcpy((dest + d), now, HDstrlen(now)); d += HDstrlen(now); dest[d++] = '\n'; - HDmemcpy((dest + d), tmp, HDstrlen(tmp)); + H5MM_memcpy((dest + d), tmp, HDstrlen(tmp)); d += HDstrlen(tmp); dest[d++] = '\n'; @@ -2777,7 +2777,7 @@ H5FD_s3comms_trim(char *dest, char *s, size_t s_len, size_t *n_written) s_len++; /* write output into dest */ - HDmemcpy(dest, s, s_len); + H5MM_memcpy(dest, s, s_len); } } diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index 15de017..7789d39 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -38,6 +38,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_SEC2_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* The description of a file belonging to this driver. The 'eoa' and 'eof' * determine the amount of hdf5 address space in use and the high-water mark * of the file (the current size of the underlying filesystem file). The @@ -50,12 +53,13 @@ static hid_t H5FD_SEC2_g = 0; * occurs), and 'op' will be set to H5F_OP_UNKNOWN. */ typedef struct H5FD_sec2_t { - H5FD_t pub; /* public stuff, must be first */ - int fd; /* the filesystem file descriptor */ - haddr_t eoa; /* end of allocated region */ - haddr_t eof; /* end of file; current file size */ - haddr_t pos; /* current file I/O position */ - H5FD_file_op_t op; /* last operation */ + H5FD_t pub; /* public stuff, must be first */ + int fd; /* the filesystem file descriptor */ + haddr_t eoa; /* end of allocated region */ + haddr_t eof; /* end of file; current file size */ + haddr_t pos; /* current file I/O position */ + H5FD_file_op_t op; /* last operation */ + hbool_t ignore_disabled_file_locks; char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -185,10 +189,20 @@ H5FL_DEFINE_STATIC(H5FD_sec2_t); static herr_t H5FD__init_package(void) { - herr_t ret_value = SUCCEED; + char * lock_env_var = NULL; /* Environment variable pointer */ + herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if (H5FD_sec2_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize sec2 VFD") @@ -306,8 +320,9 @@ H5FD__sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr #ifdef H5_HAVE_WIN32_API struct _BY_HANDLE_FILE_INFORMATION fileinfo; #endif - h5_stat_t sb; - H5FD_t * ret_value = NULL; /* Return value */ + h5_stat_t sb; + H5P_genplist_t *plist; /* Property list pointer */ + H5FD_t * ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -367,17 +382,26 @@ H5FD__sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr file->inode = sb.st_ino; #endif /* H5_HAVE_WIN32_API */ + /* Get the FAPL */ + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") + + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Retain a copy of the name used to open the file, for possible error reporting */ HDstrncpy(file->filename, name, sizeof(file->filename)); file->filename[sizeof(file->filename) - 1] = '\0'; /* Check for non-default FAPL */ if (H5P_FILE_ACCESS_DEFAULT != fapl_id) { - H5P_genplist_t *plist; /* Property list pointer */ - - /* Get the FAPL */ - if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) - HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") /* This step is for h5repart tool only. If user wants to change file driver from * family to one that uses single files (sec2, etc.) while using h5repart, this @@ -968,13 +992,15 @@ H5FD__sec2_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if (HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1002,13 +1028,15 @@ H5FD__sec2_unlock(H5FD_t *_file) HDassert(file); if (HDflock(file->fd, LOCK_UN) < 0) { - if (ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)") + if (file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDsplitter.c b/src/H5FDsplitter.c index 6af5c74..6ecafe0 100644 --- a/src/H5FDsplitter.c +++ b/src/H5FDsplitter.c @@ -210,7 +210,7 @@ H5FD_splitter_init(void) { hid_t ret_value = H5I_INVALID_HID; - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_NOAPI(H5I_INVALID_HID) H5FD_SPLITTER_LOG_CALL(FUNC); @@ -585,7 +585,7 @@ H5FD__splitter_fapl_copy(const void *_old_fa) if (NULL == new_fa_ptr) HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate log file FAPL") - HDmemcpy(new_fa_ptr, old_fa_ptr, sizeof(H5FD_splitter_fapl_t)); + H5MM_memcpy(new_fa_ptr, old_fa_ptr, sizeof(H5FD_splitter_fapl_t)); HDstrncpy(new_fa_ptr->wo_path, old_fa_ptr->wo_path, H5FD_SPLITTER_PATH_MAX); HDstrncpy(new_fa_ptr->log_file_path, old_fa_ptr->log_file_path, H5FD_SPLITTER_PATH_MAX); @@ -1096,10 +1096,11 @@ H5FD__splitter_lock(H5FD_t *_file, hbool_t rw) /* Place the lock on each file */ if (H5FD_lock(file->rw_file, rw) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTLOCK, FAIL, "unable to lock R/W file") + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock R/W file") + if (file->wo_file != NULL) if (H5FD_lock(file->wo_file, rw) < 0) - H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTLOCK, FAIL, "unable to lock W/O file") + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock W/O file") done: FUNC_LEAVE_NOAPI(ret_value) @@ -1129,10 +1130,11 @@ H5FD__splitter_unlock(H5FD_t *_file) /* Remove the lock on each file */ if (H5FD_unlock(file->rw_file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCK, FAIL, "unable to unlock R/W file") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock R/W file") + if (file->wo_file != NULL) if (H5FD_unlock(file->wo_file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCK, FAIL, "unable to unlock W/O file") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock W/O file") done: FUNC_LEAVE_NOAPI(ret_value) @@ -1298,14 +1300,14 @@ H5FD__splitter_log_error(const H5FD_splitter_t *file, const char *atfunc, const char * s; size = HDstrlen(atfunc) + HDstrlen(msg) + 3; /* ':', ' ', '\n' */ - s = (char *)HDmalloc(sizeof(char) * (size + 1)); + s = (char *)H5MM_malloc(sizeof(char) * (size + 1)); if (NULL == s) ret_value = FAIL; else if (size < (size_t)HDsnprintf(s, size + 1, "%s: %s\n", atfunc, msg)) ret_value = FAIL; else if (size != HDfwrite(s, 1, size, file->logfp)) ret_value = FAIL; - HDfree(s); + H5MM_free(s); } FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index 45ea503..4650c39 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -52,6 +52,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_STDIO_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = -1; + /* The maximum number of bytes which can be written in a single I/O operation */ static size_t H5_STDIO_MAX_IO_BYTES_g = (size_t)-1; @@ -82,7 +85,8 @@ typedef struct H5FD_stdio_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ unsigned write_access; /* Flag to indicate the file was opened with write access */ - H5FD_stdio_file_op op; /* last operation */ + hbool_t ignore_disabled_file_locks; + H5FD_stdio_file_op op; /* last operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely * identify a file. Note that Cygwin, MinGW and other Windows POSIX @@ -230,9 +234,20 @@ static const H5FD_class_t H5FD_stdio_g = { hid_t H5FD_stdio_init(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ + /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); + /* Check the use disabled file locks environment variable */ + lock_env_var = getenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && !strcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = 1; /* Override: Ignore disabled locks */ + else if (lock_env_var && (!strcmp(lock_env_var, "TRUE") || !strcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = 0; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = -1; /* Environment variable not set, or not set correctly */ + if (H5I_VFL != H5Iget_type(H5FD_STDIO_g)) H5FD_STDIO_g = H5FDregister(&H5FD_stdio_g); @@ -397,6 +412,22 @@ H5FD_stdio_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr file->eof = (haddr_t)x; } + /* Check the file locking flags in the fapl */ + if (ignore_disabled_file_locks_s != -1) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + hbool_t unused; + + /* Use the value in the property list */ + if (H5Pget_file_locking(fapl_id, &unused, &file->ignore_disabled_file_locks) < 0) { + free(file); + fclose(f); + H5Epush_ret(func, H5E_ERR_CLS, H5E_FILE, H5E_CANTGET, + "unable to get use disabled file locks property", NULL); + } + } + /* Get the file descriptor (needed for truncate and some Windows information) */ #ifdef H5_HAVE_WIN32_API file->fd = _fileno(file->fp); @@ -1104,11 +1135,13 @@ H5FD_stdio_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if (flock(file->fd, lock_flags | LOCK_NB) < 0) { - if (ENOSYS == errno) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment " - "variable to override)", - -1) else H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file lock failed", -1) + if (file->ignore_disabled_file_locks && ENOSYS == errno) + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + else + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTLOCKFILE, "file lock failed", -1) } /* end if */ /* Flush the stream */ @@ -1154,12 +1187,13 @@ H5FD_stdio_unlock(H5FD_t *_file) /* Place a non-blocking lock on the file */ if (flock(file->fd, LOCK_UN) < 0) { - if (ENOSYS == errno) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, - "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING " - "environment variable to override)", - -1) else H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file unlock failed", - -1) + if (file->ignore_disabled_file_locks && ENOSYS == errno) + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + else + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTUNLOCKFILE, "file unlock failed", -1) } /* end if */ #endif /* H5_HAVE_FLOCK */ diff --git a/src/H5Fint.c b/src/H5Fint.c index d364a04..1d7a86f 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -80,6 +80,7 @@ static int H5F__get_objects_cb(void *obj_ptr, hid_t obj_id, void *key); static herr_t H5F__build_name(const char *prefix, const char *file_name, char **full_name /*out*/); static char * H5F__getenv_prefix_name(char **env_prefix /*in,out*/); static H5F_t *H5F__new(H5F_shared_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t *lf); +static herr_t H5F__check_if_using_file_locks(H5P_genplist_t *fapl, hbool_t *use_file_locking); static herr_t H5F__build_actual_name(const H5F_t *f, const H5P_genplist_t *fapl, const char *name, char ** /*out*/ actual_name); static herr_t H5F__flush_phase1(H5F_t *f); @@ -92,6 +93,12 @@ static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); /* Package initialization variable */ hbool_t H5_PKG_INIT_VAR = FALSE; +/* Based on the value of the HDF5_USE_FILE_LOCKING environment variable. + * TRUE/FALSE have obvious meanings. FAIL means the environment variable was + * not set, so the code should ignore it and use the fapl value instead. + */ +htri_t use_locks_env_g = FAIL; + /*****************************/ /* Library Private Variables */ /*****************************/ @@ -160,6 +167,10 @@ H5F__init_package(void) if (H5I_register_type(H5I_FILE_CLS) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to initialize interface") + /* Check the file locking environment variable */ + if (H5F__parse_file_lock_env_var(&use_locks_env_g) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to parse file locking environment variable") + done: FUNC_LEAVE_NOAPI(ret_value) } /* H5F__init_package() */ @@ -247,6 +258,38 @@ done: } /* end H5F__close_cb() */ /*------------------------------------------------------------------------- + * Function: H5F__parse_file_lock_env_var + * + * Purpose: Parses the HDF5_USE_FILE_LOCKING environment variable. + * + * NOTE: This is done in a separate function so we can call it from + * the test code. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__parse_file_lock_env_var(htri_t *use_locks) +{ + char *lock_env_var = NULL; /* Environment variable pointer */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check the file locking environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if (lock_env_var && (!HDstrcmp(lock_env_var, "FALSE") || !HDstrcmp(lock_env_var, "0"))) + *use_locks = FALSE; /* Override: Never use locks */ + else if (lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "BEST_EFFORT") || + !HDstrcmp(lock_env_var, "1"))) + *use_locks = TRUE; /* Override: Always use locks */ + else + *use_locks = FAIL; /* Environment variable not set, or not set correctly */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5F__parse_file_lock_env_var() */ + +/*------------------------------------------------------------------------- * Function: H5F__set_vol_conn * * Purpose: Set the VOL connector ID and info for a file. @@ -1573,6 +1616,51 @@ H5F__dest(H5F_t *f, hbool_t flush) } /* end H5F__dest() */ /*------------------------------------------------------------------------- + * Function: H5F__check_if_using_file_locks + * + * Purpose: Determines if this file will use file locks. + * + * There are three ways that file locking can be controlled: + * + * 1) The configure/cmake option that sets the H5_USE_FILE_LOCKING + * symbol (which is used as the default fapl value). + * + * 2) The H5Pset_file_locking() API call, which will override + * the configuration default. + * + * 3) The HDF5_USE_FILE_LOCKING environment variable, which overrides + * everything above. + * + * The main reason to disable file locking is to prevent errors on file + * systems where locking is not supported or has been disabled (as is + * often the case in parallel file systems). + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5F__check_if_using_file_locks(H5P_genplist_t *fapl, hbool_t *use_file_locking) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Make sure the out parameter has a value */ + *use_file_locking = TRUE; + + /* Check the fapl property */ + if (H5P_get(fapl, H5F_ACS_USE_FILE_LOCKING_NAME, use_file_locking) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get use file locking flag") + + /* Check the environment variable */ + if (use_locks_env_g != FAIL) + *use_file_locking = (use_locks_env_g == TRUE) ? TRUE : FALSE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__check_if_using_file_locks() */ + +/*------------------------------------------------------------------------- * Function: H5F_open * * Purpose: Opens (or creates) a file. This function understands the @@ -1661,11 +1749,10 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) hbool_t set_flag = FALSE; /*set the status_flags in the superblock */ hbool_t clear = FALSE; /*clear the status_flags */ hbool_t evict_on_close; /* evict on close value from plist */ - char * lock_env_var = NULL; /*env var pointer */ - hbool_t use_file_locking; /*read from env var */ - hbool_t ci_load = FALSE; /* whether MDC ci load requested */ - hbool_t ci_write = FALSE; /* whether MDC CI write requested */ - H5F_t * ret_value = NULL; /*actual return value */ + hbool_t use_file_locking = TRUE; /* Using file locks? */ + hbool_t ci_load = FALSE; /* whether MDC ci load requested */ + hbool_t ci_write = FALSE; /* whether MDC CI write requested */ + H5F_t * ret_value = NULL; /*actual return value */ FUNC_ENTER_NOAPI(NULL) @@ -1680,15 +1767,13 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if (NULL == (drvr = H5FD_get_class(fapl_id))) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to retrieve VFL class") - /* Check the environment variable that determines if we care - * about file locking. File locking should be used unless explicitly - * disabled. - */ - lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); - if (lock_env_var && !HDstrcmp(lock_env_var, "FALSE")) - use_file_locking = FALSE; - else - use_file_locking = TRUE; + /* Get the file access property list, for future queries */ + if (NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") + + /* Check if we are using file locking */ + if (H5F__check_if_using_file_locks(a_plist, &use_file_locking) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file locking flag") /* * Opening a file is a two step process. First we try to open the @@ -1771,8 +1856,8 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if (H5FD_lock(lf, (hbool_t)((flags & H5F_ACC_RDWR) ? TRUE : FALSE)) < 0) { /* Locking failed - Closing will remove the lock */ if (H5FD_close(lf) < 0) - HDONE_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to close low-level file info") - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to lock the file") + HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "unable to close low-level file info") + HGOTO_ERROR(H5E_FILE, H5E_CANTLOCKFILE, NULL, "unable to lock the file") } /* end if */ /* Create the 'top' file structure */ @@ -1804,9 +1889,14 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) shared = file->shared; lf = shared->lf; - /* Get the file access property list, for future queries */ - if (NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") + /* Set the file locking flag. If the file is already open, the file + * requested file locking flag must match that of the open file. + */ + if (shared->nrefs == 1) + file->shared->use_file_locking = use_file_locking; + else if (shared->nrefs > 1) + if (file->shared->use_file_locking != use_file_locking) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "file locking flag values don't match") /* Check if page buffering is enabled */ if (H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &page_buf_size) < 0) @@ -1952,7 +2042,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) /* Remove the file lock for SWMR_WRITE */ if (use_file_locking && (H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)) { if (H5FD_unlock(file->shared->lf) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to unlock the file") + HGOTO_ERROR(H5E_FILE, H5E_CANTUNLOCKFILE, NULL, "unable to unlock the file") } /* end if */ } /* end if */ else { /* H5F_ACC_RDONLY: check consistency of status_flags */ @@ -2070,6 +2160,11 @@ H5F__flush_phase2(H5F_t *f, hbool_t closing) /* Sanity check arguments */ HDassert(f); + /* Inform the metadata cache that we are about to flush */ + if (H5AC_prep_for_file_flush(f) < 0) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "prep for MDC flush failed") + /* Flush the entire metadata cache */ if (H5AC_flush(f) < 0) /* Push error, but keep going*/ @@ -2102,6 +2197,11 @@ H5F__flush_phase2(H5F_t *f, hbool_t closing) H5CX_set_mpi_file_flushing(FALSE); #endif /* H5_HAVE_PARALLEL */ + /* Inform the metadata cache that we are done with the flush */ + if (H5AC_secure_from_file_flush(f) < 0) + /* Push error, but keep going*/ + HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "secure from MDC flush failed") + /* Flush out the metadata accumulator */ if (H5F__accum_flush(f->shared) < 0) /* Push error, but keep going*/ diff --git a/src/H5Fmodule.h b/src/H5Fmodule.h index 953b4da..d57926f 100644 --- a/src/H5Fmodule.h +++ b/src/H5Fmodule.h @@ -29,4 +29,17 @@ #define H5_MY_PKG_ERR H5E_FILE #define H5_MY_PKG_INIT YES +/** + * \defgroup H5F H5F + * \brief File Interface + * \todo Describe concisely what the functions in this module are about. + * + * \defgroup MDC Metadata Cache + * \ingroup H5F + * \defgroup PH5F Parallel + * \ingroup H5F + * \defgroup SWMR Single Writer Multiple Readers + * \ingroup H5F + */ + #endif /* _H5Fmodule_H */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 9f4eae6..94096af 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -304,6 +304,7 @@ struct H5F_shared_t { struct H5G_t * root_grp; /* Open root group */ H5FO_t * open_objs; /* Open objects in file */ H5UC_t * grp_btree_shared; /* Ref-counted group B-tree node info */ + hbool_t use_file_locking; /* Whether or not to use file locking */ hbool_t closing; /* File is in the process of being closed */ /* Cached VOL connector ID & info */ @@ -392,6 +393,11 @@ H5FL_EXTERN(H5F_t); /* Declare a free list to manage the H5F_shared_t struct */ H5FL_EXTERN(H5F_shared_t); +/* Whether or not to use file locking (based on the environment variable) + * FAIL means ignore the environment variable. + */ +H5_DLLVAR htri_t use_locks_env_g; + /******************************/ /* Package Private Prototypes */ /******************************/ @@ -409,6 +415,7 @@ H5_DLL herr_t H5F__start_swmr_write(H5F_t *f); H5_DLL herr_t H5F__close(H5F_t *f); H5_DLL herr_t H5F__set_libver_bounds(H5F_t *f, H5F_libver_t low, H5F_libver_t high); H5_DLL herr_t H5F__get_cont_info(const H5F_t *f, H5VL_file_cont_info_t *info); +H5_DLL herr_t H5F__parse_file_lock_env_var(htri_t *use_locks); /* File mount related routines */ H5_DLL herr_t H5F__mount(H5G_loc_t *loc, const char *name, H5F_t *child, hid_t plist_id); @@ -471,6 +478,7 @@ H5_DLL herr_t H5F__check_cached_stab_test(hid_t file_id); H5_DLL herr_t H5F__get_maxaddr_test(hid_t file_id, haddr_t *maxaddr); H5_DLL herr_t H5F__get_sbe_addr_test(hid_t file_id, haddr_t *sbe_addr); H5_DLL htri_t H5F__same_file_test(hid_t file_id1, hid_t file_id2); +H5_DLL herr_t H5F__reparse_file_lock_variable_test(void); #endif /* H5F_TESTING */ #endif /* _H5Fpkg_H */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 9b93469..ba6d67f 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -381,6 +381,7 @@ typedef struct H5F_t H5F_t; #define H5F_SET_MIN_DSET_OHDR(F, V) ((F)->shared->crt_dset_min_ohdr_flag = (V)) #define H5F_VOL_CLS(F) ((F)->shared->vol_cls) #define H5F_VOL_OBJ(F) ((F)->vol_obj) +#define H5F_USE_FILE_LOCKING(F) ((F)->shared->use_file_locking) #else /* H5F_MODULE */ #define H5F_LOW_BOUND(F) (H5F_get_low_bound(F)) #define H5F_HIGH_BOUND(F) (H5F_get_high_bound(F)) @@ -443,6 +444,7 @@ typedef struct H5F_t H5F_t; #define H5F_SET_MIN_DSET_OHDR(F, V) (H5F_set_min_dset_ohdr((F), (V))) #define H5F_VOL_CLS(F) (H5F_get_vol_cls(F)) #define H5F_VOL_OBJ(F) (H5F_get_vol_obj(F)) +#define H5F_USE_FILE_LOCKING(F) (H5F_get_use_file_locking(F)) #endif /* H5F_MODULE */ /* Macros to encode/decode offset/length's for storing in the file */ @@ -608,6 +610,11 @@ typedef struct H5F_t H5F_t; "page_buffer_min_meta_perc" /* the min metadata percentage for the page buffer cache */ #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME \ "page_buffer_min_raw_perc" /* the min raw data percentage for the page buffer cache */ +#define H5F_ACS_USE_FILE_LOCKING_NAME \ + "use_file_locking" /* whether or not we use file locks for SWMR control and to prevent multiple writers \ + */ +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME \ + "ignore_disabled_file_locks" /* whether or not we ignore "locks disabled" errors */ #ifdef H5_HAVE_PARALLEL #define H5F_ACS_MPI_PARAMS_COMM_NAME "mpi_params_comm" /* the MPI communicator */ #define H5F_ACS_MPI_PARAMS_INFO_NAME "mpi_params_info" /* the MPI info struct */ @@ -853,6 +860,7 @@ H5_DLL hbool_t H5F_get_min_dset_ohdr(const H5F_t *f); H5_DLL herr_t H5F_set_min_dset_ohdr(H5F_t *f, hbool_t minimize); H5_DLL const H5VL_class_t *H5F_get_vol_cls(const H5F_t *f); H5_DLL H5VL_object_t *H5F_get_vol_obj(const H5F_t *f); +H5_DLL hbool_t H5F_get_file_locking(const H5F_t *f); /* Functions than retrieve values set/cached from the superblock/FCPL */ H5_DLL haddr_t H5F_get_base_addr(const H5F_t *f); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index c6a5b76..fbee9c5 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -47,24 +47,20 @@ * We're assuming that these constants are used rather early in the hdf5 * session. */ -#define H5F_ACC_RDONLY (H5CHECK H5OPEN 0x0000u) /*absence of rdwr => rd-only */ -#define H5F_ACC_RDWR (H5CHECK H5OPEN 0x0001u) /*open for read and write */ -#define H5F_ACC_TRUNC (H5CHECK H5OPEN 0x0002u) /*overwrite existing files */ -#define H5F_ACC_EXCL (H5CHECK H5OPEN 0x0004u) /*fail if file already exists*/ +#define H5F_ACC_RDONLY (H5CHECK H5OPEN 0x0000u) /**< absence of rdwr => rd-only */ +#define H5F_ACC_RDWR (H5CHECK H5OPEN 0x0001u) /**< open for read and write */ +#define H5F_ACC_TRUNC (H5CHECK H5OPEN 0x0002u) /**< overwrite existing files */ +#define H5F_ACC_EXCL (H5CHECK H5OPEN 0x0004u) /**< fail if file already exists*/ /* NOTE: 0x0008u was H5F_ACC_DEBUG, now deprecated */ -#define H5F_ACC_CREAT (H5CHECK H5OPEN 0x0010u) /*create non-existing files */ +#define H5F_ACC_CREAT (H5CHECK H5OPEN 0x0010u) /**< create non-existing files */ #define H5F_ACC_SWMR_WRITE \ - (H5CHECK 0x0020u) /*indicate that this file is \ - * open for writing in a \ - * single-writer/multi-reader (SWMR) \ - * scenario. Note that the \ - * process(es) opening the file \ - * for reading must open the file \ - * with RDONLY access, and use \ - * the special "SWMR_READ" access \ - * flag. */ + (H5CHECK 0x0020u) /**< indicate that this file is open for writing in a \ + single-writer/multi-reader (SWMR) scenario. \ + Note that the process(es) opening the file for reading must \ + open the file with RDONLY access, and use the special "SWMR_READ" \ + access flag. */ #define H5F_ACC_SWMR_READ \ - (H5CHECK 0x0040u) /*indicate that this file is \ + (H5CHECK 0x0040u) /**< indicate that this file is \ * open for reading in a \ * single-writer/multi-reader (SWMR) \ * scenario. Note that the \ @@ -73,19 +69,23 @@ * open the file with the RDONLY \ * flag. */ -/* Value passed to H5Pset_elink_acc_flags to cause flags to be taken from the - * parent file. */ -#define H5F_ACC_DEFAULT (H5CHECK H5OPEN 0xffffu) /*ignore setting on lapl */ +/** + * Default property list identifier + * + * \internal Value passed to H5Pset_elink_acc_flags to cause flags to be taken from the parent file. + * \internal ignore setting on lapl + */ +#define H5F_ACC_DEFAULT (H5CHECK H5OPEN 0xffffu) /* Flags for H5Fget_obj_count() & H5Fget_obj_ids() calls */ -#define H5F_OBJ_FILE (0x0001u) /* File objects */ -#define H5F_OBJ_DATASET (0x0002u) /* Dataset objects */ -#define H5F_OBJ_GROUP (0x0004u) /* Group objects */ -#define H5F_OBJ_DATATYPE (0x0008u) /* Named datatype objects */ -#define H5F_OBJ_ATTR (0x0010u) /* Attribute objects */ +#define H5F_OBJ_FILE (0x0001u) /**< File objects */ +#define H5F_OBJ_DATASET (0x0002u) /**< Dataset objects */ +#define H5F_OBJ_GROUP (0x0004u) /**< Group objects */ +#define H5F_OBJ_DATATYPE (0x0008u) /**< Named datatype objects */ +#define H5F_OBJ_ATTR (0x0010u) /**< Attribute objects */ #define H5F_OBJ_ALL (H5F_OBJ_FILE | H5F_OBJ_DATASET | H5F_OBJ_GROUP | H5F_OBJ_DATATYPE | H5F_OBJ_ATTR) -#define H5F_OBJ_LOCAL (0x0020u) /* Restrict search to objects opened through current file ID */ -/* (as opposed to objects opened through any file ID accessing this file) */ +#define H5F_OBJ_LOCAL (0x0020u) /**< Restrict search to objects opened through current file ID + (as opposed to objects opened through any file ID accessing this file) */ #define H5F_FAMILY_DEFAULT (hsize_t)0 @@ -99,127 +99,146 @@ #define H5F_MPIO_DEBUG_KEY "H5F_mpio_debug_key" #endif /* H5_HAVE_PARALLEL */ -/* The difference between a single file and a set of mounted files */ +/** + * The difference between a single file and a set of mounted files + */ typedef enum H5F_scope_t { - H5F_SCOPE_LOCAL = 0, /*specified file handle only */ - H5F_SCOPE_GLOBAL = 1 /*entire virtual file */ + H5F_SCOPE_LOCAL = 0, /**< specified file handle only */ + H5F_SCOPE_GLOBAL = 1 /**< entire virtual file */ } H5F_scope_t; -/* Unlimited file size for H5Pset_external() */ +/** + * Unlimited file size for H5Pset_external() + */ #define H5F_UNLIMITED ((hsize_t)(-1L)) -/* How does file close behave? - * H5F_CLOSE_DEFAULT - Use the degree pre-defined by underlining VFL - * H5F_CLOSE_WEAK - file closes only after all opened objects are closed - * H5F_CLOSE_SEMI - if no opened objects, file is close; otherwise, file - close fails - * H5F_CLOSE_STRONG - if there are opened objects, close them first, then - close file +/** + * How does file close behave? */ typedef enum H5F_close_degree_t { - H5F_CLOSE_DEFAULT = 0, - H5F_CLOSE_WEAK = 1, - H5F_CLOSE_SEMI = 2, - H5F_CLOSE_STRONG = 3 + H5F_CLOSE_DEFAULT = 0, /**< Use the degree pre-defined by underlining VFL */ + H5F_CLOSE_WEAK = 1, /**< File closes only after all opened objects are closed */ + H5F_CLOSE_SEMI = 2, /**< If no opened objects, file is close; otherwise, file close fails */ + H5F_CLOSE_STRONG = 3 /**< If there are opened objects, close them first, then close file */ } H5F_close_degree_t; -/* Current "global" information about file */ +/** + * Current "global" information about file + */ +//! [H5F_info2_t_snip] typedef struct H5F_info2_t { struct { - unsigned version; /* Superblock version # */ - hsize_t super_size; /* Superblock size */ - hsize_t super_ext_size; /* Superblock extension size */ + unsigned version; /**< Superblock version # */ + hsize_t super_size; /**< Superblock size */ + hsize_t super_ext_size; /**< Superblock extension size */ } super; struct { - unsigned version; /* Version # of file free space management */ - hsize_t meta_size; /* Free space manager metadata size */ - hsize_t tot_space; /* Amount of free space in the file */ + unsigned version; /**< Version # of file free space management */ + hsize_t meta_size; /**< Free space manager metadata size */ + hsize_t tot_space; /**< Amount of free space in the file */ } free; struct { - unsigned version; /* Version # of shared object header info */ - hsize_t hdr_size; /* Shared object header message header size */ - H5_ih_info_t msgs_info; /* Shared object header message index & heap size */ + unsigned version; /**< Version # of shared object header info */ + hsize_t hdr_size; /**< Shared object header message header size */ + H5_ih_info_t msgs_info; /**< Shared object header message index & heap size */ } sohm; } H5F_info2_t; +//! [H5F_info2_t_snip] -/* - * Types of allocation requests. The values larger than H5FD_MEM_DEFAULT +/** + * Types of allocation requests. The values larger than #H5FD_MEM_DEFAULT * should not change other than adding new types to the end. These numbers * might appear in files. * - * Note: please change the log VFD flavors array if you change this - * enumeration. + * \internal Please change the log VFD flavors array if you change this + * enumeration. */ typedef enum H5F_mem_t { - H5FD_MEM_NOLIST = -1, /* Data should not appear in the free list. + H5FD_MEM_NOLIST = -1, /**< Data should not appear in the free list. * Must be negative. */ - H5FD_MEM_DEFAULT = 0, /* Value not yet set. Can also be the + H5FD_MEM_DEFAULT = 0, /**< Value not yet set. Can also be the * datatype set in a larger allocation * that will be suballocated by the library. * Must be zero. */ - H5FD_MEM_SUPER = 1, /* Superblock data */ - H5FD_MEM_BTREE = 2, /* B-tree data */ - H5FD_MEM_DRAW = 3, /* Raw data (content of datasets, etc.) */ - H5FD_MEM_GHEAP = 4, /* Global heap data */ - H5FD_MEM_LHEAP = 5, /* Local heap data */ - H5FD_MEM_OHDR = 6, /* Object header data */ - - H5FD_MEM_NTYPES /* Sentinel value - must be last */ + H5FD_MEM_SUPER = 1, /**< Superblock data */ + H5FD_MEM_BTREE = 2, /**< B-tree data */ + H5FD_MEM_DRAW = 3, /**< Raw data (content of datasets, etc.) */ + H5FD_MEM_GHEAP = 4, /**< Global heap data */ + H5FD_MEM_LHEAP = 5, /**< Local heap data */ + H5FD_MEM_OHDR = 6, /**< Object header data */ + + H5FD_MEM_NTYPES /**< Sentinel value - must be last */ } H5F_mem_t; -/* Free space section information */ +/** + * Free space section information + */ +//! [H5F_sect_info_t_snip] typedef struct H5F_sect_info_t { - haddr_t addr; /* Address of free space section */ - hsize_t size; /* Size of free space section */ + haddr_t addr; /**< Address of free space section */ + hsize_t size; /**< Size of free space section */ } H5F_sect_info_t; +//! [H5F_sect_info_t_snip] -/* Library's format versions */ +/** + * Library's format versions + */ typedef enum H5F_libver_t { H5F_LIBVER_ERROR = -1, - H5F_LIBVER_EARLIEST = 0, /* Use the earliest possible format for storing objects */ - H5F_LIBVER_V18 = 1, /* Use the latest v18 format for storing objects */ - H5F_LIBVER_V110 = 2, /* Use the latest v110 format for storing objects */ - H5F_LIBVER_V112 = 3, /* Use the latest v112 format for storing objects */ + H5F_LIBVER_EARLIEST = 0, /**< Use the earliest possible format for storing objects */ + H5F_LIBVER_V18 = 1, /**< Use the latest v18 format for storing objects */ + H5F_LIBVER_V110 = 2, /**< Use the latest v110 format for storing objects */ + H5F_LIBVER_V112 = 3, /**< Use the latest v112 format for storing objects */ H5F_LIBVER_NBOUNDS } H5F_libver_t; #define H5F_LIBVER_LATEST H5F_LIBVER_V112 -/* File space handling strategy */ +/** + * File space handling strategy + */ typedef enum H5F_fspace_strategy_t { H5F_FSPACE_STRATEGY_FSM_AGGR = - 0, /* Mechanisms: free-space managers, aggregators, and virtual file drivers */ - /* This is the library default when not set */ + 0, /**< Mechanisms: free-space managers, aggregators, and virtual file drivers + This is the library default when not set */ H5F_FSPACE_STRATEGY_PAGE = - 1, /* Mechanisms: free-space managers with embedded paged aggregation and virtual file drivers */ - H5F_FSPACE_STRATEGY_AGGR = 2, /* Mechanisms: aggregators and virtual file drivers */ - H5F_FSPACE_STRATEGY_NONE = 3, /* Mechanisms: virtual file drivers */ - H5F_FSPACE_STRATEGY_NTYPES /* must be last */ + 1, /**< Mechanisms: free-space managers with embedded paged aggregation and virtual file drivers */ + H5F_FSPACE_STRATEGY_AGGR = 2, /**< Mechanisms: aggregators and virtual file drivers */ + H5F_FSPACE_STRATEGY_NONE = 3, /**< Mechanisms: virtual file drivers */ + H5F_FSPACE_STRATEGY_NTYPES /**< Sentinel */ } H5F_fspace_strategy_t; -/* Deprecated: File space handling strategy for release 1.10.0 */ -/* They are mapped to H5F_fspace_strategy_t as defined above from release 1.10.1 onwards */ +/** + * File space handling strategy for release 1.10.0 + * + * \deprecated 1.10.1 + */ typedef enum H5F_file_space_type_t { - H5F_FILE_SPACE_DEFAULT = 0, /* Default (or current) free space strategy setting */ - H5F_FILE_SPACE_ALL_PERSIST = 1, /* Persistent free space managers, aggregators, virtual file driver */ - H5F_FILE_SPACE_ALL = 2, /* Non-persistent free space managers, aggregators, virtual file driver */ - /* This is the library default */ - H5F_FILE_SPACE_AGGR_VFD = 3, /* Aggregators, Virtual file driver */ - H5F_FILE_SPACE_VFD = 4, /* Virtual file driver */ - H5F_FILE_SPACE_NTYPES /* must be last */ + H5F_FILE_SPACE_DEFAULT = 0, /**< Default (or current) free space strategy setting */ + H5F_FILE_SPACE_ALL_PERSIST = 1, /**< Persistent free space managers, aggregators, virtual file driver */ + H5F_FILE_SPACE_ALL = 2, /**< Non-persistent free space managers, aggregators, virtual file driver + This is the library default */ + H5F_FILE_SPACE_AGGR_VFD = 3, /**< Aggregators, Virtual file driver */ + H5F_FILE_SPACE_VFD = 4, /**< Virtual file driver */ + H5F_FILE_SPACE_NTYPES /**< Sentinel */ } H5F_file_space_type_t; -/* Data structure to report the collection of read retries for metadata items with checksum */ -/* Used by public routine H5Fget_metadata_read_retry_info() */ #define H5F_NUM_METADATA_READ_RETRY_TYPES 21 + +/** + * Data structure to report the collection of read retries for metadata items with checksum as + * used by H5Fget_metadata_read_retry_info() + */ typedef struct H5F_retry_info_t { unsigned nbins; uint32_t *retries[H5F_NUM_METADATA_READ_RETRY_TYPES]; } H5F_retry_info_t; -/* Callback for H5Pset_object_flush_cb() in a file access property list */ +/** + * Callback for H5Pset_object_flush_cb() in a file access property list + */ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata); /*********************/ @@ -229,56 +248,1073 @@ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata); extern "C" { #endif +/** + * \ingroup H5F + * + * \brief Checks if a file can be opened with a given file access property + * list + * + * \param[in] container_name Name of a file + * \fapl_id + * + * \return \htri_t + * + * \details H5Fis_accessible() checks if the file specified by \p + * container_name can be opened with the file access property list + * \p fapl_id. + * + * \note The H5Fis_accessible() function enables files to be checked with a + * given file access property list, unlike H5Fis_hdf5(), which only uses + * the default file driver when opening a file. + * + * \since 1.12.0 + * + */ H5_DLL htri_t H5Fis_accessible(const char *container_name, hid_t fapl_id); -H5_DLL hid_t H5Fcreate(const char *filename, unsigned flags, hid_t create_plist, hid_t access_plist); -H5_DLL hid_t H5Fopen(const char *filename, unsigned flags, hid_t access_plist); +/** + * \example H5Fcreate.c + * After creating an HDF5 file with H5Fcreate(), we close it with + * H5Fclose(). + */ +/** + * \ingroup H5F + * + * \brief Creates an HDF5 file + * + * \param[in] filename Name of the file to create + * \param[in] flags File access flags. Allowable values are: + * - #H5F_ACC_TRUNC: Truncate file, if it already exists, + * erasing all data previously stored in the file + * - #H5F_ACC_EXCL: Fail if file already exists + * \fcpl_id + * \fapl_id + * \return \hid_t{file} + * + * \details H5Fcreate() is the primary function for creating HDF5 files; it + * creates a new HDF5 file with the specified name and property lists. + * + * The \p filename parameter specifies the name of the new file. + * + * The \p flags parameter specifies whether an existing file is to be + * overwritten. It should be set to either #H5F_ACC_TRUNC to overwrite + * an existing file or #H5F_ACC_EXCL, instructing the function to fail + * if the file already exists. + * + * New files are always created in read-write mode, so the read-write + * and read-only flags, #H5F_ACC_RDWR and #H5F_ACC_RDONLY, + * respectively, are not relevant in this function. Further note that + * a specification of #H5F_ACC_RDONLY will be ignored; the file will + * be created in read-write mode, regardless. + * + * More complex behaviors of file creation and access are controlled + * through the file creation and file access property lists, + * \p fcpl_id and \p fapl_id, respectively. The value of #H5P_DEFAULT + * for any property list value indicates that the library should use + * the default values for that appropriate property list. + * + * The return value is a file identifier for the newly-created file; + * this file identifier should be closed by calling H5Fclose() when + * it is no longer needed. + * + * \include H5Fcreate.c + * + * \note #H5F_ACC_TRUNC and #H5F_ACC_EXCL are mutually exclusive; use + * exactly one. + * + * \note An additional flag, #H5F_ACC_DEBUG, prints debug information. This + * flag can be combined with one of the above values using the bit-wise + * OR operator (\c |), but it is used only by HDF5 library developers; + * \Emph{it is neither tested nor supported for use in applications}. + * + * \attention \Bold{Special case — File creation in the case of an already-open file:} + * If a file being created is already opened, by either a previous + * H5Fopen() or H5Fcreate() call, the HDF5 library may or may not + * detect that the open file and the new file are the same physical + * file. (See H5Fopen() regarding the limitations in detecting the + * re-opening of an already-open file.)\n + * If the library detects that the file is already opened, + * H5Fcreate() will return a failure, regardless of the use of + * #H5F_ACC_TRUNC.\n + * If the library does not detect that the file is already opened + * and #H5F_ACC_TRUNC is not used, H5Fcreate() will return a failure + * because the file already exists. Note that this is correct + * behavior.\n + * But if the library does not detect that the file is already + * opened and #H5F_ACC_TRUNC is used, H5Fcreate() will truncate the + * existing file and return a valid file identifier. Such a + * truncation of a currently-opened file will almost certainly + * result in errors. While unlikely, the HDF5 library may not be + * able to detect, and thus report, such errors.\n + * Applications should avoid calling H5Fcreate() with an already + * opened file. + * + * \since 1.0.0 + * + * \see H5Fopen(), H5Fclose() + * + */ +H5_DLL hid_t H5Fcreate(const char *filename, unsigned flags, hid_t fcpl_id, hid_t fapl_id); +/** + * \ingroup H5F + * + * \brief Opens an existing HDF5 file + * + * \param[in] filename Name of the file to be opened + * \param[in] flags File access flags. Allowable values are: + * - #H5F_ACC_RDWR: Allows read and write access to file + * - #H5F_ACC_RDONLY: Allows read-only access to file + * - #H5F_ACC_RDWR \c | #H5F_ACC_SWMR_WRITE: Indicates that + * the file is open for writing in a + * single-writer/multi-writer (SWMR) scenario. + * - #H5F_ACC_RDONLY \c | #H5F_ACC_SWMR_READ: Indicates + * that the file is open for reading in a + * single-writer/multi-reader (SWMR) scenario. + * - An additional flag, #H5F_ACC_DEBUG, prints debug + * information. This flag can be combined with one of the + * above values using the bit-wise OR operator (\c |), but + * it is used only by HDF5 library developers; + * \Emph{it is neither tested nor supported} for use in + * applications. + * \fapl_id + * \return \hid_t{file} + * + * \details H5Fopen() is the primary function for accessing existing HDF5 files. + * This function opens the named file in the specified access mode and + * with the specified access property list. + * + * Note that H5Fopen() does not create a file if it does not already + * exist; see H5Fcreate(). + * + * The \p filename parameter specifies the name of the file to be + * opened. + * + * The \p fapl_id parameter specifies the file access property list. + * Use of #H5P_DEFAULT specifies that default I/O access properties + * are to be used. + * + * The \p flags parameter specifies whether the file will be opened in + * read-write or read-only mode, #H5F_ACC_RDWR or #H5F_ACC_RDONLY, + * respectively. More complex behaviors of file access are controlled + * through the file-access property list. + * + * The return value is a file identifier for the open file; this file + * identifier should be closed by calling H5Fclose() when it is no + * longer needed. + * + * \note #H5F_ACC_RDWR and #H5F_ACC_RDONLY are mutually exclusive; use + * exactly one. + * + * \attention \Bold{Special cases — Multiple opens:} A file can often be opened + * with a new H5Fopen() call without closing an already-open + * identifier established in a previous H5Fopen() or H5Fcreate() + * call. Each such H5Fopen() call will return a unique identifier + * and the file can be accessed through any of these identifiers as + * long as the identifier remains valid. In such multiply-opened + * cases, the open calls must use the same flags argument and the + * file access property lists must use the same file close degree + * property setting (see the external link discussion below and + * H5Pset_fclose_degree()).\n + * In some cases, such as files on a local Unix file system, the + * HDF5 library can detect that a file is multiply opened and will + * maintain coherent access among the file identifiers.\n + * But in many other cases, such as parallel file systems or + * networked file systems, it is not always possible to detect + * multiple opens of the same physical file. In such cases, HDF5 + * will treat the file identifiers as though they are accessing + * different files and will be unable to maintain coherent access. + * Errors are likely to result in these cases. While unlikely, the + * HDF5 library may not be able to detect, and thus report, + * such errors.\n + * It is generally recommended that applications avoid multiple + * opens of the same file. + * + * \attention \Bold{Special restriction on multiple opens of a file first + * opened by means of an external link:} When an external link is + * followed, the external file is always opened with the weak file + * close degree property setting, #H5F_CLOSE_WEAK (see + * H5Lcreate_external() and H5Pset_fclose_degree()). If the file is + * reopened with H5Fopen while it remains held open from such an + * external link call, the file access property list used in the + * open call must include the file close degree setting + * #H5F_CLOSE_WEAK or the open will fail. + * + * \version 1.10.0 The #H5F_ACC_SWMR_WRITE and #H5F_ACC_SWMR_READ flags were added. + * + * \see H5Fclose() + * + */ +H5_DLL hid_t H5Fopen(const char *filename, unsigned flags, hid_t fapl_id); +/** + * \ingroup H5F + * + * \brief Returns a new identifier for a previously-opened HDF5 file + * + * \param[in] file_id Identifier of a file for which an additional identifier + * is required + * + * \return \hid_t{file} + * + * \details H5Freopen() returns a new file identifier for an already-open HDF5 + * file, as specified by \p file_id. Both identifiers share caches and + * other information. The only difference between the identifiers is + * that the new identifier is not mounted anywhere and no files are + * mounted on it. + * + * The new file identifier should be closed by calling H5Fclose() when + * it is no longer needed. + * + * \note Note that there is no circumstance under which H5Freopen() can + * actually open a closed file; the file must already be open and have an + * active \p file_id. E.g., one cannot close a file with H5Fclose() on + * \p file_id then use H5Freopen() on \p file_id to reopen it. + * + */ H5_DLL hid_t H5Freopen(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Flushes all buffers associated with a file to storage + * + * \loc_id{object_id} + * \param[in] scope The scope of the flush action + * + * \return \herr_t + * + * \details H5Fflush() causes all buffers associated with a file to be + * immediately flushed to storage without removing the data from the + * cache. + * + * \p object_id can be any object associated with the file, including + * the file itself, a dataset, a group, an attribute, or a named + * datatype. + * + * \p scope specifies whether the scope of the flush action is + * global or local. Valid values are as follows: + * \scopes + * + * \attention HDF5 does not possess full control over buffering. H5Fflush() + * flushes the internal HDF5 buffers then asks the operating system + * (the OS) to flush the system buffers for the open files. After + * that, the OS is responsible for ensuring that the data is + * actually flushed to disk. + * + */ H5_DLL herr_t H5Fflush(hid_t object_id, H5F_scope_t scope); +/** + * \example H5Fclose.c + * After creating an HDF5 file with H5Fcreate(), we close it with + * H5Fclose(). + */ +/** + * \ingroup H5F + * + * \brief Terminates access to an HDF5 file + * + * \file_id + * \return \herr_t + * + * \details H5Fclose() terminates access to an HDF5 file (specified by + * \p file_id) by flushing all data to storage. + * + * If this is the last file identifier open for the file and no other + * access identifier is open (e.g., a dataset identifier, group + * identifier, or shared datatype identifier), the file will be fully + * closed and access will end. + * + * Use H5Fclose() as shown in the following example: + * \include H5Fclose.c + * + * \note \Bold{Delayed close:} Note the following deviation from the + * above-described behavior. If H5Fclose() is called for a file but one + * or more objects within the file remain open, those objects will remain + * accessible until they are individually closed. Thus, if the dataset + * \c data_sample is open when H5Fclose() is called for the file + * containing it, \c data_sample will remain open and accessible + * (including writable) until it is explicitly closed. The file will be + * automatically closed once all objects in the file have been closed.\n + * Be warned, however, that there are circumstances where it is not + * possible to delay closing a file. For example, an MPI-IO file close is + * a collective call; all of the processes that opened the file must + * close it collectively. The file cannot be closed at some time in the + * future by each process in an independent fashion. Another example is + * that an application using an AFS token-based file access privilege may + * destroy its AFS token after H5Fclose() has returned successfully. This + * would make any future access to the file, or any object within it, + * illegal.\n + * In such situations, applications must close all open objects in a file + * before calling H5Fclose. It is generally recommended to do so in all + * cases. + * + * \see H5Fopen() + * + */ H5_DLL herr_t H5Fclose(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Deletes an HDF5 file + * + * \param[in] filename Name of the file to delete + * \fapl_id + * + * \return \herr_t + * + * \details H5Fdelete() deletes an HDF5 file \p filename with a file access + * property list \p fapl_id. The \p fapl_id should be configured with + * the same VOL connector or VFD that was used to open the file. + * + * This API was introduced for use with the Virtual Object Layer + * (VOL). With the VOL, HDF5 "files" can map to arbitrary storage + * schemes such as object stores and relational database tables. The + * data created by these implementations may be inconvenient for a + * user to remove without a detailed knowledge of the storage scheme. + * H5Fdelete() gives VOL connector authors the ability to add + * connector-specific delete code to their connectors so that users + * can remove these "files" without detailed knowledge of the storage + * scheme. + * + * For a VOL connector, H5Fdelete() deletes the file in a way that + * makes sense for the specified VOL connector. + * + * For the native HDF5 connector, HDF5 files will be deleted via the + * VFDs, each of which will have to be modified to delete the files it + * creates. + * + * For all implementations, H5Fdelete() will first check if the file + * is an HDF5 file via H5Fis_accessible(). This is done to ensure that + * H5Fdelete() cannot be used as an arbitrary file deletion call. + * + * \since 1.12.0 + * + */ H5_DLL herr_t H5Fdelete(const char *filename, hid_t fapl_id); +/** + * \ingroup H5F + * + * \brief Returns a file creation property list identifier + * + * \file_id + * \return \hid_t{file creation property list} + * + * \details H5Fget_create_plist() returns the file creation property list + * identifier identifying the creation properties used to create this + * file. This function is useful for duplicating properties when + * creating another file. + * + * The creation property list identifier should be released with + * H5Pclose(). + * + */ H5_DLL hid_t H5Fget_create_plist(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Returns a file access property list identifier + * + * \file_id + * \return \hid_t{file access property list} + * + * \details H5Fget_access_plist() returns the file access property list + * identifier of the specified file. + * + */ H5_DLL hid_t H5Fget_access_plist(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Determines the read/write or read-only status of a file + * + * \file_id + * \param[out] intent Access mode flag as originally passed with H5Fopen() + * + * \return \herr_t + * + * \details Given the identifier of an open file, \p file_id, H5Fget_intent() + * retrieves the intended access mode" flag passed with H5Fopen() when + * the file was opened. + * + * The value of the flag is returned in \p intent. Valid values are as + * follows: + * \file_access + * + * \note The function will not return an error if intent is NULL; it will + * simply do nothing. + * + * \version 1.10.0 C function enhanced to work with SWMR functionality. + * + * \since 1.8.0 + * + */ H5_DLL herr_t H5Fget_intent(hid_t file_id, unsigned *intent); +/** + * \ingroup H5F + * + * \brief Retrieves a file's file number that uniquely identifies an open file + * + * \file_id + * \param[out] fileno A buffer to hold the file number + * + * \return \herr_t + * + * \details H5Fget_fileno() retrieves a file number for a file specified by the + * file identifier \p file_id and the pointer \p fnumber to the file + * number. + * + * \since 1.12.0 + * + */ H5_DLL herr_t H5Fget_fileno(hid_t file_id, unsigned long *fileno); +/** + * \ingroup H5F + * + * \brief Returns the number of open object identifiers for an open file + * + * \file_id or #H5F_OBJ_ALL for all currently-open HDF5 files + * \param[in] types Type of object for which identifiers are to be returned + * + * \return Returns the number of open objects if successful; otherwise returns + * a negative value. + * + * \details Given the identifier of an open file, file_id, and the desired + * object types, types, H5Fget_obj_count() returns the number of open + * object identifiers for the file. + * + * To retrieve a count of open identifiers for open objects in all + * HDF5 application files that are currently open, pass the value + * #H5F_OBJ_ALL in \p file_id. + * + * The types of objects to be counted are specified in types as + * follows: + * \obj_types + * + * Multiple object types can be combined with the + * logical \c OR operator (|). For example, the expression + * \c (#H5F_OBJ_DATASET|#H5F_OBJ_GROUP) would call for datasets and + * groups. + * + * \version 1.6.8, 1.8.2 C function return type changed to \c ssize_t. + * \version 1.6.5 #H5F_OBJ_LOCAL has been added as a qualifier on the types + * of objects to be counted. #H5F_OBJ_LOCAL restricts the + * search to objects opened through current file identifier. + * + */ H5_DLL ssize_t H5Fget_obj_count(hid_t file_id, unsigned types); +/** + *------------------------------------------------------------------------- + * \ingroup H5F + * + * \brief Returns a list of open object identifiers + * + * \file_id or #H5F_OBJ_ALL for all currently-open HDF5 files + * \param[in] types Type of object for which identifiers are to be returned + * \param[in] max_objs Maximum number of object identifiers to place into + * \p obj_id_list + * \param[out] obj_id_list Pointer to the returned buffer of open object + * identifiers + * + * \return Returns number of objects placed into \p obj_id_list if successful; + * otherwise returns a negative value. + * + * \details Given the file identifier \p file_id and the type of objects to be + * identified, types, H5Fget_obj_ids() returns the list of identifiers + * for all open HDF5 objects fitting the specified criteria. + * + * To retrieve identifiers for open objects in all HDF5 application + * files that are currently open, pass the value #H5F_OBJ_ALL in + * \p file_id. + * + * The types of object identifiers to be retrieved are specified in + * types using the codes listed for the same parameter in + * H5Fget_obj_count(). + * + * To retrieve a count of open objects, use the H5Fget_obj_count() + * function. This count can be used to set the \p max_objs parameter. + * + * \version 1.8.2 C function return type changed to \c ssize_t and \p + * max_objs parameter datatype changed to \c size_t. + * \version 1.6.8 C function return type changed to \c ssize_t and \p + * max_objs parameter datatype changed to \c size_t. + * \since 1.6.0 + * + */ H5_DLL ssize_t H5Fget_obj_ids(hid_t file_id, unsigned types, size_t max_objs, hid_t *obj_id_list); +/** + * \ingroup H5F + * + * \brief Returns pointer to the file handle from the virtual file driver + * + * \file_id + * \fapl_id{fapl} + * \param[out] file_handle Pointer to the file handle being used by the + * low-level virtual file driver + * + * \return \herr_t + * + * \details Given the file identifier \p file_id and the file access property + * list \p fapl_id, H5Fget_vfd_handle() returns a pointer to the file + * handle from the low-level file driver currently being used by the + * HDF5 library for file I/O. + * + * \note For most drivers, the value of \p fapl_id will be #H5P_DEFAULT. For + * the \c FAMILY or \c MULTI drivers, this value should be defined + * through the property list functions: H5Pset_family_offset() for the + * \c FAMILY driver and H5Pset_multi_type() for the \c MULTI driver + * + * \since 1.6.0 + * + */ H5_DLL herr_t H5Fget_vfd_handle(hid_t file_id, hid_t fapl, void **file_handle); +/** + * \ingroup H5F + * + * \brief Mounts an HDF5 file + * + * \loc_id{loc} + * \param[in] name Name of the group onto which the file specified by \p child + * is to be mounted + * \file_id{child} + * \param[in] plist File mount property list identifier. Pass #H5P_DEFAULT! + * + * \return \herr_t + * + * \details H5Fmount() mounts the file specified by \p child onto the object + * specified by \p loc and \p name using the mount properties \p plist + * If the object specified by \p loc is a dataset, named datatype or + * attribute, then the file will be mounted at the location where the + * attribute, dataset, or named datatype is attached. + * + * \note To date, no file mount properties have been defined in HDF5. The + * proper value to pass for \p plist is #H5P_DEFAULT, indicating the + * default file mount property list. + * + */ H5_DLL herr_t H5Fmount(hid_t loc, const char *name, hid_t child, hid_t plist); +/** + * \ingroup H5F + * + * \brief Unounts an HDF5 file + * + * \loc_id{loc} + * \param[in] name Name of the mount point + * + * \return \herr_t + * + * \details Given a mount point, H5Funmount() dissociates the mount point's + * file from the file mounted there. This function does not close + * either file. + * + * The mount point can be either the group in the parent or the root + * group of the mounted file (both groups have the same name). If the + * mount point was opened before the mount then it is the group in the + * parent; if it was opened after the mount then it is the root group + * of the child. + * + */ H5_DLL herr_t H5Funmount(hid_t loc, const char *name); +/** + * \ingroup H5F + * + * \brief Returns the amount of free space in a file (in bytes) + * + * \file_id + * + * \return Returns the amount of free space in the file if successful; + * otherwise returns a negative value. + * + * \details Given the identifier of an open file, \p file_id, + * H5Fget_freespace() returns the amount of space that is unused by + * any objects in the file. + * + * The interpretation of this number depends on the configured free space + * management strategy. For example, if the HDF5 library only tracks free + * space in a file from a file open or create until that file is closed, + * then this routine will report the free space that has been created + * during that interval. + * + * \since 1.6.1 + * + */ H5_DLL hssize_t H5Fget_freespace(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Returns the size of an HDF5 file (in bytes) + * + * \file_id + * \param[out] size Size of the file, in bytes + * + * \return \herr_t + * + * \details H5Fget_filesize() returns the size of the HDF5 file specified by + * \p file_id. + * + * The returned size is that of the entire file, as opposed to only + * the HDF5 portion of the file. I.e., size includes the user block, + * if any, the HDF5 portion of the file, and any data that may have + * been appended beyond the data written through the HDF5 library. + * + * \version 1.6.3 Fortran subroutine introduced in this release. + * + * \since 1.6.3 + * + */ H5_DLL herr_t H5Fget_filesize(hid_t file_id, hsize_t *size); +/** + * \ingroup H5F + * + * \brief Retrieves the file's end-of-allocation (EOA) + * + * \file_id + * \param[out] eoa The file's EOA + * + * \return \herr_t + * + * \details H5Fget_eoa() retrieves the file's EOA and returns it in the + * parameter eoa. + * + * \since 1.10.2 + * + */ H5_DLL herr_t H5Fget_eoa(hid_t file_id, haddr_t *eoa); +/** + * \ingroup H5F + * + * \brief Sets the file' EOA to the maximum of (EOA, EOF) + increment + * + * \file_id + * \param[in] increment The number of bytes to be added to the maximum of + * (EOA, EOF) + * + * \return \herr_t + * + * \details H5Fincrement_filesize() sets the file's EOA to the maximum of (EOA, + * EOF) + \p increment. The EOA is the end-of-file address stored in + * the file's superblock while EOF is the file's actual end-of-file. + * + * \since 1.10.2 + * + */ H5_DLL herr_t H5Fincrement_filesize(hid_t file_id, hsize_t increment); +/** + * \ingroup H5F + * + * \brief Retrieves a copy of the image of an existing, open file + * + * \file_id + * \param[out] buf_ptr Pointer to the buffer into which the image of the + * HDF5 file is to be copied. If \p buf_ptr is NULL, + * no data will be copied but the function’s return value + * will still indicate the buffer size required (or a + * negative value on error). + * \param[out] buf_len Size of the supplied buffer + * + * \return ssize_t + * + * \details H5Fget_file_image() retrieves a copy of the image of an existing, + * open file. This routine can be used with files opened using the + * SEC2 (or POSIX), STDIO, and Core (or Memory) virtual file drivers + * (VFDs). + * + * If the return value of H5Fget_file_image() is a positive value, it + * will be the length in bytes of the buffer required to store the + * file image. So if the file size is unknown, it can be safely + * determined with an initial H5Fget_file_image() call with buf_ptr + * set to NULL. The file image can then be retrieved with a second + * H5Fget_file_image() call with \p buf_len set to the initial call’s + * return value. + * + * While the current file size can also be retrieved with + * H5Fget_filesize(), that call may produce a larger value than is + * needed. The value returned by H5Fget_filesize() includes the user + * block, if it exists, and any unallocated space at the end of the + * file. It is safe in all situations to get the file size with + * H5Fget_file_image() and it often produces a value that is more + * appropriate for the size of a file image buffer. + * + * \note \Bold{Recommended Reading:} This function is part of the file image + * operations feature set. It is highly recommended to study the guide + * "HDF5 File Image Operations" before using this feature set.\n See the + * "See Also" section below for links to other elements of HDF5 file + * image operations. \todo Fix the references. + * + * \attention H5Pget_file_image() will fail, returning a negative value, if the + * file is too large for the supplied buffer. + * + * \see H5LTopen_file_image(), H5Pset_file_image(), H5Pget_file_image(), + * H5Pset_file_image_callbacks(), H5Pget_file_image_callbacks() + * + * \version 1.8.13 Fortran subroutine added in this release. + * + * \since 1.8.0 + * + */ H5_DLL ssize_t H5Fget_file_image(hid_t file_id, void *buf_ptr, size_t buf_len); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_mdc_config(hid_t file_id, H5AC_cache_config_t *config_ptr); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fset_mdc_config(hid_t file_id, H5AC_cache_config_t *config_ptr); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_mdc_hit_rate(hid_t file_id, double *hit_rate_ptr); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_mdc_size(hid_t file_id, size_t *max_size_ptr, size_t *min_clean_size_ptr, size_t *cur_size_ptr, int *cur_num_entries_ptr); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Freset_mdc_hit_rate_stats(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Retrieves name of file to which object belongs + * + * \obj_id + * \param[out] name Buffer for the file name + * \param[in] size Size, in bytes, of the \p name buffer + * + * \return Returns the length of the file name if successful; otherwise returns + * a negative value. + * + * \details H5Fget_name() retrieves the name of the file to which the object \p + * obj_id belongs. The object can be a file, group, dataset, + * attribute, or named datatype. + * + * Up to \p size characters of the file name are returned in \p name; + * additional characters, if any, are not returned to the user + * application. + * + * If the length of the name, which determines the required value of + * size, is unknown, a preliminary H5Fget_name() call can be made by + * setting \p name to NULL. The return value of this call will be the + * size of the file name; that value plus one (1) can then be assigned + * to size for a second H5Fget_name() call, which will retrieve the + * actual name. (The value passed in with the parameter \p size must + * be one greater than size in bytes of the actual name in order to + * accommodate the null terminator; if \p size is set to the exact + * size of the name, the last byte passed back will contain the null + * terminator and the last character will be missing from the name + * passed back to the calling application.) + * + * If an error occurs, the buffer pointed to by \p name is unchanged + * and the function returns a negative value. + * + * \since 1.6.3 + * + */ H5_DLL ssize_t H5Fget_name(hid_t obj_id, char *name, size_t size); -H5_DLL herr_t H5Fget_info2(hid_t obj_id, H5F_info2_t *finfo); +/** + * \ingroup H5F + * + * \brief Retrieves name of file to which object belongs + * + * \fgdta_obj_id + * \param[out] file_info Buffer for global file information + * + * \return \herr_t + * + * \details H5Fget_info2() returns global information for the file associated + * with the object identifier \p obj_id in the H5F_info2_t \c struct + * named \p file_info. + * + * \p obj_id is an identifier for any object in the file of interest. + * + * H5F_info2_t struct is defined in H5Fpublic.h as follows: + * \snippet this H5F_info2_t_snip + * + * The \c super sub-struct contains the following information: + * \li \c vers is the version number of the superblock. + * \li \c super_size is the size of the superblock. + * \li \c super_ext_size is the size of the superblock extension. + * + * The \c free sub-struct contains the following information: + * \li vers is the version number of the free-space manager. + * \li \c hdr_size is the size of the free-space manager header. + * \li \c tot_space is the total amount of free space in the file. + * + * The \c sohm sub-struct contains shared object header message + * information as follows: + * \li \c vers is the version number of the shared object header information. + * \li \c hdr_size is the size of the shared object header message. + * \li \c msgs_info is an H5_ih_info_t struct defined in H5public.h as + * follows: \snippet H5public.h H5_ih_info_t_snip + * \li \p index_size is the summed size of all the shared object + * header indexes. Each index might be either a B-tree or + * a list. + * \li \p heap_size is the size of the heap. + * + * + * \since 1.10.0 + * + */ +H5_DLL herr_t H5Fget_info2(hid_t obj_id, H5F_info2_t *file_info); +/** + * \ingroup SWMR + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_metadata_read_retry_info(hid_t file_id, H5F_retry_info_t *info); +/** + * \ingroup SWMR + * + * \todo Finish this! + */ H5_DLL herr_t H5Fstart_swmr_write(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Retrieves free-space section information for a file + * + * \file_id + * \param[in] type The file memory allocation type + * \param[in] nsects The number of free-space sections + * \param[out] sect_info Array of instances of H5F_sect_info_t in which + * the free-space section information is to be returned + * + * \return Returns the number of free-space sections for the specified + * free-space manager in the file; otherwise returns a negative value. + * + * \details H5Fget_free_sections() retrieves free-space section information for + * the free-space manager with type that is associated with file + * \p file_id. If type is #H5FD_MEM_DEFAULT, this routine retrieves + * free-space section information for all the free-space managers in + * the file. + * + * Valid values for \p type are the following: + * \mem_types + * + * H5F_sect_info_t is defined as follows (in H5Fpublic.h): + * \snippet this H5F_sect_info_t_snip + * + * This routine retrieves free-space section information for \p nsects + * sections or at most the maximum number of sections in the specified + * free-space manager. If the number of sections is not known, a + * preliminary H5Fget_free_sections() call can be made by setting \p + * sect_info to NULL and the total number of free-space sections for + * the specified free-space manager will be returned. Users can then + * allocate space for entries in \p sect_info, each of which is + * defined as an H5F_sect_info_t \c struct. + * + * \attention \Bold{Failure Modes:} This routine will fail when the following + * is true: + * \li The library fails to retrieve the file creation property list + * associated with \p file_id. + * \li If the parameter \p sect_info is non-null, but the parameter + * \p nsects is equal to 0. + * \li The library fails to retrieve free-space section information + * for the file. + * + * \since 1.10.0 + * + */ H5_DLL ssize_t H5Fget_free_sections(hid_t file_id, H5F_mem_t type, size_t nsects, H5F_sect_info_t *sect_info /*out*/); +/** + * \ingroup H5F + * + * \brief Clears the external link open file cache + * + * \file_id + * \return \herr_t + * + * \details H5Fclear_elink_file_cache() evicts all the cached child files in + * the specified file’s external file cache, causing them to be closed + * if there is nothing else holding them open. + * + * H5Fclear_elink_file_cache() does not close the cache itself; + * subsequent external link traversals from the parent file will again + * cache the target file. See H5Pset_elink_file_cache_size() for + * information on closing the file cache. + * + * \see H5Pset_elink_file_cache_size(), H5Pget_elink_file_cache_size() + * + * \since 1.8.7 + * + */ H5_DLL herr_t H5Fclear_elink_file_cache(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Enables the switch of version bounds setting for a file + * + * \file_id + * \param[in] low The earliest version of the library that will be used for + * writing objects + * \param[in] high The latest version of the library that will be used for + * writing objects + * + * \return \herr_t + * + * \details H5Fset_libver_bounds() enables the switch of version bounds setting + * for an open file associated with \p file_id. + * + * For the parameters \p low and \p high, see the description for + * H5Pset_libver_bounds(). + * + * \since 1.10.2 + * + */ H5_DLL herr_t H5Fset_libver_bounds(hid_t file_id, H5F_libver_t low, H5F_libver_t high); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fstart_mdc_logging(hid_t file_id); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fstop_mdc_logging(hid_t file_id); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_mdc_logging_status(hid_t file_id, /*OUT*/ hbool_t *is_enabled, /*OUT*/ hbool_t *is_currently_logging); +/** + * \ingroup SWMR + * + * \todo Finish this! + */ H5_DLL herr_t H5Fformat_convert(hid_t fid); +/** + * \ingroup H5F + * + * \brief Resets the page buffer statistics + * + * \file_id + * + * \return \herr_t + * + * \details H5Freset_page_buffering_stats() resets the page buffer statistics + * for a specified file identifier \p file_id. + * + * \since 1.10.1 + * + */ H5_DLL herr_t H5Freset_page_buffering_stats(hid_t file_id); +/** + * \ingroup H5F + * + * \brief Retrieves statistics about page access when it is enabled + * + * \file_id + * \param[out] accesses Two integer array for the number of metadata and raw + * data accesses to the page buffer + * \param[out] hits Two integer array for the number of metadata and raw data + * hits in the page buffer + * \param[out] misses Two integer array for the number of metadata and raw data + * misses in the page buffer + * \param[out] evictions Two integer array for the number of metadata and raw + * data evictions from the page buffer + * \param[out] bypasses Two integer array for the number of metadata and raw + * data accesses that bypass the page buffer + * + * \return \herr_t + * + * \details H5Fget_page_buffering_stats() retrieves page buffering statistics + * such as the number of metadata and raw data accesses (\p accesses), + * hits (\p hits), misses (\p misses), evictions (\p evictions), and + * accesses that bypass the page buffer (\p bypasses). + * + * \since 1.10.1 + * + */ H5_DLL herr_t H5Fget_page_buffering_stats(hid_t file_id, unsigned accesses[2], unsigned hits[2], unsigned misses[2], unsigned evictions[2], unsigned bypasses[2]); +/** + * \ingroup MDC + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_mdc_image_info(hid_t file_id, haddr_t *image_addr, hsize_t *image_size); +/** + * \ingroup H5F + * + * \brief Retrieves the setting for whether or not a file will create minimized + * dataset object headers + * + * \file_id + * \param[out] minimize Flag indicating whether the library will or will not + * create minimized dataset object headers + * + * \return \herr_t + * + * \details H5Fget_dset_no_attrs_hint() retrieves the no dataset attributes + * hint setting for the file specified by the file identifier \p + * file_id. This setting is used to inform the library to create + * minimized dataset object headers when \c TRUE. + * + * The setting's value is returned in the boolean pointer minimize. + * + * \since 1.10.5 + * + */ H5_DLL herr_t H5Fget_dset_no_attrs_hint(hid_t file_id, hbool_t *minimize); +/** + * \ingroup H5F + * + * \brief Sets the flag to create minimized dataset object headers + * + * \file_id + * \param[in] minimize Flag indicating whether the library will or will not + * create minimized dataset object headers + * + * \return \herr_t + * + * \details H5Fset_dset_no_attrs_hint() sets the no dataset attributes hint + * setting for the file specified by the file identifier \p file_id. + * If the boolean flag \p minimize is set to \c TRUE, then the library + * will create minimized dataset object headers in the file. + * \Bold{All} files that refer to the same file-on-disk will be + * affected by the most recent setting, regardless of the file + * identifier/handle (e.g., as returned by H5Fopen()). By setting the + * \p minimize flag to \c TRUE, the library expects that no attributes + * will be added to the dataset - attributes can be added, but they + * are appended with a continuation message, which can reduce + * performance. + * + * \attention This setting interacts with H5Pset_dset_no_attrs_hint(): if + * either is set to \c TRUE, then the created dataset's object header + * will be minimized. + * + * \since 1.10.5 + * + */ H5_DLL herr_t H5Fset_dset_no_attrs_hint(hid_t file_id, hbool_t minimize); #ifdef H5_HAVE_PARALLEL +/** + * \ingroup PH5F + * + * \todo Finish this! + */ H5_DLL herr_t H5Fset_mpi_atomicity(hid_t file_id, hbool_t flag); +/** + * \ingroup PH5F + * + * \todo Finish this! + */ H5_DLL herr_t H5Fget_mpi_atomicity(hid_t file_id, hbool_t *flag); #endif /* H5_HAVE_PARALLEL */ @@ -293,19 +1329,95 @@ H5_DLL herr_t H5Fget_mpi_atomicity(hid_t file_id, hbool_t *flag); /* Typedefs */ -/* Current "global" information about file */ +/** + * Current "global" information about file + */ +//! [H5F_info1_t_snip] typedef struct H5F_info1_t { - hsize_t super_ext_size; /* Superblock extension size */ + hsize_t super_ext_size; /**< Superblock extension size */ struct { - hsize_t hdr_size; /* Shared object header message header size */ - H5_ih_info_t msgs_info; /* Shared object header message index & heap size */ + hsize_t hdr_size; /**< Shared object header message header size */ + H5_ih_info_t msgs_info; /**< Shared object header message index & heap size */ } sohm; } H5F_info1_t; +//! [H5F_info1_t_snip] /* Function prototypes */ -H5_DLL herr_t H5Fget_info1(hid_t obj_id, H5F_info1_t *finfo); +/** + * \ingroup H5F + * + * \brief Retrieves name of file to which object belongs + * + * \fgdta_obj_id + * \param[out] file_info Buffer for global file information + * + * \return \herr_t + * + * \deprecated This function has been renamed from H5Fget_info() and is + * deprecated in favor of the macro #H5Fget_info or the function + * H5Fget_info2(). + * + * \details H5Fget_info1() returns global information for the file associated + * with the object identifier \p obj_id in the H5F_info1_t \c struct + * named \p file_info. + * + * \p obj_id is an identifier for any object in the file of interest. + * + * H5F_info1_t struct is defined in H5Fpublic.h as follows: + * \snippet this H5F_info1_t_snip + * + * \c super_ext_size is the size of the superblock extension. + * + * The \c sohm sub-struct contains shared object header message + * information as follows: + * \li \c hdr_size is the size of the shared object header message. + * \li \c msgs_info is an H5_ih_info_t struct defined in H5public.h as + * follows: \snippet H5public.h H5_ih_info_t_snip + * + * \li \p index_size is the summed size of all the shared object + * header indexes. Each index might be either a B-tree or + * a list. + * + * \version 1.10.0 C function H5Fget_info() renamed to H5Fget_info1() and + * deprecated in this release. + * + * \since 1.8.0 + * + */ +H5_DLL herr_t H5Fget_info1(hid_t obj_id, H5F_info1_t *file_info); +/** + * \ingroup H5F + * + * \brief Sets thelatest version of the library to be used for writing objects + * + * \file_id + * \param[in] latest_format Latest format flag + * + * \return \herr_t + * + * \deprecated When? + * + * \todo In which version was this function deprecated? + * + */ H5_DLL herr_t H5Fset_latest_format(hid_t file_id, hbool_t latest_format); -H5_DLL htri_t H5Fis_hdf5(const char *filename); +/** + * \ingroup H5F + * + * \brief Determines whether a file is in the HDF5 format + * + * \param[in] file_name File name + * + * \return \htri_t + * + * \deprecated When? + * + * \details H5Fis_hdf5() determines whether a file is in the HDF5 format. + * + * \todo In which version was this function deprecated? + * + */ +H5_DLL htri_t H5Fis_hdf5(const char *file_name); #endif /* H5_NO_DEPRECATED_SYMBOLS */ diff --git a/src/H5Fquery.c b/src/H5Fquery.c index ed435a8..dac6e66 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -1299,3 +1299,23 @@ H5F__get_cont_info(const H5F_t *f, H5VL_file_cont_info_t *info) done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_get_cont_info */ + +/*------------------------------------------------------------------------- + * Function: H5F_get_file_locking + * + * Purpose: Get the file locking flag for the file + * + * Return: TRUE/FALSE + * + *------------------------------------------------------------------------- + */ +hbool_t +H5F_get_file_locking(const H5F_t *f) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->use_file_locking) +} /* end H5F_get_file_locking */ diff --git a/src/H5Ftest.c b/src/H5Ftest.c index cc7ed1b..c2ce16e 100644 --- a/src/H5Ftest.c +++ b/src/H5Ftest.c @@ -253,3 +253,35 @@ H5F__same_file_test(hid_t file_id1, hid_t file_id2) done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__same_file_test() */ + +/*------------------------------------------------------------------------- + * Function: H5F__reparse_file_lock_variable_test + * + * Purpose: Re-parse the file locking environment variable. + * + * Since getenv(3) is fairly expensive, we only parse it once, + * when the library opens. This test function is used to + * re-parse the environment variable after we've changed it + * with setnev(3). + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Summer 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__reparse_file_lock_variable_test(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check the file locking environment variable */ + if (H5F__parse_file_lock_env_var(&use_locks_env_g) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to parse file locking environment variable") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__reparse_file_lock_variable_test() */ diff --git a/src/H5Osdspace.c b/src/H5Osdspace.c index b34eb76..83702eb 100644 --- a/src/H5Osdspace.c +++ b/src/H5Osdspace.c @@ -138,8 +138,11 @@ H5O__sdspace_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UN flags = *p++; /* Get or determine the type of the extent */ - if (version >= H5O_SDSPACE_VERSION_2) + if (version >= H5O_SDSPACE_VERSION_2) { sdim->type = (H5S_class_t)*p++; + if (sdim->type != H5S_SIMPLE && sdim->rank > 0) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "invalid rank for scalar or NULL dataspace") + } /* end if */ else { /* Set the dataspace type to be simple or scalar as appropriate */ if (sdim->rank > 0) @@ -248,13 +251,15 @@ H5O__sdspace_encode(H5F_t *f, uint8_t *p, const void *_mesg) *p++ = 0; /*reserved*/ } /* end else */ - /* Current & maximum dimensions */ - if (sdim->rank > 0) { - for (u = 0; u < sdim->rank; u++) - H5F_ENCODE_LENGTH(f, p, sdim->size[u]); - if (flags & H5S_VALID_MAX) { + /* Encode dataspace dimensions for simple dataspaces */ + if (H5S_SIMPLE == sdim->type) { + /* Encode current & maximum dimensions */ + if (sdim->rank > 0) { for (u = 0; u < sdim->rank; u++) - H5F_ENCODE_LENGTH(f, p, sdim->max[u]); + H5F_ENCODE_LENGTH(f, p, sdim->size[u]); + if (flags & H5S_VALID_MAX) + for (u = 0; u < sdim->rank; u++) + H5F_ENCODE_LENGTH(f, p, sdim->max[u]); } /* end if */ } /* end if */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 7588b4a..f9a730d 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -288,6 +288,29 @@ #define H5F_ACS_VOL_CONN_COPY H5P__facc_vol_copy #define H5F_ACS_VOL_CONN_CMP H5P__facc_vol_cmp #define H5F_ACS_VOL_CONN_CLOSE H5P__facc_vol_close +/* Definition for using file locking or not. The default is set + * via the configure step. + */ +#define H5F_ACS_USE_FILE_LOCKING_SIZE sizeof(hbool_t) +#if defined H5_USE_FILE_LOCKING && H5_USE_FILE_LOCKING +#define H5F_ACS_USE_FILE_LOCKING_DEF TRUE +#else +#define H5F_ACS_USE_FILE_LOCKING_DEF FALSE +#endif +#define H5F_ACS_USE_FILE_LOCKING_ENC H5P__encode_hbool_t +#define H5F_ACS_USE_FILE_LOCKING_DEC H5P__decode_hbool_t +/* Definition for whether we ignore file locking errors when we can + * tell that file locking has been disabled on the file system. + * The default is set via the configure step. + */ +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_SIZE sizeof(hbool_t) +#if defined H5_IGNORE_DISABLED_FILE_LOCKS && H5_IGNORE_DISABLED_FILE_LOCKS +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF TRUE +#else +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF FALSE +#endif +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC H5P__encode_hbool_t +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC H5P__decode_hbool_t /******************/ /* Local Typedefs */ @@ -478,6 +501,10 @@ static const unsigned H5F_def_page_buf_min_meta_perc_g = H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF; /* Default page buffer minimum metadata size */ static const unsigned H5F_def_page_buf_min_raw_perc_g = H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer mininum raw data size */ +static const hbool_t H5F_def_use_file_locking_g = + H5F_ACS_USE_FILE_LOCKING_DEF; /* Default use file locking flag */ +static const hbool_t H5F_def_ignore_disabled_file_locks_g = + H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF; /* Default ignore disabled file locks flag */ /*------------------------------------------------------------------------- * Function: H5P__facc_reg_prop @@ -764,6 +791,19 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) H5F_ACS_VOL_CONN_CLOSE) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the use file locking flag */ + if (H5P__register_real(pclass, H5F_ACS_USE_FILE_LOCKING_NAME, H5F_ACS_USE_FILE_LOCKING_SIZE, + &H5F_def_use_file_locking_g, NULL, NULL, NULL, H5F_ACS_USE_FILE_LOCKING_ENC, + H5F_ACS_USE_FILE_LOCKING_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the ignore disabled file locks flag */ + if (H5P__register_real(pclass, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, + H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_SIZE, &H5F_def_ignore_disabled_file_locks_g, + NULL, NULL, NULL, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC, + H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_reg_prop() */ @@ -4566,6 +4606,96 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_evict_on_close() */ +/*------------------------------------------------------------------------- + * Function: H5Pset_file_locking + * + * Purpose: Sets the file locking property values. + * + * Overrides the default file locking flag setting that was + * set when the library was configured. + * + * Can be overridden by the HDF5_USE_FILE_LOCKING environment + * variable. + * + * File locking is used when creating/opening a file to prevent + * problematic file accesses. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ibb", fapl_id, use_file_locking, ignore_when_disabled); + + /* Make sure this is a fapl */ + if (TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not a file access plist") + + /* Get the plist structure */ + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set values */ + if (H5P_set(plist, H5F_ACS_USE_FILE_LOCKING_NAME, &use_file_locking) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set use file locking property") + if (H5P_set(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &ignore_when_disabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set ignore disabled file locks property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_file_locking() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_file_locking + * + * Purpose: Gets the file locking property values. + * + * File locking is used when creating/opening a file to prevent + * problematic file accesses. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_file_locking(hid_t fapl_id, hbool_t *use_file_locking, hbool_t *ignore_when_disabled) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*b*b", fapl_id, use_file_locking, ignore_when_disabled); + + /* Make sure this is a fapl */ + if (TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not an access plist") + + /* Get the plist structure */ + if (NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get values */ + if (H5P_get(plist, H5F_ACS_USE_FILE_LOCKING_NAME, use_file_locking) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get use file locking property") + if (H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, ignore_when_disabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get ignore disabled file locks property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_file_locking() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 68519f8..ffcdd99 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -361,6 +361,8 @@ H5_DLL herr_t H5Pget_mdc_log_options(hid_t plist_id, hbool_t *is_enabled, c size_t *location_size, hbool_t *start_on_access); H5_DLL herr_t H5Pset_evict_on_close(hid_t fapl_id, hbool_t evict_on_close); H5_DLL herr_t H5Pget_evict_on_close(hid_t fapl_id, hbool_t *evict_on_close); +H5_DLL herr_t H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled); +H5_DLL herr_t H5Pget_file_locking(hid_t fapl_id, hbool_t *use_file_locking, hbool_t *ignore_when_disabled); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Pset_all_coll_metadata_ops(hid_t plist_id, hbool_t is_collective); H5_DLL herr_t H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collective); diff --git a/src/H5Rint.c b/src/H5Rint.c index 8f3163a..60f8131 100644 --- a/src/H5Rint.c +++ b/src/H5Rint.c @@ -746,8 +746,10 @@ H5R__copy(const H5R_ref_priv_t *src_ref, H5R_ref_priv_t *dst_ref) dst_ref->info.obj.filename = NULL; /* Set location ID and hold reference to it */ - if (H5R__set_loc_id(dst_ref, src_ref->loc_id, TRUE, TRUE) < 0) - HGOTO_ERROR(H5E_REFERENCE, H5E_CANTSET, FAIL, "cannot set reference location ID") + dst_ref->loc_id = src_ref->loc_id; + if (H5I_inc_ref(dst_ref->loc_id, TRUE) < 0) + HGOTO_ERROR(H5E_REFERENCE, H5E_CANTINC, FAIL, "incrementing location ID failed") + dst_ref->app_ref = TRUE; } done: @@ -266,7 +266,7 @@ H5SM__type_to_flag(unsigned type_id, unsigned *type_flag) switch (type_id) { case H5O_FILL_ID: type_id = H5O_FILL_NEW_ID; - /* Fall through... */ + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case H5O_SDSPACE_ID: diff --git a/src/H5Shyper.c b/src/H5Shyper.c index 56a8b14..c527cd5 100644 --- a/src/H5Shyper.c +++ b/src/H5Shyper.c @@ -2137,24 +2137,31 @@ H5S__hyper_iter_get_seq_list_opt(H5S_sel_iter_t *iter, size_t maxseq, size_t max case 0: do { DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 7: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 6: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 5: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 4: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 3: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 2: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 1: DUFF_GUTS @@ -10100,7 +10107,7 @@ H5S_select_hyperslab(H5S_t *space, H5S_seloper_t op, const hsize_t start[], cons case H5S_SEL_POINTS: /* Can't combine hyperslab operations and point selections currently */ if (op == H5S_SELECT_SET) /* Allow only "set" operation to proceed */ break; - /* Else fall through to error */ + /* FALLTHROUGH (to error) */ H5_ATTR_FALLTHROUGH case H5S_SEL_ERROR: diff --git a/src/H5Spublic.h b/src/H5Spublic.h index ad0bbd5..f0aa6dc 100644 --- a/src/H5Spublic.h +++ b/src/H5Spublic.h @@ -172,6 +172,7 @@ H5_DLL hid_t H5Sselect_project_intersection(hid_t src_space_id, hid_t dst_spa H5_DLL hid_t H5Ssel_iter_create(hid_t spaceid, size_t elmt_size, unsigned flags); H5_DLL herr_t H5Ssel_iter_get_seq_list(hid_t sel_iter_id, size_t maxseq, size_t maxbytes, size_t *nseq, size_t *nbytes, hsize_t *off, size_t *len); +H5_DLL herr_t H5Ssel_iter_reset(hid_t sel_iter_id, hid_t space_id); H5_DLL herr_t H5Ssel_iter_close(hid_t sel_iter_id); /* Symbols defined for compatibility with previous versions of the HDF5 API. diff --git a/src/H5Sselect.c b/src/H5Sselect.c index 37a065a..023ab80 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -3089,6 +3089,56 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Ssel_iter_get_seq_list() */ +/*-------------------------------------------------------------------------- + NAME + H5Ssel_iter_reset + PURPOSE + Resets a dataspace selection iterator back to an initial state. + USAGE + herr_t H5Ssel_iter_reset(sel_iter_id) + hid_t sel_iter_id; IN: ID of the dataspace selection iterator to + reset + hid_t space_id; IN: ID of the dataspace with selection to + iterate over + RETURNS + Non-negative on success / Negative on failure + DESCRIPTION + Resets a dataspace selection iterator back to an initial state so that + the iterator may be used for iteration once again. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5Ssel_iter_reset(hid_t sel_iter_id, hid_t space_id) +{ + H5S_sel_iter_t *sel_iter; + H5S_t * space; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ii", sel_iter_id, space_id); + + /* Check args */ + if (NULL == (sel_iter = (H5S_sel_iter_t *)H5I_object_verify(sel_iter_id, H5I_SPACE_SEL_ITER))) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "not a dataspace selection iterator") + if (NULL == (space = (H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, "not a dataspace") + + /* Call selection type-specific release routine */ + if (H5S_SELECT_ITER_RELEASE(sel_iter) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, + "problem releasing a selection iterator's type-specific info") + + /* Simply re-initialize iterator */ + if (H5S_select_iter_init(sel_iter, space, sel_iter->elmt_size, sel_iter->flags) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to re-initialize selection iterator") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Ssel_iter_reset() */ + /*------------------------------------------------------------------------- * Function: H5S_sel_iter_close * diff --git a/src/H5Tdbg.c b/src/H5Tdbg.c index 5d0c5cd..758430b 100644 --- a/src/H5Tdbg.c +++ b/src/H5Tdbg.c @@ -120,9 +120,9 @@ H5T__print_stats(H5T_path_t H5_ATTR_UNUSED *path, int H5_ATTR_UNUSED *nprint /*i HDfprintf(H5DEBUG(T), " %-16s %10" PRIdHSIZE " %10u %8s %8s %8s %10s\n", path->name, path->stats.nelmts, path->stats.ncalls, timestrs.user, timestrs.system, timestrs.elapsed, bandwidth); - free(timestrs.user); - free(timestrs.system); - free(timestrs.elapsed); + HDfree(timestrs.user); + HDfree(timestrs.system); + HDfree(timestrs.elapsed); } #endif diff --git a/src/H5Tref.c b/src/H5Tref.c index 7092a71..c71724f 100644 --- a/src/H5Tref.c +++ b/src/H5Tref.c @@ -513,7 +513,7 @@ H5T__ref_mem_read(H5VL_object_t H5_ATTR_UNUSED *src_file, const void *src_buf, s /* Memory-to-memory conversion to support vlen conversion */ if (NULL == dst_file) { - HDmemcpy(dst_buf, src_buf, dst_size); + H5MM_memcpy(dst_buf, src_buf, dst_size); HGOTO_DONE(ret_value); } @@ -602,7 +602,7 @@ H5T__ref_mem_write(H5VL_object_t *src_file, const void *src_buf, size_t src_size /* Memory-to-memory conversion to support vlen conversion */ if (NULL == src_file) { - HDmemcpy(dst_buf, src_buf, src_size); + H5MM_memcpy(dst_buf, src_buf, src_size); HGOTO_DONE(ret_value); } @@ -650,6 +650,7 @@ H5T__ref_mem_write(H5VL_object_t *src_file, const void *src_buf, size_t src_size /* Pass the correct encoding version for the selection depending on the * file libver bounds, this is later retrieved in H5S hyper decode */ H5CX_set_libver_bounds(src_f); + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case H5R_OBJECT2: case H5R_ATTR: diff --git a/src/H5VLnative_object.c b/src/H5VLnative_object.c index ac717b0..81ff5ea 100644 --- a/src/H5VLnative_object.c +++ b/src/H5VLnative_object.c @@ -338,7 +338,7 @@ H5VL__native_object_specific(void *obj, const H5VL_loc_params_t *loc_params, /* Lookup object */ case H5VL_OBJECT_LOOKUP: { - H5O_token_t *token = va_arg(arguments, H5O_token_t *); + H5O_token_t *token = HDva_arg(arguments, H5O_token_t *); HDassert(token); @@ -175,9 +175,9 @@ H5Z_term_package(void) H5Z_stat_table_g[i].stats[dir].errors, timestrs.user, timestrs.system, timestrs.elapsed, bandwidth); next: - free(timestrs.user); - free(timestrs.system); - free(timestrs.elapsed); + HDfree(timestrs.user); + HDfree(timestrs.system); + HDfree(timestrs.elapsed); } /* end for */ } /* end for */ } /* end if */ @@ -990,6 +990,70 @@ done: } /* end H5Z_set_local_direct() */ /*------------------------------------------------------------------------- + * Function: H5Z_ignore_filters + * + * Purpose: Determine whether filters can be ignored. + * + * Description: + * When the filters are optional (i.e., H5Z_FLAG_OPTIONAL is provided,) + * if any of the following conditions is met, the filters will be ignored: + * - dataspace is either H5S_NULL or H5S_SCALAR + * - datatype is variable-length (string or non-string) + * However, if any of these conditions exists and a filter is not + * optional, the function will produce an error. + * + * Return: Non-negative(TRUE/FALSE) on success + * Negative on failure + * + *------------------------------------------------------------------------- + */ +htri_t +H5Z_ignore_filters(hid_t dcpl_id, const H5T_t *type, const H5S_t *space) +{ + H5P_genplist_t *dc_plist; /* Dataset creation property list object */ + H5O_pline_t pline; /* Object's I/O pipeline information */ + H5S_class_t space_class; /* To check class of space */ + H5T_class_t type_class; /* To check if type is VL */ + bool bad_for_filters = FALSE; /* Suitable to have filters */ + htri_t ret_value = FALSE; /* TRUE for ignoring filters */ + + FUNC_ENTER_NOAPI(FAIL) + + if (NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + + /* Get pipeline information */ + if (H5P_peek(dc_plist, H5O_CRT_PIPELINE_NAME, &pline) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't retrieve pipeline filter") + + /* Get datatype and dataspace classes for quick access */ + space_class = H5S_GET_EXTENT_TYPE(space); + type_class = H5T_get_class(type, FALSE); + + /* These conditions are not suitable for filters */ + bad_for_filters = (H5S_NULL == space_class || H5S_SCALAR == space_class || H5T_VLEN == type_class || + (H5T_STRING == type_class && TRUE == H5T_is_variable_str(type))); + + /* When these conditions occur, if there are required filters in pline, + then report a failure, otherwise, set flag that they can be ignored */ + if (bad_for_filters) { + size_t ii; + if (pline.nused > 0) { + for (ii = 0; ii < pline.nused; ii++) { + if (!(pline.filter[ii].flags & H5Z_FLAG_OPTIONAL)) + HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "not suitable for filters") + } + + /* All filters are optional, we can ignore them */ + ret_value = TRUE; + } + } /* bad for filters */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_ignore_filters() */ + +/*------------------------------------------------------------------------- * Function: H5Z_modify * * Purpose: Modify filter parameters for specified pipeline. diff --git a/src/H5Zprivate.h b/src/H5Zprivate.h index 66054bf..21d05b0 100644 --- a/src/H5Zprivate.h +++ b/src/H5Zprivate.h @@ -26,6 +26,7 @@ typedef struct H5Z_filter_info_t H5Z_filter_info_t; /* Private headers needed by this file */ #include "H5Tprivate.h" /* Datatypes */ +#include "H5Sprivate.h" /* Dataspace */ /**************************/ /* Library Private Macros */ @@ -86,6 +87,7 @@ H5_DLL herr_t H5Z_can_apply(hid_t dcpl_id, hid_t type_id); H5_DLL herr_t H5Z_set_local(hid_t dcpl_id, hid_t type_id); H5_DLL herr_t H5Z_can_apply_direct(const struct H5O_pline_t *pline); H5_DLL herr_t H5Z_set_local_direct(const struct H5O_pline_t *pline); +H5_DLL htri_t H5Z_ignore_filters(hid_t dcpl_id, const H5T_t *type, const H5S_t *space); H5_DLL H5Z_filter_info_t *H5Z_filter_info(const struct H5O_pline_t *pline, H5Z_filter_t filter); H5_DLL htri_t H5Z_filter_in_pline(const struct H5O_pline_t *pline, H5Z_filter_t filter); H5_DLL htri_t H5Z_all_filters_avail(const struct H5O_pline_t *pline); diff --git a/src/H5Zshuffle.c b/src/H5Zshuffle.c index bd28f84..2e3ff97 100644 --- a/src/H5Zshuffle.c +++ b/src/H5Zshuffle.c @@ -177,24 +177,31 @@ H5Z__filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[] case 0: do { DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 7: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 6: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 5: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 4: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 3: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 2: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 1: DUFF_GUTS @@ -241,24 +248,31 @@ H5Z__filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[] case 0: do { DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 7: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 6: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 5: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 4: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 3: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 2: DUFF_GUTS + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 1: DUFF_GUTS diff --git a/src/H5checksum.c b/src/H5checksum.c index 7ae588e..bc36bb9 100644 --- a/src/H5checksum.c +++ b/src/H5checksum.c @@ -414,36 +414,47 @@ H5_checksum_lookup3(const void *key, size_t length, uint32_t initval) { case 12: c += ((uint32_t)k[11]) << 24; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 11: c += ((uint32_t)k[10]) << 16; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 10: c += ((uint32_t)k[9]) << 8; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 9: c += k[8]; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 8: b += ((uint32_t)k[7]) << 24; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 7: b += ((uint32_t)k[6]) << 16; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 6: b += ((uint32_t)k[5]) << 8; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 5: b += k[4]; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 4: a += ((uint32_t)k[3]) << 24; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 3: a += ((uint32_t)k[2]) << 16; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 2: a += ((uint32_t)k[1]) << 8; + /* FALLTHROUGH */ H5_ATTR_FALLTHROUGH case 1: a += k[0]; diff --git a/src/H5err.txt b/src/H5err.txt index 9fec521..24ac2ac 100644 --- a/src/H5err.txt +++ b/src/H5err.txt @@ -136,6 +136,8 @@ MINOR, FILEACC, H5E_BADFILE, Bad file ID accessed MINOR, FILEACC, H5E_TRUNCATED, File has been truncated MINOR, FILEACC, H5E_MOUNT, File mount error MINOR, FILEACC, H5E_CANTDELETEFILE, Unable to delete file +MINOR, FILEACC, H5E_CANTLOCKFILE, Unable to lock file +MINOR, FILEACC, H5E_CANTUNLOCKFILE, Unable to unlock file # Generic low-level file I/O errors MINOR, FILE, H5E_SEEKERROR, Seek failed diff --git a/src/H5mpi.c b/src/H5mpi.c index c1f9132..613a4bf 100644 --- a/src/H5mpi.c +++ b/src/H5mpi.c @@ -25,7 +25,7 @@ /****************/ /* Local Macros */ /****************/ -#define TWO_GIG_LIMIT (1 << 31) +#define TWO_GIG_LIMIT INT32_MAX #ifndef H5_MAX_MPI_COUNT #define H5_MAX_MPI_COUNT (1 << 30) #endif @@ -33,7 +33,7 @@ /*******************/ /* Local Variables */ /*******************/ -static hsize_t bigio_count = H5_MAX_MPI_COUNT; +static hsize_t bigio_count_g = H5_MAX_MPI_COUNT; /*------------------------------------------------------------------------- * Function: H5_mpi_set_bigio_count @@ -42,7 +42,7 @@ static hsize_t bigio_count = H5_MAX_MPI_COUNT; * when we utilize derived datatypes. This is of * particular interest for allowing nightly testing * - * Return: The current/previous value of bigio_count. + * Return: The current/previous value of bigio_count_g. * * Programmer: Richard Warren, March 10, 2017 * @@ -51,10 +51,10 @@ static hsize_t bigio_count = H5_MAX_MPI_COUNT; hsize_t H5_mpi_set_bigio_count(hsize_t new_count) { - hsize_t orig_count = bigio_count; + hsize_t orig_count = bigio_count_g; if ((new_count > 0) && (new_count < (hsize_t)TWO_GIG_LIMIT)) { - bigio_count = new_count; + bigio_count_g = new_count; } return orig_count; } /* end H5_mpi_set_bigio_count() */ @@ -63,9 +63,9 @@ H5_mpi_set_bigio_count(hsize_t new_count) * Function: H5_mpi_get_bigio_count * * Purpose: Allow other HDF5 library functions to access - * the current value for bigio_count. + * the current value for bigio_count_g. * - * Return: The current/previous value of bigio_count. + * Return: The current/previous value of bigio_count_g. * * Programmer: Richard Warren, October 7, 2019 * @@ -74,7 +74,7 @@ H5_mpi_set_bigio_count(hsize_t new_count) hsize_t H5_mpi_get_bigio_count(void) { - return bigio_count; + return bigio_count_g; } /*------------------------------------------------------------------------- @@ -471,8 +471,8 @@ H5_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Datat FUNC_ENTER_NOAPI(FAIL) /* Calculate how many Big MPI datatypes are needed to represent the buffer */ - num_big_types = (int)(num_elements / bigio_count); - leftover = (hsize_t)num_elements - (hsize_t)num_big_types * bigio_count; + num_big_types = (int)(num_elements / bigio_count_g); + leftover = (hsize_t)num_elements - (hsize_t)num_big_types * bigio_count_g; H5_CHECKED_ASSIGN(remaining_bytes, int, leftover, hsize_t); /* Create a contiguous datatype of size equal to the largest @@ -481,11 +481,11 @@ H5_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Datat * use type_hvector to create the type with the displacement provided */ if (0 == stride_bytes) { - if (MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)bigio_count, old_type, &inner_type))) + if (MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)bigio_count_g, old_type, &inner_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) } /* end if */ else if (MPI_SUCCESS != - (mpi_code = MPI_Type_create_hvector((int)bigio_count, 1, stride_bytes, old_type, &inner_type))) + (mpi_code = MPI_Type_create_hvector((int)bigio_count_g, 1, stride_bytes, old_type, &inner_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) /* Create a contiguous datatype of the buffer (minus the remaining < 2GB part) @@ -510,7 +510,7 @@ H5_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Datat HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) } /* end if */ else if (MPI_SUCCESS != (mpi_code = MPI_Type_create_hvector( - (int)(num_elements - (hsize_t)num_big_types * bigio_count), 1, + (int)(num_elements - (hsize_t)num_big_types * bigio_count_g), 1, stride_bytes, old_type, &leftover_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) @@ -529,7 +529,7 @@ H5_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Datat block_len[0] = 1; block_len[1] = 1; disp[0] = 0; - disp[1] = (old_extent + stride_bytes) * num_big_types * (MPI_Aint)bigio_count; + disp[1] = (old_extent + stride_bytes) * num_big_types * (MPI_Aint)bigio_count_g; if (MPI_SUCCESS != (mpi_code = MPI_Type_create_struct(2, block_len, disp, type, new_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) diff --git a/src/H5private.h b/src/H5private.h index ca005f6..1390b3f 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -298,6 +298,10 @@ * Note that Solaris Studio supports attribute, but does not support the * attributes we use. * + * When using H5_ATTR_FALLTHROUGH, you should also include a comment that + * says FALLTHROUGH to reduce warnings on compilers that don't use + * attributes but do respect fall-through comments. + * * H5_ATTR_CONST is redefined in tools/h5repack/dynlib_rpk.c to quiet * gcc warnings (it has to use the public API and can't include this * file). Be sure to update that file if the #ifdefs change here. diff --git a/src/H5public.h b/src/H5public.h index 2de4459..29a78c6 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -92,11 +92,11 @@ extern "C" { #endif /* Version numbers */ -#define H5_VERS_MAJOR 1 /* For major interface/format changes */ -#define H5_VERS_MINOR 12 /* For minor interface/format changes */ -#define H5_VERS_RELEASE 1 /* For tweaks, bug-fixes, or development */ -#define H5_VERS_SUBRELEASE "3" /* For pre-releases like snap0 */ -/* Empty string for real releases. */ +#define H5_VERS_MAJOR 1 /* For major interface/format changes */ +#define H5_VERS_MINOR 12 /* For minor interface/format changes */ +#define H5_VERS_RELEASE 1 /* For tweaks, bug-fixes, or development */ +#define H5_VERS_SUBRELEASE "3" /* For pre-releases like snap0 */ + /* Empty string for real releases. */ #define H5_VERS_INFO "HDF5 library version: 1.12.1-3" /* Full version string */ #define H5check() H5check_version(H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE) @@ -333,13 +333,15 @@ typedef enum H5_index_t { H5_INDEX_N /* Number of indices defined */ } H5_index_t; -/* +/** * Storage info struct used by H5O_info_t and H5F_info_t */ +//! [H5_ih_info_t_snip] typedef struct H5_ih_info_t { - hsize_t index_size; /* btree and/or list */ + hsize_t index_size; /**< btree and/or list */ hsize_t heap_size; } H5_ih_info_t; +//! [H5_ih_info_t_snip] /* Tokens are unique and permanent identifiers that are * used to reference HDF5 objects in a container. */ diff --git a/src/Makefile.am b/src/Makefile.am index 118c0a3..4798627 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -41,7 +41,7 @@ DISTCLEANFILES=H5pubconf.h # library sources libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5lib_settings.c H5system.c \ - H5timer.c H5trace.c \ + H5timer.c H5trace.c \ H5A.c H5Abtree2.c H5Adense.c H5Adeprec.c H5Aint.c H5Atest.c \ H5AC.c H5ACdbg.c H5ACproxy_entry.c \ H5B.c H5Bcache.c H5Bdbg.c \ diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in index fb9deec..98390b8 100644 --- a/src/libhdf5.settings.in +++ b/src/libhdf5.settings.in @@ -89,5 +89,6 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ Using memory checker: @USINGMEMCHECKER@ Memory allocation sanity checks: @MEMORYALLOCSANITYCHECK@ Function stack tracing: @CODESTACK@ + Use file locking: @DESIRED_FILE_LOCKING@ Strict file format checks: @STRICT_FORMAT_CHECKS@ Optimization instrumentation: @INSTRUMENT_LIBRARY@ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 73d0617..4cfe8c9 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -690,6 +690,26 @@ if (HDF5_ENABLE_FORMATTERS) clang_format (HDF5_TEST_use_append_chunk_FORMAT use_append_chunk) endif () +set (use_append_chunk_mirror_SOURCES ${HDF5_TEST_SOURCE_DIR}/use_append_chunk_mirror.c ${HDF5_TEST_SOURCE_DIR}/use_common.c ${HDF5_TEST_SOURCE_DIR}/use.h) +add_executable (use_append_chunk_mirror ${use_append_chunk_mirror_SOURCES}) +target_compile_options(use_append_chunk_mirror PRIVATE "${HDF5_CMAKE_C_FLAGS}") +target_include_directories (use_append_chunk_mirror PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") +if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (use_append_chunk_mirror STATIC) + target_link_libraries (use_append_chunk_mirror PRIVATE ${HDF5_TEST_LIB_TARGET}) +else () + TARGET_C_PROPERTIES (use_append_chunk_mirror SHARED) + target_link_libraries (use_append_chunk_mirror PRIVATE ${HDF5_TEST_LIBSH_TARGET}) +endif () +set_target_properties (use_append_chunk_mirror PROPERTIES FOLDER test) + +#----------------------------------------------------------------------------- +# Add Target to clang-format +#----------------------------------------------------------------------------- +if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_TEST_use_append_chunk_mirror_FORMAT use_append_chunk_mirror) +endif () + set (use_append_mchunks_SOURCES ${HDF5_TEST_SOURCE_DIR}/use_append_mchunks.c ${HDF5_TEST_SOURCE_DIR}/use_common.c ${HDF5_TEST_SOURCE_DIR}/use.h) add_executable (use_append_mchunks ${use_append_mchunks_SOURCES}) target_compile_options(use_append_mchunks PRIVATE "${HDF5_CMAKE_C_FLAGS}") diff --git a/test/accum.c b/test/accum.c index 183f955..0b9fbad 100644 --- a/test/accum.c +++ b/test/accum.c @@ -2243,7 +2243,7 @@ test_swmr_write_big(hbool_t newest_format) char *const new_argv[] = {swmr_reader, NULL}; /* Run the reader */ status = HDexecv(SWMR_READER, new_argv); - HDprintf("errno from execv = %s\n", strerror(errno)); + HDprintf("errno from execv = %s\n", HDstrerror(errno)); FAIL_STACK_ERROR; } /* end if */ diff --git a/test/btree2.c b/test/btree2.c index 4391a3b..42f9a61 100644 --- a/test/btree2.c +++ b/test/btree2.c @@ -10023,14 +10023,14 @@ main(void) if (nerrors) goto error; - puts("All v2 B-tree tests passed."); + HDputs("All v2 B-tree tests passed."); h5_cleanup(FILENAME, fapl); return 0; error: - puts("*** TESTS FAILED ***"); + HDputs("*** TESTS FAILED ***"); H5E_BEGIN_TRY { H5Pclose(fapl); } H5E_END_TRY; diff --git a/test/cache.c b/test/cache.c index 0587aa7..b7269ba 100644 --- a/test/cache.c +++ b/test/cache.c @@ -2785,6 +2785,10 @@ write_permitted_check(int * * Modifications: * + * Updated tests to accommodate the case in which the + * slist is disabled. + * JRM -- 5/14/20 + * *------------------------------------------------------------------------- */ @@ -2982,7 +2986,8 @@ check_insert_entry(unsigned paged) if (pass) { if ((cache_ptr->index_len != 4) || (cache_ptr->index_size != 4 * entry_sizes[entry_type]) || - (cache_ptr->slist_len != 4) || (cache_ptr->slist_size != 4 * entry_sizes[entry_type]) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 4) || (cache_ptr->slist_size != 4 * entry_sizes[entry_type]))) || (cache_ptr->pl_len != 0) || (cache_ptr->pl_size != (size_t)0) || (cache_ptr->pel_len != 2) || (cache_ptr->pel_size != 2 * entry_sizes[entry_type]) || (cache_ptr->LRU_list_len != 2) || (cache_ptr->LRU_list_size != 2 * entry_sizes[entry_type]) @@ -3007,10 +3012,11 @@ check_insert_entry(unsigned paged) if ((cache_ptr->insertions[entry_type] != 4) || (cache_ptr->pinned_insertions[entry_type] != 2) || (cache_ptr->pins[entry_type] != 2) || (cache_ptr->unpins[entry_type] != 0) || (cache_ptr->dirty_pins[entry_type] != 0) || (cache_ptr->max_index_len != 4) || - (cache_ptr->max_index_size != 4 * entry_sizes[entry_type]) || (cache_ptr->max_slist_len != 4) || - (cache_ptr->max_slist_size != 4 * entry_sizes[entry_type]) || (cache_ptr->max_pl_len != 0) || - (cache_ptr->max_pl_size != (size_t)0) || (cache_ptr->max_pel_len != 2) || - (cache_ptr->max_pel_size != 2 * entry_sizes[entry_type])) { + (cache_ptr->max_index_size != 4 * entry_sizes[entry_type]) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 4) || (cache_ptr->slist_size != 4 * entry_sizes[entry_type]))) || + (cache_ptr->max_pl_len != 0) || (cache_ptr->max_pl_size != (size_t)0) || + (cache_ptr->max_pel_len != 2) || (cache_ptr->max_pel_size != 2 * entry_sizes[entry_type])) { pass = FALSE; failure_mssg = "Unexpected insert results 11."; @@ -3150,6 +3156,12 @@ check_flush_cache(unsigned paged) * * Modifications: * + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. + * + * JRM -- 5/14/20 + * *------------------------------------------------------------------------- */ @@ -3157,7 +3169,6 @@ static void check_flush_cache__empty_cache(H5F_t *file_ptr) { H5C_t *cache_ptr = file_ptr->shared->cache; - herr_t result; if (cache_ptr == NULL) { @@ -3172,52 +3183,35 @@ check_flush_cache__empty_cache(H5F_t *file_ptr) /* Test behaviour on an empty cache. Can't do much sanity * checking in this case, so simply check the return values. + * + * Check of return values is done in the H5C_FLUSH_CACHE() macro. */ if (pass) { - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - - if (result < 0) { - - pass = FALSE; - failure_mssg = "flush with flags = 0x00 failed on empty cache.\n"; - } + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "flush with flags = 0x00 failed on empty cache.\n") } if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); - - if (result < 0) { - - pass = FALSE; - failure_mssg = "flush with flags = 0x04 failed on empty cache.\n"; - } + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, + "flush with flags = 0x04 failed on empty cache.\n") } if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG); - - if (result < 0) { - - pass = FALSE; - failure_mssg = "flush with flags = 0x08 failed on empty cache.\n"; - } + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG, + "flush with flags = 0x08 failed on empty cache.\n") } if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_MARKED_ENTRIES_FLAG); - - if (result < 0) { - - pass = FALSE; - failure_mssg = "flush with flags = 0x10 failed on empty cache.\n"; - } + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_MARKED_ENTRIES_FLAG, + "flush with flags = 0x10 failed on empty cache.\n") } + return; + } /* check_flush_cache__empty_cache() */ /*------------------------------------------------------------------------- @@ -4435,6 +4429,12 @@ check_flush_cache__multi_entry(H5F_t *file_ptr) * * Modifications: * + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. + * + * JRM -- 5/14/20 + * *------------------------------------------------------------------------- */ @@ -4444,7 +4444,6 @@ check_flush_cache__multi_entry_test(H5F_t *file_ptr, int test_num, unsigned int { H5C_t * cache_ptr = file_ptr->shared->cache; static char msg[128]; - herr_t result; unsigned u; size_t total_entry_size = 0; test_entry_t *base_addr; @@ -4513,11 +4512,10 @@ check_flush_cache__multi_entry_test(H5F_t *file_ptr, int test_num, unsigned int if (pass) { - result = H5C_flush_cache(file_ptr, flush_flags); + H5C_FLUSH_CACHE(file_ptr, flush_flags, "dummy failure message.\n") - if (result < 0) { + if (!pass) { - pass = FALSE; HDsnprintf(msg, (size_t)128, "flush with flags 0x%x failed in multi entry test #%d.", flush_flags, test_num); failure_mssg = msg; @@ -4571,9 +4569,9 @@ check_flush_cache__multi_entry_test(H5F_t *file_ptr, int test_num, unsigned int /* clean up the cache to prep for the next test */ if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, "dummy mssg.\n") - if (result < 0) { + if (!pass) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Flush failed on cleanup in multi entry test #%d.", test_num); @@ -4618,6 +4616,12 @@ check_flush_cache__multi_entry_test(H5F_t *file_ptr, int test_num, unsigned int * * Modifications: * + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. + * + * JRM -- 5/16/20 + * *------------------------------------------------------------------------- */ @@ -4627,7 +4631,6 @@ check_flush_cache__pe_multi_entry_test(H5F_t *file_ptr, int test_num, unsigned i { H5C_t * cache_ptr = file_ptr->shared->cache; static char msg[128]; - herr_t result; unsigned u; int j; size_t total_entry_size = 0; @@ -4703,11 +4706,10 @@ check_flush_cache__pe_multi_entry_test(H5F_t *file_ptr, int test_num, unsigned i if (pass) { - result = H5C_flush_cache(file_ptr, flush_flags); + H5C_FLUSH_CACHE(file_ptr, flush_flags, "dummy failure message.\n") - if (result < 0) { + if (!pass) { - pass = FALSE; HDsnprintf(msg, (size_t)128, "flush with flags 0x%x failed in pe multi entry test #%d.", flush_flags, test_num); failure_mssg = msg; @@ -4762,9 +4764,9 @@ check_flush_cache__pe_multi_entry_test(H5F_t *file_ptr, int test_num, unsigned i /* clean up the cache to prep for the next test */ if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, "dummy mssg.\n") - if (result < 0) { + if (!pass) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Flush failed on cleanup in pe multi entry test #%d.", test_num); @@ -8006,6 +8008,12 @@ check_flush_cache__flush_ops(H5F_t *file_ptr) * * Modifications: * + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. + * + * JRM -- 5/16/20 + * *------------------------------------------------------------------------- */ @@ -8018,7 +8026,6 @@ check_flush_cache__flush_op_test(H5F_t *file_ptr, int test_num, unsigned int flu { H5C_t * cache_ptr = file_ptr->shared->cache; static char msg[128]; - herr_t result; int i; int j; test_entry_t *base_addr; @@ -8139,9 +8146,9 @@ check_flush_cache__flush_op_test(H5F_t *file_ptr, int test_num, unsigned int flu if (pass) { - result = H5C_flush_cache(file_ptr, flush_flags); + H5C_FLUSH_CACHE(file_ptr, flush_flags, "dummy failure message") - if (result < 0) { + if (!pass) { pass = FALSE; HDsnprintf(msg, (size_t)128, "flush with flags 0x%x failed in flush op test #%d.", flush_flags, @@ -8214,74 +8221,104 @@ check_flush_cache__flush_op_test(H5F_t *file_ptr, int test_num, unsigned int flu #if 0 /* This is useful debugging code. Lets keep it around for a while. */ - if(entry_ptr->size != check[i].expected_size) { - HDfprintf(stdout, "entry_ptr->size (expected) = %d (%d).\n", - (int)(entry_ptr->size), - (int)(check[i].expected_size)); - } - if((!entry_ptr->header.destroy_in_progress) && - (check[i].in_cache) && - (entry_ptr->header.size != check[i].expected_size)) { - HDfprintf(stdout, + if ( entry_ptr->size != check[i].expected_size ) { + + HDfprintf(stdout, "entry_ptr->size (expected) = %d (%d).\n", + (int)(entry_ptr->size), + (int)(check[i].expected_size)); + } + + if ( ( ! entry_ptr->header.destroy_in_progress ) && + ( check[i].in_cache ) && + ( entry_ptr->header.size != check[i].expected_size ) ) { + + HDfprintf(stdout, "(!destroy in progress and in cache and size (expected) = %d (%d).\n", (int)(entry_ptr->header.size), - (int)(check[i].expected_size)); - } - if(entry_ptr->at_main_addr != check[i].at_main_addr) { - HDfprintf(stdout, "(%d,%d) at main addr (expected) = %d (%d).\n", - (int)(check[i].entry_type), - (int)(check[i].entry_index), + (int)(check[i].expected_size)); + } + + if ( entry_ptr->at_main_addr != check[i].at_main_addr ) { + + HDfprintf(stdout, + "(%d,%d) at main addr (expected) = %d (%d).\n", + (int)(check[i].entry_type), + (int)(check[i].entry_index), (int)(entry_ptr->at_main_addr), - (int)(check[i].at_main_addr)); + (int)(check[i].at_main_addr)); + } + + if ( entry_ptr->is_dirty != check[i].is_dirty ) { + + HDfprintf(stdout, + "entry_ptr->is_dirty (expected) = %d (%d).\n", + (int)(entry_ptr->is_dirty), + (int)(check[i].is_dirty)); + } + + if ( entry_ptr->header.is_dirty != check[i].is_dirty ) { + + HDfprintf(stdout, + "entry_ptr->header.is_dirty (expected) = %d (%d).\n", + (int)(entry_ptr->header.is_dirty), + (int)(check[i].is_dirty)); + } + + if ( entry_ptr->is_protected != check[i].is_protected ) { + + HDfprintf(stdout, + "entry_ptr->is_protected (expected) = %d (%d).\n", + (int)(entry_ptr->is_protected), + (int)(check[i].is_protected)); + } + + if ( entry_ptr->header.is_protected != check[i].is_protected ) { + + HDfprintf(stdout, + "entry_ptr->header.is_protected (expected) = %d (%d).\n", + (int)(entry_ptr->is_protected), + (int)(check[i].is_protected)); + } + + if ( entry_ptr->is_pinned != check[i].is_pinned ) { + + HDfprintf(stdout, + "entry_ptr->is_pinned (expected) = %d (%d).\n", + (int)(entry_ptr->is_pinned), + (int)(check[i].is_pinned)); + } + + if ( entry_ptr->header.is_pinned != check[i].is_pinned ) { + + HDfprintf(stdout, + "entry_ptr->header.is_pinned (expected) = %d (%d).\n", + (int)(entry_ptr->header.is_pinned), + (int)(check[i].is_pinned)); + } + + if ( entry_ptr->deserialized != check[i].expected_deserialized ) { + + HDfprintf(stdout, + "entry_ptr->deserialized (expected) = %d (%d).\n", + (int)(entry_ptr->deserialized), + (int)(check[i].expected_deserialized)); + } + + if ( entry_ptr->serialized != check[i].expected_serialized ) { + + HDfprintf(stdout, + "entry_ptr->serialized (expected) = %d (%d).\n", + (int)(entry_ptr->serialized), + (int)(check[i].expected_serialized)); + } + + if ( entry_ptr->destroyed != check[i].expected_destroyed ) { + + HDfprintf(stdout, \ + "entry_ptr->destroyed (expected) = %d (%d).\n", + (int)(entry_ptr->destroyed), + (int)(check[i].expected_destroyed)); } - if(entry_ptr->is_dirty != check[i].is_dirty) { - HDfprintf(stdout, "entry_ptr->is_dirty (expected) = %d (%d).\n", - (int)(entry_ptr->is_dirty), - (int)(check[i].is_dirty)); - } - if(entry_ptr->header.is_dirty != check[i].is_dirty) { - HDfprintf(stdout, "entry_ptr->header.is_dirty (expected) = %d (%d).\n", - (int)(entry_ptr->header.is_dirty), - (int)(check[i].is_dirty)); - } - if(entry_ptr->is_protected != check[i].is_protected) { - HDfprintf(stdout, "entry_ptr->is_protected (expected) = %d (%d).\n", - (int)(entry_ptr->is_protected), - (int)(check[i].is_protected)); - } - if(entry_ptr->header.is_protected != check[i].is_protected) { - HDfprintf(stdout, "entry_ptr->header.is_protected (expected) = %d (%d).\n", - (int)(entry_ptr->is_protected), - (int)(check[i].is_protected)); - } - if(entry_ptr->is_pinned != check[i].is_pinned) { - HDfprintf(stdout, "entry_ptr->is_pinned (expected) = %d (%d).\n", - (int)(entry_ptr->is_pinned), - (int)(check[i].is_pinned)); - } - if(entry_ptr->header.is_pinned != check[i].is_pinned) { - HDfprintf(stdout, "entry_ptr->header.is_pinned (expected) = %d (%d).\n", - (int)(entry_ptr->header.is_pinned), - (int)(check[i].is_pinned)); - } - if(entry_ptr->deserialized != - check[i].expected_deserialized) { - HDfprintf(stdout, - "entry_ptr->deserialized (expected) = %d (%d).\n", - (int)(entry_ptr->deserialized), - (int)(check[i].expected_deserialized)); - } - if(entry_ptr->serialized != check[i].expected_serialized) { - HDfprintf(stdout, - "entry_ptr->serialized (expected) = %d (%d).\n", - (int)(entry_ptr->serialized), - (int)(check[i].expected_serialized)); - } - if(entry_ptr->destroyed != check[i].expected_destroyed) { - HDfprintf(stdout, "entry_ptr->destroyed (expected) = %d (%d).\n", - (int)(entry_ptr->destroyed), - (int)(check[i].expected_destroyed)); - } #endif pass = FALSE; HDsnprintf(msg, (size_t)128, "Check2 failed on entry %d after flush op test #%d.", i, @@ -8310,11 +8347,10 @@ check_flush_cache__flush_op_test(H5F_t *file_ptr, int test_num, unsigned int flu /* clean up the cache to prep for the next test */ if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, "dummy mssg.") - if (result < 0) { + if (!pass) { - pass = FALSE; HDsnprintf(msg, (size_t)128, "Flush failed on cleanup in flush op test #%d.", test_num); failure_mssg = msg; } @@ -8377,10 +8413,16 @@ check_flush_cache__flush_op_test(H5F_t *file_ptr, int test_num, unsigned int flu * * Modifications: * - * Updated test for minor changes in the behaviour - * of H5C__flush_single_entry(). + * Updated test for minor changes in the behaviour + * of H5C__flush_single_entry(). + * + * JRM -- 2/16/15 * - * JRM -- 2/16/15 + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. + * + * JRM -- 5/16/20 * *------------------------------------------------------------------------- */ @@ -8393,7 +8435,6 @@ check_flush_cache__flush_op_eviction_test(H5F_t *file_ptr) int num_variable_entries = 10; int num_monster_entries = 31; int num_large_entries = 0; - herr_t result; test_entry_t * entry_ptr; test_entry_t * base_addr; struct expected_entry_status expected[10 + 31 + 14] = { @@ -10407,14 +10448,10 @@ check_flush_cache__flush_op_eviction_test(H5F_t *file_ptr) if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); - - if (result < 0) { + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, + "Cache flush invalidate failed after flush op eviction test") - pass = FALSE; - failure_mssg = "Cache flush invalidate failed after flush op eviction test"; - } - else if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0)) { + if ((pass) && ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0))) { pass = FALSE; failure_mssg = "Unexpected cache len/size after cleanup of flush op eviction test"; @@ -12150,6 +12187,12 @@ check_flush_cache__single_entry(H5F_t *file_ptr) * * Modifications: * + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. + * + * JRM -- 5/14/20 + * *------------------------------------------------------------------------- */ @@ -12161,7 +12204,6 @@ check_flush_cache__single_entry_test(H5F_t *file_ptr, int test_num, int entry_ty { H5C_t * cache_ptr = file_ptr->shared->cache; static char msg[128]; - herr_t result; test_entry_t *base_addr; test_entry_t *entry_ptr = NULL; @@ -12204,11 +12246,10 @@ check_flush_cache__single_entry_test(H5F_t *file_ptr, int test_num, int entry_ty if (pass) { - result = H5C_flush_cache(file_ptr, flush_flags); + H5C_FLUSH_CACHE(file_ptr, flush_flags, "dummy failure mssg.") - if (result < 0) { + if (!pass) { /* construct and set actual failure message */ - pass = FALSE; HDsnprintf(msg, (size_t)128, "flush with flags 0x%x failed in single entry test #%d.", flush_flags, test_num); @@ -12249,11 +12290,10 @@ check_flush_cache__single_entry_test(H5F_t *file_ptr, int test_num, int entry_ty /* clean up the cache to prep for the next test */ if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, "dummy failure mssg.") - if (result < 0) { + if (!pass) { /* construct and set actual failure message */ - pass = FALSE; HDsnprintf(msg, (size_t)128, "Flush failed on cleanup in single entry test #%d.", test_num); failure_mssg = msg; } @@ -12288,13 +12328,18 @@ check_flush_cache__single_entry_test(H5F_t *file_ptr, int test_num, int entry_ty * * Modifications: * - * JRM -- 5/17/06 - * Added the pop_mark_dirty_prot and pop_mark_dirty_pinned - * flags and supporting code to allow us to test the - * H5C_mark_entry_dirty() call. Use the - * call to mark the entry dirty while the entry is protected - * if pop_mark_dirty_prot is TRUE, and to mark the entry - * dirty while it is pinned if pop_mark_dirty_pinned is TRUE. + * JRM -- 5/17/06 + * Added the pop_mark_dirty_prot and pop_mark_dirty_pinned + * flags and supporting code to allow us to test the + * H5C_mark_entry_dirty() call. Use the + * call to mark the entry dirty while the entry is protected + * if pop_mark_dirty_prot is TRUE, and to mark the entry + * dirty while it is pinned if pop_mark_dirty_pinned is TRUE. + * + * JRM -- 5/14/20 + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. * *------------------------------------------------------------------------- */ @@ -12310,7 +12355,6 @@ check_flush_cache__pinned_single_entry_test(H5F_t *file_ptr, int test_num, int e H5C_t * cache_ptr = file_ptr->shared->cache; static char msg[128]; hbool_t expected_deserialized = TRUE; - herr_t result; test_entry_t *base_addr; test_entry_t *entry_ptr = NULL; @@ -12364,11 +12408,10 @@ check_flush_cache__pinned_single_entry_test(H5F_t *file_ptr, int test_num, int e if (pass) { - result = H5C_flush_cache(file_ptr, flush_flags); + H5C_FLUSH_CACHE(file_ptr, flush_flags, "dummy failure message\n") - if (result < 0) { + if (!pass) { /* construct and set the correct failure message */ - pass = FALSE; HDsnprintf(msg, (size_t)128, "flush with flags 0x%x failed in pinned single entry test #%d.", flush_flags, test_num); failure_mssg = msg; @@ -12423,11 +12466,10 @@ check_flush_cache__pinned_single_entry_test(H5F_t *file_ptr, int test_num, int e if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, "dummy mssg\n") - if (result < 0) { + if (!pass) { - pass = FALSE; HDsnprintf(msg, (size_t)128, "Flush failed on cleanup in pinned single entry test #%d.", test_num); failure_mssg = msg; @@ -13714,6 +13756,12 @@ check_pin_protected_entry(unsigned paged) * Programmer: John Mainzer * 7/7/06 * + * Modifications: + * + * Updated function to allow for disabling of the slist. + * + * JRM -- 5/18/20 + * *------------------------------------------------------------------------- */ @@ -13878,7 +13926,8 @@ check_resize_entry(unsigned paged) if (pass) { if ((cache_ptr->index_len != 1) || (cache_ptr->index_size != (LARGE_ENTRY_SIZE / 2)) || - (cache_ptr->slist_len != 1) || (cache_ptr->slist_size != (LARGE_ENTRY_SIZE / 2))) { + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 1) || (cache_ptr->slist_size != (LARGE_ENTRY_SIZE / 2))))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 3."); @@ -13952,7 +14001,8 @@ check_resize_entry(unsigned paged) if (pass) { if ((cache_ptr->index_len != 1) || (cache_ptr->index_size != LARGE_ENTRY_SIZE) || - (cache_ptr->slist_len != 1) || (cache_ptr->slist_size != LARGE_ENTRY_SIZE)) { + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 1) || (cache_ptr->slist_size != LARGE_ENTRY_SIZE)))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 4."); @@ -14008,7 +14058,8 @@ check_resize_entry(unsigned paged) if (pass) { if ((cache_ptr->index_len != 1) || (cache_ptr->index_size != (LARGE_ENTRY_SIZE / 4)) || - (cache_ptr->slist_len != 1) || (cache_ptr->slist_size != (LARGE_ENTRY_SIZE / 4))) { + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 1) || (cache_ptr->slist_size != (LARGE_ENTRY_SIZE / 4))))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 5."); @@ -14057,7 +14108,8 @@ check_resize_entry(unsigned paged) if (pass) { if ((cache_ptr->index_len != 1) || (cache_ptr->index_size != LARGE_ENTRY_SIZE) || - (cache_ptr->slist_len != 1) || (cache_ptr->slist_size != LARGE_ENTRY_SIZE)) { + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 1) || (cache_ptr->slist_size != LARGE_ENTRY_SIZE)))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 6."); @@ -14125,8 +14177,8 @@ check_resize_entry(unsigned paged) if (pass) { - if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0) || (cache_ptr->slist_len != 0) || - (cache_ptr->slist_size != 0)) { + if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0) || + ((cache_ptr->slist_enabled) && ((cache_ptr->slist_len != 0) || (cache_ptr->slist_size != 0)))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 7."); @@ -14138,8 +14190,8 @@ check_resize_entry(unsigned paged) if (pass) { - if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0) || (cache_ptr->slist_len != 0) || - (cache_ptr->slist_size != 0)) { + if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0) || + ((cache_ptr->slist_enabled) && ((cache_ptr->slist_len != 0) || (cache_ptr->slist_size != 0)))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 8."); @@ -14166,7 +14218,8 @@ check_resize_entry(unsigned paged) if (pass) { if ((cache_ptr->index_len != 3) || (cache_ptr->index_size != 3 * LARGE_ENTRY_SIZE) || - (cache_ptr->slist_len != 1) || (cache_ptr->slist_size != LARGE_ENTRY_SIZE)) { + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 1) || (cache_ptr->slist_size != LARGE_ENTRY_SIZE)))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 9."); @@ -14182,7 +14235,8 @@ check_resize_entry(unsigned paged) if (pass) { if ((cache_ptr->index_len != 4) || (cache_ptr->index_size != 4 * LARGE_ENTRY_SIZE) || - (cache_ptr->slist_len != 1) || (cache_ptr->slist_size != LARGE_ENTRY_SIZE)) { + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 1) || (cache_ptr->slist_size != LARGE_ENTRY_SIZE)))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 10."); @@ -14251,8 +14305,9 @@ check_resize_entry(unsigned paged) if ((cache_ptr->index_len != 4) || (cache_ptr->index_size != ((3 * LARGE_ENTRY_SIZE) + (LARGE_ENTRY_SIZE / 2))) || - (cache_ptr->slist_len != 2) || - (cache_ptr->slist_size != (LARGE_ENTRY_SIZE + (LARGE_ENTRY_SIZE / 2)))) { + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 2) || + (cache_ptr->slist_size != (LARGE_ENTRY_SIZE + (LARGE_ENTRY_SIZE / 2)))))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 11."); @@ -14326,7 +14381,8 @@ check_resize_entry(unsigned paged) if (pass) { if ((cache_ptr->index_len != 4) || (cache_ptr->index_size != 4 * LARGE_ENTRY_SIZE) || - (cache_ptr->slist_len != 2) || (cache_ptr->slist_size != 2 * LARGE_ENTRY_SIZE)) { + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 2) || (cache_ptr->slist_size != 2 * LARGE_ENTRY_SIZE)))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 12."); @@ -14383,8 +14439,9 @@ check_resize_entry(unsigned paged) if ((cache_ptr->index_len != 4) || (cache_ptr->index_size != ((3 * LARGE_ENTRY_SIZE) + (LARGE_ENTRY_SIZE / 4))) || - (cache_ptr->slist_len != 2) || - (cache_ptr->slist_size != (LARGE_ENTRY_SIZE + (LARGE_ENTRY_SIZE / 4)))) { + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 2) || + (cache_ptr->slist_size != (LARGE_ENTRY_SIZE + (LARGE_ENTRY_SIZE / 4)))))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 13."); @@ -14433,7 +14490,8 @@ check_resize_entry(unsigned paged) if (pass) { if ((cache_ptr->index_len != 4) || (cache_ptr->index_size != (4 * LARGE_ENTRY_SIZE)) || - (cache_ptr->slist_len != 2) || (cache_ptr->slist_size != (2 * LARGE_ENTRY_SIZE))) { + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 2) || (cache_ptr->slist_size != (2 * LARGE_ENTRY_SIZE))))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 14."); @@ -14502,7 +14560,8 @@ check_resize_entry(unsigned paged) if (pass) { if ((cache_ptr->index_len != 3) || (cache_ptr->index_size != (3 * LARGE_ENTRY_SIZE)) || - (cache_ptr->slist_len != 1) || (cache_ptr->slist_size != LARGE_ENTRY_SIZE)) { + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 1) || (cache_ptr->slist_size != LARGE_ENTRY_SIZE)))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 15."); @@ -14524,8 +14583,8 @@ check_resize_entry(unsigned paged) if (pass) { - if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0) || (cache_ptr->slist_len != 0) || - (cache_ptr->slist_size != 0)) { + if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0) || + ((cache_ptr->slist_enabled) && ((cache_ptr->slist_len != 0) || (cache_ptr->slist_size != 0)))) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 16."); @@ -14567,6 +14626,9 @@ check_resize_entry(unsigned paged) * * Modifications: * + * Updated function to allow for disabling of the slist. + * + * JRM -- 5/18/20 * *------------------------------------------------------------------------- */ @@ -14656,8 +14718,9 @@ check_evictions_enabled(unsigned paged) /* verify that it is empty */ if (pass) { - if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0) || (cache_ptr->slist_len != 0) || - (cache_ptr->slist_size != 0) || (cache_ptr->evictions_enabled != TRUE)) { + if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0) || + ((cache_ptr->slist_enabled) && ((cache_ptr->slist_len != 0) || (cache_ptr->slist_size != 0))) || + (cache_ptr->evictions_enabled != TRUE)) { pass = FALSE; HDsnprintf(msg, (size_t)128, "Unexpected cache status 1."); @@ -14700,7 +14763,7 @@ check_evictions_enabled(unsigned paged) if (pass) { if ((cache_ptr->index_len != 16) || (cache_ptr->index_size != 16 * MONSTER_ENTRY_SIZE) || - (cache_ptr->slist_len != 0) || (cache_ptr->slist_size != 0) || + ((cache_ptr->slist_enabled) && ((cache_ptr->slist_len != 0) || (cache_ptr->slist_size != 0))) || (cache_ptr->evictions_enabled != TRUE)) { pass = FALSE; @@ -14726,7 +14789,7 @@ check_evictions_enabled(unsigned paged) if (pass) { if ((cache_ptr->index_len != 16) || (cache_ptr->index_size != 16 * MONSTER_ENTRY_SIZE) || - (cache_ptr->slist_len != 0) || (cache_ptr->slist_size != 0) || + ((cache_ptr->slist_enabled) && ((cache_ptr->slist_len != 0) || (cache_ptr->slist_size != 0))) || (cache_ptr->evictions_enabled != TRUE)) { pass = FALSE; @@ -14781,7 +14844,8 @@ check_evictions_enabled(unsigned paged) if (pass) { if ((cache_ptr->index_len != 16) || (cache_ptr->index_size != 16 * MONSTER_ENTRY_SIZE) || - (cache_ptr->slist_len != 1) || (cache_ptr->slist_size != MONSTER_ENTRY_SIZE) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 1) || (cache_ptr->slist_size != MONSTER_ENTRY_SIZE))) || (cache_ptr->evictions_enabled != TRUE)) { pass = FALSE; @@ -14843,7 +14907,8 @@ check_evictions_enabled(unsigned paged) if (pass) { if ((cache_ptr->index_len != 16) || (cache_ptr->index_size != 16 * MONSTER_ENTRY_SIZE) || - (cache_ptr->slist_len != 1) || (cache_ptr->slist_size != MONSTER_ENTRY_SIZE) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 1) || (cache_ptr->slist_size != MONSTER_ENTRY_SIZE))) || (cache_ptr->evictions_enabled != FALSE)) { pass = FALSE; @@ -14869,7 +14934,8 @@ check_evictions_enabled(unsigned paged) if (pass) { if ((cache_ptr->index_len != 17) || (cache_ptr->index_size != 17 * MONSTER_ENTRY_SIZE) || - (cache_ptr->slist_len != 1) || (cache_ptr->slist_size != MONSTER_ENTRY_SIZE) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 1) || (cache_ptr->slist_size != MONSTER_ENTRY_SIZE))) || (cache_ptr->evictions_enabled != FALSE)) { pass = FALSE; @@ -14894,7 +14960,8 @@ check_evictions_enabled(unsigned paged) if (pass) { if ((cache_ptr->index_len != 18) || (cache_ptr->index_size != 18 * MONSTER_ENTRY_SIZE) || - (cache_ptr->slist_len != 2) || (cache_ptr->slist_size != 2 * MONSTER_ENTRY_SIZE) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 2) || (cache_ptr->slist_size != 2 * MONSTER_ENTRY_SIZE))) || (cache_ptr->evictions_enabled != FALSE)) { pass = FALSE; @@ -14936,7 +15003,8 @@ check_evictions_enabled(unsigned paged) if (pass) { if ((cache_ptr->index_len != 18) || (cache_ptr->index_size != 18 * MONSTER_ENTRY_SIZE) || - (cache_ptr->slist_len != 2) || (cache_ptr->slist_size != 2 * MONSTER_ENTRY_SIZE) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 2) || (cache_ptr->slist_size != 2 * MONSTER_ENTRY_SIZE))) || (cache_ptr->evictions_enabled != TRUE)) { pass = FALSE; @@ -14965,7 +15033,8 @@ check_evictions_enabled(unsigned paged) if (pass) { if ((cache_ptr->index_len != 16) || (cache_ptr->index_size != 16 * MONSTER_ENTRY_SIZE) || - (cache_ptr->slist_len != 2) || (cache_ptr->slist_size != 2 * MONSTER_ENTRY_SIZE) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 2) || (cache_ptr->slist_size != 2 * MONSTER_ENTRY_SIZE))) || (cache_ptr->evictions_enabled != TRUE)) { pass = FALSE; @@ -15069,7 +15138,8 @@ check_evictions_enabled(unsigned paged) if (pass) { if ((cache_ptr->index_len != 17) || (cache_ptr->index_size != 17 * MONSTER_ENTRY_SIZE) || - (cache_ptr->slist_len != 2) || (cache_ptr->slist_size != 2 * MONSTER_ENTRY_SIZE) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 2) || (cache_ptr->slist_size != 2 * MONSTER_ENTRY_SIZE))) || (cache_ptr->evictions_enabled != FALSE)) { pass = FALSE; @@ -15110,7 +15180,8 @@ check_evictions_enabled(unsigned paged) if (pass) { if ((cache_ptr->index_len != 16) || (cache_ptr->index_size != 16 * MONSTER_ENTRY_SIZE) || - (cache_ptr->slist_len != 3) || (cache_ptr->slist_size != 3 * MONSTER_ENTRY_SIZE) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->slist_len != 3) || (cache_ptr->slist_size != 3 * MONSTER_ENTRY_SIZE))) || (cache_ptr->evictions_enabled != TRUE)) { pass = FALSE; @@ -15205,13 +15276,19 @@ check_evictions_enabled(unsigned paged) * * Modifications: * + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). + * + * JRM -- 5/14/20 + * *------------------------------------------------------------------------- */ static unsigned check_flush_protected_err(unsigned paged) { - H5F_t *file_ptr = NULL; + H5F_t *file_ptr = NULL; + H5C_t *cache_ptr = NULL; if (paged) TESTING("flush cache with protected entry error (paged aggregation)") @@ -15231,27 +15308,41 @@ check_flush_protected_err(unsigned paged) file_ptr = setup_cache((size_t)(2 * 1024), (size_t)(1 * 1024), paged); + if (pass) { + + cache_ptr = file_ptr->shared->cache; + } + protect_entry(file_ptr, 0, 0); - if (H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET) >= 0) { + /* enable slist prior to flush */ + if ((pass) && (H5C_set_slist_enabled(cache_ptr, TRUE, FALSE) < 0)) { + + pass = FALSE; + failure_mssg = "unable to enable slist prior to flush.\n"; + } + + if ((pass) && (H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET) >= 0)) { pass = FALSE; failure_mssg = "flush succeeded on cache with protected entry.\n"; } - else { - unprotect_entry(file_ptr, 0, 0, H5C__DIRTIED_FLAG); + /* disable the slist after the flush */ + if ((pass) && (H5C_set_slist_enabled(cache_ptr, FALSE, FALSE) < 0)) { - if (H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET) < 0) { + pass = FALSE; + failure_mssg = "unable to disable slist after flush.\n"; + } - pass = FALSE; - failure_mssg = "flush failed after unprotect.\n"; - } - else { + unprotect_entry(file_ptr, 0, 0, H5C__DIRTIED_FLAG); - takedown_cache(file_ptr, FALSE, FALSE); - } + if (pass) { + + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "flush failed after unprotect.\n") } + + takedown_cache(file_ptr, FALSE, FALSE); } if (pass) { @@ -29821,10 +29912,18 @@ done: * Purpose: Verify that the order that entries with flush dependencies * is correct * - * Return: 0 on success, non-zero on failure + * Return: 0 on success, non-zero on failure * - * Programmer: Quincey Koziol - * 3/17/09 + * Programmer: Quincey Koziol + * 3/17/09 + * + * Modifications: + * + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. + * + * JRM -- 5/14/20 * *------------------------------------------------------------------------- */ @@ -29923,8 +30022,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -29938,8 +30035,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -30024,8 +30121,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -30051,8 +30146,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -30139,8 +30234,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 2, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -30170,8 +30263,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -30266,8 +30359,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 2, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -30297,8 +30388,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -30438,8 +30529,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 2, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -30487,8 +30576,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -30665,8 +30754,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 2, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -30714,8 +30801,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -30841,8 +30928,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 2, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -30880,8 +30965,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -30979,8 +31064,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 2, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -31018,8 +31101,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -31171,8 +31254,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -31223,8 +31304,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -31426,8 +31507,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 2, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -31479,8 +31558,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -31702,8 +31781,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 2, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -31755,8 +31832,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -32017,8 +32094,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 2, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -32066,8 +32141,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -32361,8 +32436,6 @@ check_flush_deps_order(unsigned paged) /* Flush the cache and verify that the entries were flushed in correct order */ { - herr_t result; /* Generic return value */ - add_flush_op(entry_type, 0, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 1, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); add_flush_op(entry_type, 2, FLUSH_OP__ORDER, entry_type, 0, FALSE, (size_t)0, &flush_order); @@ -32410,8 +32483,8 @@ check_flush_deps_order(unsigned paged) /* Reset index for tracking flush order */ flush_order = 0; - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); - if (result < 0) + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "dummy mssg") + if (!pass) CACHE_ERROR("flushing entries with flush dependendices") /* Change expected values, and verify the status of the entries @@ -33623,7 +33696,11 @@ check_entry_deletions_during_scans(unsigned paged) * * Modifications: * - * None. + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. + * + * JRM -- 5/14/20 * *------------------------------------------------------------------------- */ @@ -33633,7 +33710,6 @@ cedds__expunge_dirty_entry_in_flush_test(H5F_t *file_ptr) { H5C_t *cache_ptr = file_ptr->shared->cache; int i; - herr_t result; /* clang-format off */ struct expected_entry_status expected[36] = { @@ -33755,14 +33831,10 @@ cedds__expunge_dirty_entry_in_flush_test(H5F_t *file_ptr) if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, + "Cache flush inval failed in cedds expunge dirty entry in flush test") - if (result < 0) { - - pass = FALSE; - failure_mssg = "Cache flush invalidate failed in cedds expunge dirty entry in flush test"; - } - else if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0)) { + if ((pass) && ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0))) { pass = FALSE; failure_mssg = "Unexpected cache len/size after cedds expunge dirty entry in flush test"; @@ -33838,7 +33910,11 @@ cedds__expunge_dirty_entry_in_flush_test(H5F_t *file_ptr) * * Modifications: * - * None. + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. + * + * JRM -- 5/14/20 * *------------------------------------------------------------------------- */ @@ -33850,7 +33926,6 @@ cedds__H5C_make_space_in_cache(H5F_t *file_ptr) int i; const int num_huge_entries = 4; const int num_monster_entries = 32; - herr_t result; /* clang-format off */ struct expected_entry_status expected[36] = { @@ -34089,14 +34164,10 @@ cedds__H5C_make_space_in_cache(H5F_t *file_ptr) if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); - - if (result < 0) { + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, + "Cache flush invalidate failed after flush op eviction test") - pass = FALSE; - failure_mssg = "Cache flush invalidate failed after flush op eviction test"; - } - else if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0)) { + if ((pass) && ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0))) { pass = FALSE; failure_mssg = "Unexpected cache len/size after cleanup of flush op eviction test"; @@ -34199,7 +34270,11 @@ cedds__H5C_make_space_in_cache(H5F_t *file_ptr) * * Modifications: * - * None. + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. + * + * JRM -- 5/14/20 * *------------------------------------------------------------------------- */ @@ -34507,14 +34582,10 @@ cedds__H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t *file_ptr) if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); - - if (result < 0) { + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, + "Cache flush invalidate failed after flush op eviction test") - pass = FALSE; - failure_mssg = "Cache flush invalidate failed after flush op eviction test"; - } - else if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0)) { + if ((pass) && ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0))) { pass = FALSE; failure_mssg = "Unexpected cache len/size after cleanup of flush op eviction test"; @@ -34658,7 +34729,11 @@ cedds__H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t *file_ptr) * * Modifications: * - * None. + * Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. + * + * JRM -- 5/14/20 * *------------------------------------------------------------------------- */ @@ -34669,7 +34744,6 @@ cedds__H5C_flush_invalidate_cache__bucket_scan(H5F_t *file_ptr) H5C_t * cache_ptr = file_ptr->shared->cache; int i; int expected_hash_bucket = 0; - herr_t result; haddr_t entry_addr; test_entry_t * entry_ptr; test_entry_t * base_addr = NULL; @@ -34883,14 +34957,10 @@ cedds__H5C_flush_invalidate_cache__bucket_scan(H5F_t *file_ptr) if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); - - if (result < 0) { + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, + "Cache flush invalidate failed after flush op eviction test") - pass = FALSE; - failure_mssg = "Cache flush invalidate failed after flush op eviction test"; - } - else if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0)) { + if ((pass) && ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0))) { pass = FALSE; failure_mssg = "Unexpected cache len/size after cleanup of flush op eviction test"; @@ -35050,7 +35120,15 @@ check_stats(unsigned paged) * * Modifications: * - * None. + * Modified slist stats checks to allow for the case that + * the slist is disabled. + * + * Also added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via the + * H5C_FLUSH_CACHE macro. + * + * JRM -- 5/14/20 + * * *------------------------------------------------------------------------- */ @@ -35060,7 +35138,6 @@ check_stats__smoke_check_1(H5F_t *file_ptr) { H5C_t *cache_ptr = file_ptr->shared->cache; int i; - herr_t result; if (pass) { if (cache_ptr == NULL) { @@ -35120,13 +35197,15 @@ check_stats__smoke_check_1(H5F_t *file_ptr) failure_mssg = "Unexpected monster size entry stats in check_stats__smoke_check_1(1)."; } /* end if */ - if (pass) + if (pass) { + if ((cache_ptr->total_ht_insertions != 32) || (cache_ptr->total_ht_deletions != 0) || (cache_ptr->successful_ht_searches != 0) || (cache_ptr->total_successful_ht_search_depth != 0) || (cache_ptr->failed_ht_searches != 32) || (cache_ptr->total_failed_ht_search_depth != 48) || (cache_ptr->max_index_len != 32) || (cache_ptr->max_index_size != 2 * 1024 * 1024) || (cache_ptr->max_clean_index_size != 0) || (cache_ptr->max_dirty_index_size != 2 * 1024 * 1024) || - (cache_ptr->max_slist_len != 32) || (cache_ptr->max_slist_size != 2 * 1024 * 1024) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->max_slist_len != 32) || (cache_ptr->max_slist_size != 2 * 1024 * 1024))) || (cache_ptr->max_pl_len != 0) || (cache_ptr->max_pl_size != 0) || (cache_ptr->max_pel_len != 0) || (cache_ptr->max_pel_size != 0) || (cache_ptr->calls_to_msic != 0) || (cache_ptr->total_entries_skipped_in_msic != 0) || @@ -35138,6 +35217,7 @@ check_stats__smoke_check_1(H5F_t *file_ptr) pass = FALSE; failure_mssg = "Unexpected cache stats in check_stats__smoke_check_1(1)."; } /* end if */ + } #if H5C_COLLECT_CACHE_ENTRY_STATS if (pass) @@ -35190,17 +35270,19 @@ check_stats__smoke_check_1(H5F_t *file_ptr) failure_mssg = "Unexpected monster size entry stats in check_stats__smoke_check_1(2)."; } /* end if */ - if (pass) + if (pass) { + if ((cache_ptr->total_ht_insertions != 32) || (cache_ptr->total_ht_deletions != 0) || (cache_ptr->successful_ht_searches != 32) || (cache_ptr->total_successful_ht_search_depth != 96) || (cache_ptr->failed_ht_searches != 32) || (cache_ptr->total_failed_ht_search_depth != 48) || (cache_ptr->max_index_len != 32) || (cache_ptr->max_index_size != 2 * 1024 * 1024) || (cache_ptr->max_clean_index_size != 0) || - (cache_ptr->max_dirty_index_size != 2 * 1024 * 1024) || (cache_ptr->max_slist_len != 32) || - (cache_ptr->max_slist_size != 2 * 1024 * 1024) || (cache_ptr->max_pl_len != 1) || - (cache_ptr->max_pl_size != 64 * 1024) || (cache_ptr->max_pel_len != 0) || - (cache_ptr->max_pel_size != 0) || (cache_ptr->calls_to_msic != 0) || - (cache_ptr->total_entries_skipped_in_msic != 0) || + (cache_ptr->max_dirty_index_size != 2 * 1024 * 1024) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->max_slist_len != 32) || (cache_ptr->max_slist_size != 2 * 1024 * 1024))) || + (cache_ptr->max_pl_len != 1) || (cache_ptr->max_pl_size != 64 * 1024) || + (cache_ptr->max_pel_len != 0) || (cache_ptr->max_pel_size != 0) || + (cache_ptr->calls_to_msic != 0) || (cache_ptr->total_entries_skipped_in_msic != 0) || (cache_ptr->total_entries_scanned_in_msic != 0) || (cache_ptr->max_entries_skipped_in_msic != 0) || (cache_ptr->max_entries_scanned_in_msic != 0) || (cache_ptr->entries_scanned_to_make_space != 0) || (cache_ptr->slist_scan_restarts != 0) || @@ -35209,6 +35291,7 @@ check_stats__smoke_check_1(H5F_t *file_ptr) pass = FALSE; failure_mssg = "Unexpected cache stats in check_stats__smoke_check_1(2)."; } /* end if */ + } #if H5C_COLLECT_CACHE_ENTRY_STATS if (pass) @@ -35261,18 +35344,20 @@ check_stats__smoke_check_1(H5F_t *file_ptr) failure_mssg = "Unexpected monster size entry stats in check_stats__smoke_check_1(3)."; } /* end if */ - if (pass) + if (pass) { + if ((cache_ptr->total_ht_insertions != 33) || (cache_ptr->total_ht_deletions != 1) || (cache_ptr->successful_ht_searches != 32) || (cache_ptr->total_successful_ht_search_depth != 96) || (cache_ptr->failed_ht_searches != 33) || (cache_ptr->total_failed_ht_search_depth != 52) || (cache_ptr->max_index_len != 32) || (cache_ptr->max_index_size != 2 * 1024 * 1024) || (cache_ptr->max_clean_index_size != 2 * 1024 * 1024) || - (cache_ptr->max_dirty_index_size != 2 * 1024 * 1024) || (cache_ptr->max_slist_len != 32) || - (cache_ptr->max_slist_size != 2 * 1024 * 1024) || (cache_ptr->max_pl_len != 1) || - (cache_ptr->max_pl_size != 64 * 1024) || (cache_ptr->max_pel_len != 0) || - (cache_ptr->max_pel_size != 0) || (cache_ptr->calls_to_msic != 1) || - (cache_ptr->total_entries_skipped_in_msic != 0) || + (cache_ptr->max_dirty_index_size != 2 * 1024 * 1024) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->max_slist_len != 32) || (cache_ptr->max_slist_size != 2 * 1024 * 1024))) || + (cache_ptr->max_pl_len != 1) || (cache_ptr->max_pl_size != 64 * 1024) || + (cache_ptr->max_pel_len != 0) || (cache_ptr->max_pel_size != 0) || + (cache_ptr->calls_to_msic != 1) || (cache_ptr->total_entries_skipped_in_msic != 0) || (cache_ptr->total_entries_scanned_in_msic != 33) || (cache_ptr->max_entries_skipped_in_msic != 0) || (cache_ptr->max_entries_scanned_in_msic != 33) || (cache_ptr->entries_scanned_to_make_space != 33) || (cache_ptr->slist_scan_restarts != 0) || @@ -35281,6 +35366,7 @@ check_stats__smoke_check_1(H5F_t *file_ptr) pass = FALSE; failure_mssg = "Unexpected cache stats in check_stats__smoke_check_1(3)."; } /* end if */ + } #if H5C_COLLECT_CACHE_ENTRY_STATS if (pass) @@ -35310,14 +35396,10 @@ check_stats__smoke_check_1(H5F_t *file_ptr) if (pass) { - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, + "Cache flush invalidate failed in check_stats__smoke_check_1()") - if (result < 0) { - - pass = FALSE; - failure_mssg = "Cache flush invalidate failed in check_stats__smoke_check_1()"; - } /* end if */ - else if ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0)) { + if ((pass) && ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0))) { pass = FALSE; failure_mssg = "Unexpected cache len/size after check_stats__smoke_check_1()"; @@ -35351,18 +35433,20 @@ check_stats__smoke_check_1(H5F_t *file_ptr) failure_mssg = "Unexpected monster size entry stats in check_stats__smoke_check_1(4)."; } /* end if */ - if (pass) + if (pass) { + if ((cache_ptr->total_ht_insertions != 33) || (cache_ptr->total_ht_deletions != 33) || (cache_ptr->successful_ht_searches != 33) || (cache_ptr->total_successful_ht_search_depth != 99) || (cache_ptr->failed_ht_searches != 33) || (cache_ptr->total_failed_ht_search_depth != 52) || (cache_ptr->max_index_len != 32) || (cache_ptr->max_index_size != 2 * 1024 * 1024) || (cache_ptr->max_clean_index_size != 2 * 1024 * 1024) || - (cache_ptr->max_dirty_index_size != 2 * 1024 * 1024) || (cache_ptr->max_slist_len != 32) || - (cache_ptr->max_slist_size != 2 * 1024 * 1024) || (cache_ptr->max_pl_len != 1) || - (cache_ptr->max_pl_size != 64 * 1024) || (cache_ptr->max_pel_len != 0) || - (cache_ptr->max_pel_size != 0) || (cache_ptr->calls_to_msic != 1) || - (cache_ptr->total_entries_skipped_in_msic != 0) || + (cache_ptr->max_dirty_index_size != 2 * 1024 * 1024) || + ((cache_ptr->slist_enabled) && + ((cache_ptr->max_slist_len != 32) || (cache_ptr->max_slist_size != 2 * 1024 * 1024))) || + (cache_ptr->max_pl_len != 1) || (cache_ptr->max_pl_size != 64 * 1024) || + (cache_ptr->max_pel_len != 0) || (cache_ptr->max_pel_size != 0) || + (cache_ptr->calls_to_msic != 1) || (cache_ptr->total_entries_skipped_in_msic != 0) || (cache_ptr->total_entries_scanned_in_msic != 33) || (cache_ptr->max_entries_skipped_in_msic != 0) || (cache_ptr->max_entries_scanned_in_msic != 33) || (cache_ptr->entries_scanned_to_make_space != 33) || (cache_ptr->slist_scan_restarts != 0) || @@ -35371,6 +35455,7 @@ check_stats__smoke_check_1(H5F_t *file_ptr) pass = FALSE; failure_mssg = "Unexpected cache stats in check_stats__smoke_check_1(4)."; } /* end if */ + } #if H5C_COLLECT_CACHE_ENTRY_STATS if (pass) diff --git a/test/cache_api.c b/test/cache_api.c index acdb4ba..143cc71 100644 --- a/test/cache_api.c +++ b/test/cache_api.c @@ -1216,9 +1216,9 @@ mdc_api_call_smoke_check(int express_test, unsigned paged, hid_t fcpl_id) /* do random reads on all datasets */ n = 0; while ((pass) && (n < NUM_RANDOM_ACCESSES)) { - m = rand() % NUM_DSETS; - i = (rand() % (DSET_SIZE / CHUNK_SIZE)) * CHUNK_SIZE; - j = (rand() % (DSET_SIZE / CHUNK_SIZE)) * CHUNK_SIZE; + m = HDrand() % NUM_DSETS; + i = (HDrand() % (DSET_SIZE / CHUNK_SIZE)) * CHUNK_SIZE; + j = (HDrand() % (DSET_SIZE / CHUNK_SIZE)) * CHUNK_SIZE; /* select on disk hyperslab */ offset[0] = (hsize_t)i; /*offset of hyperslab in file*/ @@ -1328,8 +1328,8 @@ mdc_api_call_smoke_check(int express_test, unsigned paged, hid_t fcpl_id) m = 0; n = 0; while ((pass) && (n < NUM_RANDOM_ACCESSES)) { - i = (rand() % (DSET_SIZE / CHUNK_SIZE)) * CHUNK_SIZE; - j = (rand() % (DSET_SIZE / CHUNK_SIZE)) * CHUNK_SIZE; + i = (HDrand() % (DSET_SIZE / CHUNK_SIZE)) * CHUNK_SIZE; + j = (HDrand() % (DSET_SIZE / CHUNK_SIZE)) * CHUNK_SIZE; /* select on disk hyperslab */ offset[0] = (hsize_t)i; /*offset of hyperslab in file*/ diff --git a/test/cache_common.c b/test/cache_common.c index d0c267f..8fc8303 100644 --- a/test/cache_common.c +++ b/test/cache_common.c @@ -2754,7 +2754,20 @@ expunge_entry(H5F_t *file_ptr, int32_t type, int32_t idx) * Return: void * * Programmer: John Mainzer - * 6/23/04 + * 6/23/04 + * + * Changes: Added code to setup and take down the skip list before + * and after calls to H5C_flush_cache(). Do this via calls + * to the H5C_FLUSH_CACHE macro. + * + * This is necessary, as H5C_flush() is called repeatedly + * during file flush. If we setup and took down the + * skip list on H5C_flush_cache(), we would find ourselves + * doing this repeatedly -- which is contrary to the + * objective of the exercise (avoiding as many skip list + * operations as possible). + * + * JRM -- 5/14/20 * *------------------------------------------------------------------------- */ @@ -2774,22 +2787,23 @@ flush_cache(H5F_t *file_ptr, hbool_t destroy_entries, hbool_t dump_stats, hbool_ cache_ptr = file_ptr->shared->cache; - if (destroy_entries) - result = H5C_flush_cache(file_ptr, H5C__FLUSH_INVALIDATE_FLAG); + if (destroy_entries) { - else - result = H5C_flush_cache(file_ptr, H5C__NO_FLAGS_SET); + H5C_FLUSH_CACHE(file_ptr, H5C__FLUSH_INVALIDATE_FLAG, "error in H5C_flush_cache().") + } + else { - if (dump_stats) - H5C_stats(cache_ptr, "test cache", dump_detailed_stats); + H5C_FLUSH_CACHE(file_ptr, H5C__NO_FLAGS_SET, "error in H5C_flush_cache().") + } - if (result < 0) { - pass = FALSE; - failure_mssg = "error in H5C_flush_cache()."; + if (dump_stats) { + + H5C_stats(cache_ptr, "test cache", dump_detailed_stats); } - else if ((destroy_entries) && - ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0) || - (cache_ptr->clean_index_size != 0) || (cache_ptr->dirty_index_size != 0))) { + + if ((pass) && (destroy_entries) && + ((cache_ptr->index_len != 0) || (cache_ptr->index_size != 0) || + (cache_ptr->clean_index_size != 0) || (cache_ptr->dirty_index_size != 0))) { if (verbose) { @@ -3544,6 +3558,11 @@ unprotect_entry(H5F_t *file_ptr, int32_t type, int32_t idx, unsigned int flags) * Programmer: John Mainzer * 6/12/04 * + * Changes: Updated slist size == dirty index size checks to + * bypass the test if cache_ptr->slist_enabled is FALSE. + * + * JRM -- 5/8/20 + * *------------------------------------------------------------------------- */ void @@ -3587,7 +3606,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "1(i, %d, %d) ", type, tmp_idx); insert_entry(file_ptr, type, tmp_idx, H5C__NO_FLAGS_SET); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ tmp_idx--; @@ -3597,7 +3618,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "2(p, %d, %d) ", type, tmp_idx); protect_entry(file_ptr, type, tmp_idx); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ tmp_idx--; @@ -3607,7 +3630,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "3(u, %d, %d) ", type, tmp_idx); unprotect_entry(file_ptr, type, tmp_idx, H5C__NO_FLAGS_SET); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ /* (don't decrement tmp_idx) */ @@ -3617,7 +3642,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "4(r, %d, %d, %d) ", type, tmp_idx, (int)move_to_main_addr); move_entry(cache_ptr, type, tmp_idx, move_to_main_addr); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ tmp_idx--; @@ -3627,7 +3654,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "5(p, %d, %d) ", type, tmp_idx); protect_entry(file_ptr, type, tmp_idx); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ tmp_idx -= 2; @@ -3637,7 +3666,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "6(u, %d, %d) ", type, tmp_idx); unprotect_entry(file_ptr, type, tmp_idx, H5C__NO_FLAGS_SET); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ if (do_mult_ro_protects) { @@ -3648,7 +3679,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "7(p-ro, %d, %d) ", type, tmp_idx); protect_entry_ro(file_ptr, type, tmp_idx); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ tmp_idx--; @@ -3658,7 +3691,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "8(p-ro, %d, %d) ", type, tmp_idx); protect_entry_ro(file_ptr, type, tmp_idx); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ tmp_idx--; @@ -3668,7 +3703,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "9(p-ro, %d, %d) ", type, tmp_idx); protect_entry_ro(file_ptr, type, tmp_idx); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ /* (don't decrement tmp_idx) */ @@ -3678,7 +3715,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "10(u-ro, %d, %d) ", type, tmp_idx); unprotect_entry(file_ptr, type, tmp_idx, H5C__NO_FLAGS_SET); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ tmp_idx--; @@ -3688,7 +3727,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "11(u-ro, %d, %d) ", type, tmp_idx); unprotect_entry(file_ptr, type, tmp_idx, H5C__NO_FLAGS_SET); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ tmp_idx--; @@ -3698,7 +3739,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "12(u-ro, %d, %d) ", type, tmp_idx); unprotect_entry(file_ptr, type, tmp_idx, H5C__NO_FLAGS_SET); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ } /* if ( do_mult_ro_protects ) */ @@ -3707,7 +3750,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "13(p, %d, %d) ", type, idx); protect_entry(file_ptr, type, idx); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ tmp_idx = idx - lag + 2; @@ -3717,7 +3762,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "14(u, %d, %d) ", type, tmp_idx); unprotect_entry(file_ptr, type, tmp_idx, H5C__NO_FLAGS_SET); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ tmp_idx--; @@ -3727,7 +3774,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "15(p, %d, %d) ", type, tmp_idx); protect_entry(file_ptr, type, tmp_idx); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ if (do_destroys) { @@ -3739,7 +3788,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "16(u, %d, %d) ", type, tmp_idx); unprotect_entry(file_ptr, type, tmp_idx, H5C__NO_FLAGS_SET); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); break; case 1: @@ -3748,7 +3799,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "17(u, %d, %d) ", type, tmp_idx); unprotect_entry(file_ptr, type, tmp_idx, H5C__NO_FLAGS_SET); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ else { if (verbose) @@ -3756,7 +3809,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t unprotect_entry(file_ptr, type, tmp_idx, (dirty_unprotects ? H5C__DIRTIED_FLAG : H5C__NO_FLAGS_SET)); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end else */ break; @@ -3765,7 +3820,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "19(u-del, %d, %d) ", type, tmp_idx); unprotect_entry(file_ptr, type, tmp_idx, H5C__DELETED_FLAG); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); break; case 3: @@ -3774,7 +3831,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t HDfprintf(stdout, "20(u-del, %d, %d) ", type, tmp_idx); unprotect_entry(file_ptr, type, tmp_idx, H5C__DELETED_FLAG); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ else { if (verbose) @@ -3783,7 +3842,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t unprotect_entry(file_ptr, type, tmp_idx, (dirty_destroys ? H5C__DIRTIED_FLAG : H5C__NO_FLAGS_SET) | H5C__DELETED_FLAG); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end else */ break; @@ -3801,7 +3862,9 @@ row_major_scan_forward(H5F_t *file_ptr, int32_t max_index, int32_t lag, hbool_t unprotect_entry(file_ptr, type, tmp_idx, (dirty_unprotects ? H5C__DIRTIED_FLAG : H5C__NO_FLAGS_SET)); - HDassert(cache_ptr->slist_size == cache_ptr->dirty_index_size); + + HDassert((!cache_ptr->slist_enabled) || + (cache_ptr->slist_size == cache_ptr->dirty_index_size)); } /* end if */ } /* end elsef */ diff --git a/test/cache_common.h b/test/cache_common.h index f466f8e..aa32858 100644 --- a/test/cache_common.h +++ b/test/cache_common.h @@ -116,6 +116,62 @@ #define MAX_ADDR (haddr_t)(NOTIFY_ALT_BASE_ADDR + (NOTIFY_ENTRY_SIZE * NUM_NOTIFY_ENTRIES)) #define ADDR_SPACE_SIZE (haddr_t)(MAX_ADDR - BASE_ADDR) +/*********************************************************************** + * + * Macro: H5C_FLUSH_CACHE + * + * Purpose: Wrap a call to H5C_flush_cache() in calls to + * H5C_set_slist_enabled() to setup and take down the slist. + * + * This is necessary, as H5C_flush_cache() needs the + * slist to be active. Further, since it is called + * repeatedly during file flush, it would be inefficient + * for it to setup the slist on entry, and take it down + * on exit. + * + * Note that the slist need not be empty if the flags + * indicate a partial flush (i.e. + * H5C__FLUSH_MARKED_ENTRIES_FLAG). Compute clear_slist + * and pass it into H5C_set_slist_enabled as appropriate. + * + * On error, set pass to FALSE, and set failure_mssg + * to the supplied error message. + * + * Return: N/A + * + * Programmer: John Mainzer + * 5/14/20 + * + * Changes: None. + * + ***********************************************************************/ + +#define H5C_FLUSH_CACHE(file, flags, fail_mssg) \ + { \ + hbool_t clear_slist; \ + herr_t rslt; \ + \ + clear_slist = ((flags & H5C__FLUSH_MARKED_ENTRIES_FLAG) != 0); \ + \ + rslt = H5C_set_slist_enabled((file)->shared->cache, TRUE, FALSE); \ + \ + if (rslt >= 0) { \ + \ + rslt = H5C_flush_cache((file), (flags)); \ + } \ + \ + if (rslt >= 0) { \ + \ + rslt = H5C_set_slist_enabled((file)->shared->cache, FALSE, clear_slist); \ + } \ + \ + if (rslt < 0) { \ + \ + pass = FALSE; \ + failure_mssg = (fail_mssg); \ + } \ + } /* H5C_FLUSH_CACHE */ + #define MAX_PINS \ 8 /* Maximum number of entries that can be \ * directly pinned by a single entry. \ diff --git a/test/cache_tagging.c b/test/cache_tagging.c index dbb7c81..b2975d1 100644 --- a/test/cache_tagging.c +++ b/test/cache_tagging.c @@ -343,12 +343,20 @@ evict_entries(hid_t fid) /* Mark all entries investigated */ mark_all_entries_investigated(fid); + /* setup the skip list prior to calling H5C_flush_cache() */ + if (H5C_set_slist_enabled(f->shared->cache, TRUE, FALSE) < 0) + TEST_ERROR; + /* Evict all we can from the cache to examine full tag creation tree */ /* This function will likely return failure since the root group * is still protected. Thus, don't check its return value. */ H5C_flush_cache(f, H5C__FLUSH_INVALIDATE_FLAG); + /* shutdown the slist -- allow it to be non-empty */ + if (H5C_set_slist_enabled(f->shared->cache, FALSE, TRUE) < 0) + TEST_ERROR; + return 0; error: diff --git a/test/chunk_info.c b/test/chunk_info.c index 7774873..cc35bfe 100644 --- a/test/chunk_info.c +++ b/test/chunk_info.c @@ -421,7 +421,7 @@ verify_idx_nchunks(hid_t dset, hid_t dspace, H5D_chunk_index_t exp_idx_type, hsi /* Ensure the correct chunk indexing scheme is used */ if (idx_type != exp_idx_type) { char msg[256]; - sprintf(msg, "Should be using %s.\n", index_type_str(idx_type)); + HDsprintf(msg, "Should be using %s.\n", index_type_str(idx_type)); FAIL_PUTS_ERROR(msg); } @@ -542,7 +542,7 @@ test_get_chunk_info_highest_v18(hid_t fapl) #ifdef H5_HAVE_FILTER_DEFLATE /* Allocate input (compressed) buffer */ - inbuf = malloc(z_dst_nbytes); + inbuf = HDmalloc(z_dst_nbytes); /* Set chunk size to the compressed chunk size and the chunk point to the compressed data chunk */ @@ -554,20 +554,20 @@ test_get_chunk_info_highest_v18(hid_t fapl) /* Check for various zlib errors */ if (Z_BUF_ERROR == ret) { - fprintf(stderr, "overflow"); + HDfprintf(stderr, "overflow"); TEST_ERROR } else if (Z_MEM_ERROR == ret) { - fprintf(stderr, "deflate memory error"); + HDfprintf(stderr, "deflate memory error"); TEST_ERROR } else if (Z_OK != ret) { - fprintf(stderr, "other deflate error"); + HDfprintf(stderr, "other deflate error"); TEST_ERROR } #else /* Allocate input (non-compressed) buffer */ - inbuf = malloc(CHK_SIZE); + inbuf = HDmalloc(CHK_SIZE); HDmemcpy(inbuf, direct_buf, CHK_SIZE); #endif /* end H5_HAVE_FILTER_DEFLATE */ @@ -1638,7 +1638,7 @@ test_basic_query(hid_t fapl) TEST_ERROR /* Remove the test file */ - remove(filename); + HDremove(filename); PASSED(); return SUCCEED; @@ -2040,7 +2040,7 @@ test_flt_msk_with_skip_compress(hid_t fapl) TEST_ERROR /* Remove the test file */ - remove(filename); + HDremove(filename); PASSED(); return SUCCEED; diff --git a/test/cmpd_dset.c b/test/cmpd_dset.c index 6e5dd07..37217e7 100644 --- a/test/cmpd_dset.c +++ b/test/cmpd_dset.c @@ -302,7 +302,7 @@ test_compound(char *filename, hid_t fapl) s1[i].c[1] != s2[i].c[1] || s1[i].c[2] != s2[i].c[2] || s1[i].c[3] != s2[i].c[3] || s1[i].d != s2[i].d || s1[i].e != s2[i].e) { H5_FAILED(); - puts(" Incorrect values read from the file"); + HDputs(" Incorrect values read from the file"); goto error; } } @@ -339,7 +339,7 @@ test_compound(char *filename, hid_t fapl) s1[i].c[1] != s3[i].c[1] || s1[i].c[2] != s3[i].c[2] || s1[i].c[3] != s3[i].c[3] || s1[i].d != s3[i].d || s1[i].e != s3[i].e) { H5_FAILED(); - puts(" Incorrect values read from the file"); + HDputs(" Incorrect values read from the file"); goto error; } } @@ -369,7 +369,7 @@ test_compound(char *filename, hid_t fapl) for (i = 0; i < NX * NY; i++) { if (s1[i].b != s4[i].b || s1[i].d != s4[i].d) { H5_FAILED(); - puts(" Incorrect values read from the file"); + HDputs(" Incorrect values read from the file"); goto error; } } @@ -413,7 +413,7 @@ test_compound(char *filename, hid_t fapl) s1[i].c[1] != s5[i].c[1] || s1[i].c[2] != s5[i].c[2] || s1[i].c[3] != s5[i].c[3] || s1[i].d != s5[i].d || s1[i].e != s5[i].e) { H5_FAILED(); - puts(" Incorrect values read from the file"); + HDputs(" Incorrect values read from the file"); goto error; } } @@ -423,7 +423,7 @@ test_compound(char *filename, hid_t fapl) if (s5[i].pre != 1000 + 4 * i || s5[i].mid1 != 1001 + 4 * i || s5[i].mid2 != 1002 + 4 * i || s5[i].post != 1003 + 4 * i) { H5_FAILED(); - puts(" Memory values were clobbered"); + HDputs(" Memory values were clobbered"); goto error; } } @@ -482,7 +482,7 @@ test_compound(char *filename, hid_t fapl) s1[i].c[1] != s6[i].c[1] || s1[i].c[2] != s6[i].c[2] || s1[i].c[3] != s6[i].c[3] || s1[i].d != s6[i].d || s1[i].e != s6[i].e) { H5_FAILED(); - puts(" Incorrect values read from the file"); + HDputs(" Incorrect values read from the file"); goto error; } } @@ -492,7 +492,7 @@ test_compound(char *filename, hid_t fapl) if (s6[i].pre != 1000 + 4 * i || s6[i].mid1 != 1001 + 4 * i || s6[i].mid2 != 1002 + 4 * i || s6[i].post != 1003 + 4 * i) { H5_FAILED(); - puts(" Memory values were clobbered"); + HDputs(" Memory values were clobbered"); goto error; } } @@ -561,7 +561,7 @@ test_compound(char *filename, hid_t fapl) s2[i].c[1] != s1[i].c[1] || s2[i].c[2] != s1[i].c[2] || s2[i].c[3] != s1[i].c[3] || s2[i].d != s1[i].d || s2[i].e != s1[i].e) { H5_FAILED(); - puts(" Incorrect values read from file"); + HDputs(" Incorrect values read from file"); goto error; } } @@ -590,7 +590,7 @@ test_compound(char *filename, hid_t fapl) /* Read the dataset */ s8 = (s1_t *)HDcalloc((size_t)(h_size[0] * h_size[1]), sizeof(s1_t)); - assert(s8); + HDassert(s8); if (H5Dread(dataset, s1_tid, s8_m_sid, s8_f_sid, H5P_DEFAULT, s8) < 0) { goto error; } @@ -604,7 +604,7 @@ test_compound(char *filename, hid_t fapl) if (ps8->a != ps1->a || ps8->b != ps1->b || ps8->c[0] != ps1->c[0] || ps8->c[1] != ps1->c[1] || ps8->c[2] != ps1->c[2] || ps8->c[3] != ps1->c[3] || ps8->d != ps1->d || ps8->e != ps1->e) { H5_FAILED(); - puts(" Incorrect values read from file"); + HDputs(" Incorrect values read from file"); goto error; } } @@ -643,7 +643,7 @@ test_compound(char *filename, hid_t fapl) ps2->c[1] != ps1->c[1] || ps2->c[2] != ps1->c[2] || ps2->c[3] != ps1->c[3] || ps2->d != ps1->d || ps2->e != ps1->e) { H5_FAILED(); - puts(" Memory values clobbered"); + HDputs(" Memory values clobbered"); goto error; } } @@ -652,7 +652,7 @@ test_compound(char *filename, hid_t fapl) ps2->c[1] != (unsigned)(-1) || ps2->c[2] != (unsigned)(-1) || ps2->c[3] != (unsigned)(-1) || ps2->d != (unsigned)(-1) || ps2->e != (unsigned)(-1)) { H5_FAILED(); - puts(" Incorrect values read from file"); + HDputs(" Incorrect values read from file"); goto error; } } @@ -691,7 +691,7 @@ test_compound(char *filename, hid_t fapl) ps5->c[2] != ps1->c[2] || ps5->c[3] != ps1->c[3] || ps5->mid2 != (unsigned)(-1) || ps5->d != ps1->d || ps5->e != ps1->e || ps5->post != (unsigned)(-1)) { H5_FAILED(); - puts(" Memory values clobbered"); + HDputs(" Memory values clobbered"); goto error; } } @@ -702,7 +702,7 @@ test_compound(char *filename, hid_t fapl) ps5->c[3] != (unsigned)(-1) || ps5->mid2 != (unsigned)(-1) || ps5->d != (unsigned)(-1) || ps5->e != (unsigned)(-1) || ps5->post != (unsigned)(-1)) { H5_FAILED(); - puts(" Incorrect values read from file"); + HDputs(" Incorrect values read from file"); goto error; } } @@ -723,7 +723,7 @@ test_compound(char *filename, hid_t fapl) h_size[0] = 2 * NX / 3 - f_offset[0]; h_size[1] = 2 * NY / 3 - f_offset[1]; s11 = (s4_t *)HDmalloc((size_t)h_size[0] * (size_t)h_size[1] * sizeof(s4_t)); - assert(s11); + HDassert(s11); /* Initialize */ for (i = 0; i < h_size[0] * h_size[1]; i++) { @@ -751,7 +751,7 @@ test_compound(char *filename, hid_t fapl) ps1->c[1] != 8 * (i * NY + j) + 3 || ps1->c[2] != 8 * (i * NY + j) + 4 || ps1->c[3] != 8 * (i * NY + j) + 5 || ps1->e != 8 * (i * NY + j) + 7) { H5_FAILED(); - puts(" Write clobbered values"); + HDputs(" Write clobbered values"); goto error; } @@ -759,14 +759,14 @@ test_compound(char *filename, hid_t fapl) j < f_offset[1] + h_size[1]) { if (ps1->b != (unsigned)(-1) || ps1->d != (unsigned)(-1)) { H5_FAILED(); - puts(" Wrong values written or read"); + HDputs(" Wrong values written or read"); goto error; } } else { if (ps1->b != 8 * (i * NY + j) + 1 || ps1->d != 8 * (i * NY + j) + 6) { H5_FAILED(); - puts(" Write clobbered values"); + HDputs(" Write clobbered values"); goto error; } } @@ -792,7 +792,7 @@ test_compound(char *filename, hid_t fapl) return 0; error: - puts("*** DATASET TESTS FAILED ***"); + HDputs("*** DATASET TESTS FAILED ***"); /* Release resources */ if (s1) @@ -1669,7 +1669,7 @@ test_hdf5_dst_subset(char *filename, hid_t fapl) return 0; error: - puts("*** DATASET TESTS FAILED ***"); + HDputs("*** DATASET TESTS FAILED ***"); return 1; } @@ -1684,7 +1684,7 @@ error: for (_i = 0; _i < PACK_NMEMBS; _i++) \ HDprintf(" %d", order[_i]); \ HDprintf("\n Inner compound order = %d, location = %d\n", sub_cmpd_order, order[sub_cmpd_order]); \ - fflush(stdout); \ + HDfflush(stdout); \ goto error; \ } @@ -1719,7 +1719,7 @@ test_pack_ooo(void) * the compound */ unsigned i, j; /* Indices */ - HDsrand((unsigned)time(NULL)); + HDsrand((unsigned)HDtime(NULL)); /* Initialize "free_order" array to indicate that all slots in order are * free */ @@ -2233,7 +2233,7 @@ main(int argc, char *argv[]) /* Turn off optimized compound converter? */ if (argc > 1) { - if (argc > 2 || strcmp("--noopt", argv[1])) { + if (argc > 2 || HDstrcmp("--noopt", argv[1])) { HDfprintf(stderr, "usage: %s [--noopt]\n", argv[0]); HDexit(EXIT_FAILURE); } diff --git a/test/dangle.c b/test/dangle.c index 4728923..d7c7dfc 100644 --- a/test/dangle.c +++ b/test/dangle.c @@ -660,7 +660,7 @@ main(void) int nerrors = 0; /* Run tests w/weak file close */ - puts("Testing dangling objects with weak file close:"); + HDputs("Testing dangling objects with weak file close:"); nerrors += test_dangle_dataset(H5F_CLOSE_WEAK); nerrors += test_dangle_group(H5F_CLOSE_WEAK); nerrors += test_dangle_datatype1(H5F_CLOSE_WEAK); @@ -668,7 +668,7 @@ main(void) nerrors += test_dangle_attribute(H5F_CLOSE_WEAK); /* Run tests w/semi file close */ - puts("Testing dangling objects with semi file close:"); + HDputs("Testing dangling objects with semi file close:"); nerrors += test_dangle_dataset(H5F_CLOSE_SEMI); nerrors += test_dangle_group(H5F_CLOSE_SEMI); nerrors += test_dangle_datatype1(H5F_CLOSE_SEMI); @@ -676,7 +676,7 @@ main(void) nerrors += test_dangle_attribute(H5F_CLOSE_SEMI); /* Run tests w/strong file close */ - puts("Testing dangling objects with strong file close:"); + HDputs("Testing dangling objects with strong file close:"); nerrors += test_dangle_dataset(H5F_CLOSE_STRONG); nerrors += test_dangle_group(H5F_CLOSE_STRONG); nerrors += test_dangle_datatype1(H5F_CLOSE_STRONG); @@ -689,11 +689,11 @@ main(void) /* Check for errors */ if (nerrors) goto error; - puts("All dangling ID tests passed."); + HDputs("All dangling ID tests passed."); return 0; error: - puts("***** DANGLING ID TESTS FAILED *****"); + HDputs("***** DANGLING ID TESTS FAILED *****"); return 1; } diff --git a/test/direct_chunk.c b/test/direct_chunk.c index 388f2cc..00fc06f 100644 --- a/test/direct_chunk.c +++ b/test/direct_chunk.c @@ -2320,7 +2320,7 @@ main(void) need_comma = TRUE; } /* end if */ HDprintf(":\n"); - fflush(stdout); + HDfflush(stdout); nerrors += test_single_chunk(config); } /* end for */ diff --git a/test/dsets.c b/test/dsets.c index e81a33e..a7b67ef 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -110,6 +110,8 @@ const char *FILENAME[] = {"dataset", /* 0 */ #define DSET_FLETCHER32_NAME_3 "fletcher32_3" #define DSET_SHUF_DEF_FLET_NAME "shuffle+deflate+fletcher32" #define DSET_SHUF_DEF_FLET_NAME_2 "shuffle+deflate+fletcher32_2" +#define DSET_OPTIONAL_SCALAR "dataset_with_scalar_space" +#define DSET_OPTIONAL_VLEN "dataset_with_vlen_type" #ifdef H5_HAVE_FILTER_SZIP #define DSET_SZIP_NAME "szip" #define DSET_SHUF_SZIP_FLET_NAME "shuffle+szip+fletcher32" @@ -373,13 +375,13 @@ test_create(hid_t file) dims[0] = 256; dims[1] = 512; space = H5Screate_simple(2, dims, NULL); - assert(space >= 0); + HDassert(space >= 0); /* Create a small data space for compact dataset */ small_dims[0] = 16; small_dims[1] = 8; small_space = H5Screate_simple(2, small_dims, NULL); - assert(space >= 0); + HDassert(space >= 0); /* * Create a dataset using the default dataset creation properties. We're @@ -445,13 +447,13 @@ test_create(hid_t file) * layout. */ create_parms = H5Pcreate(H5P_DATASET_CREATE); - assert(create_parms >= 0); + HDassert(create_parms >= 0); /* Attempt to create a dataset with invalid chunk sizes */ csize[0] = dims[0] * 2; csize[1] = dims[1] * 2; status = H5Pset_chunk(create_parms, 2, csize); - assert(status >= 0); + HDassert(status >= 0); H5E_BEGIN_TRY { dataset = H5Dcreate2(file, DSET_CHUNKED_NAME, H5T_NATIVE_DOUBLE, space, H5P_DEFAULT, create_parms, @@ -467,7 +469,7 @@ test_create(hid_t file) csize[0] = 5; csize[1] = 100; status = H5Pset_chunk(create_parms, 2, csize); - assert(status >= 0); + HDassert(status >= 0); dataset = H5Dcreate2(file, DSET_CHUNKED_NAME, H5T_NATIVE_DOUBLE, space, H5P_DEFAULT, create_parms, H5P_DEFAULT); @@ -489,11 +491,11 @@ test_create(hid_t file) * Create a compact dataset, then close it. */ create_parms = H5Pcreate(H5P_DATASET_CREATE); - assert(create_parms >= 0); + HDassert(create_parms >= 0); status = H5Pset_layout(create_parms, H5D_COMPACT); - assert(status >= 0); + HDassert(status >= 0); status = H5Pset_alloc_time(create_parms, H5D_ALLOC_TIME_EARLY); - assert(status >= 0); + HDassert(status >= 0); dataset = H5Dcreate2(file, DSET_COMPACT_NAME, H5T_NATIVE_DOUBLE, small_space, H5P_DEFAULT, create_parms, H5P_DEFAULT); @@ -565,7 +567,7 @@ test_simple_io(const char *env_h5_drvr, hid_t fapl) /* Create a small conversion buffer to test strip mining */ tconv_buf = HDmalloc((size_t)1000); xfer = H5Pcreate(H5P_DATASET_XFER); - assert(xfer >= 0); + HDassert(xfer >= 0); if (H5Pset_buffer(xfer, (size_t)1000, tconv_buf, NULL) < 0) goto error; @@ -4334,9 +4336,9 @@ test_nbit_compound_3(hid_t file) /* Check that the values read are the same as the values written */ for (i = 0; i < (size_t)size[0]; i++) { - if (new_data[i].i != orig_data[i].i || strcmp(new_data[i].str, orig_data[i].str) != 0 || - strcmp(new_data[i].vl_str, orig_data[i].vl_str) != 0 || new_data[i].v.len != orig_data[i].v.len || - new_data[i].r != orig_data[i].r) { + if (new_data[i].i != orig_data[i].i || HDstrcmp(new_data[i].str, orig_data[i].str) != 0 || + HDstrcmp(new_data[i].vl_str, orig_data[i].vl_str) != 0 || + new_data[i].v.len != orig_data[i].v.len || new_data[i].r != orig_data[i].r) { H5_FAILED(); HDprintf(" Read different values than written.\n"); HDprintf(" At index %lu\n", (unsigned long)i); @@ -6115,6 +6117,115 @@ error: } /* end test_can_apply2() */ /*------------------------------------------------------------------------- + * Function: test_optional_filters + * + * Purpose: Tests that H5Dcreate2 will not fail when a combination of + * type, space, etc... doesn't work for a filter and filter is + * optional. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Binh-Minh Ribler + * 24 July 2020 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_optional_filters(hid_t file) +{ + unsigned int level = 9; + unsigned int cd_values[1] = {level}; + size_t cd_nelmts = 1; + hsize_t dim1d[1]; /* Dataspace dimensions */ + hid_t dsid = H5I_INVALID_HID; /* Dataset ID */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t strtid = H5I_INVALID_HID; /* Datatype ID for string */ + hid_t vlentid = H5I_INVALID_HID; /* Datatype ID for vlen */ + hid_t dcplid = H5I_INVALID_HID; /* Dataspace creation property list ID */ + + TESTING("dataset with optional filters"); + + /* Create dcpl with special filter */ + if ((dcplid = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + /* Create the datatype */ + if ((strtid = H5Tcreate(H5T_STRING, H5T_VARIABLE)) < 0) + TEST_ERROR; + + /* Create the data space */ + if ((sid = H5Screate(H5S_SCALAR)) < 0) + TEST_ERROR; + + /* The filter is optional. */ + if (H5Pset_filter(dcplid, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, cd_nelmts, cd_values) < 0) + TEST_ERROR; + + /* Create dataset with optional filter */ + if ((dsid = H5Dcreate2(file, DSET_OPTIONAL_SCALAR, strtid, sid, H5P_DEFAULT, dcplid, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Close dataset */ + if (H5Dclose(dsid) < 0) + TEST_ERROR; + + /* Close dataspace */ + if (H5Sclose(sid) < 0) + TEST_ERROR; + + /* Close datatype */ + if (H5Tclose(strtid) < 0) + TEST_ERROR; + + /* Set dataspace dimensions */ + dim1d[0] = DIM1; + + /* Create a non-scalar dataspace */ + if ((sid = H5Screate_simple(1, dim1d, NULL)) < 0) + TEST_ERROR; + + /* Create a vlen datatype */ + if ((vlentid = H5Tvlen_create(H5T_NATIVE_INT)) < 0) + TEST_ERROR; + + /* Create dataset with optional filter */ + if ((dsid = H5Dcreate2(file, DSET_OPTIONAL_VLEN, vlentid, sid, H5P_DEFAULT, dcplid, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Close dataset */ + if (H5Dclose(dsid) < 0) + TEST_ERROR; + + /* Close dataspace */ + if (H5Sclose(sid) < 0) + TEST_ERROR; + + /* Close datatype */ + if (H5Tclose(vlentid) < 0) + TEST_ERROR; + + /* Close dataset creation property list */ + if (H5Pclose(dcplid) < 0) + TEST_ERROR; + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Dclose(dsid); + H5Sclose(sid); + H5Pclose(dcplid); + H5Tclose(strtid); + H5Tclose(vlentid); + } + H5E_END_TRY; + return FAIL; +} /* end test_optional_filters() */ + +/*------------------------------------------------------------------------- * Function: test_can_apply_szip * * Purpose: Tests library behavior when szip filter indicates it can't @@ -7721,7 +7832,7 @@ test_random_chunks_real(const char *testname, hbool_t early_alloc, hid_t fapl) TESTING(testname); - assert(NPOINTS < 100); + HDassert(NPOINTS < 100); h5_fixname(FILENAME[6], fapl, filename, sizeof filename); @@ -8243,13 +8354,13 @@ test_deprec(hid_t file) dims[0] = 256; dims[1] = 512; space = H5Screate_simple(2, dims, NULL); - assert(space >= 0); + HDassert(space >= 0); /* Create a small data space for compact dataset */ small_dims[0] = 16; small_dims[1] = 8; small_space = H5Screate_simple(2, small_dims, NULL); - assert(space >= 0); + HDassert(space >= 0); /* * Create a dataset using the default dataset creation properties. We're @@ -8302,7 +8413,7 @@ test_deprec(hid_t file) * layout. */ create_parms = H5Pcreate(H5P_DATASET_CREATE); - assert(create_parms >= 0); + HDassert(create_parms >= 0); /* Add the deflate filter, if available */ #if defined H5_HAVE_FILTER_DEFLATE @@ -8338,7 +8449,7 @@ test_deprec(hid_t file) csize[0] = dims[0] * 2; csize[1] = dims[1] * 2; status = H5Pset_chunk(create_parms, 2, csize); - assert(status >= 0); + HDassert(status >= 0); H5E_BEGIN_TRY { dataset = H5Dcreate1(file, DSET_DEPREC_NAME_CHUNKED, H5T_NATIVE_DOUBLE, space, create_parms); @@ -8353,7 +8464,7 @@ test_deprec(hid_t file) csize[0] = 5; csize[1] = 100; status = H5Pset_chunk(create_parms, 2, csize); - assert(status >= 0); + HDassert(status >= 0); if ((dataset = H5Dcreate1(file, DSET_DEPREC_NAME_CHUNKED, H5T_NATIVE_DOUBLE, space, create_parms)) < 0) goto error; @@ -8378,11 +8489,11 @@ test_deprec(hid_t file) * Create a compact dataset, then close it. */ create_parms = H5Pcreate(H5P_DATASET_CREATE); - assert(create_parms >= 0); + HDassert(create_parms >= 0); status = H5Pset_layout(create_parms, H5D_COMPACT); - assert(status >= 0); + HDassert(status >= 0); status = H5Pset_alloc_time(create_parms, H5D_ALLOC_TIME_EARLY); - assert(status >= 0); + HDassert(status >= 0); if ((dataset = H5Dcreate1(file, DSET_DEPREC_NAME_COMPACT, H5T_NATIVE_DOUBLE, small_space, create_parms)) < 0) @@ -15121,6 +15232,7 @@ main(void) nerrors += (test_missing_filter(file) < 0 ? 1 : 0); nerrors += (test_can_apply(file) < 0 ? 1 : 0); nerrors += (test_can_apply2(file) < 0 ? 1 : 0); + nerrors += (test_optional_filters(file) < 0 ? 1 : 0); nerrors += (test_set_local(my_fapl) < 0 ? 1 : 0); nerrors += (test_can_apply_szip(file) < 0 ? 1 : 0); nerrors += (test_compare_dcpl(file) < 0 ? 1 : 0); diff --git a/test/dt_arith.c b/test/dt_arith.c index 466554c..7b2274c 100644 --- a/test/dt_arith.c +++ b/test/dt_arith.c @@ -359,7 +359,7 @@ static int without_hardware_g = 0; \ for (n = 0; n < 2; n++) { \ if (n == 1) { \ - memset(value, 0, SRC_SIZE * sizeof(unsigned char)); \ + HDmemset(value, 0, SRC_SIZE * sizeof(unsigned char)); \ /* -0 */ \ H5T__bit_set(value, (size_t)(SRC_PREC - 1), (size_t)1, TRUE); \ CHANGE_ORDER(value, SRC_ORDR, SRC_SIZE); /*change order for big endian*/ \ @@ -566,7 +566,7 @@ generates_sigfpe(void) HDfflush(stdout); HDfflush(stderr); - if ((pid = fork()) < 0) { + if ((pid = HDfork()) < 0) { HDperror("fork"); HDexit(EXIT_FAILURE); } @@ -580,7 +580,7 @@ generates_sigfpe(void) HDexit(EXIT_SUCCESS); } - while (pid != waitpid(pid, &status, 0)) + while (pid != HDwaitpid(pid, &status, 0)) /*void*/; if (WIFEXITED(status) && 0 == WEXITSTATUS(status)) { HDputs("Floating-point overflow cases will be tested."); @@ -2388,7 +2388,7 @@ test_conv_int_1(const char *name, hid_t src, hid_t dst) } /* Make certain that there isn't some weird number of destination bits */ - assert(dst_nbits % 8 == 0); + HDassert(dst_nbits % 8 == 0); /* Are the two results the same? */ for (k = (dst_size - (dst_nbits / 8)); k < dst_size; k++) @@ -2945,12 +2945,12 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) */ HDfflush(stdout); HDfflush(stderr); - if ((child_pid = fork()) < 0) { + if ((child_pid = HDfork()) < 0) { HDperror("fork"); return 1; } else if (child_pid > 0) { - while (child_pid != waitpid(child_pid, &status, 0)) /*void*/ + while (child_pid != HDwaitpid(child_pid, &status, 0)) /*void*/ ; if (WIFEXITED(status) && 255 == WEXITSTATUS(status)) { return 0; /*child exit after catching SIGFPE*/ @@ -3019,7 +3019,7 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) if (sizeof(float) == sizeof(double)) HDputs("Sizeof(float)==sizeof(double) - some tests may not be sensible."); if (OTHER == src_type || OTHER == dst_type) { - if (!strcmp(name, "noop")) + if (!HDstrcmp(name, "noop")) HDsnprintf(str, sizeof(str), "Testing %s %s -> %s conversions", name, src_type_name, dst_type_name); else if (run_test == TEST_SPECIAL) @@ -3038,7 +3038,7 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) goto error; } else { - if (!strcmp(name, "noop")) + if (!HDstrcmp(name, "noop")) HDsnprintf(str, sizeof(str), "Testing %s %s -> %s conversions", name, src_type_name, dst_type_name); else if (run_test == TEST_SPECIAL) @@ -4347,7 +4347,7 @@ test_conv_int_fp(const char *name, int run_test, hid_t src, hid_t dst) } /* Make certain that there isn't some weird number of destination bits */ - assert(dst_nbits % 8 == 0); + HDassert(dst_nbits % 8 == 0); /* For Intel machines, the size of "long double" is 12 bytes, precision * is 80 bits; for AMD processors, the size of "long double" is 16 bytes, @@ -5006,7 +5006,7 @@ run_fp_tests(const char *name) { int nerrors = 0; - if (!strcmp(name, "noop")) { + if (!HDstrcmp(name, "noop")) { nerrors += test_conv_flt_1("noop", TEST_NOOP, H5T_NATIVE_FLOAT, H5T_NATIVE_FLOAT); nerrors += test_conv_flt_1("noop", TEST_NOOP, H5T_NATIVE_DOUBLE, H5T_NATIVE_DOUBLE); #if H5_SIZEOF_LONG_DOUBLE != 0 @@ -5243,7 +5243,7 @@ run_fp_int_conv(const char *name) #endif #if H5_SIZEOF_LONG_LONG != H5_SIZEOF_LONG - if (!strcmp(name, "hw")) { /* Hardware conversion */ + if (!HDstrcmp(name, "hw")) { /* Hardware conversion */ nerrors += test_conv_int_fp(name, test_values, H5T_NATIVE_FLOAT, H5T_NATIVE_LLONG); nerrors += test_conv_int_fp(name, test_values, H5T_NATIVE_DOUBLE, H5T_NATIVE_LLONG); } diff --git a/test/dtypes.c b/test/dtypes.c index 157e3fa..13cf6e8 100644 --- a/test/dtypes.c +++ b/test/dtypes.c @@ -1891,7 +1891,7 @@ test_compound_9(void) goto error; } /* end if */ - if (rdata.i1 != wdata.i1 || rdata.i2 != wdata.i2 || strcmp(rdata.str, wdata.str)) { + if (rdata.i1 != wdata.i1 || rdata.i2 != wdata.i2 || HDstrcmp(rdata.str, wdata.str)) { H5_FAILED(); AT(); HDprintf("incorrect read data\n"); @@ -2097,7 +2097,7 @@ test_compound_10(void) t1 = rdata[i].text.p; t2 = wdata[i].text.p; - if (strcmp((char *)t1, (char *)t2)) { + if (HDstrcmp((char *)t1, (char *)t2)) { H5_FAILED(); AT(); HDprintf("incorrect VL read data\n"); @@ -2502,7 +2502,7 @@ test_compound_12(void) H5E_END_TRY; if (ret >= 0) { H5_FAILED(); - puts(" Tries to cut off the last member. Should have failed."); + HDputs(" Tries to cut off the last member. Should have failed."); goto error; } @@ -3054,7 +3054,7 @@ test_compound_14(void) goto error; } /* end if */ - if (rdata1.c1 != wdata1.c1 || rdata1.c2 != wdata1.c2 || strcmp(rdata1.str, wdata1.str)) { + if (rdata1.c1 != wdata1.c1 || rdata1.c2 != wdata1.c2 || HDstrcmp(rdata1.str, wdata1.str)) { H5_FAILED(); AT(); HDprintf("incorrect read data\n"); @@ -3885,7 +3885,7 @@ test_query(void) HDprintf("Can't get name for enum member\n"); goto error; } /* end if */ - if (strcmp("YELLOW", enum_name)) { + if (HDstrcmp("YELLOW", enum_name)) { H5_FAILED(); HDprintf("Incorrect name for enum member\n"); goto error; diff --git a/test/earray.c b/test/earray.c index 79bf0f1..4ef79ee 100644 --- a/test/earray.c +++ b/test/earray.c @@ -759,7 +759,7 @@ test_create(hid_t fapl, H5EA_create_t *cparam, earray_test_param_t H5_ATTR_UNUSE } #else /* NDEBUG */ SKIPPED(); - puts(" Not tested when assertions are disabled"); + HDputs(" Not tested when assertions are disabled"); #endif /* NDEBUG */ /* @@ -2427,12 +2427,12 @@ main(void) switch (curr_test) { /* "Normal" testing parameters */ case EARRAY_TEST_NORMAL: - puts("Testing with normal parameters"); + HDputs("Testing with normal parameters"); break; /* "Re-open array" testing parameters */ case EARRAY_TEST_REOPEN: - puts("Testing with reopen array flag set"); + HDputs("Testing with reopen array flag set"); tparam.reopen_array = EARRAY_TEST_REOPEN; break; @@ -2461,31 +2461,31 @@ main(void) switch (curr_iter) { /* "Forward" testing parameters */ case EARRAY_ITER_FW: - puts("Testing with forward iteration"); + HDputs("Testing with forward iteration"); tparam.eiter = &ea_iter_fw; break; /* "Reverse" testing parameters */ case EARRAY_ITER_RV: - puts("Testing with reverse iteration"); + HDputs("Testing with reverse iteration"); tparam.eiter = &ea_iter_rv; break; /* "Random" testing parameters */ case EARRAY_ITER_RND: - puts("Testing with random iteration"); + HDputs("Testing with random iteration"); tparam.eiter = &ea_iter_rnd; break; /* "Random #2" testing parameters */ case EARRAY_ITER_RND2: - puts("Testing with random #2 iteration"); + HDputs("Testing with random #2 iteration"); tparam.eiter = &ea_iter_rnd2; break; /* "Cyclic" testing parameters */ case EARRAY_ITER_CYC: - puts("Testing with cyclic iteration"); + HDputs("Testing with cyclic iteration"); tparam.eiter = &ea_iter_cyc; break; diff --git a/test/enum.c b/test/enum.c index eafce72..4a13358 100644 --- a/test/enum.c +++ b/test/enum.c @@ -768,11 +768,11 @@ main(void) if (nerrors) goto error; - puts("All enum tests passed."); + HDputs("All enum tests passed."); h5_cleanup(FILENAME, fapl); return 0; error: - puts("*** ENUM TESTS FAILED ***"); + HDputs("*** ENUM TESTS FAILED ***"); return 1; } diff --git a/test/external.c b/test/external.c index 376204a..d106555 100644 --- a/test/external.c +++ b/test/external.c @@ -476,12 +476,12 @@ __add_external_files(hid_t dcpl_id, unsigned int n_external_files, off_t offset, for (i = 0; i < n_external_files; i++) { if (HDsnprintf(exname, AEF_EXNAME_MAX_LEN, "ext%d.data", i + 1) > AEF_EXNAME_MAX_LEN) { HDfprintf(stderr, "External file %d overflows name buffer\n", i + 1); - fflush(stderr); + HDfflush(stderr); return -1; } if (H5Pset_external(dcpl_id, exname, offset, max_ext_size) < 0) { HDfprintf(stderr, "Problem adding external file %s\n", exname); - fflush(stderr); + HDfflush(stderr); return -1; } } diff --git a/test/farray.c b/test/farray.c index b6a6c31..46889a3 100644 --- a/test/farray.c +++ b/test/farray.c @@ -483,7 +483,7 @@ test_create(hid_t fapl, H5FA_create_t *cparam, farray_test_param_t H5_ATTR_UNUSE } #else /* NDEBUG */ SKIPPED(); - puts(" Not tested when assertions are disabled"); + HDputs(" Not tested when assertions are disabled"); #endif /* NDEBUG */ /* diff --git a/test/filenotclosed.c b/test/filenotclosed.c index 13ce9fd..330224d 100644 --- a/test/filenotclosed.c +++ b/test/filenotclosed.c @@ -80,7 +80,7 @@ main(void) contig_addr_vfd = (hbool_t)(HDstrcmp(env_h5_drvr, "split") && HDstrcmp(env_h5_drvr, "multi")); if (!contig_addr_vfd) { SKIPPED(); - puts(" Temporary skipped for a spilt/multi driver"); + HDputs(" Temporary skipped for a spilt/multi driver"); HDexit(EXIT_SUCCESS); } diff --git a/test/fillval.c b/test/fillval.c index 05570f0..2decbfa 100644 --- a/test/fillval.c +++ b/test/fillval.c @@ -201,7 +201,7 @@ test_getset(void) H5E_END_TRY; if (fill_i != 0) { H5_FAILED(); - puts(" H5Pget_fill_value() should return default 0"); + HDputs(" H5Pget_fill_value() should return default 0"); goto error; } @@ -220,8 +220,8 @@ test_getset(void) goto error; if (fill_ss.v1 != fill_ss_rd.v1 || fill_ss.v2 != fill_ss_rd.v2) { H5_FAILED(); - puts(" Failed to get fill value using same data type that was "); - puts(" used to set the fill value."); + HDputs(" Failed to get fill value using same data type that was "); + HDputs(" used to set the fill value."); goto error; } @@ -232,8 +232,8 @@ test_getset(void) goto error; if (fill_ss.v1 != fill_si.v1 || fill_ss.v2 != fill_si.v2) { H5_FAILED(); - puts(" Failed to get fill value using a data type other than what"); - puts(" was used to set the fill value."); + HDputs(" Failed to get fill value using a data type other than what"); + HDputs(" was used to set the fill value."); goto error; } @@ -246,7 +246,7 @@ test_getset(void) goto error; if (fill_si.v1 != fill_ss.v1 || fill_si.v2 != fill_ss.v2) { H5_FAILED(); - puts(" Resetting the fill value was unsuccessful."); + HDputs(" Resetting the fill value was unsuccessful."); goto error; } @@ -662,12 +662,12 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) goto error; if (alloc_time != H5D_ALLOC_TIME_LATE) { H5_FAILED(); - puts(" Got non-H5D_ALLOC_TIME_LATE space allocation time."); + HDputs(" Got non-H5D_ALLOC_TIME_LATE space allocation time."); HDprintf(" Got %d\n", alloc_time); } if (fill_time != H5D_FILL_TIME_ALLOC) { H5_FAILED(); - puts(" Got non-H5D_FILL_TIME_ALLOC fill value write time."); + HDputs(" Got non-H5D_FILL_TIME_ALLOC fill value write time."); HDprintf(" Got %d\n", fill_time); } if (H5Dclose(dset3) < 0) @@ -682,7 +682,7 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) goto error; if (layout == H5D_CONTIGUOUS && allocation != H5D_SPACE_STATUS_NOT_ALLOCATED) { H5_FAILED(); - puts(" Got allocated space instead of unallocated."); + HDputs(" Got allocated space instead of unallocated."); HDprintf(" Got %d\n", allocation); goto error; } @@ -694,12 +694,12 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) goto error; if (alloc_time != H5D_ALLOC_TIME_LATE) { H5_FAILED(); - puts(" Got non-H5D_ALLOC_TIME_LATE space allocation time."); + HDputs(" Got non-H5D_ALLOC_TIME_LATE space allocation time."); HDprintf(" Got %d\n", alloc_time); } if (fill_time != H5D_FILL_TIME_NEVER) { H5_FAILED(); - puts(" Got non-H5D_FILL_TIME_NEVER fill value write time."); + HDputs(" Got non-H5D_FILL_TIME_NEVER fill value write time."); HDprintf(" Got %d\n", fill_time); } if (H5Dclose(dset4) < 0) @@ -717,7 +717,7 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) if (!H5_FLT_ABS_EQUAL(rd_c.a, 0) || !H5_DBL_ABS_EQUAL(rd_c.y, fill_ctype.y) || rd_c.x != 0 || rd_c.z != '\0') { H5_FAILED(); - puts(" Got wrong fill value"); + HDputs(" Got wrong fill value"); HDprintf(" Got rd_c.a=%f, rd_c.y=%f and rd_c.x=%d, rd_c.z=%c\n", (double)rd_c.a, rd_c.y, rd_c.x, rd_c.z); } @@ -746,14 +746,14 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) goto error; if (alloc_time != H5D_ALLOC_TIME_EARLY) { H5_FAILED(); - puts(" Got non-H5D_ALLOC_TIME_EARLY space allocation time."); + HDputs(" Got non-H5D_ALLOC_TIME_EARLY space allocation time."); HDprintf(" Got %d\n", alloc_time); } if (H5Pget_fill_time(dcpl, &fill_time) < 0) goto error; if (fill_time != H5D_FILL_TIME_NEVER) { H5_FAILED(); - puts(" Got non-H5D_FILL_TIME_NEVER fill value write time."); + HDputs(" Got non-H5D_FILL_TIME_NEVER fill value write time."); HDprintf(" Got %d\n", fill_time); } if (H5Dclose(dset5) < 0) @@ -786,14 +786,14 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) goto error; if (alloc_time != H5D_ALLOC_TIME_EARLY) { H5_FAILED(); - puts(" Got non-H5D_ALLOC_TIME_EARLY space allocation time."); + HDputs(" Got non-H5D_ALLOC_TIME_EARLY space allocation time."); HDprintf(" Got %d\n", alloc_time); } if (H5Pget_fill_time(dcpl, &fill_time) < 0) goto error; if (fill_time != H5D_FILL_TIME_ALLOC) { H5_FAILED(); - puts(" Got non-H5D_FILL_TIME_ALLOC fill value write time."); + HDputs(" Got non-H5D_FILL_TIME_ALLOC fill value write time."); HDprintf(" Got %d\n", fill_time); } if (H5Dclose(dset6) < 0) @@ -811,7 +811,7 @@ test_create(hid_t fapl, const char *base_name, H5D_layout_t layout) if (!H5_FLT_ABS_EQUAL(rd_c.a, 0) || !H5_DBL_ABS_EQUAL(rd_c.y, fill_ctype.y) || rd_c.x != 0 || rd_c.z != '\0') { H5_FAILED(); - puts(" Got wrong fill value"); + HDputs(" Got wrong fill value"); HDprintf(" Got rd_c.a=%f, rd_c.y=%f and rd_c.x=%d, rd_c.z=%c\n", (double)rd_c.a, rd_c.y, rd_c.x, rd_c.z); } @@ -1649,7 +1649,7 @@ test_extend_cases(hid_t file, hid_t _dcpl, const char *dset_name, hsize_t *ch_si } /* end if */ else { /* Sanity check */ - assert(dtype_class == H5T_COMPOUND); + HDassert(dtype_class == H5T_COMPOUND); /* Initialize specific values for this datatype */ val_size = sizeof(comp_vl_datatype); @@ -1701,7 +1701,7 @@ test_extend_cases(hid_t file, hid_t _dcpl, const char *dset_name, hsize_t *ch_si } /* end for */ /* Check for overflow */ - assert((nelmts * val_size) == (hsize_t)((size_t)(nelmts * val_size))); + HDassert((nelmts * val_size) == (hsize_t)((size_t)(nelmts * val_size))); /* Allocate & initialize buffer */ buf = HDmalloc((size_t)(nelmts * val_size)); @@ -2141,7 +2141,7 @@ test_extend(hid_t fapl, const char *base_name, H5D_layout_t layout) */ if (H5D_CONTIGUOUS == layout) { SKIPPED(); - puts(" Not implemented yet -- needs H5S_SELECT_DIFF operator"); + HDputs(" Not implemented yet -- needs H5S_SELECT_DIFF operator"); goto skip; } #endif @@ -2259,7 +2259,7 @@ test_compatible(void) goto error; if (dims[0] != 8 || dims[1] != 8) { H5_FAILED(); - puts(" Got a different dimension size than what was set."); + HDputs(" Got a different dimension size than what was set."); HDprintf(" Got dims[0]=%ld, dims[1]=%ld, set 8x8\n", (long)dims[0], (long)dims[1]); goto error; } @@ -2271,7 +2271,7 @@ test_compatible(void) goto error; if (val_rd != 0) { H5_FAILED(); - puts(" Got a different value than what was set."); + HDputs(" Got a different value than what was set."); HDprintf(" Got %ld, set 0\n", (long)val_rd); goto error; } @@ -2312,7 +2312,7 @@ test_compatible(void) goto error; if (dims[0] != 8 || dims[1] != 8) { H5_FAILED(); - puts(" Got a different dimension size than what was set."); + HDputs(" Got a different dimension size than what was set."); HDprintf(" Got dims[0]=%ld, dims[1]=%ld, set 8x8\n", (long)dims[0], (long)dims[1]); goto error; } @@ -2324,7 +2324,7 @@ test_compatible(void) goto error; if (val_rd != fill_val) { H5_FAILED(); - puts(" Got a different value than what was set."); + HDputs(" Got a different value than what was set."); HDprintf(" Got %ld, set %ld\n", (long)val_rd, (long)fill_val); goto error; } @@ -2659,11 +2659,11 @@ main(int argc, char *argv[]) if (argc >= 2) { test_contig = test_chunk = test_compact = 0; for (argno = 1; argno < argc; argno++) { - if (!strcmp(argv[argno], "contiguous")) + if (!HDstrcmp(argv[argno], "contiguous")) test_contig = 1; - else if (!strcmp(argv[argno], "chunked")) + else if (!HDstrcmp(argv[argno], "chunked")) test_chunk = 1; - else if (!strcmp(argv[argno], "compact")) + else if (!HDstrcmp(argv[argno], "compact")) test_compact = 1; else { HDfprintf(stderr, "usage: %s [contiguous] [chunked] [compact]\n", argv[0]); @@ -2693,11 +2693,11 @@ main(int argc, char *argv[]) /* Set the FAPL for the type of format */ if (new_format) { - puts("\nTesting with new file format:"); + HDputs("\nTesting with new file format:"); my_fapl = fapl2; } /* end if */ else { - puts("Testing with old file format:"); + HDputs("Testing with old file format:"); my_fapl = fapl; } /* end else */ diff --git a/test/filter_fail.c b/test/filter_fail.c index dca9a20..d7f784c 100644 --- a/test/filter_fail.c +++ b/test/filter_fail.c @@ -171,7 +171,7 @@ test_filter_write(char *file_name, hid_t my_fapl, hbool_t cache_enabled) H5E_END_TRY; if (ret >= 0) { H5_FAILED(); - puts(" Data writing is supposed to fail because the chunk can't be written to file."); + HDputs(" Data writing is supposed to fail because the chunk can't be written to file."); TEST_ERROR } } @@ -189,7 +189,7 @@ test_filter_write(char *file_name, hid_t my_fapl, hbool_t cache_enabled) H5E_END_TRY; if (ret >= 0) { H5_FAILED(); - puts(" Dataset is supposed to fail because the chunk can't be flushed to file."); + HDputs(" Dataset is supposed to fail because the chunk can't be flushed to file."); TEST_ERROR } } diff --git a/test/flushrefreshTest.cmake b/test/flushrefreshTest.cmake index 6faf37b..e280b8b 100644 --- a/test/flushrefreshTest.cmake +++ b/test/flushrefreshTest.cmake @@ -41,7 +41,7 @@ endif () message (STATUS "COMMAND: ${TEST_PROGRAM} ${TEST_ARGS}") if (TEST_LIBRARY_DIRECTORY) - if (WIN32 OR MINGW) + if (WIN32) set (ENV{PATH} "$ENV{PATH};${TEST_LIBRARY_DIRECTORY}") else () set (ENV{LD_LIBRARY_PATH} "$ENV{LD_LIBRARY_PATH}:${TEST_LIBRARY_DIRECTORY}") diff --git a/test/gen_bounds.c b/test/gen_bounds.c index 6933626..d7e9b8c 100644 --- a/test/gen_bounds.c +++ b/test/gen_bounds.c @@ -27,8 +27,60 @@ * that they can or cannot read particular file format. */ +/* + * Add two routines gen_ref_files() and gen_sel_files() to generate the + * following test files: + * + * (1) gen_ref_files(): + * bounds_ref_earliest_latest.h5 + * bounds_ref_latest_latest.h5 + * bounds_ref_v110_v110.h5 + * bounds_ref_v18_v18.h5 + * (1) gen_sel_files(): + * bounds_sel_earliest_latest.h5 + * bounds_sel_latest_latest.h5 + * bounds_sel_v112_v112.h5 + * bounds_sel_v110_v110.h5 + * + * These test files will be copied to 1.12, 1.10 and 1.8 libraries for + * compatibility testing. + */ #include "h5test.h" +/* + * Defines for gen_ref_files() + */ +/* File names used for references */ +#define FILENAME_REF_E_L "bounds_ref_earliest_latest.h5" +#define FILENAME_REF_L_L "bounds_ref_latest_latest.h5" +#define FILENAME_REF_V112_V112 "bounds_ref_v112_v112.h5" +#define FILENAME_REF_V110_V110 "bounds_ref_v110_v110.h5" +#define FILENAME_REF_V18_V18 "bounds_ref_v18_v18.h5" + +/* Dataset names for references */ +#define REVISED_REFS_DSET "Revised_refs_dset" +#define OLD_REF_OBJ_DSET "Old_ref_object_dset" +#define OLD_REF_REG_DSET "Old_ref_region_dset" + +#define GROUP "Group" +#define ATTR "Attr" +#define DATASET "Dataset" +#define POWER32 4294967296 /* 2^32 */ + +/* + * Defines for gen_sel_files() + */ +/* File names for hyperslab/point selections */ +#define FILENAME_SEL_E_L "bounds_sel_earliest_latest.h5" +#define FILENAME_SEL_L_L "bounds_sel_latest_latest.h5" +#define FILENAME_SEL_V112_V112 "bounds_sel_v112_v112.h5" +#define FILENAME_SEL_V110_V110 "bounds_sel_v110_v110.h5" + +/* Dataset names for hyperslab/point selections */ +#define SEL_EX_REG_DSET "Sel_ex32_reg_dset" +#define SEL_EX_IRR_DSET "Sel_ex32_irr_dset" +#define SEL_EX_PT_DSET "Sel_ex32_pt_dset" + /*********************************************************************** * gen_earliest_latest() creates file "bounds_earliest_latest.h5" * @@ -692,6 +744,472 @@ error: return FAIL; } /* gen_v18_v18 */ +/*********************************************************************** + * gen_sel_files() is used to create the following test files: + * bounds_sel_earliest_latest.h5 + * bounds_sel_latest_latest.h5 + * bounds_sel_v112_v112.h5 + * bounds_sel_v110_v110.h5 + * + * File contents for: + * --bounds_sel_earliest_latest.h5 + * --bounds_sel_latest_latest.h5 + * --bounds_sel_v112_v112.h5 + * --each file contains 3 datasets with old region reference type + * (1) Sel_ex32_reg_dset: + * --regular hyperslab selection exceeding 32 bits integer limit + * (2) Sel_ex32_irr_dset: + * --irregular hyperslab selection exceeding 32 bits integer limit + * (3) Sel_ex32_pt_dset: + * --point selection exceeding 32 bits integer limit + * + * File contents for: + * --bounds_ref_v110_v110.h5 + * (1) Sel_ex32_reg_dset: a dataset with old region reference type + * --regular hyperslab selection exceeding 32 bits integer limit + * (2) Sel_ex32_irr_dset: does not exist, cannot be created + * (3) Sel_ex32_pt_dset: does not exist, cannot be created + * + * Return: SUCCEED/FAIL + * + ***********************************************************************/ +static herr_t +gen_sel_files(const char *filename, H5F_libver_t low_bound, H5F_libver_t high_bound) +{ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t did = H5I_INVALID_HID; /* Dataset ID */ + hsize_t numparticles = 8388608; + hsize_t total_particles = numparticles * 513; + hsize_t vdsdims[1] = {total_particles}; /* Dataset dimension size */ + hsize_t coord[4]; /* Point selection */ + hsize_t ref_start; /* Starting location of hyperslab */ + hsize_t ref_stride; /* Stride of hyperslab */ + hsize_t ref_count; /* Element count of hyperslab */ + hsize_t ref_block; /* Block size of hyperslab */ + hid_t ref_sid = H5I_INVALID_HID; /* Dataspace ID for the reference dataset */ + hid_t ref_did = H5I_INVALID_HID; /* Dataset ID for the reference dataset */ + hsize_t ref_dims[1] = {1}; /* Dimension for reference dataset */ + hdset_reg_ref_t ref_wbuf[1]; /* Buffer for dataset region reference */ + + /* + * Create test file, attribute, group and dataset + */ + + /* Create the test file */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create a dataset */ + if ((sid = H5Screate_simple(1, vdsdims, NULL)) < 0) + TEST_ERROR; + + if ((did = H5Dcreate2(fid, DATASET, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + if (H5Sclose(sid) < 0) + TEST_ERROR; + + if (H5Dclose(did) < 0) + TEST_ERROR; + + if (H5Fclose(fid) < 0) + TEST_ERROR; + + /* Create file access property list */ + if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + /* Set to use the low/high bounds in fapl */ + if (H5Pset_libver_bounds(fapl, low_bound, high_bound) < 0) + TEST_ERROR; + + /* Open the file with fapl */ + if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + TEST_ERROR; + + /* Open the dataset */ + if ((did = H5Dopen2(fid, DATASET, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Get the dataset's dataspace */ + if ((sid = H5Dget_space(did)) < 0) + TEST_ERROR; + + /* Create dataspace for the reference dataset */ + ref_dims[0] = 1; + if ((ref_sid = H5Screate_simple(1, ref_dims, NULL)) < 0) + TEST_ERROR; + + /* Generate regular hyperslab exceeding 32 */ + ref_start = 0; + ref_count = 2; + ref_block = 4; + ref_stride = POWER32; + + if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, &ref_start, &ref_stride, &ref_count, &ref_block) < 0) + TEST_ERROR; + + /* Should succeed for v110 and above */ + if (high_bound >= H5F_LIBVER_V110) { + + /* Create the first reference */ + if (H5Rcreate(&ref_wbuf[0], fid, DATASET, H5R_DATASET_REGION, sid) < 0) + TEST_ERROR; + + /* Create the reference datset */ + if ((ref_did = H5Dcreate2(fid, SEL_EX_REG_DSET, H5T_STD_REF_DSETREG, ref_sid, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write to the reference datset */ + if (H5Dwrite(ref_did, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, ref_wbuf) < 0) + TEST_ERROR; + + if (H5Dclose(ref_did) < 0) + TEST_ERROR; + } + + /* Generate irregular hyperslab exceeding 32 */ + ref_start = 8; + ref_count = 5; + ref_block = 2; + ref_stride = POWER32; + if (H5Sselect_hyperslab(sid, H5S_SELECT_OR, &ref_start, &ref_stride, &ref_count, &ref_block) < 0) + TEST_ERROR; + + /* Should succeed for v112 and above */ + if (high_bound >= H5F_LIBVER_V112) { + + /* Create the second reference */ + if (H5Rcreate(&ref_wbuf[0], fid, DATASET, H5R_DATASET_REGION, sid) < 0) + TEST_ERROR; + + /* Create the reference datset */ + if ((ref_did = H5Dcreate2(fid, SEL_EX_IRR_DSET, H5T_STD_REF_DSETREG, ref_sid, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write to the reference datset */ + if (H5Dwrite(ref_did, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, ref_wbuf) < 0) + TEST_ERROR; + + if (H5Dclose(ref_did) < 0) + TEST_ERROR; + } + + /* Generate point selection exceeding 32 */ + coord[0] = 5; + coord[1] = 15; + coord[2] = POWER32 + 1; + coord[3] = 19; + + if (H5Sselect_elements(sid, H5S_SELECT_SET, (size_t)4, coord) < 0) + TEST_ERROR; + + /* Should succeed for v112 and above */ + if (high_bound >= H5F_LIBVER_V112) { + + /* Create the third reference */ + if (H5Rcreate(&ref_wbuf[0], fid, DATASET, H5R_DATASET_REGION, sid) < 0) + TEST_ERROR; + + /* Create the reference datset */ + if ((ref_did = H5Dcreate2(fid, SEL_EX_PT_DSET, H5T_STD_REF_DSETREG, ref_sid, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Write to the reference datset */ + if (H5Dwrite(ref_did, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, ref_wbuf) < 0) + TEST_ERROR; + + if (H5Dclose(ref_did) < 0) + TEST_ERROR; + } + + /* Closing */ + if (H5Sclose(ref_sid) < 0) + TEST_ERROR; + + if (H5Dclose(did) < 0) + TEST_ERROR; + + if (H5Sclose(sid) < 0) + TEST_ERROR; + + if (H5Fclose(fid) < 0) + TEST_ERROR; + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Dclose(ref_did); + H5Sclose(ref_sid); + H5Dclose(did); + H5Sclose(sid); + H5Pclose(fapl); + H5Fclose(fid); + } + H5E_END_TRY; + + return FAIL; + +} /* gen_sel_files() */ + +/*********************************************************************** + * gen_ref_files() is used to create the following test files: + * bounds_ref_earliest_latest.h5 + * bounds_ref_latest_latest.h5 + * bounds_ref_v110_v110.h5 + * bounds_ref_v18_v18.h5 + * + * File contents for: + * --bounds_ref_earliest_latest.h5 + * --bounds_ref_latest_latest.h5 + * --bounds_ref_v112_v112.h5 + * (1) Revised_refs_dset: a dataset created with the revised reference type + * --attribute reference + * --object reference + * --dataset region reference + * (2) Old_ref_object_dset: + * --a dataset created with the old object reference type + * (3) Old_ref_region_dset: + * --a dataset created with the old dataset region reference type + * + * File contents for: + * --bounds_ref_v110_v110.h5 + * --bounds_ref_v18_v18.h5 + * (1) Old_ref_object_dset: + * --a dataset created with the old object reference type + * (2) Old_ref_region_dset: + * --a dataset created with the old dataset region reference type + * + * Return: SUCCEED/FAIL + * + ***********************************************************************/ +static herr_t +gen_ref_files(const char *filename, H5F_libver_t low_bound, H5F_libver_t high_bound) +{ + hid_t fid = H5I_INVALID_HID; /* File ID */ + hid_t gid = H5I_INVALID_HID; /* Group ID */ + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + hid_t aid = H5I_INVALID_HID; /* Attribute ID */ + hid_t asid = H5I_INVALID_HID; /* Dataspace ID for attribute */ + hid_t sid = H5I_INVALID_HID; /* Dataspace ID */ + hid_t did = H5I_INVALID_HID; /* Dataset ID */ + hsize_t dims[1] = {100}; /* Dimension size */ + unsigned * dwbuf = NULL; /* Buffer for writing data */ + hid_t ref_sid = H5I_INVALID_HID; /* Dataspace ID for the reference dataset */ + hid_t ref_did = H5I_INVALID_HID; /* Dataset ID for the reference dataset */ + hsize_t rev_ref_dims[1] = {3}; /* Dimension size for the reference dataset */ + H5R_ref_t rev_ref_wbuf[3]; /* Buffer for storing the revised references */ + hobj_ref_t old_ref_obj_wbuf[1]; /* Buffer for storing the old reference object */ + hdset_reg_ref_t old_ref_reg_wbuf[1]; /* Buffer for storing the old dataset region reference */ + hsize_t old_ref_dims[] = {1}; /* Dimension size for the reference dataset */ + hsize_t start[1]; /* Starting location of hyperslab */ + hsize_t stride[1]; /* Stride of hyperslab */ + hsize_t count[1]; /* Element count of hyperslab */ + hsize_t block[1]; /* Block size of hyperslab */ + unsigned i; /* Local index variable */ + + /* + * Create test file, attribute, group and dataset + */ + + if ((dwbuf = HDcalloc(sizeof(unsigned), 100)) == NULL) + TEST_ERROR; + + /* Create the test file */ + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace for the attribute */ + if ((asid = H5Screate(H5S_SCALAR)) < 0) + TEST_ERROR; + + /* Create an attribute to the root group */ + if ((aid = H5Acreate2(fid, ATTR, H5T_NATIVE_UINT, asid, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create a group */ + if ((gid = H5Gcreate2(fid, GROUP, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace for the dataset */ + if ((sid = H5Screate_simple(1, dims, NULL)) < 0) + TEST_ERROR; + + /* Create a dataset in the group */ + if ((did = H5Dcreate2(gid, DATASET, H5T_NATIVE_UINT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Initialize data to write */ + for (i = 0; i < 100; i++) + dwbuf[i] = i * 3; + + /* Write data to disk */ + if (H5Dwrite(did, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dwbuf) < 0) + TEST_ERROR; + + /* Closing */ + if (H5Dclose(did) < 0) + TEST_ERROR; + if (H5Sclose(sid) < 0) + TEST_ERROR; + if (H5Sclose(asid) < 0) + TEST_ERROR; + if (H5Aclose(aid) < 0) + TEST_ERROR; + if (H5Gclose(gid) < 0) + TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; + + if (dwbuf) { + HDfree(dwbuf); + dwbuf = NULL; + } + + /* Create file access property list */ + if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + /* Set to use the low/high bounds in fapl */ + if (H5Pset_libver_bounds(fapl, low_bound, high_bound) < 0) + TEST_ERROR; + + /* Open the file with fapl */ + if ((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + TEST_ERROR; + + /* + * Create the revised and old references in the file + */ + + /* Retrieve dataspace for the existing dataset */ + if ((did = H5Dopen2(fid, "/Group/Dataset", H5P_DEFAULT)) < 0) + TEST_ERROR; + if ((sid = H5Dget_space(did)) < 0) + TEST_ERROR; + + /* Select 15 2x1 hyperslabs for the dataset region reference */ + start[0] = 2; + stride[0] = 5; + count[0] = 15; + block[0] = 2; + if (H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, stride, count, block) < 0) + TEST_ERROR; + + if (high_bound >= H5F_LIBVER_V112) { + + /* Create dataspace for the reference dataset */ + if ((ref_sid = H5Screate_simple(1, rev_ref_dims, NULL)) < 0) + TEST_ERROR; + + /* Create a dataset with the revised reference type */ + ref_did = + H5Dcreate2(fid, REVISED_REFS_DSET, H5T_STD_REF, ref_sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + /* Store the reference to "Attr" */ + if (H5Rcreate_attr(fid, "/", "Attr", H5P_DEFAULT, &rev_ref_wbuf[0]) < 0) + TEST_ERROR; + + /* Store the reference to /Group */ + if (H5Rcreate_object(fid, "/Group", H5P_DEFAULT, &rev_ref_wbuf[1]) < 0) + TEST_ERROR; + + /* Store the dataset region referenced to /Group/Dataset */ + if (H5Rcreate_region(fid, "/Group/Dataset", sid, H5P_DEFAULT, &rev_ref_wbuf[2]) < 0) + TEST_ERROR; + + /* Write to the reference dataset */ + if (H5Dwrite(ref_did, H5T_STD_REF, H5S_ALL, H5S_ALL, H5P_DEFAULT, rev_ref_wbuf) < 0) + TEST_ERROR; + + /* Destroy references */ + for (i = 0; i < 3; i++) + if (H5Rdestroy(&rev_ref_wbuf[i]) < 0) + TEST_ERROR; + + /* Closing */ + if (H5Dclose(ref_did) < 0) + TEST_ERROR; + + if (H5Sclose(ref_sid) < 0) + TEST_ERROR; + } + + /* Create dataspace for the reference dataset */ + if ((ref_sid = H5Screate_simple(1, old_ref_dims, NULL)) < 0) + TEST_ERROR; + + /* Create a dataset with the old object reference type */ + if ((ref_did = H5Dcreate2(fid, OLD_REF_OBJ_DSET, H5T_STD_REF_OBJ, ref_sid, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create reference to /Group */ + if (H5Rcreate(&old_ref_obj_wbuf[0], fid, "/Group", H5R_OBJECT, -1) < 0) + TEST_ERROR; + + /* Write to the reference dataset */ + if (H5Dwrite(ref_did, H5T_STD_REF_OBJ, H5S_ALL, H5S_ALL, H5P_DEFAULT, old_ref_obj_wbuf) < 0) + TEST_ERROR; + + /* Close the dataset */ + if (H5Dclose(ref_did) < 0) + TEST_ERROR; + + /* Create a dataset with the old dataset region reference type */ + if ((ref_did = H5Dcreate2(fid, OLD_REF_REG_DSET, H5T_STD_REF_DSETREG, ref_sid, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataset region reference */ + if (H5Rcreate(&old_ref_reg_wbuf[0], fid, "/Group/Dataset", H5R_DATASET_REGION, sid) < 0) + TEST_ERROR; + + /* Write selection to the reference dataset */ + if (H5Dwrite(ref_did, H5T_STD_REF_DSETREG, H5S_ALL, H5S_ALL, H5P_DEFAULT, old_ref_reg_wbuf) < 0) + TEST_ERROR; + + /* Closing */ + if (H5Dclose(ref_did) < 0) + TEST_ERROR; + if (H5Sclose(ref_sid) < 0) + TEST_ERROR; + + if (H5Dclose(did) < 0) + TEST_ERROR; + if (H5Sclose(sid) < 0) + TEST_ERROR; + if (H5Pclose(fapl) < 0) + TEST_ERROR; + if (H5Fclose(fid) < 0) + TEST_ERROR; + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Dclose(ref_did); + H5Sclose(ref_sid); + H5Dclose(did); + H5Sclose(sid); + H5Pclose(fapl); + H5Fclose(fid); + HDfree(dwbuf); + } + H5E_END_TRY; + + return FAIL; + +} /* gen_ref_files() */ + int main(void) { @@ -715,6 +1233,50 @@ main(void) if (gen_v18_v18() < 0) TEST_ERROR; + /* + * Files generated via gen_ref_files() + */ + + /* Generate bounds_ref_earliest_latest.h5 */ + if (gen_ref_files(FILENAME_REF_E_L, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0) + TEST_ERROR; + + /* Generate bounds_ref_latest_latest.h5 */ + if (gen_ref_files(FILENAME_REF_L_L, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + TEST_ERROR; + + /* Generate bounds_ref_v112_v112.h5 */ + if (gen_ref_files(FILENAME_REF_V112_V112, H5F_LIBVER_V112, H5F_LIBVER_V112) < 0) + TEST_ERROR; + + /* Generate bounds_ref_v110_v110.h5 */ + if (gen_ref_files(FILENAME_REF_V110_V110, H5F_LIBVER_V110, H5F_LIBVER_V110) < 0) + TEST_ERROR; + + /* Generate bounds_ref_v18_v18.h5 */ + if (gen_ref_files(FILENAME_REF_V18_V18, H5F_LIBVER_V18, H5F_LIBVER_V18) < 0) + TEST_ERROR; + + /* + * Files generated via gen_sel_files() + */ + + /* Generate bounds_sel_earliest_latest.h5 */ + if (gen_sel_files(FILENAME_SEL_E_L, H5F_LIBVER_EARLIEST, H5F_LIBVER_LATEST) < 0) + TEST_ERROR; + + /* Generate bounds_sel_latest_latest.h5 */ + if (gen_sel_files(FILENAME_SEL_L_L, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + TEST_ERROR; + + /* Generate bounds_sel_v112_v112.h5 */ + if (gen_sel_files(FILENAME_SEL_V112_V112, H5F_LIBVER_V112, H5F_LIBVER_V112) < 0) + TEST_ERROR; + + /* Generate bounds_sel_v110_v110.h5 */ + if (gen_sel_files(FILENAME_SEL_V110_V110, H5F_LIBVER_V110, H5F_LIBVER_V110) < 0) + TEST_ERROR; + return EXIT_SUCCESS; error: diff --git a/test/gen_cross.c b/test/gen_cross.c index 37c6dcf..030cc65 100644 --- a/test/gen_cross.c +++ b/test/gen_cross.c @@ -928,7 +928,7 @@ create_deflate_dsets_float(hid_t fid, hid_t fsid, hid_t msid) #else /* H5_HAVE_FILTER_DEFLATE */ const char *not_supported = "Deflate filter is not enabled. Can't create the dataset."; - puts(not_supported); + HDputs(not_supported); #endif /* H5_HAVE_FILTER_DEFLATE */ return 0; diff --git a/test/gen_filters.c b/test/gen_filters.c index 502e041..0d107a9 100644 --- a/test/gen_filters.c +++ b/test/gen_filters.c @@ -243,12 +243,12 @@ main(void) if (nerrors) goto error; - printf("All tests passed.\n"); + HDprintf("All tests passed.\n"); return 0; error: nerrors = MAX(1, nerrors); - printf("***** %d GEN_FILTERS FAILURES *****\n", nerrors); + HDprintf("***** %d GEN_FILTERS FAILURES *****\n", nerrors); return 1; } diff --git a/test/getname.c b/test/getname.c index 61feee6..0cf9768 100644 --- a/test/getname.c +++ b/test/getname.c @@ -3787,7 +3787,7 @@ main(void) if (nerrors) goto error; - puts("All getname tests passed."); + HDputs("All getname tests passed."); h5_cleanup(FILENAME, fapl); @@ -3797,7 +3797,7 @@ error: H5E_BEGIN_TRY { H5Fclose(file_id); } H5E_END_TRY; - puts("***** GET NAME TESTS FAILED *****"); + HDputs("***** GET NAME TESTS FAILED *****"); return 1; } diff --git a/test/gheap.c b/test/gheap.c index 51e1dd2..1a8a4c0 100644 --- a/test/gheap.c +++ b/test/gheap.c @@ -42,9 +42,9 @@ nerrors++; \ if (nerrors <= GHEAP_REPEATED_ERR_LIM) { \ H5_FAILED(); \ - puts(MSG); \ + HDputs(MSG); \ if (nerrors == GHEAP_REPEATED_ERR_LIM) \ - puts(" Suppressing further errors..."); \ + HDputs(" Suppressing further errors..."); \ } /* end if */ \ } /* end GHEAP_REPEATED_ERR */ @@ -226,7 +226,7 @@ test_2(hid_t fapl) HDputs(" Unable to read object"); nerrors++; } - else if (memcmp(in, out, size)) { + else if (HDmemcmp(in, out, size)) { H5_FAILED(); HDputs(" Value read doesn't match value written"); nerrors++; diff --git a/test/h5test.c b/test/h5test.c index 33af30d..dbd1a90 100644 --- a/test/h5test.c +++ b/test/h5test.c @@ -2267,3 +2267,61 @@ done: HDfree(dup_buf); return ret_value; } /* end h5_duplicate_file_by_bytes() */ + +/*------------------------------------------------------------------------- + * Function: h5_check_if_file_locking_enabled + * + * Purpose: Checks if file locking is enabled on this file system. + * + * Return: SUCCEED/FAIL + * are_enabled will be FALSE if file locking is disabled on + * the file system of if there were errors. + * + *------------------------------------------------------------------------- + */ +herr_t +h5_check_if_file_locking_enabled(hbool_t *is_enabled) +{ + const char *filename = "locking_test_file"; + int pmode = O_RDWR | O_CREAT | O_TRUNC; + int fd = -1; + + *is_enabled = TRUE; + + if ((fd = HDopen(filename, pmode, H5_POSIX_CREATE_MODE_RW)) < 0) + goto error; + + /* Test HDflock() to see if it works */ + if (HDflock(fd, LOCK_EX | LOCK_NB) < 0) { + if (ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. This is most frequently used on + * Lustre. If we also want to check for disabled NFS locks + * we'll need to check for ENOLCK, too. That isn't done by + * default here since that could also represent an actual + * error condition. + */ + errno = 0; + *is_enabled = FALSE; + } + else + goto error; + } + if (HDflock(fd, LOCK_UN) < 0) + goto error; + + if (HDclose(fd) < 0) + goto error; + if (HDremove(filename) < 0) + goto error; + + return SUCCEED; + +error: + *is_enabled = FALSE; + if (fd > -1) { + HDclose(fd); + HDremove(filename); + } + return FAIL; +} /* end h5_check_if_file_locking_enabled() */ diff --git a/test/h5test.h b/test/h5test.h index df996c2..66929cd 100644 --- a/test/h5test.h +++ b/test/h5test.h @@ -258,6 +258,7 @@ H5TEST_DLL H5VL_class_t *h5_get_dummy_vol_class(void); H5TEST_DLL const char * h5_get_version_string(H5F_libver_t libver); H5TEST_DLL int h5_compare_file_bytes(char *fname1, char *fname2); H5TEST_DLL int h5_duplicate_file_by_bytes(const char *orig, const char *dest); +H5TEST_DLL herr_t h5_check_if_file_locking_enabled(hbool_t *are_enabled); /* Functions that will replace components of a FAPL */ H5TEST_DLL herr_t h5_get_vfd_fapl(hid_t fapl_id); diff --git a/test/hdfs.c b/test/hdfs.c index ee66314..a59cdb6 100644 --- a/test/hdfs.c +++ b/test/hdfs.c @@ -303,7 +303,7 @@ *---------------------------------------------------------------------------- */ #define JSVERIFY_STR(expected, actual, reason) \ - if (strcmp((actual), (expected)) != 0) { \ + if (HDstrcmp((actual), (expected)) != 0) { \ JSERR_STR((expected), (actual), (reason)); \ goto error; \ } /* JSVERIFY_STR */ @@ -348,7 +348,7 @@ *---------------------------------------------------------------------------- */ #define JSVERIFY_STR(actual, expected, reason) \ - if (strcmp((actual), (expected)) != 0) { \ + if (HDstrcmp((actual), (expected)) != 0) { \ JSERR_STR((expected), (actual), (reason)); \ goto error; \ } /* JSVERIFY_STR */ @@ -414,8 +414,8 @@ test_fapl_config_validation(void) #ifndef H5_HAVE_LIBHDFS TESTING("HDFS fapl configuration validation"); SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); + HDputs(" HDFS VFD is not enabled"); + HDfflush(stdout); return 0; #else @@ -564,9 +564,9 @@ test_fapl_config_validation(void) JSVERIFY(config.version, fa_fetch.version, "version number mismatch") JSVERIFY(config.namenode_port, fa_fetch.namenode_port, "namenode port mismatch") JSVERIFY(config.stream_buffer_size, fa_fetch.stream_buffer_size, "streambuffer size mismatch") - JSVERIFY_STR(config.namenode_name, fa_fetch.namenode_name, NULL) - JSVERIFY_STR(config.user_name, fa_fetch.user_name, NULL) - JSVERIFY_STR(config.kerberos_ticket_cache, fa_fetch.kerberos_ticket_cache, NULL) + JSVERIFY_STR(config.namenode_name, fa_fetch.namenode_name, "node name mismatch") + JSVERIFY_STR(config.user_name, fa_fetch.user_name, "user name mismatch") + JSVERIFY_STR(config.kerberos_ticket_cache, fa_fetch.kerberos_ticket_cache, "kerberos ticket cache mismatch") } /*----------------------------- @@ -620,8 +620,8 @@ test_hdfs_fapl(void) #ifndef H5_HAVE_LIBHDFS TESTING("HDFS fapl "); SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); + HDputs(" HDFS VFD is not enabled"); + HDfflush(stdout); return 0; #else @@ -703,8 +703,8 @@ test_vfd_open(void) #ifndef H5_HAVE_LIBHDFS TESTING("HDFS VFD-level open"); SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); + HDputs(" HDFS VFD is not enabled"); + HDfflush(stdout); return 0; #else @@ -956,8 +956,8 @@ test_eof_eoa(void) #ifndef H5_HAVE_LIBHDFS TESTING("HDFS eof/eoa gets and sets"); SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); + HDputs(" HDFS VFD is not enabled"); + HDfflush(stdout); return 0; #else @@ -996,7 +996,7 @@ test_eof_eoa(void) /* verify as found */ - JSVERIFY(5458199, H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), NULL) + JSVERIFY(5458199, H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), "EOF mismatch") JSVERIFY(H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), H5FDget_eof(fd_shakespeare, H5FD_MEM_DRAW), "mismatch between DEFAULT and RAW memory types") JSVERIFY(0, H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), "EoA should be unset by H5FDopen") @@ -1066,8 +1066,8 @@ test_H5FDread_without_eoa_set_fails(void) #ifndef H5_HAVE_LIBHDFS TESTING("HDFS VFD read-eoa temporal coupling library limitation"); SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); + HDputs(" HDFS VFD is not enabled"); + HDfflush(stdout); return 0; #else @@ -1165,8 +1165,8 @@ test_read(void) #ifndef H5_HAVE_LIBHDFS TESTING("HDFS VFD read/range-gets"); SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); + HDputs(" HDFS VFD is not enabled"); + HDfflush(stdout); return 0; #else @@ -1272,7 +1272,7 @@ test_read(void) HADDR_UNDEF); /* Demonstrate success with "automatic" value */ FAIL_IF(NULL == file_raven) - JSVERIFY(6464, H5FDget_eof(file_raven, H5FD_MEM_DEFAULT), NULL) + JSVERIFY(6464, H5FDget_eof(file_raven, H5FD_MEM_DEFAULT), "EOF mismatch") /********* * TESTS * @@ -1376,8 +1376,8 @@ test_noops_and_autofails(void) #ifndef H5_HAVE_LIBHDFS TESTING("HDFS VFD always-fail and no-op routines"); SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); + HDputs(" HDFS VFD is not enabled"); + HDfflush(stdout); return 0; #else @@ -1432,8 +1432,8 @@ test_noops_and_autofails(void) /* no-op calls to `lock()` and `unlock()` */ JSVERIFY(SUCCEED, H5FDlock(file, TRUE), "lock always succeeds; has no effect") - JSVERIFY(SUCCEED, H5FDlock(file, FALSE), NULL) - JSVERIFY(SUCCEED, H5FDunlock(file), NULL) + JSVERIFY(SUCCEED, H5FDlock(file, FALSE), "lock issue") + JSVERIFY(SUCCEED, H5FDunlock(file), "unlock issue") /* Lock/unlock with null file or similar error crashes tests. * HDassert in calling heirarchy, `H5FD[un]lock()` and `H5FD_[un]lock()` */ @@ -1528,8 +1528,8 @@ test_H5F_integration(void) #ifndef H5_HAVE_LIBHDFS TESTING("HDFS file access through HD5F library (H5F API)"); SKIPPED(); - puts(" HDFS VFD is not enabled"); - fflush(stdout); + HDputs(" HDFS VFD is not enabled"); + HDfflush(stdout); return 0; #else @@ -1597,7 +1597,7 @@ error: #if HDFS_TEST_DEBUG HDprintf("\nerror!"); - fflush(stdout); + HDfflush(stdout); #endif /* HDFS_TEST_DEBUG */ if (fapl_id >= 0) { diff --git a/test/hyperslab.c b/test/hyperslab.c index 43992de..5c7b78f 100644 --- a/test/hyperslab.c +++ b/test/hyperslab.c @@ -177,7 +177,7 @@ test_fill(size_t nx, size_t ny, size_t nz, size_t di, size_t dj, size_t dk, size } /* end else */ HDsprintf(s, "Testing hyperslab fill %-11s variable hyperslab", dim); HDprintf("%-70s", s); - fflush(stdout); + HDfflush(stdout); /* Allocate array */ if (NULL == (dst = (uint8_t *)HDcalloc((size_t)1, nx * ny * nz))) @@ -366,7 +366,7 @@ test_copy(int mode, size_t nx, size_t ny, size_t nz, size_t di, size_t dj, size_ HDsprintf(s, "Testing hyperslab copy %-11s %s", dim, sub); HDprintf("%-70s", s); - fflush(stdout); + HDfflush(stdout); /* * Allocate arrays @@ -575,7 +575,7 @@ test_multifill(size_t nx) hsize_t i, j; HDprintf("%-70s", "Testing multi-byte fill value"); - fflush(stdout); + HDfflush(stdout); /* Initialize the source and destination */ if (NULL == (src = (struct a_struct *)HDmalloc(nx * sizeof(*src)))) @@ -687,7 +687,7 @@ test_endian(size_t nx) hsize_t i, j; HDprintf("%-70s", "Testing endian conversion by stride"); - fflush(stdout); + HDfflush(stdout); /* Initialize arrays */ if (NULL == (src = (uint8_t *)HDmalloc(nx * 4))) @@ -773,7 +773,7 @@ test_transpose(size_t nx, size_t ny) HDsprintf(s, "Testing 2d transpose by stride %4lux%-lud", (unsigned long)nx, (unsigned long)ny); HDprintf("%-70s", s); - fflush(stdout); + HDfflush(stdout); /* Initialize */ if (NULL == (src = (int *)HDmalloc(nx * ny * sizeof(*src)))) @@ -872,7 +872,7 @@ test_sub_super(size_t nx, size_t ny) HDsprintf(s, "Testing image sampling %4lux%-4lu to %4lux%-4lu ", (unsigned long)(2 * nx), (unsigned long)(2 * ny), (unsigned long)nx, (unsigned long)ny); HDprintf("%-70s", s); - fflush(stdout); + HDfflush(stdout); /* Initialize */ if (NULL == (full = (uint8_t *)HDmalloc(4 * nx * ny))) @@ -922,7 +922,7 @@ test_sub_super(size_t nx, size_t ny) HDsprintf(s, "Testing image sampling %4lux%-4lu to %4lux%-4lu ", (unsigned long)nx, (unsigned long)ny, (unsigned long)(2 * nx), (unsigned long)(2 * ny)); HDprintf("%-70s", s); - fflush(stdout); + HDfflush(stdout); /* Setup stride */ size[0] = nx; diff --git a/test/istore.c b/test/istore.c index a724359..c600c31 100644 --- a/test/istore.c +++ b/test/istore.c @@ -619,13 +619,13 @@ main(int argc, char *argv[]) else { int i; for (i = 1, size_of_test = 0; i < argc; i++) { - if (!strcmp(argv[i], "small")) { + if (!HDstrcmp(argv[i], "small")) { size_of_test |= TEST_SMALL; } - else if (!strcmp(argv[i], "medium")) { + else if (!HDstrcmp(argv[i], "medium")) { size_of_test |= TEST_MEDIUM; } - else if (!strcmp(argv[i], "large")) { + else if (!HDstrcmp(argv[i], "large")) { size_of_test |= TEST_LARGE; } else { diff --git a/test/lheap.c b/test/lheap.c index 17e3082..27e025e 100644 --- a/test/lheap.c +++ b/test/lheap.c @@ -103,7 +103,7 @@ main(void) if (j > 4) buf[j] = '\0'; - if (UFAIL == (obj[i] = H5HL_insert(f, heap, strlen(buf) + 1, buf))) { + if (UFAIL == (obj[i] = H5HL_insert(f, heap, HDstrlen(buf) + 1, buf))) { H5_FAILED(); H5Eprint2(H5E_DEFAULT, stdout); goto error; @@ -155,7 +155,7 @@ main(void) goto error; } - if (strcmp(s, buf)) { + if (HDstrcmp(s, buf)) { H5_FAILED(); HDprintf(" i=%d, heap offset=%lu\n", i, (unsigned long)(obj[i])); HDprintf(" got: \"%s\"\n", s); diff --git a/test/links.c b/test/links.c index 43cd83d..f0932dc 100644 --- a/test/links.c +++ b/test/links.c @@ -14138,7 +14138,7 @@ UD_cb_create(const char *link_name, hid_t loc_group, const void *udata, size_t u if (lcpl_id < 0) TEST_ERROR - if (HDstrcmp(link_name, UD_CB_LINK_NAME) && strcmp(link_name, NEW_UD_CB_LINK_NAME)) + if (HDstrcmp(link_name, UD_CB_LINK_NAME) && HDstrcmp(link_name, NEW_UD_CB_LINK_NAME)) TEST_ERROR if (HDstrcmp((const char *)udata, UD_CB_TARGET)) TEST_ERROR @@ -14165,7 +14165,7 @@ UD_cb_traverse(const char *link_name, hid_t cur_group, const void *udata, size_t if (udata_size > 0 && !udata) TEST_ERROR - if (HDstrcmp(link_name, UD_CB_LINK_NAME) && strcmp(link_name, NEW_UD_CB_LINK_NAME)) + if (HDstrcmp(link_name, UD_CB_LINK_NAME) && HDstrcmp(link_name, NEW_UD_CB_LINK_NAME)) TEST_ERROR if (HDstrcmp((const char *)udata, UD_CB_TARGET)) TEST_ERROR @@ -14537,7 +14537,7 @@ lapl_udata(hid_t fapl, hbool_t new_format) TEST_ERROR /* Now use the same ud link to access group_b */ - strcpy(group_b_name, "group_b"); + HDstrcpy(group_b_name, "group_b"); if (H5Pset(plist_id, DEST_PROP_NAME, group_b_name) < 0) TEST_ERROR @@ -14851,7 +14851,7 @@ ud_link_errors(hid_t fapl, hbool_t new_format) H5E_END_TRY; /* Create a user-defined link to the group. */ - strcpy(group_name, "/group"); + HDstrcpy(group_name, "/group"); if (H5Lcreate_ud(fid, "/ud_link", (H5L_type_t)UD_CBFAIL_TYPE, &group_name, HDstrlen(group_name) + 1, H5P_DEFAULT, H5P_DEFAULT) < 0) FAIL_STACK_ERROR diff --git a/test/links_env.c b/test/links_env.c index 0e5d1c0..2cfa3fa 100644 --- a/test/links_env.c +++ b/test/links_env.c @@ -112,7 +112,7 @@ external_link_env(hid_t fapl, hbool_t new_format) /* Should be able to find the target file from pathnames set via HDF5_EXT_PREFIX */ if (gid < 0) { H5_FAILED(); - puts(" Should have found the file in tmp_links_env directory."); + HDputs(" Should have found the file in tmp_links_env directory."); goto error; } @@ -6112,7 +6112,7 @@ test_mf_bug1(const char *env_h5_drvr, hid_t fapl) /* Free memb_name */ for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) - free(memb_name[mt]); + HDfree(memb_name[mt]); } /* end else */ /* Close memb_fapl */ @@ -7685,7 +7685,7 @@ set_multi_split(hid_t fapl, hsize_t pagesize, hbool_t is_multi_or_split) /* Free memb_name */ for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) - free(memb_name[mt]); + HDfree(memb_name[mt]); return 0; @@ -8841,7 +8841,7 @@ test_page_alignment(const char *env_h5_drvr, hid_t fapl) /* Free memb_name */ for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) - free(memb_name[mt]); + HDfree(memb_name[mt]); /* Close memb_fapl */ if (H5Pclose(memb_fapl) < 0) diff --git a/test/mirror_vfd.c b/test/mirror_vfd.c index 325999b..43141e0 100644 --- a/test/mirror_vfd.c +++ b/test/mirror_vfd.c @@ -63,8 +63,8 @@ static unsigned int g_verbosity = DEFAULT_VERBOSITY; #define LOGPRINT(lvl, ...) \ do { \ if ((lvl) <= g_verbosity) { \ - fprintf(g_log_stream, __VA_ARGS__); \ - fflush(g_log_stream); \ + HDfprintf(g_log_stream, __VA_ARGS__); \ + HDfflush(g_log_stream); \ } \ } while (0) @@ -150,7 +150,7 @@ _populate_filepath(const char *dirname, const char *_basename, hid_t fapl_id, ch } if (HDsnprintf(_path, H5FD_SPLITTER_PATH_MAX, "%s%s%s", dirname, - (dirname[strlen(dirname)] == '/') ? "" : "/", /* slash iff needed */ + (dirname[HDstrlen(dirname)] == '/') ? "" : "/", /* slash iff needed */ _basename) > H5FD_SPLITTER_PATH_MAX) { TEST_ERROR; } diff --git a/test/mtime.c b/test/mtime.c index c28b81a..a8ad6ae 100644 --- a/test/mtime.c +++ b/test/mtime.c @@ -147,7 +147,7 @@ main(void) H5_FAILED(); /* If this fails, examine H5Omtime.c. Modification time is very * system dependent (e.g., on Windows DST must be hardcoded). */ - puts(" Old modification time incorrect"); + HDputs(" Old modification time incorrect"); goto error; } if (H5Fclose(file) < 0) diff --git a/test/objcopy.c b/test/objcopy.c index 383bd05..876e7de 100644 --- a/test/objcopy.c +++ b/test/objcopy.c @@ -4494,7 +4494,7 @@ test_copy_dataset_compressed(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, hid #ifndef H5_HAVE_FILTER_DEFLATE SKIPPED(); - puts(" Deflation filter not available"); + HDputs(" Deflation filter not available"); #else /* H5_HAVE_FILTER_DEFLATE */ /* set initial data values */ for (i = 0; i < DIM_SIZE_1; i++) @@ -4919,7 +4919,7 @@ test_copy_dataset_no_edge_filt(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, h #ifndef H5_HAVE_FILTER_DEFLATE SKIPPED(); - puts(" Deflation filter not available"); + HDputs(" Deflation filter not available"); #else /* H5_HAVE_FILTER_DEFLATE */ /* set initial data values */ for (i = 0; i < DIM_SIZE_1; i++) @@ -7273,7 +7273,7 @@ test_copy_dataset_compressed_vl(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, #ifndef H5_HAVE_FILTER_DEFLATE SKIPPED(); - puts(" Deflation filter not available"); + HDputs(" Deflation filter not available"); #else /* H5_HAVE_FILTER_DEFLATE */ /* set initial data values */ for (i = 0; i < DIM_SIZE_1; i++) { diff --git a/test/ohdr.c b/test/ohdr.c index 87d394b..14d021b 100644 --- a/test/ohdr.c +++ b/test/ohdr.c @@ -121,8 +121,12 @@ test_cont(char *filename, hid_t fapl) FAIL_STACK_ERROR if (1 != H5O_link(&oh_locB, 1)) FAIL_STACK_ERROR + if (H5AC_prep_for_file_flush(f) < 0) + FAIL_STACK_ERROR if (H5AC_flush(f) < 0) FAIL_STACK_ERROR + if (H5AC_secure_from_file_flush(f) < 0) + FAIL_STACK_ERROR if (H5O__expunge_chunks_test(&oh_locA) < 0) FAIL_STACK_ERROR @@ -1814,8 +1818,12 @@ main(void) FAIL_STACK_ERROR if (1 != H5O_link(&oh_loc, 1)) FAIL_STACK_ERROR + if (H5AC_prep_for_file_flush(f) < 0) + FAIL_STACK_ERROR if (H5AC_flush(f) < 0) FAIL_STACK_ERROR + if (H5AC_secure_from_file_flush(f) < 0) + FAIL_STACK_ERROR if (H5AC_expunge_entry(f, H5AC_OHDR, oh_loc.addr, H5AC__NO_FLAGS_SET) < 0) FAIL_STACK_ERROR if (NULL == H5O_msg_read(&oh_loc, H5O_MTIME_NEW_ID, &ro)) @@ -1831,8 +1839,12 @@ main(void) time_new = 33333333; if (H5O_msg_write(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new) < 0) FAIL_STACK_ERROR + if (H5AC_prep_for_file_flush(f) < 0) + FAIL_STACK_ERROR if (H5AC_flush(f) < 0) FAIL_STACK_ERROR + if (H5AC_secure_from_file_flush(f) < 0) + FAIL_STACK_ERROR if (H5AC_expunge_entry(f, H5AC_OHDR, oh_loc.addr, H5AC__NO_FLAGS_SET) < 0) FAIL_STACK_ERROR if (NULL == H5O_msg_read(&oh_loc, H5O_MTIME_NEW_ID, &ro)) @@ -1862,8 +1874,12 @@ main(void) if (H5O_msg_create(&oh_loc, H5O_MTIME_ID, 0, 0, &time_new) < 0) FAIL_STACK_ERROR } /* end for */ + if (H5AC_prep_for_file_flush(f) < 0) + FAIL_STACK_ERROR if (H5AC_flush(f) < 0) FAIL_STACK_ERROR + if (H5AC_secure_from_file_flush(f) < 0) + FAIL_STACK_ERROR if (H5AC_expunge_entry(f, H5AC_OHDR, oh_loc.addr, H5AC__NO_FLAGS_SET) < 0) FAIL_STACK_ERROR @@ -1905,8 +1921,12 @@ main(void) time_new = (i + 1) * 1000 + 10; if (H5O_msg_create(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new) < 0) FAIL_STACK_ERROR + if (H5AC_prep_for_file_flush(f) < 0) + FAIL_STACK_ERROR if (H5AC_flush(f) < 0) FAIL_STACK_ERROR + if (H5AC_secure_from_file_flush(f) < 0) + FAIL_STACK_ERROR if (H5AC_expunge_entry(f, H5AC_OHDR, oh_loc.addr, H5AC__NO_FLAGS_SET) < 0) FAIL_STACK_ERROR } /* end for */ @@ -1934,8 +1954,12 @@ main(void) time_new = 22222222; if (H5O_msg_create(&oh_loc, H5O_MTIME_NEW_ID, H5O_MSG_FLAG_CONSTANT, 0, &time_new) < 0) FAIL_STACK_ERROR + if (H5AC_prep_for_file_flush(f) < 0) + FAIL_STACK_ERROR if (H5AC_flush(f) < 0) FAIL_STACK_ERROR + if (H5AC_secure_from_file_flush(f) < 0) + FAIL_STACK_ERROR if (H5AC_expunge_entry(f, H5AC_OHDR, oh_loc.addr, H5AC__NO_FLAGS_SET) < 0) FAIL_STACK_ERROR if (NULL == H5O_msg_read(&oh_loc, H5O_MTIME_NEW_ID, &ro)) diff --git a/test/page_buffer.c b/test/page_buffer.c index ca33872..abbae32 100644 --- a/test/page_buffer.c +++ b/test/page_buffer.c @@ -349,7 +349,7 @@ set_multi_split(const char *env_h5_drvr, hid_t fapl, hsize_t pagesize) /* Free memb_name */ for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) - free(memb_name[mt]); + HDfree(memb_name[mt]); } /* end if */ diff --git a/test/pool.c b/test/pool.c index 05304e5..ef2a627 100644 --- a/test/pool.c +++ b/test/pool.c @@ -784,11 +784,11 @@ main(void) if (nerrors) goto error; - puts("All memory pool tests passed."); + HDputs("All memory pool tests passed."); return 0; error: - puts("*** TESTS FAILED ***"); + HDputs("*** TESTS FAILED ***"); return 1; } diff --git a/test/ros3.c b/test/ros3.c index 7ecba0e..654082b 100644 --- a/test/ros3.c +++ b/test/ros3.c @@ -318,7 +318,7 @@ jserr_str(const char *expected, const char *actual, const char *reason) *---------------------------------------------------------------------------- */ #define JSVERIFY_STR(expected, actual, reason) \ - if (strcmp((actual), (expected)) != 0) { \ + if (HDstrcmp((actual), (expected)) != 0) { \ JSERR_STR((expected), (actual), (reason)); \ goto error; \ } /* JSVERIFY_STR */ @@ -363,7 +363,7 @@ jserr_str(const char *expected, const char *actual, const char *reason) *---------------------------------------------------------------------------- */ #define JSVERIFY_STR(actual, expected, reason) \ - if (strcmp((actual), (expected)) != 0) { \ + if (HDstrcmp((actual), (expected)) != 0) { \ JSERR_STR((expected), (actual), (reason)); \ goto error; \ } /* JSVERIFY_STR */ @@ -560,8 +560,8 @@ test_fapl_config_validation(void) if (FALSE == s3_test_bucket_defined) { SKIPPED(); - puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); - fflush(stdout); + HDputs(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + HDfflush(stdout); return 0; } @@ -835,8 +835,8 @@ test_vfd_open(void) if (FALSE == s3_test_bucket_defined) { SKIPPED(); - puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); - fflush(stdout); + HDputs(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + HDfflush(stdout); return 0; } @@ -969,15 +969,15 @@ test_eof_eoa(void) if (s3_test_credentials_loaded == 0) { SKIPPED(); - puts(" s3 credentials are not loaded"); - fflush(stdout); + HDputs(" s3 credentials are not loaded"); + HDfflush(stdout); return 0; } if (FALSE == s3_test_bucket_defined) { SKIPPED(); - puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); - fflush(stdout); + HDputs(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + HDfflush(stdout); return 0; } @@ -1077,15 +1077,15 @@ test_H5FDread_without_eoa_set_fails(void) if (s3_test_credentials_loaded == 0) { SKIPPED(); - puts(" s3 credentials are not loaded"); - fflush(stdout); + HDputs(" s3 credentials are not loaded"); + HDfflush(stdout); return 0; } if (FALSE == s3_test_bucket_defined) { SKIPPED(); - puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); - fflush(stdout); + HDputs(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + HDfflush(stdout); return 0; } @@ -1248,15 +1248,15 @@ test_read(void) if (s3_test_credentials_loaded == 0) { SKIPPED(); - puts(" s3 credentials are not loaded"); - fflush(stdout); + HDputs(" s3 credentials are not loaded"); + HDfflush(stdout); return 0; } if (FALSE == s3_test_bucket_defined) { SKIPPED(); - puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); - fflush(stdout); + HDputs(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + HDfflush(stdout); return 0; } @@ -1389,8 +1389,8 @@ test_noops_and_autofails(void) if (FALSE == s3_test_bucket_defined) { SKIPPED(); - puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); - fflush(stdout); + HDputs(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + HDfflush(stdout); return 0; } @@ -1515,15 +1515,15 @@ test_cmp(void) if (s3_test_credentials_loaded == 0) { SKIPPED(); - puts(" s3 credentials are not loaded"); - fflush(stdout); + HDputs(" s3 credentials are not loaded"); + HDfflush(stdout); return 0; } if (FALSE == s3_test_bucket_defined) { SKIPPED(); - puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); - fflush(stdout); + HDputs(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + HDfflush(stdout); return 0; } @@ -1636,15 +1636,15 @@ test_H5F_integration(void) if (s3_test_credentials_loaded == 0) { SKIPPED(); - puts(" s3 credentials are not loaded"); - fflush(stdout); + HDputs(" s3 credentials are not loaded"); + HDfflush(stdout); return 0; } if (FALSE == s3_test_bucket_defined) { SKIPPED(); - puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); - fflush(stdout); + HDputs(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + HDfflush(stdout); return 0; } @@ -1691,7 +1691,7 @@ error: * CLEANUP * ***********/ HDprintf("\nerror!"); - fflush(stdout); + HDfflush(stdout); if (fapl_id >= 0) { H5E_BEGIN_TRY { (void)H5Pclose(fapl_id); } diff --git a/test/s3comms.c b/test/s3comms.c index 862914e..c6173fd 100644 --- a/test/s3comms.c +++ b/test/s3comms.c @@ -311,7 +311,7 @@ jserr_str(const char *expected, const char *actual, const char *reason) *---------------------------------------------------------------------------- */ #define JSVERIFY_STR(expected, actual, reason) \ - if (strcmp((actual), (expected)) != 0) { \ + if (HDstrcmp((actual), (expected)) != 0) { \ JSERR_STR((expected), (actual), (reason)); \ goto error; \ } /* JSVERIFY_STR */ @@ -359,7 +359,7 @@ jserr_str(const char *expected, const char *actual, const char *reason) *---------------------------------------------------------------------------- */ #define JSVERIFY_STR(actual, expected, reason) \ - if (strcmp((actual), (expected)) != 0) { \ + if (HDstrcmp((actual), (expected)) != 0) { \ JSERR_STR((expected), (actual), (reason)); \ goto error; \ } /* JSVERIFY_STR */ @@ -1281,10 +1281,10 @@ test_HMAC_SHA256(void) cases[i].msg); if (cases[i].ret == SUCCEED) { #ifdef VERBOSE - if (0 != strncmp(cases[i].exp, dest, HDstrlen(cases[i].exp))) { + if (0 != HDstrncmp(cases[i].exp, dest, HDstrlen(cases[i].exp))) { /* print out how wrong things are, and then fail */ - dest = (char *)realloc(dest, cases[i].dest_size + 1); + dest = (char *)HDrealloc(dest, cases[i].dest_size + 1); HDassert(dest != NULL); dest[cases[i].dest_size] = 0; HDfprintf(stdout, "ERROR:\n!!! \"%s\"\n != \"%s\"\n", cases[i].exp, dest); @@ -1293,17 +1293,17 @@ test_HMAC_SHA256(void) #else /* VERBOSE not defined */ /* simple pass/fail test */ - JSVERIFY(0, strncmp(cases[i].exp, dest, HDstrlen(cases[i].exp)), NULL); + JSVERIFY(0, HDstrncmp(cases[i].exp, dest, HDstrlen(cases[i].exp)), NULL); #endif /* VERBOSE */ } - free(dest); + HDfree(dest); } PASSED(); return 0; error: - free(dest); + HDfree(dest); return -1; } /* end test_HMAC_SHA256() */ @@ -1369,9 +1369,9 @@ test_nlowercase(void) JSVERIFY(SUCCEED, H5FD_s3comms_nlowercase(dest, cases[i].in, cases[i].len), cases[i].in) if (cases[i].len > 0) { - JSVERIFY(0, strncmp(dest, cases[i].exp, cases[i].len), NULL) + JSVERIFY(0, HDstrncmp(dest, cases[i].exp, cases[i].len), NULL) } - free(dest); + HDfree(dest); } /* end for each testcase */ JSVERIFY(FAIL, H5FD_s3comms_nlowercase(NULL, cases[0].in, cases[0].len), "null distination should fail") @@ -1380,7 +1380,7 @@ test_nlowercase(void) return 0; error: - free(dest); + HDfree(dest); return -1; } /* end test_nlowercase() */ @@ -1741,7 +1741,7 @@ test_percent_encode_char(void) JSVERIFY(SUCCEED, H5FD_s3comms_percent_encode_char(dest, (const unsigned char)cases[i].c, &dest_len), NULL) JSVERIFY(cases[i].exp_len, dest_len, NULL) - JSVERIFY(0, strncmp(dest, cases[i].exp, dest_len), NULL) + JSVERIFY(0, HDstrncmp(dest, cases[i].exp, dest_len), NULL) JSVERIFY_STR(cases[i].exp, dest, NULL) } @@ -1780,8 +1780,8 @@ test_s3r_get_filesize(void) */ if (FALSE == s3_test_bucket_defined) { SKIPPED(); - puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); - fflush(stdout); + HDputs(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + HDfflush(stdout); return 0; } @@ -1840,14 +1840,14 @@ test_s3r_open(void) if (s3_test_credentials_loaded == 0) { SKIPPED(); - puts(" s3 credentials are not loaded"); - fflush(stdout); + HDputs(" s3 credentials are not loaded"); + HDfflush(stdout); return 0; } if (FALSE == s3_test_bucket_defined) { SKIPPED(); - puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); - fflush(stdout); + HDputs(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + HDfflush(stdout); return 0; } @@ -1873,7 +1873,7 @@ test_s3r_open(void) FAIL_IF(purl->port == NULL); FAIL_IF(5 < HDsnprintf(purl->port, 5, "9000")) } - else if (strcmp(purl->port, "9000") != 0) { + else if (HDstrcmp(purl->port, "9000") != 0) { FAIL_IF(5 < HDsnprintf(purl->port, 5, "9000")) } else { @@ -2044,8 +2044,8 @@ test_s3r_read(void) */ if (FALSE == s3_test_bucket_defined) { SKIPPED(); - puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); - fflush(stdout); + HDputs(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + HDfflush(stdout); return 0; } @@ -2105,7 +2105,7 @@ test_s3r_read(void) JSVERIFY(SUCCEED, H5FD_s3comms_s3r_read(handle, (haddr_t)6370, (size_t)0, buffer), NULL) JSVERIFY( 0, - strncmp( + HDstrncmp( buffer, "And my soul from out that shadow that lies floating on the floor\nShall be lifted—nevermore!\n", 94), @@ -2122,7 +2122,7 @@ test_s3r_read(void) H5FD_s3comms_s3r_read(handle, (haddr_t)6400, (size_t)100, /* 6400+100 > 6464 */ buffer), NULL) - JSVERIFY(0, strcmp("", buffer), NULL) + JSVERIFY(0, HDstrcmp("", buffer), NULL) /************************ * read starts past eof * @@ -2132,14 +2132,14 @@ test_s3r_read(void) H5FD_s3comms_s3r_read(handle, (haddr_t)1200699, /* 1200699 > 6464 */ (size_t)100, buffer), NULL) - JSVERIFY(0, strcmp("", buffer), NULL) + JSVERIFY(0, HDstrcmp("", buffer), NULL) /********************** * read starts on eof * **********************/ JSVERIFY(FAIL, H5FD_s3comms_s3r_read(handle, (haddr_t)6464, (size_t)0, buffer), NULL) - JSVERIFY(0, strcmp("", buffer), NULL) + JSVERIFY(0, HDstrcmp("", buffer), NULL) /************* * TEAR DOWN * @@ -2229,10 +2229,10 @@ test_signing_key(void) JSVERIFY(SUCCEED, H5FD_s3comms_signing_key(key, cases[i].secret_key, cases[i].region, cases[i].when), NULL) - JSVERIFY(0, strncmp((const char *)cases[i].exp, (const char *)key, SHA256_DIGEST_LENGTH), + JSVERIFY(0, HDstrncmp((const char *)cases[i].exp, (const char *)key, SHA256_DIGEST_LENGTH), (const char *)cases[i].exp) - free(key); + HDfree(key); key = NULL; } @@ -2255,7 +2255,7 @@ test_signing_key(void) JSVERIFY(FAIL, H5FD_s3comms_signing_key(key, cases[0].secret_key, cases[0].region, NULL), "time string cannot be NULL") - free(key); + HDfree(key); key = NULL; PASSED(); @@ -2263,7 +2263,7 @@ test_signing_key(void) error: if (key != NULL) { - free(key); + HDfree(key); } return -1; @@ -2411,9 +2411,9 @@ test_trim(void) JSVERIFY(SUCCEED, H5FD_s3comms_trim(dest, str, cases[i].in_len, &dest_len), NULL) JSVERIFY(cases[i].exp_len, dest_len, cases[i].in) if (dest_len > 0) { - JSVERIFY(0, strncmp(cases[i].exp, dest, dest_len), cases[i].exp) + JSVERIFY(0, HDstrncmp(cases[i].exp, dest, dest_len), cases[i].exp) } - free(str); + HDfree(str); str = NULL; } /* end for each testcase */ @@ -2424,9 +2424,9 @@ test_trim(void) HDassert(str == NULL); str = (char *)HDmalloc(sizeof(char *) * 11); HDassert(str != NULL); - memcpy(str, "some text ", 11); /* string with null terminator */ + HDmemcpy(str, "some text ", 11); /* string with null terminator */ JSVERIFY(FAIL, H5FD_s3comms_trim(NULL, str, 10, &dest_len), "destination for trim cannot be NULL"); - free(str); + HDfree(str); str = NULL; PASSED(); @@ -2434,7 +2434,7 @@ test_trim(void) error: if (str != NULL) { - free(str); + HDfree(str); } return -1; @@ -2525,9 +2525,9 @@ test_uriencode(void) H5FD_s3comms_uriencode(dest, cases[i].str, str_len, cases[i].encode_slash, &dest_written), NULL); JSVERIFY(HDstrlen(cases[i].expected), dest_written, NULL) - JSVERIFY(0, strncmp(dest, cases[i].expected, dest_written), cases[i].expected); + JSVERIFY(0, HDstrncmp(dest, cases[i].expected, dest_written), cases[i].expected); - free(dest); + HDfree(dest); dest = NULL; } /* end for each testcase */ @@ -2543,7 +2543,7 @@ test_uriencode(void) JSVERIFY(FAIL, H5FD_s3comms_uriencode(dest, NULL, 5, false, &dest_written), "source string cannot be NULL"); - free(dest); + HDfree(dest); dest = NULL; PASSED(); @@ -2551,7 +2551,7 @@ test_uriencode(void) error: if (dest != NULL) { - free(dest); + HDfree(dest); } return -1; diff --git a/test/set_extent.c b/test/set_extent.c index 133ba48..2ad4fd5 100644 --- a/test/set_extent.c +++ b/test/set_extent.c @@ -2621,7 +2621,7 @@ test_random_rank4_vl(hid_t fapl, hid_t dcpl, hbool_t do_fillvalue, hbool_t disab TEST_ERROR if (H5Treclaim(type, mspace, H5P_DEFAULT, wbuf) < 0) TEST_ERROR - free(fill_value.p); + HDfree(fill_value.p); if (H5Sclose(mspace) < 0) TEST_ERROR if (H5Pclose(my_dcpl) < 0) diff --git a/test/swmr.c b/test/swmr.c index 44f4277..61d312b 100644 --- a/test/swmr.c +++ b/test/swmr.c @@ -87,7 +87,7 @@ static int test_file_lock_concur(hid_t fapl); static int test_file_lock_swmr_concur(hid_t fapl); /* Test file lock environment variable */ -static int test_file_lock_env_var(hid_t fapl); +static int test_file_locking(hid_t in_fapl, hbool_t turn_locking_on, hbool_t env_var_override); /* Tests for SWMR VFD flag */ static int test_swmr_vfd_flag(void); @@ -4239,8 +4239,11 @@ test_file_lock_same(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags--single process access"); + /* Set locking in the fapl */ if ((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if (H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -4400,7 +4403,7 @@ test_file_lock_swmr_same(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags + SWMR flags--single process access"); - /* Get a copy of the parameter in_fapl */ + /* Set locking in the fapl */ if ((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR @@ -4702,8 +4705,11 @@ test_file_lock_concur(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags--concurrent access"); + /* Set locking in the fapl */ if ((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if (H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -5080,8 +5086,11 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combintations of flags + SWMR flags--concurrent access"); + /* Set locking in the fapl */ if ((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if (H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[2], fapl, filename, sizeof(filename)); @@ -6031,38 +6040,61 @@ error: /**************************************************************** ** -** test_file_lock_swmr_concur(): low-level file test routine. -** With the implementation of file locking, this test checks file -** open with different combinations of flags + SWMR flags. -** This is for concurrent access. +** test_file_locking(): +** Tests various combinations of file locking flags and +** and environment variables. ** *****************************************************************/ static int -test_file_lock_env_var(hid_t in_fapl) +test_file_locking(hid_t in_fapl, hbool_t turn_locking_on, hbool_t env_var_override) { #if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) SKIPPED(); HDputs(" Test skipped due to fork or waitpid not defined."); return 0; #else - hid_t fid = -1; /* File ID */ - hid_t fapl = -1; /* File access property list */ - char filename[NAME_BUF_SIZE]; /* file name */ - pid_t childpid = 0; /* Child process ID */ - int child_status; /* Status passed to waitpid */ - int child_wait_option = 0; /* Options passed to waitpid */ - int out_pdf[2]; - int notify = 0; - - TESTING("File locking environment variable"); + hid_t fid = -1; /* File ID */ + hid_t fapl = -1; /* File access property list */ + char filename[NAME_BUF_SIZE]; /* file name */ + pid_t childpid = 0; /* Child process ID */ + int child_status; /* Status passed to waitpid */ + int child_wait_option = 0; /* Options passed to waitpid */ + int out_pdf[2]; + int notify = 0; + int exit_status = 0; + herr_t ret; + + if (turn_locking_on && env_var_override) + TESTING("File locking: ON w/ env var override") + else if (turn_locking_on && !env_var_override) + TESTING("File locking: ON") + else if (!turn_locking_on && env_var_override) + TESTING("File locking: OFF w/ env var override") + else + TESTING("File locking: OFF") - /* Set the environment variable */ - if (HDsetenv("HDF5_USE_FILE_LOCKING", "FALSE", TRUE) < 0) + /* Copy the incoming fapl */ + if ((fapl = H5Pcopy(in_fapl)) < 0) TEST_ERROR - if ((fapl = H5Pcopy(in_fapl)) < 0) + /* Set locking in the fapl */ + if (H5Pset_file_locking(fapl, turn_locking_on ? TRUE : FALSE, TRUE) < 0) TEST_ERROR + /* If requested, set the environment variable */ + if (env_var_override) { + if (HDsetenv("HDF5_USE_FILE_LOCKING", turn_locking_on ? "FALSE" : "TRUE", TRUE) < 0) + TEST_ERROR + if (H5F__reparse_file_lock_variable_test() < 0) + TEST_ERROR + } + else { + if (HDsetenv("HDF5_USE_FILE_LOCKING", "", TRUE) < 0) + TEST_ERROR + if (H5F__reparse_file_lock_variable_test() < 0) + TEST_ERROR + } + /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -6074,10 +6106,8 @@ test_file_lock_env_var(hid_t in_fapl) if (H5Fclose(fid) < 0) TEST_ERROR - /* Open a file for read-only and then read-write. This would - * normally fail due to the file locking scheme but should - * pass when the environment variable is set to disable file - * locking. + /* Open a file for read-only and then read-write. This will fail + * when the locking scheme is turned on. */ /* Create 1 pipe */ @@ -6105,16 +6135,23 @@ test_file_lock_env_var(hid_t in_fapl) HDexit(EXIT_FAILURE); } - /* Open the test file */ - if ((child_fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) - TEST_ERROR + /* Open and close the test file */ + H5E_BEGIN_TRY + { + child_fid = H5Fopen(filename, H5F_ACC_RDWR, fapl); + ret = H5Fclose(child_fid); + } + H5E_END_TRY; /* Close the pipe */ if (HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); - HDexit(EXIT_SUCCESS); - } /* end if */ + if (H5I_INVALID_HID == child_fid || FAIL == ret) + HDexit(EXIT_FAILURE); + else + HDexit(EXIT_SUCCESS); + } /* end child process work */ /* close unused read end for out_pdf */ if (HDclose(out_pdf[0]) < 0) @@ -6137,15 +6174,28 @@ test_file_lock_env_var(hid_t in_fapl) if (HDwaitpid(childpid, &child_status, child_wait_option) < 0) TEST_ERROR - /* Check if child terminated normally */ - if (WIFEXITED(child_status)) { - /* Check exit status of the child */ - if (WEXITSTATUS(child_status) != 0) - TEST_ERROR - } /* end if */ + /* Check exit status of the child */ + if (WIFEXITED(child_status)) + exit_status = WEXITSTATUS(child_status); else TEST_ERROR + /* The child process should have passed or failed as follows: + * + * locks on: FAIL + * locks off: PASS + * locks on, env var override: PASS + * locks off, env var override: FAIL + */ + if (turn_locking_on && !env_var_override && (0 == exit_status)) + TEST_ERROR + else if (!turn_locking_on && !env_var_override && (0 != exit_status)) + TEST_ERROR + else if (turn_locking_on && env_var_override && (0 != exit_status)) + TEST_ERROR + else if (!turn_locking_on && env_var_override && (0 == exit_status)) + TEST_ERROR + /* Close the file */ if (H5Fclose(fid) < 0) TEST_ERROR @@ -6170,7 +6220,81 @@ error: #endif /* !(defined(H5_HAVE_FORK && defined(H5_HAVE_WAITPID)) */ -} /* end test_file_lock_env_var() */ +} /* end test_file_locking() */ + +/**************************************************************** +** +** test_different_lock_flags(): +** Tests opening a file multiple times with different lock +** flags. +** +*****************************************************************/ +static int +test_different_lock_flags(hid_t in_fapl) +{ + hid_t fid1 = H5I_INVALID_HID; /* File ID */ + hid_t fid2 = H5I_INVALID_HID; /* File ID */ + hid_t fid3 = H5I_INVALID_HID; /* File ID */ + hid_t fapl_id = H5I_INVALID_HID; /* File access property list */ + char filename[NAME_BUF_SIZE]; /* File name */ + + TESTING("Using different lock flags") + + /* Copy the incoming fapl */ + if ((fapl_id = H5Pcopy(in_fapl)) < 0) + TEST_ERROR + + /* Set locking in the fapl */ + if (H5Pset_file_locking(fapl_id, TRUE, TRUE) < 0) + TEST_ERROR + + /* Set the filename to use for this test (dependent on fapl) */ + h5_fixname(FILENAME[1], fapl_id, filename, sizeof(filename)); + + /* Create the test file */ + if ((fid1 = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) + TEST_ERROR + + /* Open the test file with the same flags (should pass) */ + if ((fid2 = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR + + /* Unset locking in the fapl */ + if (H5Pset_file_locking(fapl_id, FALSE, FALSE) < 0) + TEST_ERROR + + /* Open the test file with different flags (should FAIL) */ + H5E_BEGIN_TRY { fid3 = H5Fopen(filename, H5F_ACC_RDWR, fapl_id); } + H5E_END_TRY; + if (H5I_INVALID_HID != fid3) + FAIL_PUTS_ERROR("Should not have been able to open a file with different locking flags") + + /* Close the files */ + if (H5Fclose(fid1) < 0) + TEST_ERROR + if (H5Fclose(fid2) < 0) + TEST_ERROR + + /* Close the copied property list */ + if (H5Pclose(fapl_id) < 0) + TEST_ERROR + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY + { + H5Pclose(fapl_id); + H5Fclose(fid1); + H5Fclose(fid2); + H5Fclose(fid3); + } + H5E_END_TRY; + + return -1; +} /* end test_different_lock_flags() */ static int test_swmr_vfd_flag(void) @@ -7025,11 +7149,12 @@ error: int main(void) { - int nerrors = 0; /* The # of errors */ - hid_t fapl = -1; /* File access property list ID */ - char * driver = NULL; /* VFD string (from env variable) */ - char * lock_env_var = NULL; /* file locking env var pointer */ - hbool_t use_file_locking; /* read from env var */ + int nerrors = 0; /* The # of errors */ + hid_t fapl = -1; /* File access property list ID */ + char * driver = NULL; /* VFD string (from env variable) */ + char * lock_env_var = NULL; /* file locking env var pointer */ + hbool_t use_file_locking; /* read from env var */ + hbool_t file_locking_enabled = FALSE; /* Checks if the file system supports locks */ /* Skip this test if SWMR I/O is not supported for the VFD specified * by the environment variable. @@ -7050,6 +7175,13 @@ main(void) else use_file_locking = TRUE; + /* Check if file locking is enabled on this file system */ + if (use_file_locking) + if (h5_check_if_file_locking_enabled(&file_locking_enabled) < 0) { + HDprintf("Error when determining if file locks are enabled\n"); + return EXIT_FAILURE; + } + /* Set up */ h5_reset(); @@ -7091,7 +7223,7 @@ main(void) nerrors += test_append_flush_dataset_fixed(fapl); nerrors += test_append_flush_dataset_multiple(fapl); - if (use_file_locking) { + if (use_file_locking && file_locking_enabled) { /* * Tests for: * file open flags--single process access @@ -7118,10 +7250,19 @@ main(void) if (NULL == driver || !HDstrcmp(driver, "") || !HDstrcmp(driver, "sec2")) nerrors += test_swmr_vfd_flag(); - /* This test changes the HDF5_USE_FILE_LOCKING environment variable - * so it should be run last. + /* Test multiple opens via different locking flags */ + if (use_file_locking && file_locking_enabled) + nerrors += test_different_lock_flags(fapl); + + /* These tests change the HDF5_USE_FILE_LOCKING environment variable + * so they should be run last. */ - nerrors += test_file_lock_env_var(fapl); + if (use_file_locking && file_locking_enabled) { + nerrors += test_file_locking(fapl, TRUE, TRUE); + nerrors += test_file_locking(fapl, TRUE, FALSE); + nerrors += test_file_locking(fapl, FALSE, TRUE); + nerrors += test_file_locking(fapl, FALSE, FALSE); + } if (nerrors) goto error; diff --git a/test/tarray.c b/test/tarray.c index ba0982f..d278f51 100644 --- a/test/tarray.c +++ b/test/tarray.c @@ -2222,5 +2222,5 @@ test_array(void) void cleanup_array(void) { - remove(FILENAME); + HDremove(FILENAME); } /* end cleanup_array() */ diff --git a/test/tcoords.c b/test/tcoords.c index e92bdde..a52851d 100644 --- a/test/tcoords.c +++ b/test/tcoords.c @@ -87,10 +87,10 @@ test_singleEnd_selElements(hid_t file, hbool_t is_chunked) } /* Construct dataset's name */ - memset(dset_name, 0, (size_t)NAME_LEN); - strcat(dset_name, SINGLE_END_DSET); + HDmemset(dset_name, 0, (size_t)NAME_LEN); + HDstrcat(dset_name, SINGLE_END_DSET); if (is_chunked) - strcat(dset_name, "_chunked"); + HDstrcat(dset_name, "_chunked"); did = H5Dcreate2(file, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, plid, H5P_DEFAULT); CHECK(did, FAIL, "H5Dcreate2"); @@ -259,10 +259,10 @@ test_singleEnd_selHyperslab(hid_t file, hbool_t is_chunked) hsize_t mem3_block[4] = {1, 3, 6, 1}; /* Construct dataset's name */ - memset(dset_name, 0, NAME_LEN); - strcat(dset_name, SINGLE_END_DSET); + HDmemset(dset_name, 0, NAME_LEN); + HDstrcat(dset_name, SINGLE_END_DSET); if (is_chunked) - strcat(dset_name, "_chunked"); + HDstrcat(dset_name, "_chunked"); /* Dataspace for the dataset in file */ sid = H5Screate_simple(4, da_dims, da_dims); @@ -436,10 +436,10 @@ test_multiple_ends(hid_t file, hbool_t is_chunked) } /* Construct dataset's name */ - memset(dset_name, 0, NAME_LEN); - strcat(dset_name, MULTI_ENDS_SEL_HYPER_DSET); + HDmemset(dset_name, 0, NAME_LEN); + HDstrcat(dset_name, MULTI_ENDS_SEL_HYPER_DSET); if (is_chunked) - strcat(dset_name, "_chunked"); + HDstrcat(dset_name, "_chunked"); did = H5Dcreate2(file, dset_name, H5T_NATIVE_INT, sid, H5P_DEFAULT, plid, H5P_DEFAULT); CHECK(did, FAIL, "H5Dcreate2"); @@ -687,5 +687,5 @@ test_coords(void) void cleanup_coords(void) { - remove(FILENAME); + HDremove(FILENAME); } diff --git a/test/testframe.c b/test/testframe.c index 22f694e..bf6a0e8 100644 --- a/test/testframe.c +++ b/test/testframe.c @@ -442,15 +442,15 @@ GetTestExpress(void) /* set it here for now. Should be done in something like h5test_init(). */ if (TestExpress == -1) { - env_val = getenv("HDF5TestExpress"); + env_val = HDgetenv("HDF5TestExpress"); if (env_val == NULL) SetTestExpress(1); - else if (strcmp(env_val, "0") == 0) + else if (HDstrcmp(env_val, "0") == 0) SetTestExpress(0); - else if (strcmp(env_val, "1") == 0) + else if (HDstrcmp(env_val, "1") == 0) SetTestExpress(1); - else if (strcmp(env_val, "2") == 0) + else if (HDstrcmp(env_val, "2") == 0) SetTestExpress(2); else SetTestExpress(3); @@ -521,7 +521,7 @@ ParseTestVerbosity(char *argv) else if (*argv == 'h') SetTestVerbosity(VERBO_HI); else - SetTestVerbosity(atoi(argv)); + SetTestVerbosity(HDatoi(argv)); } /* diff --git a/test/testhdf5.c b/test/testhdf5.c index 152719a..ab79f28 100644 --- a/test/testhdf5.c +++ b/test/testhdf5.c @@ -82,7 +82,7 @@ main(int argc, char *argv[]) TestSummary(); /* Clean up test files, if allowed */ - if (GetTestCleanup() && !getenv("HDF5_NOCLEANUP")) + if (GetTestCleanup() && !HDgetenv("HDF5_NOCLEANUP")) TestCleanup(); /* Release test infrastructure */ diff --git a/test/testmeta.c b/test/testmeta.c index fee1954..c3044a6 100644 --- a/test/testmeta.c +++ b/test/testmeta.c @@ -83,7 +83,7 @@ main(void) for (i = 0; i < NEXTARRAYS; i++) { /* Create dataset */ - sprintf(name, "/ExtArray%06d", i); + HDsprintf(name, "/ExtArray%06d", i); dataset_id = H5Dcreate2(file_id, name, H5T_NATIVE_FLOAT, dataspace_id, H5P_DEFAULT, prop_id, H5P_DEFAULT); @@ -103,7 +103,7 @@ main(void) /* Removed print statement as it would lock system resources on Windows */ /* * HDprintf("\rWriting Object #%d of %d", j+1, NDATAOBJECTS); - * fflush(stdout); + * HDfflush(stdout); */ floatval = (float)j; diff --git a/test/tfile.c b/test/tfile.c index e41cbbd..3d75f9b 100644 --- a/test/tfile.c +++ b/test/tfile.c @@ -4195,7 +4195,7 @@ set_multi_split(hid_t fapl, hsize_t pagesize, hbool_t split) /* Free memb_name */ for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) - free(memb_name[mt]); + HDfree(memb_name[mt]); return 0; diff --git a/test/tgenprop.c b/test/tgenprop.c index 910153c..c1d9cac 100644 --- a/test/tgenprop.c +++ b/test/tgenprop.c @@ -2172,5 +2172,5 @@ test_genprop(void) void cleanup_genprop(void) { - remove(FILENAME); + HDremove(FILENAME); } diff --git a/test/th5o.c b/test/th5o.c index 934a3dc..8b67159 100644 --- a/test/th5o.c +++ b/test/th5o.c @@ -247,7 +247,7 @@ test_h5o_open_by_addr(void) { hid_t fid; /* HDF5 File ID */ hid_t grp, dset, dtype, dspace; /* Object identifiers */ - H5L_info2_t li; /* Buffer for H5Lget_info */ + H5L_info2_t li; /* Buffer for H5Lget_info2 */ haddr_t grp_addr; /* Addresses for objects */ haddr_t dset_addr; haddr_t dtype_addr; @@ -1767,5 +1767,5 @@ test_h5o(void) void cleanup_h5o(void) { - remove(TEST_FILENAME); + HDremove(TEST_FILENAME); } diff --git a/test/th5s.c b/test/th5s.c index 07cb0ab..04ac2e6 100644 --- a/test/th5s.c +++ b/test/th5s.c @@ -1936,7 +1936,7 @@ test_h5s_encode_irregular_hyper(H5F_libver_t low, H5F_libver_t high) break; default: - assert(0); + HDassert(0); break; } @@ -3400,9 +3400,9 @@ test_h5s(void) void cleanup_h5s(void) { - remove(DATAFILE); - remove(NULLFILE); - remove(BASICFILE); - remove(ZEROFILE); - remove(VERBFNAME); + HDremove(DATAFILE); + HDremove(NULLFILE); + HDremove(BASICFILE); + HDremove(ZEROFILE); + HDremove(VERBFNAME); } diff --git a/test/thread_id.c b/test/thread_id.c index ce60893..d8ddaf8 100644 --- a/test/thread_id.c +++ b/test/thread_id.c @@ -73,7 +73,7 @@ pthread_barrier_init(pthread_barrier_t *barrier, const pthread_barrierattr_t *at if (attr != NULL) return EINVAL; - memset(barrier, 0, sizeof(*barrier)); + HDmemset(barrier, 0, sizeof(*barrier)); barrier->count = count; @@ -96,7 +96,7 @@ barrier_lock(pthread_barrier_t *barrier) int rc; if ((rc = pthread_mutex_lock(&barrier->mtx)) != 0) { - my_errx(EXIT_FAILURE, "%s: pthread_mutex_lock: %s", __func__, strerror(rc)); + my_errx(EXIT_FAILURE, "%s: pthread_mutex_lock: %s", __func__, HDstrerror(rc)); } } @@ -106,7 +106,7 @@ barrier_unlock(pthread_barrier_t *barrier) int rc; if ((rc = pthread_mutex_unlock(&barrier->mtx)) != 0) { - my_errx(EXIT_FAILURE, "%s: pthread_mutex_unlock: %s", __func__, strerror(rc)); + my_errx(EXIT_FAILURE, "%s: pthread_mutex_unlock: %s", __func__, HDstrerror(rc)); } } @@ -203,16 +203,16 @@ atomic_printf(const char *fmt, ...) va_list ap; ssize_t nprinted, nwritten; - va_start(ap, fmt); - nprinted = vsnprintf(buf, sizeof(buf), fmt, ap); - va_end(ap); + HDva_start(ap, fmt); + nprinted = HDvsnprintf(buf, sizeof(buf), fmt, ap); + HDva_end(ap); if (nprinted == -1) my_err(EXIT_FAILURE, "%s.%d: vsnprintf", __func__, __LINE__); else if (nprinted >= (ssize_t)sizeof(buf)) my_errx(EXIT_FAILURE, "%s.%d: vsnprintf overflowed", __func__, __LINE__); - nwritten = write(STDOUT_FILENO, buf, (size_t)nprinted); + nwritten = HDwrite(STDOUT_FILENO, buf, (size_t)nprinted); if (nwritten < nprinted) { my_errx(EXIT_FAILURE, "%s.%d: write error or short write", __func__, __LINE__); } diff --git a/test/timer.c b/test/timer.c index e907655..24f3245 100644 --- a/test/timer.c +++ b/test/timer.c @@ -146,7 +146,7 @@ test_timer_system_user(void) */ if (timer.initial.system < (double)0.0f || timer.initial.user < (double)0.0f) { SKIPPED(); - printf("NOTE: No suitable way to get system/user times on this platform.\n"); + HDprintf("NOTE: No suitable way to get system/user times on this platform.\n"); return 0; } @@ -383,7 +383,7 @@ main(void) h5_reset(); - printf("Testing platform-independent timer functionality.\n"); + HDprintf("Testing platform-independent timer functionality.\n"); nerrors += test_time_formatting() < 0 ? 1 : 0; nerrors += test_timer_system_user() < 0 ? 1 : 0; @@ -391,11 +391,12 @@ main(void) nerrors += test_timer_functionality() < 0 ? 1 : 0; if (nerrors) { - printf("***** %d platform-independent timer TEST%s FAILED! *****\n", nerrors, nerrors > 1 ? "S" : ""); + HDprintf("***** %d platform-independent timer TEST%s FAILED! *****\n", nerrors, + nerrors > 1 ? "S" : ""); return 1; } else { - printf("All platform-independent timer tests passed.\n"); + HDprintf("All platform-independent timer tests passed.\n"); return 0; } } diff --git a/test/titerate.c b/test/titerate.c index bcf19b9..050dcec 100644 --- a/test/titerate.c +++ b/test/titerate.c @@ -966,7 +966,7 @@ find_err_msg_cb(unsigned H5_ATTR_UNUSED n, const H5E_error2_t *err_desc, void *_ return H5_ITER_ERROR; /* If the searched error message is found, stop the iteration */ - if (err_desc->desc != NULL && strcmp(err_desc->desc, searched_err->message) == 0) { + if (err_desc->desc != NULL && HDstrcmp(err_desc->desc, searched_err->message) == 0) { searched_err->found = true; status = H5_ITER_STOP; } diff --git a/test/tmisc.c b/test/tmisc.c index a6db198..2d842be 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -2837,7 +2837,7 @@ test_misc16(void) if (HDstrlen(wdata[i]) != HDstrlen(rdata[i])) { TestErrPrintf( "Line %u: VL data length don't match!, strlen(wdata[%d])=%d, strlen(rdata[%d])=%d\n", - (unsigned)__LINE__, (int)i, (int)strlen(wdata[i]), (int)i, (int)strlen(rdata[i])); + (unsigned)__LINE__, (int)i, (int)HDstrlen(wdata[i]), (int)i, (int)HDstrlen(rdata[i])); continue; } /* end if */ if (HDstrcmp(wdata[i], rdata[i]) != 0) { @@ -2922,7 +2922,7 @@ test_misc17(void) if (HDstrlen(wdata[i]) != HDstrlen(rdata[i])) { TestErrPrintf( "Line %u: VL data length don't match!, strlen(wdata[%d])=%d, strlen(rdata[%d])=%d\n", - (unsigned)__LINE__, (int)i, (int)strlen(wdata[i]), (int)i, (int)strlen(rdata[i])); + (unsigned)__LINE__, (int)i, (int)HDstrlen(wdata[i]), (int)i, (int)HDstrlen(rdata[i])); continue; } /* end if */ if (HDstrcmp(wdata[i], rdata[i]) != 0) { diff --git a/test/trefer.c b/test/trefer.c index 8aa8bc2..530a409 100644 --- a/test/trefer.c +++ b/test/trefer.c @@ -21,17 +21,18 @@ #include "testhdf5.h" -#define FILE_REF_PARAM "trefer_param.h5" -#define FILE_REF_OBJ "trefer_obj.h5" -#define FILE_REF_VL_OBJ "trefer_vl_obj.h5" -#define FILE_REF_REG "trefer_reg.h5" -#define FILE_REF_REG_1D "trefer_reg_1d.h5" -#define FILE_REF_OBJ_DEL "trefer_obj_del.h5" -#define FILE_REF_GRP "trefer_grp.h5" -#define FILE_REF_ATTR "trefer_attr.h5" -#define FILE_REF_EXT1 "trefer_ext1.h5" -#define FILE_REF_EXT2 "trefer_ext2.h5" -#define FILE_REF_COMPAT "trefer_compat.h5" +#define FILE_REF_PARAM "trefer_param.h5" +#define FILE_REF_OBJ "trefer_obj.h5" +#define FILE_REF_VL_OBJ "trefer_vl_obj.h5" +#define FILE_REF_CMPND_OBJ "trefer_cmpnd_obj.h5" +#define FILE_REF_REG "trefer_reg.h5" +#define FILE_REF_REG_1D "trefer_reg_1d.h5" +#define FILE_REF_OBJ_DEL "trefer_obj_del.h5" +#define FILE_REF_GRP "trefer_grp.h5" +#define FILE_REF_ATTR "trefer_attr.h5" +#define FILE_REF_EXT1 "trefer_ext1.h5" +#define FILE_REF_EXT2 "trefer_ext2.h5" +#define FILE_REF_COMPAT "trefer_compat.h5" /* 1-D dataset with fixed dimensions */ #define SPACE1_RANK 1 @@ -56,6 +57,15 @@ typedef struct s1_t { float c; } s1_t; +/* Compound datatype with reference */ +typedef struct s2_t { + H5R_ref_t ref0; /* reference */ + H5R_ref_t ref1; /* reference */ + H5R_ref_t ref2; /* reference */ + H5R_ref_t ref3; /* reference */ + unsigned int dim_idx; /* dimension index of the dataset */ +} s2_t; + #define GROUPNAME "/group" #define GROUPNAME2 "group2" #define GROUPNAME3 "group3" @@ -805,6 +815,297 @@ test_reference_vlen_obj(void) /**************************************************************** ** +** test_reference_cmpnd_obj(): Test basic H5R (reference) object reference +** within a compound type. +** Tests references to various kinds of objects +** +****************************************************************/ +static void +test_reference_cmpnd_obj(void) +{ + hid_t fid1; /* HDF5 File IDs */ + hid_t dataset, /* Dataset ID */ + dset2; /* Dereferenced dataset ID */ + hid_t group; /* Group ID */ + hid_t sid1; /* Dataspace ID */ + hid_t tid1; /* Datatype ID */ + hsize_t dims1[] = {SPACE1_DIM1}; + hsize_t cmpnd_dims[] = {1}; + hid_t dapl_id; /* Dataset access property list */ + H5R_ref_t *wbuf, /* buffer to write to disk */ + *rbuf; /* buffer read from disk */ + unsigned * ibuf, *obuf; + unsigned i, j; /* Counters */ + H5O_type_t obj_type; /* Object type */ + herr_t ret; /* Generic return value */ + s2_t cmpnd_wbuf, cmpnd_rbuf; + + /* Output message about test being performed */ + MESSAGE(5, ("Testing Object Reference Functions within compound type\n")); + + /* Allocate write & read buffers */ + wbuf = HDcalloc(sizeof(H5R_ref_t), SPACE1_DIM1); + rbuf = HDcalloc(sizeof(H5R_ref_t), SPACE1_DIM1); + ibuf = HDcalloc(sizeof(unsigned), SPACE1_DIM1); + obuf = HDcalloc(sizeof(unsigned), SPACE1_DIM1); + + for (i = 0; i < SPACE1_DIM1; i++) + obuf[i] = i * 3; + + /* Create file */ + fid1 = H5Fcreate(FILE_REF_CMPND_OBJ, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + CHECK(fid1, H5I_INVALID_HID, "H5Fcreate"); + + /* Create dataspace for datasets */ + sid1 = H5Screate_simple(SPACE1_RANK, dims1, NULL); + CHECK(sid1, H5I_INVALID_HID, "H5Screate_simple"); + + /* Create dataset access property list */ + dapl_id = H5Pcreate(H5P_DATASET_ACCESS); + CHECK(dapl_id, H5I_INVALID_HID, "H5Pcreate"); + + /* Create a group */ + group = H5Gcreate2(fid1, "Group1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + CHECK(group, H5I_INVALID_HID, "H5Gcreate2"); + + /* Create a dataset (inside Group1) */ + dataset = H5Dcreate2(group, "Dataset1", H5T_NATIVE_UINT, sid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + CHECK(dataset, H5I_INVALID_HID, "H5Dcreate2"); + + /* Write selection to disk */ + ret = H5Dwrite(dataset, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, obuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close Dataset */ + ret = H5Dclose(dataset); + CHECK(ret, FAIL, "H5Dclose"); + + /* Create another dataset (inside Group1) */ + dataset = H5Dcreate2(group, "Dataset2", H5T_NATIVE_UCHAR, sid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + CHECK(dataset, FAIL, "H5Dcreate2"); + + /* Close Dataset */ + ret = H5Dclose(dataset); + CHECK(ret, FAIL, "H5Dclose"); + + /* Close disk dataspace */ + ret = H5Sclose(sid1); + CHECK(ret, FAIL, "H5Sclose"); + + /* Create a datatype to refer to */ + tid1 = H5Tcreate(H5T_COMPOUND, sizeof(s1_t)); + CHECK(tid1, H5I_INVALID_HID, "H5Tcreate"); + + /* Insert fields */ + ret = H5Tinsert(tid1, "a", HOFFSET(s1_t, a), H5T_NATIVE_INT); + CHECK(ret, FAIL, "H5Tinsert"); + + ret = H5Tinsert(tid1, "b", HOFFSET(s1_t, b), H5T_NATIVE_INT); + CHECK(ret, FAIL, "H5Tinsert"); + + ret = H5Tinsert(tid1, "c", HOFFSET(s1_t, c), H5T_NATIVE_FLOAT); + CHECK(ret, FAIL, "H5Tinsert"); + + /* Save datatype for later */ + ret = H5Tcommit2(group, "Datatype1", tid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + CHECK(ret, FAIL, "H5Tcommit2"); + + /* Close datatype */ + ret = H5Tclose(tid1); + CHECK(ret, FAIL, "H5Tclose"); + + /* Close group */ + ret = H5Gclose(group); + CHECK(ret, FAIL, "H5Gclose"); + + /* Create compound type */ + tid1 = H5Tcreate(H5T_COMPOUND, sizeof(s2_t)); + CHECK(tid1, H5I_INVALID_HID, "H5Tcreate"); + + /* Insert fields */ + ret = H5Tinsert(tid1, "ref0", HOFFSET(s2_t, ref0), H5T_STD_REF); + CHECK(ret, FAIL, "H5Tinsert"); + + ret = H5Tinsert(tid1, "ref1", HOFFSET(s2_t, ref1), H5T_STD_REF); + CHECK(ret, FAIL, "H5Tinsert"); + + ret = H5Tinsert(tid1, "ref2", HOFFSET(s2_t, ref2), H5T_STD_REF); + CHECK(ret, FAIL, "H5Tinsert"); + + ret = H5Tinsert(tid1, "ref3", HOFFSET(s2_t, ref3), H5T_STD_REF); + CHECK(ret, FAIL, "H5Tinsert"); + + ret = H5Tinsert(tid1, "dim_idx", HOFFSET(s2_t, dim_idx), H5T_NATIVE_INT); + CHECK(ret, FAIL, "H5Tinsert"); + + /* Create dataspace for datasets */ + sid1 = H5Screate_simple(SPACE1_RANK, cmpnd_dims, NULL); + CHECK(sid1, H5I_INVALID_HID, "H5Screate_simple"); + + /* Create a dataset */ + dataset = H5Dcreate2(fid1, "Dataset3", tid1, sid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + CHECK(dataset, H5I_INVALID_HID, "H5Dcreate2"); + + /* Create reference to dataset */ + ret = H5Rcreate_object(fid1, "/Group1/Dataset1", H5P_DEFAULT, &wbuf[0]); + CHECK(ret, FAIL, "H5Rcreate_object"); + ret = H5Rget_obj_type3(&wbuf[0], H5P_DEFAULT, &obj_type); + CHECK(ret, FAIL, "H5Rget_obj_type3"); + VERIFY(obj_type, H5O_TYPE_DATASET, "H5Rget_obj_type3"); + + /* Create reference to dataset */ + ret = H5Rcreate_object(fid1, "/Group1/Dataset2", H5P_DEFAULT, &wbuf[1]); + CHECK(ret, FAIL, "H5Rcreate_object"); + ret = H5Rget_obj_type3(&wbuf[1], H5P_DEFAULT, &obj_type); + CHECK(ret, FAIL, "H5Rget_obj_type3"); + VERIFY(obj_type, H5O_TYPE_DATASET, "H5Rget_obj_type3"); + + /* Create reference to group */ + ret = H5Rcreate_object(fid1, "/Group1", H5P_DEFAULT, &wbuf[2]); + CHECK(ret, FAIL, "H5Rcreate_object"); + ret = H5Rget_obj_type3(&wbuf[2], H5P_DEFAULT, &obj_type); + CHECK(ret, FAIL, "H5Rget_obj_type3"); + VERIFY(obj_type, H5O_TYPE_GROUP, "H5Rget_obj_type3"); + + /* Create reference to named datatype */ + ret = H5Rcreate_object(fid1, "/Group1/Datatype1", H5P_DEFAULT, &wbuf[3]); + CHECK(ret, FAIL, "H5Rcreate_object"); + ret = H5Rget_obj_type3(&wbuf[3], H5P_DEFAULT, &obj_type); + CHECK(ret, FAIL, "H5Rget_obj_type3"); + VERIFY(obj_type, H5O_TYPE_NAMED_DATATYPE, "H5Rget_obj_type3"); + + /* Store dimensions */ + cmpnd_wbuf.ref0 = wbuf[0]; + cmpnd_wbuf.ref1 = wbuf[1]; + cmpnd_wbuf.ref2 = wbuf[2]; + cmpnd_wbuf.ref3 = wbuf[3]; + cmpnd_wbuf.dim_idx = SPACE1_DIM1; + + /* Write selection to disk */ + ret = H5Dwrite(dataset, tid1, H5S_ALL, H5S_ALL, H5P_DEFAULT, &cmpnd_wbuf); + CHECK(ret, FAIL, "H5Dwrite"); + + /* Close disk dataspace */ + ret = H5Sclose(sid1); + CHECK(ret, FAIL, "H5Sclose"); + + /* Close Dataset */ + ret = H5Dclose(dataset); + CHECK(ret, FAIL, "H5Dclose"); + + /* Close datatype */ + ret = H5Tclose(tid1); + CHECK(ret, FAIL, "H5Tclose"); + + /* Close file */ + ret = H5Fclose(fid1); + CHECK(ret, FAIL, "H5Fclose"); + + /* Re-open the file */ + fid1 = H5Fopen(FILE_REF_CMPND_OBJ, H5F_ACC_RDWR, H5P_DEFAULT); + CHECK(fid1, H5I_INVALID_HID, "H5Fopen"); + + /* Open the dataset */ + dataset = H5Dopen2(fid1, "/Dataset3", H5P_DEFAULT); + CHECK(dataset, H5I_INVALID_HID, "H5Dopen2"); + + tid1 = H5Dget_type(dataset); + CHECK(tid1, H5I_INVALID_HID, "H5Dget_type"); + + /* Read selection from disk */ + ret = H5Dread(dataset, tid1, H5S_ALL, H5S_ALL, H5P_DEFAULT, &cmpnd_rbuf); + CHECK(ret, FAIL, "H5Dread"); + + VERIFY(cmpnd_rbuf.dim_idx, SPACE1_DIM1, "H5Dread"); + rbuf[0] = cmpnd_rbuf.ref0; + rbuf[1] = cmpnd_rbuf.ref1; + rbuf[2] = cmpnd_rbuf.ref2; + rbuf[3] = cmpnd_rbuf.ref3; + + /* Close datatype */ + ret = H5Tclose(tid1); + CHECK(ret, FAIL, "H5Tclose"); + + /* Open dataset object */ + dset2 = H5Ropen_object(&rbuf[0], H5P_DEFAULT, dapl_id); + CHECK(dset2, H5I_INVALID_HID, "H5Ropen_object"); + + /* Check information in referenced dataset */ + sid1 = H5Dget_space(dset2); + CHECK(sid1, H5I_INVALID_HID, "H5Dget_space"); + + ret = (int)H5Sget_simple_extent_npoints(sid1); + VERIFY(ret, SPACE1_DIM1, "H5Sget_simple_extent_npoints"); + + /* Read from disk */ + ret = H5Dread(dset2, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, ibuf); + CHECK(ret, FAIL, "H5Dread"); + + for (i = 0; i < SPACE1_DIM1; i++) + VERIFY(ibuf[i], i * 3, "Data"); + + /* Close dereferenced Dataset */ + ret = H5Dclose(dset2); + CHECK(ret, FAIL, "H5Dclose"); + + /* Open group object. GAPL isn't supported yet. But it's harmless to pass in */ + group = H5Ropen_object(&rbuf[2], H5P_DEFAULT, H5P_DEFAULT); + CHECK(group, H5I_INVALID_HID, "H5Ropen_object"); + + /* Close group */ + ret = H5Gclose(group); + CHECK(ret, FAIL, "H5Gclose"); + + /* Open datatype object. TAPL isn't supported yet. But it's harmless to pass in */ + tid1 = H5Ropen_object(&rbuf[3], H5P_DEFAULT, H5P_DEFAULT); + CHECK(tid1, H5I_INVALID_HID, "H5Ropen_object"); + + /* Verify correct datatype */ + { + H5T_class_t tclass; + + tclass = H5Tget_class(tid1); + VERIFY(tclass, H5T_COMPOUND, "H5Tget_class"); + + ret = H5Tget_nmembers(tid1); + VERIFY(ret, 3, "H5Tget_nmembers"); + } + + /* Close datatype */ + ret = H5Tclose(tid1); + CHECK(ret, FAIL, "H5Tclose"); + + /* Close Dataset */ + ret = H5Dclose(dataset); + CHECK(ret, FAIL, "H5Dclose"); + + /* Close dataset access property list */ + ret = H5Pclose(dapl_id); + CHECK(ret, FAIL, "H5Pclose"); + + /* Close file */ + ret = H5Fclose(fid1); + CHECK(ret, FAIL, "H5Fclose"); + + /* Destroy references */ + for (j = 0; j < SPACE1_DIM1; j++) { + ret = H5Rdestroy(&wbuf[j]); + CHECK(ret, FAIL, "H5Rdestroy"); + } + for (j = 0; j < SPACE1_DIM1; j++) { + ret = H5Rdestroy(&rbuf[j]); + CHECK(ret, FAIL, "H5Rdestroy"); + } + + /* Free memory buffers */ + HDfree(wbuf); + HDfree(rbuf); + HDfree(ibuf); + HDfree(obuf); +} /* test_reference_cmpnd_obj() */ + +/**************************************************************** +** ** test_reference_region(): Test basic H5R (reference) object reference code. ** Tests references to various kinds of objects ** @@ -3150,9 +3451,10 @@ test_reference(void) /* Output message about test being performed */ MESSAGE(5, ("Testing References\n")); - test_reference_params(); /* Test for correct parameter checking */ - test_reference_obj(); /* Test basic H5R object reference code */ - test_reference_vlen_obj(); /* Test reference within vlen */ + test_reference_params(); /* Test for correct parameter checking */ + test_reference_obj(); /* Test basic H5R object reference code */ + test_reference_vlen_obj(); /* Test reference within vlen */ + test_reference_cmpnd_obj(); /* Test reference within compound type */ /* Loop through all the combinations of low/high version bounds */ for (low = H5F_LIBVER_EARLIEST; low < H5F_LIBVER_NBOUNDS; low++) { @@ -3198,6 +3500,7 @@ cleanup_reference(void) HDremove(FILE_REF_PARAM); HDremove(FILE_REF_OBJ); HDremove(FILE_REF_VL_OBJ); + HDremove(FILE_REF_CMPND_OBJ); HDremove(FILE_REF_REG); HDremove(FILE_REF_REG_1D); HDremove(FILE_REF_OBJ_DEL); diff --git a/test/trefer_deprec.c b/test/trefer_deprec.c index 6191526..d2a403f 100644 --- a/test/trefer_deprec.c +++ b/test/trefer_deprec.c @@ -1406,9 +1406,9 @@ test_reference_group(void) /* Create bottom dataset */ did = H5Dcreate2(gid, DSETNAME2, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - assert(did > 0); + HDassert(did > 0); ret = H5Dclose(did); - assert(ret >= 0); + HDassert(ret >= 0); ret = H5Gclose(gid); CHECK(ret, FAIL, "H5Gclose"); diff --git a/test/tselect.c b/test/tselect.c index 261388a..2aeba77 100644 --- a/test/tselect.c +++ b/test/tselect.c @@ -5239,7 +5239,7 @@ test_select_hyper_union_stagger(void) CHECK(error, FAIL, "H5Fclose"); /* Initialize intput buffer */ - memset(data_out, 0, 7 * 7 * sizeof(int)); + HDmemset(data_out, 0, 7 * 7 * sizeof(int)); /* Open file */ file_id = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT); @@ -5537,7 +5537,7 @@ test_select_hyper_valid_combination(void) /* Output message about test being performed */ MESSAGE(6, ("Testing Selection Combination Validity\n")); - assert(SPACE9_DIM2 >= POINT1_NPOINTS); + HDassert(SPACE9_DIM2 >= POINT1_NPOINTS); /* Create dataspace for single point selection */ single_pt_sid = H5Screate_simple(SPACE9_RANK, dims2D, NULL); @@ -8377,7 +8377,7 @@ test_shape_same(void) /* Output message about test being performed */ MESSAGE(6, ("Testing Same Shape Comparisons\n")); - assert(SPACE9_DIM2 >= POINT1_NPOINTS); + HDassert(SPACE9_DIM2 >= POINT1_NPOINTS); /* Create dataspace for "all" selection */ all_sid = H5Screate_simple(SPACE9_RANK, dims, NULL); @@ -14648,7 +14648,7 @@ test_internal_consistency(void) /* Output message about test being performed */ MESSAGE(6, ("Testing Consistency of Internal States\n")); - assert(SPACE9_DIM2 >= POINT1_NPOINTS); + HDassert(SPACE9_DIM2 >= POINT1_NPOINTS); /* Create dataspace for "all" selection */ all_sid = H5Screate_simple(SPACE9_RANK, dims, NULL); @@ -15107,6 +15107,14 @@ test_sel_iter(void) iter_id = H5Ssel_iter_create(sid, (size_t)1, (unsigned)sel_iter_flags); CHECK(iter_id, FAIL, "H5Ssel_iter_create"); + /* Try resetting selection iterator with bad parameters */ + H5E_BEGIN_TRY { ret = H5Ssel_iter_reset(H5I_INVALID_HID, sid); } + H5E_END_TRY; + VERIFY(ret, FAIL, "H5Ssel_iter_reset"); + H5E_BEGIN_TRY { ret = H5Ssel_iter_reset(iter_id, H5I_INVALID_HID); } + H5E_END_TRY; + VERIFY(ret, FAIL, "H5Ssel_iter_reset"); + /* Try retrieving sequences, with bad parameters */ H5E_BEGIN_TRY { /* Invalid ID */ @@ -15257,6 +15265,167 @@ test_sel_iter(void) CHECK(ret, FAIL, "H5Ssel_iter_close"); } /* end for */ + /* Create selection iterator object */ + iter_id = H5Ssel_iter_create(sid, (size_t)1, (unsigned)sel_iter_flags); + CHECK(iter_id, FAIL, "H5Ssel_iter_create"); + + /* Test iterators on various basic selection types using + * H5Ssel_iter_reset instead of creating multiple iterators */ + for (sel_type = H5S_SEL_NONE; sel_type <= H5S_SEL_ALL; sel_type = (H5S_sel_type)(sel_type + 1)) { + switch (sel_type) { + case H5S_SEL_NONE: /* "None" selection */ + ret = H5Sselect_none(sid); + CHECK(ret, FAIL, "H5Sselect_none"); + break; + + case H5S_SEL_POINTS: /* Point selection */ + /* Select sequence of ten points */ + coord1[0][0] = 0; + coord1[0][1] = 9; + coord1[1][0] = 1; + coord1[1][1] = 2; + coord1[2][0] = 2; + coord1[2][1] = 4; + coord1[3][0] = 0; + coord1[3][1] = 6; + coord1[4][0] = 1; + coord1[4][1] = 8; + coord1[5][0] = 2; + coord1[5][1] = 10; + coord1[6][0] = 0; + coord1[6][1] = 11; + coord1[7][0] = 1; + coord1[7][1] = 4; + coord1[8][0] = 2; + coord1[8][1] = 1; + coord1[9][0] = 0; + coord1[9][1] = 3; + ret = H5Sselect_elements(sid, H5S_SELECT_SET, (size_t)POINT1_NPOINTS, + (const hsize_t *)coord1); + CHECK(ret, FAIL, "H5Sselect_elements"); + break; + + case H5S_SEL_HYPERSLABS: /* Hyperslab selection */ + /* Select regular hyperslab */ + start[0] = 3; + start[1] = 0; + stride[0] = 2; + stride[1] = 2; + count[0] = 2; + count[1] = 5; + block[0] = 1; + block[1] = 1; + ret = H5Sselect_hyperslab(sid, H5S_SELECT_SET, start, stride, count, block); + CHECK(ret, FAIL, "H5Sselect_hyperslab"); + break; + + case H5S_SEL_ALL: /* "All" selection */ + ret = H5Sselect_all(sid); + CHECK(ret, FAIL, "H5Sselect_all"); + break; + + case H5S_SEL_ERROR: + case H5S_SEL_N: + default: + HDassert(0 && "Can't occur"); + break; + } /* end switch */ + + /* Try retrieving no sequences, with 0 for maxseq & maxbytes */ + ret = H5Ssel_iter_get_seq_list(iter_id, (size_t)0, (size_t)1, &nseq, &nbytes, off, len); + CHECK(ret, FAIL, "H5Ssel_iter_get_seq_list"); + VERIFY(nseq, 0, "H5Ssel_iter_get_seq_list"); + VERIFY(nbytes, 0, "H5Ssel_iter_get_seq_list"); + ret = H5Ssel_iter_get_seq_list(iter_id, (size_t)1, (size_t)0, &nseq, &nbytes, off, len); + CHECK(ret, FAIL, "H5Ssel_iter_get_seq_list"); + VERIFY(nseq, 0, "H5Ssel_iter_get_seq_list"); + VERIFY(nbytes, 0, "H5Ssel_iter_get_seq_list"); + + /* Reset iterator */ + ret = H5Ssel_iter_reset(iter_id, sid); + CHECK(ret, FAIL, "H5Ssel_iter_reset"); + + /* Try retrieving all sequences */ + ret = H5Ssel_iter_get_seq_list(iter_id, (size_t)SEL_ITER_MAX_SEQ, (size_t)(1024 * 1024), &nseq, + &nbytes, off, len); + CHECK(ret, FAIL, "H5Ssel_iter_get_seq_list"); + + /* Check results from retrieving sequence list */ + switch (sel_type) { + case H5S_SEL_NONE: /* "None" selection */ + VERIFY(nseq, 0, "H5Ssel_iter_get_seq_list"); + VERIFY(nbytes, 0, "H5Ssel_iter_get_seq_list"); + break; + + case H5S_SEL_POINTS: /* Point selection */ + VERIFY(nseq, 10, "H5Ssel_iter_get_seq_list"); + VERIFY(nbytes, 10, "H5Ssel_iter_get_seq_list"); + break; + + case H5S_SEL_HYPERSLABS: /* Hyperslab selection */ + VERIFY(nseq, 10, "H5Ssel_iter_get_seq_list"); + VERIFY(nbytes, 10, "H5Ssel_iter_get_seq_list"); + break; + + case H5S_SEL_ALL: /* "All" selection */ + VERIFY(nseq, 1, "H5Ssel_iter_get_seq_list"); + VERIFY(nbytes, 72, "H5Ssel_iter_get_seq_list"); + break; + + case H5S_SEL_ERROR: + case H5S_SEL_N: + default: + HDassert(0 && "Can't occur"); + break; + } /* end switch */ + + /* Reset iterator */ + ret = H5Ssel_iter_reset(iter_id, sid); + CHECK(ret, FAIL, "H5Ssel_iter_reset"); + + /* Try retrieving all sequences again */ + ret = H5Ssel_iter_get_seq_list(iter_id, (size_t)SEL_ITER_MAX_SEQ, (size_t)(1024 * 1024), &nseq, + &nbytes, off, len); + CHECK(ret, FAIL, "H5Ssel_iter_get_seq_list"); + + /* Check results from retrieving sequence list */ + switch (sel_type) { + case H5S_SEL_NONE: /* "None" selection */ + VERIFY(nseq, 0, "H5Ssel_iter_get_seq_list"); + VERIFY(nbytes, 0, "H5Ssel_iter_get_seq_list"); + break; + + case H5S_SEL_POINTS: /* Point selection */ + VERIFY(nseq, 10, "H5Ssel_iter_get_seq_list"); + VERIFY(nbytes, 10, "H5Ssel_iter_get_seq_list"); + break; + + case H5S_SEL_HYPERSLABS: /* Hyperslab selection */ + VERIFY(nseq, 10, "H5Ssel_iter_get_seq_list"); + VERIFY(nbytes, 10, "H5Ssel_iter_get_seq_list"); + break; + + case H5S_SEL_ALL: /* "All" selection */ + VERIFY(nseq, 1, "H5Ssel_iter_get_seq_list"); + VERIFY(nbytes, 72, "H5Ssel_iter_get_seq_list"); + break; + + case H5S_SEL_ERROR: + case H5S_SEL_N: + default: + HDassert(0 && "Can't occur"); + break; + } /* end switch */ + + /* Reset iterator */ + ret = H5Ssel_iter_reset(iter_id, sid); + CHECK(ret, FAIL, "H5Ssel_iter_reset"); + } /* end for */ + + /* Close selection iterator */ + ret = H5Ssel_iter_close(iter_id); + CHECK(ret, FAIL, "H5Ssel_iter_close"); + /* Point selection which will merge into smaller # of sequences */ coord1[0][0] = 0; coord1[0][1] = 9; @@ -15292,6 +15461,17 @@ test_sel_iter(void) VERIFY(nseq, 6, "H5Ssel_iter_get_seq_list"); VERIFY(nbytes, 10, "H5Ssel_iter_get_seq_list"); + /* Reset iterator */ + ret = H5Ssel_iter_reset(iter_id, sid); + CHECK(ret, FAIL, "H5Ssel_iter_reset"); + + /* Try retrieving all sequences again */ + ret = H5Ssel_iter_get_seq_list(iter_id, (size_t)SEL_ITER_MAX_SEQ, (size_t)(1024 * 1024), &nseq, + &nbytes, off, len); + CHECK(ret, FAIL, "H5Ssel_iter_get_seq_list"); + VERIFY(nseq, 6, "H5Ssel_iter_get_seq_list"); + VERIFY(nbytes, 10, "H5Ssel_iter_get_seq_list"); + /* Close selection iterator */ ret = H5Ssel_iter_close(iter_id); CHECK(ret, FAIL, "H5Ssel_iter_close"); @@ -15330,6 +15510,17 @@ test_sel_iter(void) VERIFY(nseq, 6, "H5Ssel_iter_get_seq_list"); VERIFY(nbytes, 20, "H5Ssel_iter_get_seq_list"); + /* Reset iterator */ + ret = H5Ssel_iter_reset(iter_id, sid); + CHECK(ret, FAIL, "H5Ssel_iter_reset"); + + /* Try retrieving all sequences again */ + ret = H5Ssel_iter_get_seq_list(iter_id, (size_t)SEL_ITER_MAX_SEQ, (size_t)(1024 * 1024), &nseq, + &nbytes, off, len); + CHECK(ret, FAIL, "H5Ssel_iter_get_seq_list"); + VERIFY(nseq, 6, "H5Ssel_iter_get_seq_list"); + VERIFY(nbytes, 20, "H5Ssel_iter_get_seq_list"); + /* Close selection iterator */ ret = H5Ssel_iter_close(iter_id); CHECK(ret, FAIL, "H5Ssel_iter_close"); diff --git a/test/ttsafe.c b/test/ttsafe.c index d2085b9..2b98ded 100644 --- a/test/ttsafe.c +++ b/test/ttsafe.c @@ -134,7 +134,7 @@ main(int argc, char *argv[]) TestSummary(); /* Clean up test files, if allowed */ - if (GetTestCleanup() && !getenv("HDF5_NOCLEANUP")) + if (GetTestCleanup() && !HDgetenv("HDF5_NOCLEANUP")) TestCleanup(); /* Release test infrastructure */ diff --git a/test/ttsafe_cancel.c b/test/ttsafe_cancel.c index 1e40c58..2ec9a20 100644 --- a/test/ttsafe_cancel.c +++ b/test/ttsafe_cancel.c @@ -65,46 +65,46 @@ tts_cancel(void) /* make thread scheduling global */ ret = pthread_attr_init(&attribute); - assert(ret == 0); + HDassert(ret == 0); #ifdef H5_HAVE_SYSTEM_SCOPE_THREADS ret = pthread_attr_setscope(&attribute, PTHREAD_SCOPE_SYSTEM); - assert(ret == 0); + HDassert(ret == 0); #endif /* H5_HAVE_SYSTEM_SCOPE_THREADS */ /* Initialize mutex & condition variables */ ret = pthread_mutex_init(&mutex, NULL); - assert(ret == 0); + HDassert(ret == 0); ret = pthread_cond_init(&cond, NULL); - assert(ret == 0); + HDassert(ret == 0); /* * Create a hdf5 file using H5F_ACC_TRUNC access, default file * creation plist and default file access plist */ cancel_file = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - assert(cancel_file >= 0); + HDassert(cancel_file >= 0); ret = pthread_create(&childthread, &attribute, tts_cancel_thread, NULL); - assert(ret == 0); + HDassert(ret == 0); tts_cancel_barrier(); ret = pthread_cancel(childthread); - assert(ret == 0); + HDassert(ret == 0); dataset = H5Dopen2(cancel_file, DATASETNAME, H5P_DEFAULT); - assert(dataset >= 0); + HDassert(dataset >= 0); ret = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, &buffer); - assert(ret >= 0); + HDassert(ret >= 0); if (buffer != 11) TestErrPrintf("operation unsuccessful with value at %d instead of 11\n", buffer); ret = H5Dclose(dataset); - assert(ret >= 0); + HDassert(ret >= 0); ret = H5Fclose(cancel_file); - assert(ret >= 0); + HDassert(ret >= 0); /* Destroy the thread attribute */ ret = pthread_attr_destroy(&attribute); - assert(ret == 0); + HDassert(ret == 0); } /* end tts_cancel() */ void * diff --git a/test/ttsafe_dcreate.c b/test/ttsafe_dcreate.c index cedafd0..a5a22be 100644 --- a/test/ttsafe_dcreate.c +++ b/test/ttsafe_dcreate.c @@ -140,7 +140,7 @@ tts_dcreate_creator(void *_thread_data) hsize_t dimsf[1]; /* dataset dimensions */ struct thread_info thread_data; - memcpy(&thread_data, _thread_data, sizeof(struct thread_info)); + HDmemcpy(&thread_data, _thread_data, sizeof(struct thread_info)); /* define dataspace for dataset */ dimsf[0] = 1; diff --git a/test/tunicode.c b/test/tunicode.c index 28f819b..2fed891 100644 --- a/test/tunicode.c +++ b/test/tunicode.c @@ -532,7 +532,7 @@ test_attrname(hid_t fid, const char *string) CHECK(attr_id, FAIL, "H5Acreate2"); size = H5Aget_name(attr_id, (size_t)MAX_STRING_LENGTH, read_buf); CHECK(size, FAIL, "H5Aget_name"); - ret = strcmp(read_buf, string); + ret = HDstrcmp(read_buf, string); VERIFY(ret, 0, "strcmp"); read_buf[0] = '\0'; @@ -541,7 +541,7 @@ test_attrname(hid_t fid, const char *string) CHECK(ret, FAIL, "H5Awrite"); ret = H5Aread(attr_id, dtype_id, read_buf); CHECK(ret, FAIL, "H5Aread"); - ret = strcmp(read_buf, string); + ret = HDstrcmp(read_buf, string); VERIFY(ret, 0, "strcmp"); /* Clean up */ @@ -682,7 +682,7 @@ test_enum(hid_t H5_ATTR_UNUSED fid, const char *string) VERIFY(val, E1_WHITE, "H5Tenum_valueof"); ret = H5Tenum_nameof(type_id, &val, readbuf, (size_t)MAX_STRING_LENGTH); CHECK(ret, FAIL, "H5Tenum_nameof"); - ret = strcmp(readbuf, string); + ret = HDstrcmp(readbuf, string); VERIFY(ret, 0, "strcmp"); /* Close the datatype */ @@ -709,7 +709,7 @@ test_opaque(hid_t H5_ATTR_UNUSED fid, const char *string) /* Read the tag back. */ read_buf = H5Tget_tag(type_id); - ret = strcmp(read_buf, string); + ret = HDstrcmp(read_buf, string); VERIFY(ret, 0, "H5Tget_tag"); H5free_memory(read_buf); diff --git a/test/tvlstr.c b/test/tvlstr.c index c8e65b5..c2d2f2d 100644 --- a/test/tvlstr.c +++ b/test/tvlstr.c @@ -203,7 +203,7 @@ test_vlstrings_basic(void) for (i = 0; i < SPACE1_DIM1; i++) { if (HDstrlen(wdata[i]) != HDstrlen(rdata[i])) { TestErrPrintf("VL data length don't match!, strlen(wdata[%d])=%d, strlen(rdata[%d])=%d\n", (int)i, - (int)strlen(wdata[i]), (int)i, (int)HDstrlen(rdata[i])); + (int)HDstrlen(wdata[i]), (int)i, (int)HDstrlen(rdata[i])); continue; } /* end if */ if (HDstrcmp(wdata[i], rdata[i]) != 0) { @@ -307,7 +307,7 @@ test_vlstrings_special(void) for (i = 0; i < SPACE1_DIM1; i++) { if (HDstrlen(wdata[i]) != HDstrlen(rdata[i])) { TestErrPrintf("VL data length don't match!, strlen(wdata[%d])=%d, strlen(rdata[%d])=%d\n", (int)i, - (int)strlen(wdata[i]), (int)i, (int)HDstrlen(rdata[i])); + (int)HDstrlen(wdata[i]), (int)i, (int)HDstrlen(rdata[i])); continue; } /* end if */ if ((wdata[i] == NULL && rdata[i] != NULL) || (rdata[i] == NULL && wdata[i] != NULL)) { @@ -536,7 +536,7 @@ test_compact_vlstring(void) for (i = 0; i < SPACE1_DIM1; i++) { if (HDstrlen(wdata[i]) != HDstrlen(rdata[i])) { TestErrPrintf("VL data length don't match!, strlen(wdata[%d])=%d, strlen(rdata[%d])=%d\n", (int)i, - (int)strlen(wdata[i]), (int)i, (int)HDstrlen(rdata[i])); + (int)HDstrlen(wdata[i]), (int)i, (int)HDstrlen(rdata[i])); continue; } /* end if */ if (HDstrcmp(wdata[i], rdata[i]) != 0) { diff --git a/test/tvltypes.c b/test/tvltypes.c index 50cfbd7..47b28d3 100644 --- a/test/tvltypes.c +++ b/test/tvltypes.c @@ -2676,13 +2676,13 @@ test_vltypes_fill_value(void) } break; case H5D_VIRTUAL: - assert(0 && "Invalid layout type!"); + HDassert(0 && "Invalid layout type!"); break; case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: - assert(0 && "Unknown layout type!"); + HDassert(0 && "Unknown layout type!"); break; } /* end switch */ @@ -2784,13 +2784,13 @@ test_vltypes_fill_value(void) break; case H5D_VIRTUAL: - assert(0 && "Invalid layout type!"); + HDassert(0 && "Invalid layout type!"); break; case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: - assert(0 && "Unknown layout type!"); + HDassert(0 && "Unknown layout type!"); break; } /* end switch */ @@ -2969,13 +2969,13 @@ test_vltypes_fill_value(void) break; case H5D_VIRTUAL: - assert(0 && "Invalid layout type!"); + HDassert(0 && "Invalid layout type!"); break; case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: - assert(0 && "Unknown layout type!"); + HDassert(0 && "Unknown layout type!"); break; } /* end switch */ @@ -3241,5 +3241,5 @@ test_vltypes(void) void cleanup_vltypes(void) { - remove(FILENAME); + HDremove(FILENAME); } diff --git a/test/twriteorder.c b/test/twriteorder.c index 83a615c..9c8a145 100644 --- a/test/twriteorder.c +++ b/test/twriteorder.c @@ -141,21 +141,21 @@ parse_option(int argc, char *const argv[]) HDexit(EXIT_SUCCESS); break; case 'b': /* number of planes to write/read */ - if ((blocksize_g = atoi(optarg)) <= 0) { + if ((blocksize_g = HDatoi(optarg)) <= 0) { HDfprintf(stderr, "bad blocksize %s, must be a positive integer\n", optarg); usage(progname_g); Hgoto_error(-1); }; break; case 'n': /* number of planes to write/read */ - if ((nlinkedblock_g = atoi(optarg)) < 2) { + if ((nlinkedblock_g = HDatoi(optarg)) < 2) { HDfprintf(stderr, "bad number of linked blocks %s, must be greater than 1.\n", optarg); usage(progname_g); Hgoto_error(-1); }; break; case 'p': /* number of planes to write/read */ - if ((part_size_g = atoi(optarg)) <= 0) { + if ((part_size_g = HDatoi(optarg)) <= 0) { HDfprintf(stderr, "bad partition size %s, must be a positive integer\n", optarg); usage(progname_g); Hgoto_error(-1); @@ -404,7 +404,7 @@ main(int argc, char *argv[]) Hgoto_error(1); }; }; - mypid = getpid(); + mypid = HDgetpid(); /* ============= */ /* launch reader */ @@ -440,7 +440,7 @@ main(int argc, char *argv[]) /* If readwrite, collect exit code of child process */ /* ================================================ */ if (launch_g == UC_READWRITE) { - if ((tmppid = waitpid(childpid, &child_status, child_wait_option)) < 0) { + if ((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) { HDperror("waitpid"); Hgoto_error(1); } diff --git a/test/unlink.c b/test/unlink.c index 8129cc9..2f68a69 100644 --- a/test/unlink.c +++ b/test/unlink.c @@ -575,7 +575,7 @@ test_filespace(hid_t fapl) size_t rdcc_nbytes; double rdcc_w0; - puts("Testing file space gets reused:"); + HDputs("Testing file space gets reused:"); /* Open file */ h5_fixname(FILENAME[4], fapl, filename, sizeof filename); diff --git a/test/use_append_mchunks.c b/test/use_append_mchunks.c index 0b2c409..94adfb0 100644 --- a/test/use_append_mchunks.c +++ b/test/use_append_mchunks.c @@ -165,12 +165,12 @@ main(int argc, char *argv[]) /* Fork process */ /* ============ */ if (UC_opts.launch == UC_READWRITE) { - if ((childpid = fork()) < 0) { - perror("fork"); + if ((childpid = HDfork()) < 0) { + HDperror("fork"); Hgoto_error(1); } } - mypid = getpid(); + mypid = HDgetpid(); /* ============= */ /* launch reader */ @@ -239,8 +239,8 @@ main(int argc, char *argv[]) /* If readwrite, collect exit code of child process */ /* ================================================ */ if (UC_opts.launch == UC_READWRITE) { - if ((tmppid = waitpid(childpid, &child_status, child_wait_option)) < 0) { - perror("waitpid"); + if ((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) { + HDperror("waitpid"); Hgoto_error(1); } diff --git a/test/use_common.c b/test/use_common.c index ccebd0b..7a3c95a 100644 --- a/test/use_common.c +++ b/test/use_common.c @@ -68,7 +68,7 @@ parse_option(int argc, char *const argv[], options_t *opts) switch (c) { case 'h': usage(opts->progname); - exit(EXIT_SUCCESS); + HDexit(EXIT_SUCCESS); break; case 'f': /* usecase data file name */ opts->filename = HDstrdup(optarg); @@ -2168,8 +2168,6 @@ test_ros3(void) hid_t driver_id = -1; /* ID for this VFD */ unsigned long driver_flags = 0; /* VFD feature flags */ char filename[1024]; /* filename */ - void * os_file_handle = NULL; /* OS file handle */ - hsize_t file_size; /* file size */ H5FD_ros3_fapl_t test_ros3_fa; H5FD_ros3_fapl_t ros3_fa_0 = { /* version = */ H5FD_CURR_ROS3_FAPL_T_VERSION, @@ -2201,9 +2199,9 @@ test_ros3(void) /* need a macro to compare instances of H5FD_ros3_fapl_t */ if ((test_ros3_fa.version != ros3_fa_0.version) || (test_ros3_fa.authenticate != ros3_fa_0.authenticate) || - (strcmp(test_ros3_fa.aws_region, ros3_fa_0.aws_region) != 0) || - (strcmp(test_ros3_fa.secret_id, ros3_fa_0.secret_id) != 0) || - (strcmp(test_ros3_fa.secret_key, ros3_fa_0.secret_key) != 0)) + (HDstrcmp(test_ros3_fa.aws_region, ros3_fa_0.aws_region) != 0) || + (HDstrcmp(test_ros3_fa.secret_id, ros3_fa_0.secret_id) != 0) || + (HDstrcmp(test_ros3_fa.secret_key, ros3_fa_0.secret_key) != 0)) TEST_ERROR; h5_fixname(FILENAME[10], fapl_id, filename, sizeof(filename)); @@ -2252,7 +2250,7 @@ error: AT(); \ HDfprintf(stderr, mesg); \ H5Eprint2(H5E_DEFAULT, stderr); \ - fflush(stderr); \ + HDfflush(stderr); \ ret_value = -1; \ goto done; \ } @@ -2464,7 +2462,7 @@ run_splitter_test(const struct splitter_dataset_def *data, hbool_t ignore_wo_err } /* Verify existence of logfile if appropriate */ - logfile = fopen(vfd_config->log_file_path, "r"); + logfile = HDfopen(vfd_config->log_file_path, "r"); if ((TRUE == provide_logfile_path && NULL == logfile) || (FALSE == provide_logfile_path && NULL != logfile)) { SPLITTER_TEST_FAULT("no logfile when one was expected\n"); @@ -2485,7 +2483,7 @@ done: } if (logfile != NULL) - fclose(logfile); + HDfclose(logfile); HDfree(vfd_config); HDfree(filename_rw); diff --git a/testpar/t_mdset.c b/testpar/t_mdset.c index a4bc26d..bf4ce73 100644 --- a/testpar/t_mdset.c +++ b/testpar/t_mdset.c @@ -1456,10 +1456,8 @@ read_attribute(hid_t obj_id, int this_type, int num) if (this_type == is_group) { HDsprintf(attr_name, "Group Attribute %d", num); aid = H5Aopen(obj_id, attr_name, H5P_DEFAULT); - if (MAINPROCESS) { - H5Aread(aid, H5T_NATIVE_INT, &in_num); - vrfy_errors = dataset_vrfy(NULL, NULL, NULL, group_block, &in_num, &num); - } + H5Aread(aid, H5T_NATIVE_INT, &in_num); + vrfy_errors = dataset_vrfy(NULL, NULL, NULL, group_block, &in_num, &num); H5Aclose(aid); } else if (this_type == is_dset) { @@ -1467,10 +1465,8 @@ read_attribute(hid_t obj_id, int this_type, int num) for (i = 0; i < 8; i++) out_data[i] = i; aid = H5Aopen(obj_id, attr_name, H5P_DEFAULT); - if (MAINPROCESS) { - H5Aread(aid, H5T_NATIVE_INT, in_data); - vrfy_errors = dataset_vrfy(NULL, NULL, NULL, dset_block, in_data, out_data); - } + H5Aread(aid, H5T_NATIVE_INT, in_data); + vrfy_errors = dataset_vrfy(NULL, NULL, NULL, dset_block, in_data, out_data); H5Aclose(aid); } diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake index af73b75..81d619c 100644 --- a/tools/test/h5repack/CMakeTests.cmake +++ b/tools/test/h5repack/CMakeTests.cmake @@ -51,6 +51,7 @@ ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_named_dtypes.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_nested_8bit_enum.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_nested_8bit_enum_deflated.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_HDFFV-10590_CVE-2018-17432.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_nbit.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_objs.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_refs.h5 @@ -1549,6 +1550,11 @@ # Note: this test is experimental for sharing test file among tools ADD_H5_TEST (HDFFV-7840 "TEST" h5diff_attr1.h5) +# test CVE-2018-17432 fix + set (arg h5repack_HDFFV-10590_CVE-2018-17432.h5 h5repack_HDFFV-10590_CVE-2018-17432_out.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6) + set (TESTTYPE "TEST") + ADD_H5_FILTER_TEST (HDFFV-10590 "" ${TESTTYPE} 1 ${arg}) + # tests for metadata block size option ('-M') ADD_H5_TEST_META (meta_short h5repack_layout.h5 -M 8192) ADD_H5_TEST_META (meta_long h5repack_layout.h5 --metadata_block_size=8192) diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in index a95a22e..3764081 100644 --- a/tools/test/h5repack/h5repack.sh.in +++ b/tools/test/h5repack/h5repack.sh.in @@ -128,6 +128,8 @@ $SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5 $SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5 ########h5diff/testfile######## $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5 +########test#HDFFV-10590######## +$SRC_H5REPACK_TESTFILES/h5repack_HDFFV-10590_CVE-2018-17432.h5 ########tools/testfiles#for#external#links######## $SRC_TOOLS_TESTFILES/tsoftlinks.h5 $SRC_TOOLS_TESTFILES/textlinkfar.h5 @@ -865,6 +867,34 @@ TOOLTESTV() rm -f $outfile } +# Same as TOOLTEST, but expects h5repack fails +# +TOOLTEST_FAIL() +{ + infile=$1 + outfile=$2 + expect="$TESTDIR/$2-$1.ddl" + actual="$TESTDIR/$2-$1.out" + actual_err="$TESTDIR/$2-$1.err" + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) >$actual + RET=$? + if [ $RET == 0 ] ; then + nerrors="`expr $nerrors + 1`" + echo " FAILED" + else + echo " PASSED" + fi + rm -f $outfile +} + # This is same as TOOLTESTV() with comparing display output # with actual filename swapped # @@ -1681,6 +1711,10 @@ TOOLTEST HDFFV-5932 h5repack_attr_refs.h5 # Note: this test is experimental for sharing test file among tools TOOLTEST HDFFV-7840 h5diff_attr1.h5 +# test HDFFV-10590 +arg="h5repack_HDFFV-10590_CVE-2018-17432.h5 h5repack_HDFFV-10590_CVE-2018-17432_out.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6" +TOOLTEST_FAIL $arg + # tests for metadata block size option TOOLTEST_META meta_short h5repack_layout.h5 -M 8192 TOOLTEST_META meta_long h5repack_layout.h5 --metadata_block_size=8192 diff --git a/tools/test/h5repack/testfiles/h5repack_HDFFV-10590_CVE-2018-17432.h5 b/tools/test/h5repack/testfiles/h5repack_HDFFV-10590_CVE-2018-17432.h5 Binary files differnew file mode 100644 index 0000000..7a815ba --- /dev/null +++ b/tools/test/h5repack/testfiles/h5repack_HDFFV-10590_CVE-2018-17432.h5 |