diff options
| author | Dana Robinson <derobins@hdfgroup.org> | 2020-08-15 21:20:18 (GMT) |
|---|---|---|
| committer | Dana Robinson <derobins@hdfgroup.org> | 2020-08-15 21:20:18 (GMT) |
| commit | c638d93f3e660ce669a36e50a02473aac126953d (patch) | |
| tree | b39f5688adc827da7d000aaa5a5264d29c2ee919 | |
| parent | 8e7c24f5148e91335bf744583a6a407014014391 (diff) | |
| parent | e9305abf48426f265d7ec3ae54538e3ded8889ea (diff) | |
| download | hdf5-c638d93f3e660ce669a36e50a02473aac126953d.zip hdf5-c638d93f3e660ce669a36e50a02473aac126953d.tar.gz hdf5-c638d93f3e660ce669a36e50a02473aac126953d.tar.bz2 | |
Merge pull request #2770 in HDFFV/hdf5 from ~DEROBINS/hdf5_der:1_10_normalization to hdf5_1_10
* commit 'e9305abf48426f265d7ec3ae54538e3ded8889ea':
Fixes missing chunk_info entry in CMake files
Even more normalization with develop
More normalizations with develop
Misc normalizations with develop
Normalization of perform directory with develop
Brings monotonic timer changes from develop
Brings Mirror VFD to 1.10 from develop
Brings splitter VFD from develop
Brings file locking changes from develop
200 files changed, 15806 insertions, 2308 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index ef4caca..5d1295f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -137,6 +137,7 @@ set (HDF5_CPP_LIB_CORENAME "hdf5_cpp") set (HDF5_HL_LIB_CORENAME "hdf5_hl") set (HDF5_HL_CPP_LIB_CORENAME "hdf5_hl_cpp") set (HDF5_TOOLS_LIB_CORENAME "hdf5_tools") +set (HDF5_UTILS_LIB_CORENAME "hdf5_utils") set (HDF5_F90_LIB_CORENAME "hdf5_fortran") set (HDF5_F90_C_LIB_CORENAME "hdf5_f90cstub") set (HDF5_F90_TEST_LIB_CORENAME "hdf5_test_fortran") @@ -156,6 +157,7 @@ set (HDF5_CPP_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_CPP_LIB_COREN set (HDF5_HL_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_HL_LIB_CORENAME}") set (HDF5_HL_CPP_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_HL_CPP_LIB_CORENAME}") set (HDF5_TOOLS_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_TOOLS_LIB_CORENAME}") +set (HDF5_UTILS_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_UTILS_LIB_CORENAME}") set (HDF5_F90_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_F90_LIB_CORENAME}") set (HDF5_F90_C_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_F90_C_LIB_CORENAME}") set (HDF5_F90_TEST_LIB_NAME "${HDF5_EXTERNAL_LIB_PREFIX}${HDF5_F90_TEST_LIB_CORENAME}") @@ -175,6 +177,7 @@ set (HDF5_CPP_LIB_TARGET "${HDF5_CPP_LIB_CORENAME}-static") set (HDF5_HL_LIB_TARGET "${HDF5_HL_LIB_CORENAME}-static") set (HDF5_HL_CPP_LIB_TARGET "${HDF5_HL_CPP_LIB_CORENAME}-static") set (HDF5_TOOLS_LIB_TARGET "${HDF5_TOOLS_LIB_CORENAME}-static") +set (HDF5_UTILS_LIB_TARGET "${HDF5_UTILS_LIB_CORENAME}-static") set (HDF5_F90_LIB_TARGET "${HDF5_F90_LIB_CORENAME}-static") set (HDF5_F90_C_LIB_TARGET "${HDF5_F90_C_LIB_CORENAME}-static") set (HDF5_F90_TEST_LIB_TARGET "${HDF5_F90_TEST_LIB_CORENAME}-static") @@ -190,6 +193,7 @@ set (HDF5_CPP_LIBSH_TARGET "${HDF5_CPP_LIB_CORENAME}-shared") set (HDF5_HL_LIBSH_TARGET "${HDF5_HL_LIB_CORENAME}-shared") set (HDF5_HL_CPP_LIBSH_TARGET "${HDF5_HL_CPP_LIB_CORENAME}-shared") set (HDF5_TOOLS_LIBSH_TARGET "${HDF5_TOOLS_LIB_CORENAME}-shared") +set (HDF5_UTILS_LIBSH_TARGET "${HDF5_UTILS_LIB_CORENAME}-shared") set (HDF5_F90_LIBSH_TARGET "${HDF5_F90_LIB_CORENAME}-shared") set (HDF5_F90_C_LIBSH_TARGET "${HDF5_F90_C_LIB_CORENAME}-shared") set (HDF5_F90_TEST_LIBSH_TARGET "${HDF5_F90_TEST_LIB_CORENAME}-shared") @@ -212,6 +216,7 @@ set (HDF5_HL_TOOLS_DIR ${HDF5_SOURCE_DIR}/hl/tools) set (HDF5_TOOLS_DIR ${HDF5_SOURCE_DIR}/tools) set (HDF5_TOOLS_SRC_DIR ${HDF5_SOURCE_DIR}/tools/src) set (HDF5_PERFORM_SRC_DIR ${HDF5_SOURCE_DIR}/tools/src/perform) +set (HDF5_UTILS_DIR ${HDF5_SOURCE_DIR}/utils) set (HDF5_F90_SRC_DIR ${HDF5_SOURCE_DIR}/fortran) set (HDF5_JAVA_JNI_SRC_DIR ${HDF5_SOURCE_DIR}/java/src/jni) set (HDF5_JAVA_HDF5_SRC_DIR ${HDF5_SOURCE_DIR}/java/src/hdf) @@ -886,6 +891,16 @@ if (BUILD_TESTING) endif () #----------------------------------------------------------------------------- +# Option to build HDF5 Utilities +#----------------------------------------------------------------------------- +if (EXISTS "${HDF5_SOURCE_DIR}/utils" AND IS_DIRECTORY "${HDF5_SOURCE_DIR}/utils") + option (HDF5_BUILD_UTILS "Build HDF5 Utils" ON) + if (HDF5_BUILD_UTILS) + add_subdirectory (utils) + endif () +endif () + +#----------------------------------------------------------------------------- # Option to build HDF5 Tools #----------------------------------------------------------------------------- if (EXISTS "${HDF5_SOURCE_DIR}/tools" AND IS_DIRECTORY "${HDF5_SOURCE_DIR}/tools") @@ -190,6 +190,7 @@ ./examples/h5_chunk_read.c ./examples/h5_compound.c ./examples/h5_crtgrpd.c +./examples/h5_debug_trace.c ./examples/h5_subset.c ./examples/h5_cmprss.c ./examples/h5_rdwt.c @@ -682,6 +683,9 @@ ./src/H5FDint.c ./src/H5FDlog.c ./src/H5FDlog.h +./src/H5FDmirror.c +./src/H5FDmirror.h +./src/H5FDmirror_priv.h ./src/H5FDmodule.h ./src/H5FDmpi.c ./src/H5FDmpi.h @@ -699,6 +703,8 @@ ./src/H5FDsec2.c ./src/H5FDsec2.h ./src/H5FDspace.c +./src/H5FDsplitter.c +./src/H5FDsplitter.h ./src/H5FDstdio.c ./src/H5FDstdio.h ./src/H5FDtest.c @@ -1001,6 +1007,7 @@ ./test/cache_image.c ./test/cache_logging.c ./test/cache_tagging.c +./test/chunk_info.c ./test/cmpd_dset.c ./test/cork.c ./test/corrupt_stab_msg.h5 @@ -1066,6 +1073,7 @@ ./test/gen_deflate.c ./test/gen_file_image.c ./test/gen_filespace.c +./test/gen_filters.c ./test/gen_mergemsg.c ./test/gen_new_array.c ./test/gen_new_fill.c @@ -1104,6 +1112,7 @@ ./test/corrupted_name_len.h5 ./test/mergemsg.h5 ./test/mf.c +./test/mirror_vfd.c ./test/mount.c ./test/mtime.c ./test/multi_file_v16-r.h5 @@ -1161,6 +1170,7 @@ ./test/testhdf5.c ./test/testhdf5.h ./test/testlibinfo.sh.in +./test/test_mirror.sh.in ./test/test_usecases.sh.in ./test/testmeta.c ./test/testswmr.sh.in @@ -1174,6 +1184,7 @@ ./test/theap.c ./test/thread_id.c ./test/tid.c +./test/timer.c ./test/titerate.c ./test/tlayouto.h5 ./test/tmeta.c @@ -1202,19 +1213,18 @@ ./test/unlink.c ./test/unregister.c ./test/use_append_chunk.c +./test/use_append_chunk_mirror.c ./test/use_append_mchunks.c ./test/use_common.c ./test/use_disable_mdc_flushes.c ./test/use.h -./test/vfd.c -./test/gen_filters.c -./test/chunk_info.c ./test/vds.c ./test/vds_env.c ./test/vds_swmr.h ./test/vds_swmr_gen.c ./test/vds_swmr_reader.c ./test/vds_swmr_writer.c +./test/vfd.c ./test/testfiles/err_compat_1 ./test/testfiles/err_compat_2 @@ -2728,6 +2738,18 @@ ./tools/test/perform/sio_standalone.h ./tools/test/perform/zip_perf.c +# Utils directory +./utils/COPYING +./utils/Makefile.am + +# Mirror VFD utilities +./utils/mirror_vfd/Makefile.am +./utils/mirror_vfd/mirror_remote.c +./utils/mirror_vfd/mirror_remote.h +./utils/mirror_vfd/mirror_server.c +./utils/mirror_vfd/mirror_server_stop.c +./utils/mirror_vfd/mirror_writer.c + # high level libraries ./hl/COPYING ./hl/Makefile.am @@ -3456,6 +3478,8 @@ ./tools/test/misc/vds/CMakeLists.txt ./tools/test/perform/CMakeLists.txt ./tools/test/perform/CMakeTests.cmake +./utils/CMakeLists.txt +./utils/mirror_vfd/CMakeLists.txt # CMake-specific User Scripts ./config/cmake/CTestScript.cmake diff --git a/Makefile.am b/Makefile.am index 19522f7..d31f7c7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -85,9 +85,9 @@ else TOOLS_DIR= endif -SUBDIRS = src $(TESTSERIAL_DIR) $(TESTPARALLEL_DIR) bin $(TOOLS_DIR) . \ +SUBDIRS = src $(TESTSERIAL_DIR) $(TESTPARALLEL_DIR) bin utils $(TOOLS_DIR) . \ $(CXX_DIR) $(FORTRAN_DIR) $(JAVA_DIR) $(HDF5_HL_DIR) -DIST_SUBDIRS = src test testpar tools . c++ fortran hl examples java +DIST_SUBDIRS = src test testpar utils tools . c++ fortran hl examples java # Some files generated during configure that should be cleaned DISTCLEANFILES=config/stamp1 config/stamp2 @@ -190,10 +190,10 @@ trace: # Run tests with different Virtual File Drivers. # Currently, only invoke check-vfd in the test directory. check-vfd: - for d in src test; do \ - if test $$d != .; then \ - (cd $$d && $(MAKE) $(AM_MAKEFLAGS) $@) || exit 1; \ - fi; \ + for d in src utils test; do \ + if test $$d != .; then \ + (cd $$d && $(MAKE) $(AM_MAKEFLAGS) $@) || exit 1; \ + fi; \ done # Automake wants to make config.status depend on configure. This @@ -43,6 +43,7 @@ $Source = ""; "H5D_space_status_t" => "Ds", "H5D_vds_view_t" => "Dv", "H5FD_mpio_xfer_t" => "Dt", + "H5FD_splitter_vfd_config_t" => "Dr", "herr_t" => "e", "H5E_direction_t" => "Ed", "H5E_error_t" => "Ee", @@ -126,6 +127,7 @@ $Source = ""; "H5FD_ros3_fapl_t" => "x", "H5FD_hdfs_fapl_t" => "x", "H5FD_file_image_callbacks_t" => "x", + "H5FD_mirror_fapl_t" => "x", "H5G_iterate_t" => "x", "H5G_info_t" => "x", "H5I_free_t" => "x", diff --git a/c++/src/H5FaccProp.cpp b/c++/src/H5FaccProp.cpp index 1657351..6128d65 100644 --- a/c++/src/H5FaccProp.cpp +++ b/c++/src/H5FaccProp.cpp @@ -685,6 +685,53 @@ unsigned FileAccPropList::getGcReferences() const } //-------------------------------------------------------------------------- +// Function: FileAccPropList::setFileLocking +///\brief Sets file locking flags. +///\param use_file_locking - IN: Flag that determines if file locks should +// be used or not. +///\param ignore_when_disabled - IN: Flag that determines if file locks +// should be be used when disabled on the file system or not. +///\exception H5::PropListIException +///\par Description +/// For information, please refer to the H5Pset_file_locking API in +/// the HDF5 C Reference Manual. +// Programmer Dana Robinson - 2020 +//-------------------------------------------------------------------------- +void FileAccPropList::setFileLocking(hbool_t use_file_locking, hbool_t ignore_when_disabled) const +{ + herr_t ret_value = H5Pset_file_locking(id, use_file_locking, ignore_when_disabled); + + if (ret_value < 0) + { + throw PropListIException("FileAccPropList::setFileLocking", "H5Pset_file_locking failed"); + } +} + +//-------------------------------------------------------------------------- +// Function: FileAccPropList::getFileLocking +///\brief Gets file locking flags. +///\param use_file_locking - OUT: Flag that determines if file locks +// should be used or not. +///\param ignore_when_disabled - OUT: Flag that determines if file locks +// should be be used when disabled on the file system or not. +///\exception H5::PropListIException +///\par Description +/// For information, please refer to the H5Pget_file_locking API in +/// the HDF5 C Reference Manual. +// Programmer Dana Robinson - 2020 +//-------------------------------------------------------------------------- +void FileAccPropList::getFileLocking(hbool_t& use_file_locking, hbool_t& ignore_when_disabled) const +{ + herr_t ret_value = H5Pget_file_locking(id, &use_file_locking, &ignore_when_disabled); + + if (ret_value < 0) + { + throw PropListIException("FileAccPropList::getFileLocking", "H5Pget_file_locking failed"); + } +} + + +//-------------------------------------------------------------------------- // Function: FileAccPropList::setLibverBounds ///\brief Sets bounds on versions of library format to be used when creating /// or writing objects. diff --git a/c++/src/H5FaccProp.h b/c++/src/H5FaccProp.h index 58f049e..ee90fa4 100644 --- a/c++/src/H5FaccProp.h +++ b/c++/src/H5FaccProp.h @@ -126,6 +126,12 @@ class H5_DLLCPP FileAccPropList : public PropList { // Returns garbage collecting references setting. unsigned getGcReferences() const; + // Sets file locking parameters. + void setFileLocking(hbool_t use_file_locking, hbool_t ignore_when_disabled) const; + + // Gets file locking parameters. + void getFileLocking(hbool_t& use_file_locking, hbool_t& ignore_when_disabled) const; + // Sets bounds on versions of library format to be used when creating // or writing objects. void setLibverBounds(H5F_libver_t libver_low, H5F_libver_t libver_high) const; diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 11bf39c..ab121e9 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -61,6 +61,31 @@ if (HDF5_ENABLE_CODESTACK) endif () MARK_AS_ADVANCED (HDF5_ENABLE_CODESTACK) +# ---------------------------------------------------------------------- +# Check if they would like to use file locking by default +#----------------------------------------------------------------------------- +option (HDF5_USE_FILE_LOCKING "Use file locking by default (mainly for SWMR)" ON) +if (HDF5_USE_FILE_LOCKING) + set (${HDF_PREFIX}_USE_FILE_LOCKING 1) +endif () + +# ---------------------------------------------------------------------- +# Check if they would like to ignore file locks when disabled on a file system +#----------------------------------------------------------------------------- +option (HDF5_IGNORE_DISABLED_FILE_LOCKS "Ignore file locks when disabled on file system" ON) +if (HDF5_IGNORE_DISABLED_FILE_LOCKS) + set (${HDF_PREFIX}_IGNORE_DISABLED_FILE_LOCKS 1) +endif () + +# Set the libhdf5.settings file variable +if (HDF5_IGNORE_DISABLED_FILE_LOCKS AND HDF5_USE_FILE_LOCKING) + set (HDF5_FILE_LOCKING_SETTING "best-effort") +elseif (HDF5_IGNORE_DISABLED_FILE_LOCKS) + set (HDF5_FILE_LOCKING_SETTING "yes") +else () + set (HDF5_FILE_LOCKING_SETTING "no") +endif () + #----------------------------------------------------------------------------- # Are we going to use HSIZE_T #----------------------------------------------------------------------------- @@ -171,6 +196,23 @@ option (HDF5_ENABLE_ROS3_VFD "Build the ROS3 Virtual File Driver" OFF) endif () endif () +# ---------------------------------------------------------------------- +# Check whether we can build the Mirror VFD +# Header-check flags set in config/cmake_ext_mod/ConfigureChecks.cmake +# ---------------------------------------------------------------------- +option (HDF5_ENABLE_MIRROR_VFD "Build the Mirror Virtual File Driver" OFF) +if (H5FD_ENABLE_MIRROR_VFD) + if ( ${HDF_PREFIX}_HAVE_NETINET_IN_H AND + ${HDF_PREFIX}_HAVE_NETDB_H AND + ${HDF_PREFIX}_HAVE_ARPA_INET_H AND + ${HDF_PREFIX}_HAVE_SYS_SOCKET_H AND + ${HDF_PREFIX}_HAVE_FORK) + set (${HDF_PREFIX}_HAVE_MIRROR_VFD 1) + else() + message(STATUS "The socket-based Mirror VFD was requested but cannot be built. System prerequisites are not met.") + endif() +endif() + #----------------------------------------------------------------------------- # Check if C has __float128 extension #----------------------------------------------------------------------------- diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index 6cbfd29..5ccf460 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -104,6 +104,9 @@ /* Define if the compiler understands the __func__ keyword */ #cmakedefine H5_HAVE_C99_FUNC @H5_HAVE_C99_FUNC@ +/* Define to 1 if you have the <arpa/inet.h> header file. */ +#cmakedefine H5_HAVE_ARPA_INET_H @H5_HAVE_ARPA_INET_H@ + /* Define to 1 if you have the `clock_gettime' function. */ #cmakedefine H5_HAVE_CLOCK_GETTIME @H5_HAVE_CLOCK_GETTIME@ @@ -264,12 +267,12 @@ /* Define to 1 if you have the `lstat' function. */ #cmakedefine H5_HAVE_LSTAT @H5_HAVE_LSTAT@ -/* Define to 1 if you have the <mach/mach_time.h> header file. */ -#cmakedefine H5_HAVE_MACH_MACH_TIME_H @H5_HAVE_MACH_MACH_TIME_H@ - /* Define to 1 if you have the <memory.h> header file. */ #cmakedefine H5_HAVE_MEMORY_H @H5_HAVE_MEMORY_H@ +/* Define if we can build the Mirror VFD */ +#cmakedefine H5_HAVE_MIRROR_VFD @H5_HAVE_MIRROR_VFD@ + /* Define if we have MPE support */ #cmakedefine H5_HAVE_MPE @H5_HAVE_MPE@ @@ -282,6 +285,12 @@ /* Define if MPI_Info_c2f and MPI_Info_f2c exists */ #cmakedefine H5_HAVE_MPI_MULTI_LANG_Info @H5_HAVE_MPI_MULTI_LANG_Info@ +/* Define to 1 if you have the <netdb.h> header file. */ +#cmakedefine H5_HAVE_NETDB_H @H5_HAVE_NETDB_H@ + +/* Define to 1 if you have the <netinet/in.h> header file. */ +#cmakedefine H5_HAVE_NETINET_IN_H @H5_HAVE_NETINET_IN_H@ + /* Define to 1 if you have the <openssl/evp.h> header file. */ #cmakedefine H5_HAVE_OPENSSL_EVP_H @H5_HAVE_OPENSSL_EVP_H@ @@ -409,9 +418,6 @@ /* Define to 1 if you have the <sys/stat.h> header file. */ #cmakedefine H5_HAVE_SYS_STAT_H @H5_HAVE_SYS_STAT_H@ -/* Define to 1 if you have the <sys/timeb.h> header file. */ -#cmakedefine H5_HAVE_SYS_TIMEB_H @H5_HAVE_SYS_TIMEB_H@ - /* Define to 1 if you have the <sys/time.h> header file. */ #cmakedefine H5_HAVE_SYS_TIME_H @H5_HAVE_SYS_TIME_H@ @@ -480,6 +486,9 @@ /* Define if the compiler understands __inline__ */ #cmakedefine H5_HAVE___INLINE__ @H5_HAVE___INLINE__@ +/* Define if the library will ignore file locks when disabled */ +#cmakedefine H5_IGNORE_DISABLED_FILE_LOCKS @H5_IGNORE_DISABLED_FILE_LOCKS@ + /* Define if the high-level library headers should be included in hdf5.h */ #cmakedefine H5_INCLUDE_HL @H5_INCLUDE_HL@ @@ -714,6 +723,9 @@ /* Define using v1.10 public API symbols by default */ #cmakedefine H5_USE_110_API_DEFAULT @H5_USE_110_API_DEFAULT@ +/* Define if the library will use file locking */ +#cmakedefine H5_FILE_LOCKING @H5_USE_FILE_LOCKING@ + /* Define if a memory checking tool will be used on the library, to cause library to be very picky about memory operations and also disable the internal free list manager code. */ diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in index e9a8395..ebcbd61 100644 --- a/config/cmake/libhdf5.settings.cmake.in +++ b/config/cmake/libhdf5.settings.cmake.in @@ -76,6 +76,7 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ I/O filters (external): @EXTERNAL_FILTERS@ MPE: @H5_HAVE_LIBLMPE@ Direct VFD: @H5_HAVE_DIRECT@ + Mirror VFD: @H5_HAVE_MIRROR_VFD@ (Read-Only) S3 VFD: @H5_HAVE_ROS3_VFD@ (Read-Only) HDFS VFD: @H5_HAVE_LIBHDFS@ dmalloc: @H5_HAVE_LIBDMALLOC@ @@ -84,5 +85,6 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ Using memory checker: @HDF5_ENABLE_USING_MEMCHECKER@ Memory allocation sanity checks: @HDF5_MEMORY_ALLOC_SANITY_CHECK@ Function Stack Tracing: @HDF5_ENABLE_CODESTACK@ + Use file locking: @HDF5_FILE_LOCKING_SETTING@ Strict File Format Checks: @HDF5_STRICT_FORMAT_CHECKS@ Optimization Instrumentation: @HDF5_Enable_Instrument@ diff --git a/config/cmake_ext_mod/ConfigureChecks.cmake b/config/cmake_ext_mod/ConfigureChecks.cmake index 93b977e..0ddb6d0 100644 --- a/config/cmake_ext_mod/ConfigureChecks.cmake +++ b/config/cmake_ext_mod/ConfigureChecks.cmake @@ -137,15 +137,11 @@ CHECK_INCLUDE_FILE_CONCAT ("stddef.h" ${HDF_PREFIX}_HAVE_STDDEF_H) CHECK_INCLUDE_FILE_CONCAT ("stdint.h" ${HDF_PREFIX}_HAVE_STDINT_H) CHECK_INCLUDE_FILE_CONCAT ("unistd.h" ${HDF_PREFIX}_HAVE_UNISTD_H) -# Darwin -CHECK_INCLUDE_FILE_CONCAT ("mach/mach_time.h" ${HDF_PREFIX}_HAVE_MACH_MACH_TIME_H) - # Windows CHECK_INCLUDE_FILE_CONCAT ("io.h" ${HDF_PREFIX}_HAVE_IO_H) if (NOT CYGWIN) CHECK_INCLUDE_FILE_CONCAT ("winsock2.h" ${HDF_PREFIX}_HAVE_WINSOCK2_H) endif () -CHECK_INCLUDE_FILE_CONCAT ("sys/timeb.h" ${HDF_PREFIX}_HAVE_SYS_TIMEB_H) if (CMAKE_SYSTEM_NAME MATCHES "OSF") CHECK_INCLUDE_FILE_CONCAT ("sys/sysinfo.h" ${HDF_PREFIX}_HAVE_SYS_SYSINFO_H) @@ -166,6 +162,8 @@ CHECK_INCLUDE_FILE_CONCAT ("memory.h" ${HDF_PREFIX}_HAVE_MEMORY_H) CHECK_INCLUDE_FILE_CONCAT ("dlfcn.h" ${HDF_PREFIX}_HAVE_DLFCN_H) CHECK_INCLUDE_FILE_CONCAT ("inttypes.h" ${HDF_PREFIX}_HAVE_INTTYPES_H) CHECK_INCLUDE_FILE_CONCAT ("netinet/in.h" ${HDF_PREFIX}_HAVE_NETINET_IN_H) +CHECK_INCLUDE_FILE_CONCAT ("netdb.h" ${HDF_PREFIX}_HAVE_NETDB_H) +CHECK_INCLUDE_FILE_CONCAT ("arpa/inet.h" ${HDF_PREFIX}_HAVE_ARPA_INET_H) # _Bool type support CHECK_INCLUDE_FILE_CONCAT (stdbool.h ${HDF_PREFIX}_HAVE_STDBOOL_H) diff --git a/configure.ac b/configure.ac index db0d1f0..93cb2e3 100644 --- a/configure.ac +++ b/configure.ac @@ -1147,6 +1147,7 @@ AC_CHECK_HEADERS([stddef.h setjmp.h features.h]) AC_CHECK_HEADERS([dirent.h]) AC_CHECK_HEADERS([stdint.h], [C9x=yes]) AC_CHECK_HEADERS([stdbool.h]) +AC_CHECK_HEADERS([netdb.h netinet/in.h arpa/inet.h]) ## Darwin AC_SUBST([H5_IS_DARWIN]) @@ -2315,6 +2316,56 @@ case "X-$OPTIMIZATION" in esac ## ---------------------------------------------------------------------- +## Check if file locking should be used +## +AC_MSG_CHECKING([enable file locking]) +AC_ARG_ENABLE([file-locking], + [AS_HELP_STRING([--enable-file-locking=(yes|no|best-effort)], + [Sets the default for whether or not to use file + locking when opening files. Can be overridden + with the HDF5_USE_FILE_LOCKING environment variable + and the H5Pset_file_locking() API call. + best-effort attempts to use file locking but does + not fail when file locks have been disabled on + the file system (useful with Lustre). + [default=best-effort] + ])], + [DESIRED_FILE_LOCKING=$enableval]) + +## Set defaults +if test "X-$DESIRED_FILE_LOCKING" = X- ; then + DESIRED_FILE_LOCKING=best-effort +fi + +## Allow this variable to be substituted in +## other files (src/libhdf5.settings.in, etc.) +AC_SUBST([DESIRED_FILE_LOCKING]) +AC_SUBST([USE_FILE_LOCKING]) +AC_SUBST([IGNORE_DISABLED_FILE_LOCKS]) + +case "X-$DESIRED_FILE_LOCKING" in + X-best-effort) + AC_MSG_RESULT([best-effort]) + AC_DEFINE([USE_FILE_LOCKING], [1], + [Define if the library will use file locking]) + AC_DEFINE([IGNORE_DISABLED_FILE_LOCKS], [1], + [Define if the library will ignore file locks when disabled]) + ;; + X-yes) + AC_MSG_RESULT([yes]) + AC_DEFINE([USE_FILE_LOCKING], [1], + [Define if the library will use file locking]) + ;; + X-no) + AC_MSG_RESULT([no]) + ;; + *) + AC_MSG_ERROR([Unrecognized value: $USE_FILE_LOCKING]) + ;; +esac + + +## ---------------------------------------------------------------------- ## Enable/disable internal package-level debugging output ## AC_MSG_CHECKING([for internal debug output]) @@ -2878,6 +2929,50 @@ fi AM_CONDITIONAL([DIRECT_VFD_CONDITIONAL], [test "X$DIRECT_VFD" = "Xyes"]) ## ---------------------------------------------------------------------- +## Check whether the Mirror VFD can be built. +## Auto-enabled if the required libraries are present. +## +AC_SUBST([MIRROR_VFD]) + +## Default is no Mirror VFD +MIRROR_VFD=no + +AC_ARG_ENABLE([mirror-vfd], + [AS_HELP_STRING([--enable-mirror-vfd], + [Build the socket-based Mirror virtual file driver (VFD). + [default=no]])], + [MIRROR_VFD=$enableval], [MIRROR_VFD=no]) + +if test "X$MIRROR_VFD" = "Xyes"; then + + AC_CHECK_HEADERS([arpa/inet.h],, [unset MIRROR_VFD]) + AC_CHECK_HEADERS([netinet/in.h],, [unset MIRROR_VFD]) + AC_CHECK_HEADERS([netdb.h],, [unset MIRROR_VFD]) + AC_CHECK_HEADERS([sys/socket.h],, [unset MIRROR_VFD]) + AC_CHECK_FUNC([fork], [], [unset MIRROR_VFD]) + + AC_MSG_CHECKING([if the Mirror virtual file driver (VFD) can be built]) + if test "X$MIRROR_VFD" = "Xyes"; then + AC_DEFINE([HAVE_MIRROR_VFD], [1], + [Define whether the Mirror virtual file driver (VFD) will be compiled]) + AC_MSG_RESULT([yes]) + else + AC_MSG_RESULT([no]) + MIRROR_VFD=no + AC_MSG_ERROR([The Mirror VFD cannot be built. + Missing one or more of: arpa/inet.h, netinet/in.h, + netdb.h, sys/socket.h, fork().]) + fi +else + AC_MSG_CHECKING([if the Mirror virtual file driver (VFD) is enabled]) + AC_MSG_RESULT([no]) + MIRROR_VFD=no +fi + +## Mirror VFD files built only if able. +AM_CONDITIONAL([MIRROR_VFD_CONDITIONAL], [test "X$MIRROR_VFD" = "Xyes"]) + +## ---------------------------------------------------------------------- ## Check if Read-Only S3 virtual file driver is enabled by --enable-ros3-vfd ## AC_SUBST([ROS3_VFD]) @@ -3746,9 +3841,12 @@ AC_CONFIG_FILES([src/libhdf5.settings test/testvds_env.sh test/testvdsswmr.sh test/test_filter_plugin.sh + test/test_mirror.sh test/test_usecases.sh testpar/Makefile testpar/testpflush.sh + utils/Makefile + utils/mirror_vfd/Makefile tools/Makefile tools/lib/Makefile tools/libtest/Makefile diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt index 8383ba2..bc9a445 100644 --- a/examples/CMakeLists.txt +++ b/examples/CMakeLists.txt @@ -29,6 +29,7 @@ set (examples h5_extlink h5_elink_unix2win h5_shared_mesg + h5_debug_trace h5_vds h5_vds-exc h5_vds-exclim diff --git a/examples/Makefile.am b/examples/Makefile.am index 12ec52a..adc86ed 100644 --- a/examples/Makefile.am +++ b/examples/Makefile.am @@ -35,22 +35,23 @@ EXAMPLE_PROG = h5_write h5_read h5_extend_write h5_chunk_read h5_compound \ h5_crtgrpd h5_subset h5_cmprss h5_rdwt h5_crtgrpar h5_extend \ h5_crtatt h5_crtgrp h5_crtdat \ h5_group h5_select h5_attribute h5_mount h5_reference h5_drivers \ - h5_ref2reg h5_extlink h5_elink_unix2win h5_shared_mesg h5_vds h5_vds-exc \ - h5_vds-exclim h5_vds-eiger h5_vds-simpleIO h5_vds-percival \ + h5_ref2reg h5_extlink h5_elink_unix2win h5_shared_mesg h5_debug_trace \ + h5_vds h5_vds-exc h5_vds-exclim h5_vds-eiger h5_vds-simpleIO h5_vds-percival \ h5_vds-percival-unlim h5_vds-percival-unlim-maxmin TEST_SCRIPT=testh5cc.sh TEST_EXAMPLES_SCRIPT=$(INSTALL_SCRIPT_FILES) # Install files # List all file that should be installed in examples directory -INSTALL_FILES = h5_write.c h5_read.c h5_extend_write.c h5_chunk_read.c \ - h5_crtgrpd.c h5_subset.c h5_cmprss.c h5_rdwt.c h5_crtgrpar.c \ - h5_extend.c h5_crtatt.c h5_crtgrp.c h5_crtdat.c \ - h5_compound.c h5_group.c h5_select.c h5_attribute.c h5_mount.c \ - h5_reference.c h5_drivers.c h5_extlink.c h5_elink_unix2win.c \ - h5_ref2reg.c h5_shared_mesg.c ph5example.c h5_vds.c h5_vds-exc.c \ - h5_vds-exclim.c h5_vds-eiger.c h5_vds-simpleIO.c h5_vds-percival.c \ - h5_vds-percival-unlim.c h5_vds-percival-unlim-maxmin.c +INSTALL_FILES = h5_write.c h5_read.c h5_extend_write.c h5_chunk_read.c h5_compound.c \ + h5_crtgrpd.c h5_subset.c h5_cmprss.c h5_rdwt.c h5_crtgrpar.c h5_extend.c \ + h5_crtatt.c h5_crtgrp.c h5_crtdat.c \ + h5_group.c h5_select.c h5_attribute.c h5_mount.c h5_drivers.c \ + h5_reference.c h5_ref2reg.c \ + h5_extlink.c h5_elink_unix2win.c h5_shared_mesg.c h5_debug_trace.c \ + ph5example.c \ + h5_vds.c h5_vds-exc.c h5_vds-exclim.c h5_vds-eiger.c h5_vds-simpleIO.c \ + h5_vds-percival.c h5_vds-percival-unlim.c h5_vds-percival-unlim-maxmin.c diff --git a/examples/h5_debug_trace.c b/examples/h5_debug_trace.c new file mode 100644 index 0000000..5da55ed --- /dev/null +++ b/examples/h5_debug_trace.c @@ -0,0 +1,144 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* This example demonstrates debug trace output. + * + * Debug/trace/performance output is not tested as a regular part of our + * testing so this program gives a quick check that it's all working. + * + * Preconditions: + * + * You need to set an environment variable named HDF5_DEBUG to have a value + * of "+all trace ttimes". In the bash shell, you'd use: + * + * export HDF5_DEBUG="+all trace ttimes" + * + * When you are done with this test program, you can set the variable back + * to "-all" to suppress trace output. + * + * Usage: + * + * Compile and run the test program, then inspect the output. You should see + * trace information for each HDF5 function call that increase over time. + * Each time stamp is in seconds and designated with an '@' sign. The + * elapsed time for the function call is given in seconds in the [dt= ] + * part. + * + * You will also get summary output for the shuffle filter performance and + * data type conversion performance. These will include the elapsed time + * (always) and the system and user times (if available on your system). On + * fast machines, these numbers may be 0.0. Adjust the loop variables in + * the program as needed to generate reasonable output. + */ + +#include <stdio.h> +#include <stdlib.h> + +#include "hdf5.h" + +#define BUF_SIZE 1048576 +#define N_LOOPS 64 + +#define TESTFILE "h5_debug_trace_out.h5" + +int +main(int argc, char **argv) +{ + int i, j; + int *data; + + hid_t fid; + hid_t pid; + hid_t did; + hid_t sid; + + hsize_t dims[1] = { BUF_SIZE }; + hsize_t chunk_sizes[1] = { 1024 }; + + herr_t err; + + /*************************************************************************/ + + /* Warn the user about trace deluge to come */ + + printf("Testing debug/trace/performance data generation\n"); + printf("\n"); + printf("This test should generate a large amount of trace data\n"); + printf("\n"); + printf("*** BEGIN TRACE OUTPUT ***\n"); + printf("\n"); + fflush(stdout); + + /* This will emit H5Tconvert() performance information */ + + for(i = 0; i < N_LOOPS; i++) { + + /* The buffer has to be large enough to hold the conversion output */ + data = (int *)malloc(BUF_SIZE * sizeof(double)); + + for(j = 0; j < BUF_SIZE; j++) { + data[j] = j; + } + + err = H5Tconvert(H5T_NATIVE_INT, H5T_NATIVE_DOUBLE, BUF_SIZE, data, + NULL, H5P_DEFAULT); + + if(err < 0) { + fprintf(stderr, "ERROR: Conversion failed\n"); + free(data); + return err; + } + + free(data); + + } + + + /* This will emit H5Z performance information */ + + data = (int *)malloc(BUF_SIZE * sizeof(int)); + + for(i = 0; i < BUF_SIZE; i++) { + data[i] = i; + } + + fid = H5Fcreate(TESTFILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + + pid = H5Pcreate(H5P_DATASET_CREATE); + err = H5Pset_chunk(pid, 1, chunk_sizes); + err = H5Pset_shuffle(pid); + + sid = H5Screate_simple(1, dims, dims); + did = H5Dcreate2(fid, "somedata", H5T_NATIVE_INT, sid, H5P_DEFAULT, pid, H5P_DEFAULT); + err = H5Dwrite(did, H5T_NATIVE_INT, sid, sid, H5P_DEFAULT, data); + + H5Sclose(sid); + H5Dclose(did); + H5Pclose(pid); + H5Fclose(fid); + + free(data); + + /* Finished */ + fflush(stdout); + printf("\n"); + printf("*** END TRACE OUTPUT ***\n"); + printf("\n"); + + remove(TESTFILE); + + return 0; +} + diff --git a/examples/run-c-ex.sh.in b/examples/run-c-ex.sh.in index b5ec561..1990ad9 100644 --- a/examples/run-c-ex.sh.in +++ b/examples/run-c-ex.sh.in @@ -145,6 +145,11 @@ then rm h5_extlink &&\ RunTest h5_elink_unix2win &&\ rm h5_elink_unix2win &&\ + OLD_DEBUG_STRING=$HDF5_DEBUG &&\ + export HDF5_DEBUG="+all +trace +ttimes" &&\ + RunTest h5_debug_trace &&\ + HDF5_DEBUG=$OLD_DEBUG_STRING &&\ + rm h5_debug_trace &&\ RunTest h5_shared_mesg &&\ rm h5_shared_mesg &&\ RunTest h5_vds-eiger &&\ diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index aa9861d..c6f053f 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -8115,5 +8115,107 @@ END SUBROUTINE h5pget_virtual_dsetname_f END SUBROUTINE h5pset_dset_no_attrs_hint_f +!****s* H5P (F03)/h5pget_file_locking_f_F03 +! +! NAME +! h5pget_file_locking_f +! +! PURPOSE +! Gets the file locking properties. File locking is mainly used to help +! enforce SWMR semantics. +! +! INPUTS +! fapl_id - Target file access property list identifier. +! +! OUTPUTS +! use_file_locking - Whether or not to use file locks. +! ignore_disabled_locks - Whether or not to ignore file locks when locking +! is disabled on a file system. +! hdferr - error code: +! 0 on success and -1 on failure +! +! AUTHOR +! Dana Robinson +! Summer 2020 +! +! Fortran2003 Interface: + SUBROUTINE h5pget_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, hdferr) + IMPLICIT NONE + INTEGER(HID_T) , INTENT(IN) :: fapl_id + LOGICAL , INTENT(OUT) :: use_file_locking + LOGICAL , INTENT(OUT) :: ignore_disabled_locks + INTEGER , INTENT(OUT) :: hdferr +!***** + LOGICAL(C_BOOL) :: c_use_flag + LOGICAL(C_BOOL) :: c_ignore_flag + + INTERFACE + INTEGER FUNCTION h5pget_file_locking(fapl_id, use_file_locking, ignore_disabled_locks) BIND(C, NAME='H5Pget_file_locking') + IMPORT :: HID_T, C_BOOL + IMPLICIT NONE + INTEGER(HID_T), INTENT(IN), VALUE :: fapl_id + LOGICAL(C_BOOL), INTENT(OUT) :: use_file_locking + LOGICAL(C_BOOL), INTENT(OUT) :: ignore_disabled_locks + END FUNCTION h5pget_file_locking + END INTERFACE + + hdferr = INT(h5pget_file_locking(fapl_id, c_use_flag, c_ignore_flag)) + + ! Transfer value of C C_BOOL type to Fortran LOGICAL + use_file_locking = c_use_flag + ignore_disabled_locks = c_ignore_flag + + END SUBROUTINE h5pget_file_locking_f + +!****s* H5P (F03)/h5pset_file_locking_f_F03 +! +! NAME +! h5pset_file_locking_f +! +! PURPOSE +! Sets the file locking properties. File locking is mainly used to help +! enforce SWMR semantics. +! +! INPUTS +! fapl_id - Target file access property list identifier. +! use_file_locking - Whether or not to use file locks. +! ignore_disabled_locks - Whether or not to ignore file locks when locking +! is disabled on a file system. +! hdferr - error code: +! 0 on success and -1 on failure +! +! AUTHOR +! Dana Robinson +! Summer 2020 +! +! Fortran2003 Interface: + SUBROUTINE h5pset_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, hdferr) + IMPLICIT NONE + INTEGER(HID_T) , INTENT(IN) :: fapl_id + LOGICAL , INTENT(IN) :: use_file_locking + LOGICAL , INTENT(IN) :: ignore_disabled_locks + INTEGER , INTENT(OUT) :: hdferr +!***** + LOGICAL(C_BOOL) :: c_use_flag + LOGICAL(C_BOOL) :: c_ignore_flag + + INTERFACE + INTEGER FUNCTION h5pset_file_locking(fapl_id, use_file_locking, ignore_disabled_locks) BIND(C, NAME='H5Pset_file_locking') + IMPORT :: HID_T, C_BOOL + IMPLICIT NONE + INTEGER(HID_T), INTENT(IN), VALUE :: fapl_id + LOGICAL(C_BOOL), INTENT(IN), VALUE :: use_file_locking + LOGICAL(C_BOOL), INTENT(IN), VALUE :: ignore_disabled_locks + END FUNCTION h5pset_file_locking + END INTERFACE + + ! Transfer value of Fortran LOGICAL to C C_BOOL type + c_use_flag = use_file_locking + c_ignore_flag = ignore_disabled_locks + + hdferr = INT(h5pset_file_locking(fapl_id, c_use_flag, c_ignore_flag)) + + END SUBROUTINE h5pset_file_locking_f + END MODULE H5P diff --git a/fortran/src/hdf5_fortrandll.def.in b/fortran/src/hdf5_fortrandll.def.in index b9e2314..8438f2a 100644 --- a/fortran/src/hdf5_fortrandll.def.in +++ b/fortran/src/hdf5_fortrandll.def.in @@ -333,6 +333,8 @@ H5P_mp_H5PGET_VIRTUAL_FILENAME_F H5P_mp_H5PGET_VIRTUAL_DSETNAME_F H5P_mp_H5PGET_DSET_NO_ATTRS_HINT_F H5P_mp_H5PSET_DSET_NO_ATTRS_HINT_F +H5P_mp_H5PSET_FILE_LOCKING_F +H5P_mp_H5PGET_FILE_LOCKING_F ; Parallel @H5_NOPAREXP@H5P_mp_H5PSET_FAPL_MPIO_F @H5_NOPAREXP@H5P_mp_H5PGET_FAPL_MPIO_F diff --git a/fortran/test/fortranlib_test.F90 b/fortran/test/fortranlib_test.F90 index 92f9279..214f3c2 100644 --- a/fortran/test/fortranlib_test.F90 +++ b/fortran/test/fortranlib_test.F90 @@ -186,6 +186,10 @@ PROGRAM fortranlibtest CALL test_chunk_cache (cleanup, ret_total_error) CALL write_test_status(ret_total_error, ' Dataset chunk cache configuration', total_error) + ret_total_error = 0 + CALL test_misc_properties(cleanup, ret_total_error) + CALL write_test_status(ret_total_error, ' Miscellaneous properties', total_error) + ! ! '=========================================' ! 'Testing ATTRIBUTE interface ' diff --git a/fortran/test/tH5P.F90 b/fortran/test/tH5P.F90 index 7fe3971..19bee75 100644 --- a/fortran/test/tH5P.F90 +++ b/fortran/test/tH5P.F90 @@ -724,4 +724,77 @@ SUBROUTINE test_chunk_cache(cleanup, total_error) END SUBROUTINE test_chunk_cache +!------------------------------------------------------------------------- +! Function: test_misc_properties +! +! Purpose: Tests setting and getting of miscellaneous properties. Does +! not test the underlying functionality as that is done in +! the C library tests. +! +! Tests APIs: +! H5P_GET/SET_FILE_LOCKING_F +! +! Return: Success: 0 +! Failure: -1 +! +!------------------------------------------------------------------------- +! +SUBROUTINE test_misc_properties(cleanup, total_error) + + IMPLICIT NONE + LOGICAL, INTENT(IN) :: cleanup + INTEGER, INTENT(INOUT) :: total_error + + INTEGER(hid_t) :: fapl_id = -1 ! Local fapl + LOGICAL :: use_file_locking ! (H5Pset/get_file_locking_f) + LOGICAL :: ignore_disabled_locks ! (H5Pset/get_file_locking_f) + INTEGER :: error + + ! Create a default fapl + CALL H5Pcreate_f(H5P_FILE_ACCESS_F, fapl_id, error) + CALL check("H5Pcreate_f", error, total_error) + + ! Test H5Pset/get_file_locking_f + ! true values + use_file_locking = .TRUE. + ignore_disabled_locks = .TRUE. + CALL h5pset_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, error) + CALL check("h5pset_set_file_locking_f", error, total_error) + use_file_locking = .FALSE. + ignore_disabled_locks = .FALSE. + CALL h5pget_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, error) + CALL check("h5pget_set_file_locking_f", error, total_error) + if(use_file_locking .neqv. .TRUE.) then + total_error = total_error + 1 + write(*,*) "Got wrong use_file_locking flag from h5pget_file_locking_f" + endif + if(ignore_disabled_locks .neqv. .TRUE.) then + total_error = total_error + 1 + write(*,*) "Got wrong ignore_disabled_locks flag from h5pget_file_locking_f" + endif + + ! false values + use_file_locking = .FALSE. + ignore_disabled_locks = .FALSE. + CALL h5pset_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, error) + CALL check("h5pset_set_file_locking_f", error, total_error) + use_file_locking = .TRUE. + ignore_disabled_locks = .TRUE. + CALL h5pget_file_locking_f(fapl_id, use_file_locking, ignore_disabled_locks, error) + CALL check("h5pget_set_file_locking_f", error, total_error) + if(use_file_locking .neqv. .FALSE.) then + total_error = total_error + 1 + write(*,*) "Got wrong use_file_locking flag from h5pget_file_locking_f" + endif + if(ignore_disabled_locks .neqv. .FALSE.) then + total_error = total_error + 1 + write(*,*) "Got wrong ignore_disabled_locks flag from h5pget_file_locking_f" + endif + + ! Close the fapl + CALL H5Pclose_f(fapl_id, error) + CALL check("H5Pclose_f", error, total_error) + +END SUBROUTINE test_misc_properties + END MODULE TH5P diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 7681c48..4c9713b 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -6462,6 +6462,51 @@ public class H5 implements java.io.Serializable { public synchronized static native void H5Pset_evict_on_close(long fapl_id, boolean evict_on_close) throws HDF5LibraryException; + /** + * H5Pget_use_file_locking retrieves whether we are using file locking. + * + * @param fapl_id + * IN: File access property list identifier + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * + **/ + public synchronized static native boolean H5Pget_use_file_locking(long fapl_id) + throws HDF5LibraryException; + + /** + * H5Pget_use_file_locking retrieves whether we ignore file locks when they are disabled. + * + * @param fapl_id + * IN: File access property list identifier + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * + **/ + public synchronized static native boolean H5Pget_ignore_disabled_file_locking(long fapl_id) + throws HDF5LibraryException; + + /** + * H5Pset_file_locking sets parameters related to file locking. + * + * @param fapl_id + * IN: File access property list identifier + * + * @param use_file_locking + * IN: Whether the library will use file locking when opening files (mainly for SWMR semantics). + * + * @param ignore_when_disabled + * IN: Whether file locking will be ignored when disabled on a file system (useful for Lustre). + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * + **/ + public synchronized static native void H5Pset_file_locking(long fapl_id, boolean use_file_locking, boolean ignore_when_disabled) + throws HDF5LibraryException; + // Dataset creation property list (DCPL) routines // /** diff --git a/java/src/jni/h5pFAPLImp.c b/java/src/jni/h5pFAPLImp.c index 9ae8775..dbfbb5a 100644 --- a/java/src/jni/h5pFAPLImp.c +++ b/java/src/jni/h5pFAPLImp.c @@ -1377,6 +1377,78 @@ done: /* * Class: hdf_hdf5lib_H5 + * Method: H5Pset_file_locking + * Signature: (JZZ)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Pset_1file_1locking + (JNIEnv *env, jclass clss, jlong fapl_id, jboolean use_file_locking, jboolean ignore_when_disabled) +{ + hbool_t use_file_locking_val = TRUE; + hbool_t ignore_when_disabled_val = TRUE; + + UNUSED(clss); + + use_file_locking_val = (use_file_locking == JNI_TRUE) ? TRUE : FALSE; + ignore_when_disabled_val = (ignore_when_disabled == JNI_TRUE) ? TRUE : FALSE; + + if (H5Pset_file_locking((hid_t)fapl_id, use_file_locking_val, ignore_when_disabled_val) < 0) + H5_LIBRARY_ERROR(ENVONLY); + +done: + return; +} /* end Java_hdf_hdf5lib_H5_H5Pset_1file_1locking */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_use_file_locking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1use_1file_1locking + (JNIEnv *env, jclass clss, jlong fapl_id) +{ + hbool_t use_file_locking_val = TRUE; + hbool_t unused = TRUE; + jboolean bval = JNI_FALSE; + + UNUSED(clss); + + if (H5Pget_file_locking((hid_t)fapl_id, &use_file_locking_val, &unused) < 0) + H5_LIBRARY_ERROR(ENVONLY); + + bval = (use_file_locking_val == TRUE) ? JNI_TRUE : JNI_FALSE; + +done: + return bval; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1use_1file_1locking */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_ignore_disabled_file_locking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1ignore_1disabled_1file_1locking + (JNIEnv *env, jclass clss, jlong fapl_id) +{ + hbool_t ignore_when_disabled_val = TRUE; + hbool_t unused = TRUE; + jboolean bval = JNI_FALSE; + + UNUSED(clss); + + if (H5Pget_file_locking((hid_t)fapl_id, &unused, &ignore_when_disabled_val) < 0) + H5_LIBRARY_ERROR(ENVONLY); + + bval = (ignore_when_disabled_val == TRUE) ? JNI_TRUE : JNI_FALSE; + +done: + return bval; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1ignore_1disabled_1file_1locking */ + +/* + * Class: hdf_hdf5lib_H5 * Method: H5Pset_metadata_read_attempts * Signature: (JJ)V */ diff --git a/java/src/jni/h5pFAPLImp.h b/java/src/jni/h5pFAPLImp.h index 9b353e6..b9b4556 100644 --- a/java/src/jni/h5pFAPLImp.h +++ b/java/src/jni/h5pFAPLImp.h @@ -392,6 +392,33 @@ Java_hdf_hdf5lib_H5_H5Pget_1evict_1on_1close /* * Class: hdf_hdf5lib_H5 + * Method: H5Pset_file_locking + * Signature: (JZZ)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Pset_1file_1locking +(JNIEnv *env, jclass clss, jlong fapl_id, jboolean use_file_locking, jboolean ignore_when_disabled); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_use_file_locking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1use_1file_1locking +(JNIEnv *env, jclass clss, jlong fapl_id); + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_ignore_disabled_file_locking + * Signature: (J)Z + */ +JNIEXPORT jboolean JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1ignore_1disabled_1file_1locking +(JNIEnv *env, jclass clss, jlong fapl_id); + +/* + * Class: hdf_hdf5lib_H5 * Method: H5Pset_metadata_read_attempts * Signature: (JJ)V */ diff --git a/java/test/TestH5Pfapl.java b/java/test/TestH5Pfapl.java index 10a79dd..4233580 100644 --- a/java/test/TestH5Pfapl.java +++ b/java/test/TestH5Pfapl.java @@ -1398,4 +1398,36 @@ public class TestH5Pfapl { fail("H5P_evict_on_close: " + err); } } + + @Test + public void testH5P_file_locking() { + boolean use_file_locking = false; + boolean ignore_disabled_file_locking = false; + try { + // false values (usually not the default) + H5.H5Pset_file_locking(fapl_id, false, false); + use_file_locking = H5.H5Pget_use_file_locking(fapl_id); + ignore_disabled_file_locking = H5.H5Pget_ignore_disabled_file_locking(fapl_id); + assertFalse("H5P_file_locking", use_file_locking); + assertFalse("H5P_file_locking", ignore_disabled_file_locking); + + // true values (typically the default) + H5.H5Pset_file_locking(fapl_id, true, true); + use_file_locking = H5.H5Pget_use_file_locking(fapl_id); + ignore_disabled_file_locking = H5.H5Pget_ignore_disabled_file_locking(fapl_id); + assertTrue("H5P_file_locking", use_file_locking); + assertTrue("H5P_file_locking", ignore_disabled_file_locking); + } + catch (HDF5PropertyListInterfaceException err) { + // parallel is not supported + if (err.getMinorErrorNumber() != HDF5Constants.H5E_UNSUPPORTED) { + err.printStackTrace(); + fail("H5P_test_file_locking: " + err); + } + } + catch (Throwable err) { + err.printStackTrace(); + fail("H5P_test_file_locking: " + err); + } + } } diff --git a/java/test/testfiles/JUnit-TestH5Pfapl.txt b/java/test/testfiles/JUnit-TestH5Pfapl.txt index c4f37d0..c34ac1c 100644 --- a/java/test/testfiles/JUnit-TestH5Pfapl.txt +++ b/java/test/testfiles/JUnit-TestH5Pfapl.txt @@ -28,6 +28,7 @@ JUnit version 4.11 .testH5Pset_elink_fapl .testH5P_hyper_vector_size .testH5P_gc_references +.testH5P_file_locking .testH5P_family_offset .testH5P_fapl_core .testH5P_fapl_muti @@ -37,5 +38,5 @@ JUnit version 4.11 Time: XXXX -OK (35 tests) +OK (36 tests) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 04501b1..ef85584 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -251,9 +251,72 @@ New Features (ADB - 2019/04/15, HDFFV-10741) + - Add file locking configure and CMake options + + HDF5 1.10.0 introduced a file locking scheme, primarily to help + enforce SWMR setup. Formerly, the only user-level control of the scheme + was via the HDF5_USE_FILE_LOCKING environment variable. + + This change introduces configure-time options that control whether + or not file locking will be used and whether or not the library + ignores errors when locking has been disabled on the file system + (useful on some HPC Lustre installations). + + In both the Autotools and CMake, the settings have the effect of changing + the default property list settings (see the H5Pset/get_file_locking() + entry, below). + + The yes/no/best-effort file locking configure setting has also been + added to the libhdf5.settings file. + + Autotools: + + An --enable-file-locking=(yes|no|best-effort) option has been added. + + yes: Use file locking. + no: Do not use file locking. + best-effort: Use file locking and ignore "disabled" errors. + + CMake: + + Two self-explanatory options have been added: + + HDF5_USE_FILE_LOCKING + HDF5_IGNORE_DISABLED_FILE_LOCKS + + Setting both of these to ON is the equivalent to the Autotools' + best-effort setting. + + NOTE: + The precedence order of the various file locking control mechanisms is: + + 1) HDF5_USE_FILE_LOCKING environment variable (highest) + + 2) H5Pset_file_locking() + + 3) configure/CMake options (which set the property list defaults) + + 4) library defaults (currently best-effort) + + (DER - 2020/07/30, HDFFV-11092) + Library: -------- + - Add Mirror VFD + + Use TCP/IP sockets to perform write-only (W/O) file I/O on a remote + machine. Must be used in conjunction with the Splitter VFD. + + (JOS - 2020/03/13, TBD) + + - Add Splitter VFD + + Maintain separate R/W and W/O channels for "concurrent" file writes + to two files using a single HDF5 file handle. + + (JOS - 2020/03/13, TBD) + - Add S3 and HDFS VFDs to HDF5 maintenance Fix windows requirements and java tests. Windows requires CMake 3.13. @@ -292,11 +355,49 @@ New Features (JTH - 2019/10/07) + - Add BEST_EFFORT value to HDF5_USE_FILE_LOCKING environment variable + + This change adds a BEST_EFFORT to the TRUE/FALSE, 1/0 settings that + were previously accepted. This option turns on file locking but + ignores locking errors when the library detects that file locking + has been disabled on a file system (useful on some HPC Lustre + installations). + + The capitalization of BEST_EFFORT is mandatory. + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + - Parallel Library: + - Add H5Pset/get_file_locking() API calls + + This change adds new API calls which can be used to set or get the + file locking parameters. The single API call sets both the "use file + locking" flag and the "ignore disabled file locking" flag. + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + + Parallel Library: ----------------- - + Fortran Library: + ---------------- + - Add wrappers for H5Pset/get_file_locking() API calls + + h5pget_file_locking_f() + h5pset_file_locking_f() + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) + C++ Library: ------------ - Added new wrappers for H5Pset/get_create_intermediate_group() @@ -305,6 +406,15 @@ New Features (BMR - 2019/04/22, HDFFV-10622) + - Add wrappers for H5Pset/get_file_locking() API calls + + FileAccPropList::setFileLocking() + FileAccPropList::getFileLocking() + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) Java Library: ---------------- @@ -312,6 +422,19 @@ New Features (JTH - 2019/04/30) + - Add wrappers for H5Pset/get_file_locking() API calls + + H5Pset_file_locking() + H5Pget_use_file_locking() + H5Pget_ignore_disabled_file_locking() + + Unlike the C++ and Fortran wrappers, there are separate getters for the + two file locking settings, each of which returns a boolean value. + + See the configure option discussion for HDFFV-11092 (above) for more + information on the file locking feature and how it's controlled. + + (DER - 2020/07/30, HDFFV-11092) Tools: ------ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 9ff16c9..34afd42 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -230,6 +230,7 @@ set (H5FD_SOURCES ${HDF5_SRC_DIR}/H5FDs3comms.c ${HDF5_SRC_DIR}/H5FDsec2.c ${HDF5_SRC_DIR}/H5FDspace.c + ${HDF5_SRC_DIR}/H5FDsplitter.c ${HDF5_SRC_DIR}/H5FDstdio.c ${HDF5_SRC_DIR}/H5FDtest.c ${HDF5_SRC_DIR}/H5FDwindows.c @@ -248,6 +249,7 @@ set (H5FD_HDRS ${HDF5_SRC_DIR}/H5FDros3.h ${HDF5_SRC_DIR}/H5FDs3comms.h ${HDF5_SRC_DIR}/H5FDsec2.h + ${HDF5_SRC_DIR}/H5FDsplitter.h ${HDF5_SRC_DIR}/H5FDstdio.h ${HDF5_SRC_DIR}/H5FDwindows.h ) diff --git a/src/H5EAdblock.c b/src/H5EAdblock.c index d926fd5..d540a3c 100644 --- a/src/H5EAdblock.c +++ b/src/H5EAdblock.c @@ -15,7 +15,7 @@ * * Created: H5EAdblock.c * Sep 11 2008 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Data block routines for extensible arrays. * @@ -92,7 +92,6 @@ H5FL_DEFINE_STATIC(H5EA_dblock_t); * Return: Non-NULL pointer to data block on success/NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 11 2008 * *------------------------------------------------------------------------- @@ -153,7 +152,6 @@ END_FUNC(PKG) /* end H5EA__dblock_alloc() */ * Return: Valid file address on success/HADDR_UNDEF on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -248,7 +246,6 @@ END_FUNC(PKG) /* end H5EA__dblock_create() */ * Return: Super block index on success/Can't fail * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 11 2008 * *------------------------------------------------------------------------- @@ -285,7 +282,6 @@ END_FUNC(PKG) /* end H5EA__dblock_sblk_idx() */ * Return: Non-NULL pointer to data block on success/NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 18 2008 * *------------------------------------------------------------------------- @@ -348,7 +344,6 @@ END_FUNC(PKG) /* end H5EA__dblock_protect() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 11 2008 * *------------------------------------------------------------------------- @@ -379,7 +374,6 @@ END_FUNC(PKG) /* end H5EA__dblock_unprotect() */ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 22 2008 * *------------------------------------------------------------------------- @@ -442,7 +436,6 @@ END_FUNC(PKG) /* end H5EA__dblock_delete() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 11 2008 * *------------------------------------------------------------------------- diff --git a/src/H5EAhdr.c b/src/H5EAhdr.c index ec40298..8b4d5f7 100644 --- a/src/H5EAhdr.c +++ b/src/H5EAhdr.c @@ -15,7 +15,7 @@ * * Created: H5EAhdr.c * Aug 26 2008 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Array header routines for extensible arrays. * @@ -110,7 +110,6 @@ H5FL_SEQ_DEFINE_STATIC(H5EA_sblk_info_t); * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 26 2008 * *------------------------------------------------------------------------- @@ -180,7 +179,6 @@ END_FUNC(PKG) /* end H5EA__hdr_alloc() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 18 2008 * *------------------------------------------------------------------------- @@ -245,7 +243,6 @@ END_FUNC(PKG) /* end H5EA__hdr_init() */ * Return: Non-NULL pointer to buffer for elements on success/NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 16 2008 * *------------------------------------------------------------------------- @@ -312,7 +309,6 @@ END_FUNC(PKG) /* end H5EA__hdr_alloc_elmts() */ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 18 2008 * *------------------------------------------------------------------------- @@ -349,7 +345,6 @@ END_FUNC(PKG) /* end H5EA__hdr_free_elmts() */ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Jun 17 2008 * *------------------------------------------------------------------------- @@ -464,7 +459,6 @@ END_FUNC(PKG) /* end H5EA__hdr_create() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 26 2008 * *------------------------------------------------------------------------- @@ -497,7 +491,6 @@ END_FUNC(PKG) /* end H5EA__hdr_incr() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 26 2008 * *------------------------------------------------------------------------- @@ -533,7 +526,6 @@ END_FUNC(PKG) /* end H5EA__hdr_decr() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 26 2008 * *------------------------------------------------------------------------- @@ -559,7 +551,6 @@ END_FUNC(PKG) /* end H5EA__hdr_fuse_incr() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 26 2008 * *------------------------------------------------------------------------- @@ -589,7 +580,6 @@ END_FUNC(PKG) /* end H5EA__hdr_fuse_decr() */ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -619,7 +609,6 @@ END_FUNC(PKG) /* end H5EA__hdr_modified() */ * Return: Non-NULL pointer to header on success/NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Jul 31 2013 * *------------------------------------------------------------------------- @@ -677,7 +666,6 @@ END_FUNC(PKG) /* end H5EA__hdr_protect() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 1 2013 * *------------------------------------------------------------------------- @@ -708,7 +696,6 @@ END_FUNC(PKG) /* end H5EA__hdr_unprotect() */ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 26 2008 * *------------------------------------------------------------------------- @@ -765,7 +752,6 @@ END_FUNC(PKG) /* end H5EA__hdr_delete() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 11 2008 * *------------------------------------------------------------------------- diff --git a/src/H5EAiblock.c b/src/H5EAiblock.c index 1b5957a..c45d15a 100644 --- a/src/H5EAiblock.c +++ b/src/H5EAiblock.c @@ -15,7 +15,7 @@ * * Created: H5EAiblock.c * Sep 9 2008 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Index block routines for extensible arrays. * @@ -42,7 +42,7 @@ #include "H5EApkg.h" /* Extensible Arrays */ #include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ -#include "H5VMprivate.h" /* Vectors and arrays */ +#include "H5VMprivate.h" /* Vectors and arrays */ /****************/ @@ -98,7 +98,6 @@ H5FL_SEQ_DEFINE_STATIC(haddr_t); * Return: Non-NULL pointer to index block on success/NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -129,11 +128,6 @@ H5EA__iblock_alloc(H5EA_hdr_t *hdr)) iblock->nsblks = H5EA_SBLK_FIRST_IDX(hdr->cparam.sup_blk_min_data_ptrs); iblock->ndblk_addrs = 2 * ((size_t)hdr->cparam.sup_blk_min_data_ptrs - 1); iblock->nsblk_addrs = hdr->nsblks - iblock->nsblks; -#ifdef QAK -HDfprintf(stderr, "%s: iblock->nsblks = %u\n", FUNC, iblock->nsblks); -HDfprintf(stderr, "%s: iblock->ndblk_addrs = %Zu\n", FUNC, iblock->ndblk_addrs); -HDfprintf(stderr, "%s: iblock->nsblk_addrs = %Zu\n", FUNC, iblock->nsblk_addrs); -#endif /* QAK */ /* Allocate buffer for elements in index block */ if(hdr->cparam.idx_blk_elmts > 0) @@ -169,7 +163,6 @@ END_FUNC(PKG) /* end H5EA__iblock_alloc() */ * Return: Valid file address on success/HADDR_UNDEF on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -183,10 +176,6 @@ H5EA__iblock_create(H5EA_hdr_t *hdr, hbool_t *stats_changed)) haddr_t iblock_addr; /* Extensible array index block address */ hbool_t inserted = FALSE; /* Whether the header was inserted into cache */ -#ifdef QAK -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* QAK */ - /* Sanity check */ HDassert(hdr); HDassert(stats_changed); @@ -197,9 +186,6 @@ HDfprintf(stderr, "%s: Called\n", FUNC); /* Set size of index block on disk */ iblock->size = H5EA_IBLOCK_SIZE(iblock); -#ifdef QAK -HDfprintf(stderr, "%s: iblock->size = %Zu\n", FUNC, iblock->size); -#endif /* QAK */ /* Allocate space for the index block on disk */ if(HADDR_UNDEF == (iblock_addr = H5MF_alloc(hdr->f, H5FD_MEM_EARRAY_IBLOCK, (hsize_t)iblock->size))) @@ -284,7 +270,6 @@ END_FUNC(PKG) /* end H5EA__iblock_create() */ * Return: Non-NULL pointer to index block on success/NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -296,10 +281,6 @@ H5EA__iblock_protect(H5EA_hdr_t *hdr, unsigned flags)) /* Local variables */ H5EA_iblock_t *iblock = NULL; /* Pointer to index block */ -#ifdef QAK -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* QAK */ - /* Sanity check */ HDassert(hdr); @@ -340,7 +321,6 @@ END_FUNC(PKG) /* end H5EA__iblock_protect() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -351,10 +331,6 @@ H5EA__iblock_unprotect(H5EA_iblock_t *iblock, unsigned cache_flags)) /* Local variables */ -#ifdef QAK -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* QAK */ - /* Sanity check */ HDassert(iblock); @@ -375,7 +351,6 @@ END_FUNC(PKG) /* end H5EA__iblock_unprotect() */ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 9 2008 * *------------------------------------------------------------------------- @@ -387,10 +362,6 @@ H5EA__iblock_delete(H5EA_hdr_t *hdr)) /* Local variables */ H5EA_iblock_t *iblock = NULL; /* Pointer to index block */ -#ifdef QAK -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* QAK */ - /* Sanity check */ HDassert(hdr); HDassert(H5F_addr_defined(hdr->idx_blk_addr)); @@ -459,7 +430,6 @@ END_FUNC(PKG) /* end H5EA__iblock_delete() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Sep 11 2008 * *------------------------------------------------------------------------- diff --git a/src/H5EAint.c b/src/H5EAint.c index 2baf1f4..ef8cd7a 100644 --- a/src/H5EAint.c +++ b/src/H5EAint.c @@ -15,7 +15,7 @@ * * Created: H5EAint.c * Jun 17 2008 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Internal routines for extnsible arrays. * @@ -86,7 +86,6 @@ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Mar 26 2009 * *------------------------------------------------------------------------- @@ -116,7 +115,6 @@ END_FUNC(PKG) /* end H5EA__create_flush_depend() */ * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Mar 26 2009 * *------------------------------------------------------------------------- diff --git a/src/H5EAsblock.c b/src/H5EAsblock.c index fb7c458..b5b9d94 100644 --- a/src/H5EAsblock.c +++ b/src/H5EAsblock.c @@ -42,7 +42,7 @@ #include "H5EApkg.h" /* Extensible Arrays */ #include "H5FLprivate.h" /* Free Lists */ #include "H5MFprivate.h" /* File memory management */ -#include "H5VMprivate.h" /* Vectors and arrays */ +#include "H5VMprivate.h" /* Vectors and arrays */ /****************/ diff --git a/src/H5EAstat.c b/src/H5EAstat.c index 509d3f8..68e0b1e 100644 --- a/src/H5EAstat.c +++ b/src/H5EAstat.c @@ -15,7 +15,7 @@ * * Created: H5EAstat.c * Sep 11 2008 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Extensible array metadata statistics functions. * @@ -87,7 +87,6 @@ * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Aug 21 2008 * *------------------------------------------------------------------------- @@ -98,10 +97,6 @@ H5EA_get_stats(const H5EA_t *ea, H5EA_stat_t *stats)) /* Local variables */ -#ifdef QAK -HDfprintf(stderr, "%s: Called\n", FUNC); -#endif /* QAK */ - /* * Check arguments. */ diff --git a/src/H5EAtest.c b/src/H5EAtest.c index 8926d6a..a0802bc 100644 --- a/src/H5EAtest.c +++ b/src/H5EAtest.c @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Quincey Koziol <koziol@hdfgroup.org> +/* Programmer: Quincey Koziol * Thursday, August 28, 2008 * * Purpose: Extensible array testing functions. @@ -38,7 +38,7 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5EApkg.h" /* Extensible Arrays */ #include "H5FLprivate.h" /* Free Lists */ -#include "H5VMprivate.h" /* Vector functions */ +#include "H5VMprivate.h" /* Vector functions */ /****************/ @@ -60,9 +60,6 @@ /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ @@ -74,86 +71,6 @@ hbool_t H5_PKG_INIT_VAR = FALSE; /*******************/ -/* File ID class */ -static const H5I_class_t H5I_FILE_CLS[1] = {{ - H5I_FILE, /* ID class value */ - 0, /* Class flags */ - 0, /* # of reserved IDs for class */ - (H5I_free_t)H5F__close_cb /* Callback routine for closing objects of this class */ -}}; - - -/*-------------------------------------------------------------------------- -NAME - H5F__init_package -- Initialize interface-specific information -USAGE - herr_t H5F__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. - ---------------------------------------------------------------------------*/ -herr_t -H5F__init_package(void) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* - * Initialize the atom group for the file IDs. - */ - if(H5I_register_type(H5I_FILE_CLS) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to initialize interface") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5F__init_package() */ - - -/*------------------------------------------------------------------------- - * Function: H5F_term_package - * - * Purpose: Terminate this interface: free all memory and reset global - * variables to their initial values. Release all ID groups - * associated with this interface. - * - * Return: Success: Positive if anything was done that might - * have affected other interfaces; - * zero otherwise. - * - * Failure: Never fails - *------------------------------------------------------------------------- - */ -int -H5F_term_package(void) -{ - int n = 0; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - if(H5_PKG_INIT_VAR) { - if(H5I_nmembers(H5I_FILE) > 0) { - (void)H5I_clear_type(H5I_FILE, FALSE, FALSE); - n++; /*H5I*/ - } /* end if */ - else { - /* Make certain we've cleaned up all the shared file objects */ - H5F_sfile_assert_num(0); - - /* Destroy the file object id group */ - n += (H5I_dec_type_ref(H5I_FILE) > 0); - - /* Mark closed */ - if(0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end else */ - } /* end if */ - - FUNC_LEAVE_NOAPI(n) -} /* end H5F_term_package() */ - /*------------------------------------------------------------------------- * Function: H5Fget_create_plist @@ -1436,7 +1436,7 @@ done: * constant H5P_DEFAULT). The bytes to be written come from the * buffer BUF. * - * Return: SNon-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1642,7 +1642,7 @@ H5FDlock(H5FD_t *file, hbool_t rw) /* Call private function */ if(H5FD_lock(file, rw) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file lock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "file lock request failed") done: FUNC_LEAVE_API(ret_value) @@ -1672,7 +1672,7 @@ H5FD_lock(H5FD_t *file, hbool_t rw) /* Dispatch to driver */ if(file->cls->lock && (file->cls->lock)(file, rw) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "driver lock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "driver lock request failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -1704,7 +1704,7 @@ H5FDunlock(H5FD_t *file) /* Call private function */ if(H5FD_unlock(file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file unlock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "file unlock request failed") done: FUNC_LEAVE_API(ret_value) @@ -1733,7 +1733,7 @@ H5FD_unlock(H5FD_t *file) /* Dispatch to driver */ if(file->cls->unlock && (file->cls->unlock)(file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "driver unlock request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "driver unlock request failed") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDcore.c b/src/H5FDcore.c index e4a3207..40d838c 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -36,6 +36,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_CORE_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* The skip list node type. Represents a region in the file. */ typedef struct H5FD_core_region_t { haddr_t start; /* Start address of the region */ @@ -55,6 +58,7 @@ typedef struct H5FD_core_t { size_t increment; /* multiples for mem allocation */ hbool_t backing_store; /* write to file name on flush */ size_t bstore_page_size; /* backing store page size */ + hbool_t ignore_disabled_file_locks; int fd; /* backing store file descriptor */ /* Information for determining uniqueness of a file with a backing store */ #ifndef H5_HAVE_WIN32_API @@ -407,9 +411,19 @@ done: static herr_t H5FD__init_package(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ herr_t ret_value = SUCCEED; - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC + + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if(lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ if(H5FD_core_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize core VFD") @@ -691,6 +705,16 @@ H5FD__core_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr /* Save file image callbacks */ file->fi_callbacks = file_image_info.callbacks; + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + if(fd >= 0) { /* Retrieve information for determining uniqueness of file */ #ifdef H5_HAVE_WIN32_API @@ -1509,18 +1533,20 @@ H5FD_core_lock(H5FD_t *_file, hbool_t rw) * descriptor, this is a no-op. */ if(file->fd >= 0) { - /* Set exclusive or shared lock based on rw status */ lock_flags = rw ? LOCK_EX : LOCK_SH; /* Place a non-blocking lock on the file */ if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") } /* end if */ - } /* end if */ done: @@ -1549,16 +1575,17 @@ H5FD_core_unlock(H5FD_t *_file) HDassert(file); - if(file->fd >= 0) { - + if(file->fd >= 0) if(HDflock(file->fd, LOCK_UN) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ - - } /* end if */ + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDdirect.c b/src/H5FDdirect.c index 34c4346..4046050 100644 --- a/src/H5FDdirect.c +++ b/src/H5FDdirect.c @@ -38,6 +38,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_DIRECT_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* File operations */ #define OP_UNKNOWN 0 #define OP_READ 1 @@ -71,6 +74,7 @@ typedef struct H5FD_direct_t { haddr_t pos; /*current file I/O position */ int op; /*last operation */ H5FD_direct_fapl_t fa; /*file access properties */ + hbool_t ignore_disabled_file_locks; #ifndef H5_HAVE_WIN32_API /* * On most systems the combination of device and i-node number uniquely @@ -193,10 +197,20 @@ DESCRIPTION static herr_t H5FD__init_package(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if(lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if(H5FD_direct_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize direct VFD") @@ -518,6 +532,16 @@ H5FD_direct_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxadd file->fa.fbsize = fa->fbsize; file->fa.cbsize = fa->cbsize; + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Try to decide if data alignment is required. The reason to check it here * is to handle correctly the case that the file is in a different file system * than the one where the program is running. @@ -1335,16 +1359,27 @@ static herr_t H5FD_direct_lock(H5FD_t *_file, hbool_t rw) { H5FD_direct_t *file = (H5FD_direct_t*)_file; /* VFD file struct */ - const int lock = rw ? LOCK_EX : LOCK_SH; + int lock_flags; /* file locking flags */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT HDassert(file); - /* Place the lock with non-blocking */ - if(HDflock(file->fd, lock | LOCK_NB) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock file") + /* Set exclusive or shared lock based on rw status */ + lock_flags = rw ? LOCK_EX : LOCK_SH; + + /* Place a non-blocking lock on the file */ + if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } + else + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -1372,8 +1407,16 @@ H5FD_direct_unlock(H5FD_t *_file) HDassert(file); - if(HDflock(file->fd, LOCK_UN) < 0) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to flock (unlock) file") + if(HDflock(file->fd, LOCK_UN) < 0) { + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } + else + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c index d110ef7..154878e 100644 --- a/src/H5FDfamily.c +++ b/src/H5FDfamily.c @@ -1372,9 +1372,9 @@ H5FD_family_lock(H5FD_t *_file, hbool_t rw) for(v = 0; v < u; v++) { if(H5FD_unlock(file->memb[v]) < 0) /* Push error, but keep going */ - HDONE_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + HDONE_ERROR(H5E_IO, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock member files") } /* end for */ - HGOTO_ERROR(H5E_IO, H5E_CANTLOCK, FAIL, "unable to lock member files") + HGOTO_ERROR(H5E_IO, H5E_CANTLOCKFILE, FAIL, "unable to lock member files") } /* end if */ done: @@ -1406,7 +1406,7 @@ H5FD_family_unlock(H5FD_t *_file) for(u = 0; u < file->nmembs; u++) if(file->memb[u]) if(H5FD_unlock(file->memb[u]) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCK, FAIL, "unable to unlock member files") + HGOTO_ERROR(H5E_IO, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock member files") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDlog.c b/src/H5FDlog.c index f649bc4..5d1b536 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * Programmer: Quincey Koziol * Monday, April 17, 2000 * * Purpose: The POSIX unbuffered file driver using only the HDF5 public @@ -40,6 +40,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_LOG_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* Driver-specific file access properties */ typedef struct H5FD_log_fapl_t { char *logfile; /* Allocated log file name */ @@ -80,6 +83,7 @@ typedef struct H5FD_log_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ H5FD_file_op_t op; /* last operation */ + hbool_t ignore_disabled_file_locks; char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -151,71 +155,71 @@ typedef struct H5FD_log_t { * which can be addressed entirely by the second * argument of the file seek function. */ -#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) -#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) +#define MAXADDR (((haddr_t)1 << (8 * sizeof(HDoff_t) - 1)) - 1) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF == (A) || ((A) & ~(haddr_t)MAXADDR)) #define SIZE_OVERFLOW(Z) ((Z) & ~(hsize_t)MAXADDR) #define REGION_OVERFLOW(A,Z) (ADDR_OVERFLOW(A) || SIZE_OVERFLOW(Z) || \ - HADDR_UNDEF==(A)+(Z) || \ - (HDoff_t)((A)+(Z))<(HDoff_t)(A)) + HADDR_UNDEF == (A) + (Z) || \ + (HDoff_t)((A) + (Z)) < (HDoff_t)(A)) /* Prototypes */ -static herr_t H5FD_log_term(void); -static void *H5FD_log_fapl_get(H5FD_t *file); -static void *H5FD_log_fapl_copy(const void *_old_fa); -static herr_t H5FD_log_fapl_free(void *_fa); -static H5FD_t *H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, +static herr_t H5FD__log_term(void); +static void *H5FD__log_fapl_get(H5FD_t *file); +static void *H5FD__log_fapl_copy(const void *_old_fa); +static herr_t H5FD__log_fapl_free(void *_fa); +static H5FD_t *H5FD__log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); -static herr_t H5FD_log_close(H5FD_t *_file); -static int H5FD_log_cmp(const H5FD_t *_f1, const H5FD_t *_f2); -static herr_t H5FD_log_query(const H5FD_t *_f1, unsigned long *flags); -static haddr_t H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); +static herr_t H5FD__log_close(H5FD_t *_file); +static int H5FD__log_cmp(const H5FD_t *_f1, const H5FD_t *_f2); +static herr_t H5FD__log_query(const H5FD_t *_f1, unsigned long *flags); +static haddr_t H5FD__log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); static herr_t H5FD__log_free(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size); -static haddr_t H5FD_log_get_eoa(const H5FD_t *_file, H5FD_mem_t type); -static herr_t H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); -static haddr_t H5FD_log_get_eof(const H5FD_t *_file, H5FD_mem_t type); -static herr_t H5FD_log_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle); -static herr_t H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, +static haddr_t H5FD__log_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD__log_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__log_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle); +static herr_t H5FD__log_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, void *buf); -static herr_t H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, +static herr_t H5FD__log_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, size_t size, const void *buf); -static herr_t H5FD_log_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); -static herr_t H5FD_log_lock(H5FD_t *_file, hbool_t rw); -static herr_t H5FD_log_unlock(H5FD_t *_file); +static herr_t H5FD__log_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD__log_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD__log_unlock(H5FD_t *_file); static const H5FD_class_t H5FD_log_g = { - "log", /*name */ - MAXADDR, /*maxaddr */ - H5F_CLOSE_WEAK, /* fc_degree */ - H5FD_log_term, /*terminate */ - NULL, /*sb_size */ - NULL, /*sb_encode */ - NULL, /*sb_decode */ - sizeof(H5FD_log_fapl_t), /*fapl_size */ - H5FD_log_fapl_get, /*fapl_get */ - H5FD_log_fapl_copy, /*fapl_copy */ - H5FD_log_fapl_free, /*fapl_free */ - 0, /*dxpl_size */ - NULL, /*dxpl_copy */ - NULL, /*dxpl_free */ - H5FD_log_open, /*open */ - H5FD_log_close, /*close */ - H5FD_log_cmp, /*cmp */ - H5FD_log_query, /*query */ - NULL, /*get_type_map */ - H5FD_log_alloc, /*alloc */ - H5FD__log_free, /*free */ - H5FD_log_get_eoa, /*get_eoa */ - H5FD_log_set_eoa, /*set_eoa */ - H5FD_log_get_eof, /*get_eof */ - H5FD_log_get_handle, /*get_handle */ - H5FD_log_read, /*read */ - H5FD_log_write, /*write */ - NULL, /*flush */ - H5FD_log_truncate, /*truncate */ - H5FD_log_lock, /*lock */ - H5FD_log_unlock, /*unlock */ - H5FD_FLMAP_DICHOTOMY /*fl_map */ + "log", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD__log_term, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + sizeof(H5FD_log_fapl_t), /* fapl_size */ + H5FD__log_fapl_get, /* fapl_get */ + H5FD__log_fapl_copy, /* fapl_copy */ + H5FD__log_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD__log_open, /* open */ + H5FD__log_close, /* close */ + H5FD__log_cmp, /* cmp */ + H5FD__log_query, /* query */ + NULL, /* get_type_map */ + H5FD__log_alloc, /* alloc */ + H5FD__log_free, /* free */ + H5FD__log_get_eoa, /* get_eoa */ + H5FD__log_set_eoa, /* set_eoa */ + H5FD__log_get_eof, /* get_eof */ + H5FD__log_get_handle, /* get_handle */ + H5FD__log_read, /* read */ + H5FD__log_write, /* write */ + NULL, /* flush */ + H5FD__log_truncate, /* truncate */ + H5FD__log_lock, /* lock */ + H5FD__log_unlock, /* unlock */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ }; /* Declare a free list to manage the H5FD_log_t struct */ @@ -234,10 +238,20 @@ H5FL_DEFINE_STATIC(H5FD_log_t); static herr_t H5FD__init_package(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if(lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if(H5FD_log_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize log VFD") @@ -279,7 +293,7 @@ done: /*--------------------------------------------------------------------------- - * Function: H5FD_log_term + * Function: H5FD__log_term * * Purpose: Shut down the VFD * @@ -291,15 +305,15 @@ done: *--------------------------------------------------------------------------- */ static herr_t -H5FD_log_term(void) +H5FD__log_term(void) { - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* Reset VFL ID */ H5FD_LOG_g = 0; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5FD_log_term() */ +} /* end H5FD__log_term() */ /*------------------------------------------------------------------------- @@ -352,7 +366,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5FD_log_fapl_get + * Function: H5FD__log_fapl_get * * Purpose: Returns a file access property list which indicates how the * specified file is being accessed. The return list could be @@ -368,22 +382,22 @@ done: *------------------------------------------------------------------------- */ static void * -H5FD_log_fapl_get(H5FD_t *_file) +H5FD__log_fapl_get(H5FD_t *_file) { H5FD_log_t *file = (H5FD_log_t *)_file; void *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* Set return value */ - ret_value = H5FD_log_fapl_copy(&(file->fa)); + ret_value = H5FD__log_fapl_copy(&(file->fa)); FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_fapl_get() */ +} /* end H5FD__log_fapl_get() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_fapl_copy + * Function: H5FD__log_fapl_copy * * Purpose: Copies the log-specific file access properties. * @@ -396,13 +410,13 @@ H5FD_log_fapl_get(H5FD_t *_file) *------------------------------------------------------------------------- */ static void * -H5FD_log_fapl_copy(const void *_old_fa) +H5FD__log_fapl_copy(const void *_old_fa) { const H5FD_log_fapl_t *old_fa = (const H5FD_log_fapl_t*)_old_fa; H5FD_log_fapl_t *new_fa = NULL; /* New FAPL info */ void *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(old_fa); @@ -430,11 +444,11 @@ done: } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_fapl_copy() */ +} /* end H5FD__log_fapl_copy() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_fapl_free + * Function: H5FD__log_fapl_free * * Purpose: Frees the log-specific file access properties. * @@ -446,11 +460,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_fapl_free(void *_fa) +H5FD__log_fapl_free(void *_fa) { H5FD_log_fapl_t *fa = (H5FD_log_fapl_t*)_fa; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* Free the fapl information */ if(fa->logfile) @@ -458,11 +472,11 @@ H5FD_log_fapl_free(void *_fa) H5MM_xfree(fa); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5FD_log_fapl_free() */ +} /* end H5FD__log_fapl_free() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_open + * Function: H5FD__log_open * * Purpose: Create and/or opens a file as an HDF5 file. * @@ -477,7 +491,7 @@ H5FD_log_fapl_free(void *_fa) *------------------------------------------------------------------------- */ static H5FD_t * -H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) +H5FD__log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) { H5FD_log_t *file = NULL; H5P_genplist_t *plist; /* Property list */ @@ -487,15 +501,12 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) #ifdef H5_HAVE_WIN32_API struct _BY_HANDLE_FILE_INFORMATION fileinfo; #endif -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval timeval_start; - struct timeval open_timeval_diff; - struct timeval stat_timeval_diff; -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_t open_timer; /* Timer for open() call */ + H5_timer_t stat_timer; /* Timer for stat() call */ h5_stat_t sb; H5FD_t *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC /* Sanity check on file offsets */ HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t)); @@ -523,54 +534,36 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) if(NULL == (fa = (const H5FD_log_fapl_t *)H5P_peek_driver_info(plist))) HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, NULL, "bad VFL driver info") -#ifdef H5_HAVE_GETTIMEOFDAY - if(fa->flags & H5FD_LOG_TIME_OPEN) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Start timer for open() call */ + if(fa->flags & H5FD_LOG_TIME_OPEN) { + H5_timer_init(&open_timer); + H5_timer_start(&open_timer); + } /* end if */ + /* Open the file */ if((fd = HDopen(name, o_flags, H5_POSIX_CREATE_MODE_RW)) < 0) { int myerrno = errno; HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to open file: name = '%s', errno = %d, error message = '%s', flags = %x, o_flags = %x", name, myerrno, HDstrerror(myerrno), flags, (unsigned)o_flags); } /* end if */ -#ifdef H5_HAVE_GETTIMEOFDAY - if(fa->flags & H5FD_LOG_TIME_OPEN) { - struct timeval timeval_stop; - HDgettimeofday(&timeval_stop, NULL); + /* Stop timer for open() call */ + if(fa->flags & H5FD_LOG_TIME_OPEN) + H5_timer_stop(&open_timer); - /* Calculate the elapsed gettimeofday time */ - open_timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - open_timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(open_timeval_diff.tv_usec < 0) { - open_timeval_diff.tv_usec += 1000000; - open_timeval_diff.tv_sec--; - } /* end if */ + /* Start timer for stat() call */ + if(fa->flags & H5FD_LOG_TIME_STAT) { + H5_timer_init(&stat_timer); + H5_timer_start(&stat_timer); } /* end if */ -#endif /* H5_HAVE_GETTIMEOFDAY */ -#ifdef H5_HAVE_GETTIMEOFDAY - if(fa->flags & H5FD_LOG_TIME_STAT) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ /* Get the file stats */ if(HDfstat(fd, &sb) < 0) HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, NULL, "unable to fstat file") -#ifdef H5_HAVE_GETTIMEOFDAY - if(fa->flags & H5FD_LOG_TIME_STAT) { - struct timeval timeval_stop; - HDgettimeofday(&timeval_stop, NULL); - - /* Calculate the elapsed gettimeofday time */ - stat_timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - stat_timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(stat_timeval_diff.tv_usec < 0) { - stat_timeval_diff.tv_usec += 1000000; - stat_timeval_diff.tv_sec--; - } /* end if */ - } /* end if */ -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Stop timer for stat() call */ + if(fa->flags & H5FD_LOG_TIME_STAT) + H5_timer_stop(&stat_timer); /* Create the new file struct */ if(NULL == (file = H5FL_CALLOC(H5FD_log_t))) @@ -631,15 +624,31 @@ H5FD_log_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) else file->logfp = stderr; -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_OPEN) - HDfprintf(file->logfp, "Open took: (%f s)\n", (double)open_timeval_diff.tv_sec + ((double)open_timeval_diff.tv_usec / (double)1000000.0f)); - if(file->fa.flags & H5FD_LOG_TIME_STAT) - HDfprintf(file->logfp, "Stat took: (%f s)\n", (double)stat_timeval_diff.tv_sec + ((double)stat_timeval_diff.tv_usec / (double)1000000.0f)); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Log the timer values */ + if(file->fa.flags & H5FD_LOG_TIME_OPEN) { + H5_timevals_t open_times; /* Elapsed time for open() call */ + + H5_timer_get_times(open_timer, &open_times); + HDfprintf(file->logfp, "Open took: (%f s)\n", open_times.elapsed); + } /* end if */ + if(file->fa.flags & H5FD_LOG_TIME_STAT) { + H5_timevals_t stat_times; /* Elapsed time for stat() call */ + H5_timer_get_times(stat_timer, &stat_times); + HDfprintf(file->logfp, "Stat took: (%f s)\n", stat_times.elapsed); + } /* end if */ } /* end if */ + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Check for non-default FAPL */ if(H5P_FILE_ACCESS_DEFAULT != fapl_id) { /* This step is for h5repart tool only. If user wants to change file driver from @@ -664,11 +673,11 @@ done: } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_open() */ +} /* end H5FD__log_open() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_close + * Function: H5FD__log_close * * Purpose: Closes an HDF5 file. * @@ -681,30 +690,30 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_close(H5FD_t *_file) +H5FD__log_close(H5FD_t *_file) { H5FD_log_t *file = (H5FD_log_t *)_file; -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval timeval_start, timeval_stop; -#endif /* H5_HAVE_GETTIMEOFDAY */ - herr_t ret_value = SUCCEED; /* Return value */ + H5_timer_t close_timer; /* Timer for close() call */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC /* Sanity check */ HDassert(file); -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags&H5FD_LOG_TIME_CLOSE) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Start timer for close() call */ + if(file->fa.flags & H5FD_LOG_TIME_CLOSE) { + H5_timer_init(&close_timer); + H5_timer_start(&close_timer); + } /* end if */ + /* Close the underlying file */ if(HDclose(file->fd) < 0) HSYS_GOTO_ERROR(H5E_IO, H5E_CANTCLOSEFILE, FAIL, "unable to close file") -#ifdef H5_HAVE_GETTIMEOFDAY + + /* Stop timer for close() call */ if(file->fa.flags&H5FD_LOG_TIME_CLOSE) - HDgettimeofday(&timeval_stop, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_stop(&close_timer); /* Dump I/O information */ if(file->fa.flags != 0) { @@ -712,20 +721,12 @@ H5FD_log_close(H5FD_t *_file) haddr_t last_addr; unsigned char last_val; -#ifdef H5_HAVE_GETTIMEOFDAY if(file->fa.flags & H5FD_LOG_TIME_CLOSE) { - struct timeval timeval_diff; - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(timeval_diff.tv_usec < 0) { - timeval_diff.tv_usec += 1000000; - timeval_diff.tv_sec--; - } /* end if */ - HDfprintf(file->logfp, "Close took: (%f s)\n", (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f)); + H5_timevals_t close_times; /* Elapsed time for close() call */ + + H5_timer_get_times(close_timer, &close_times); + HDfprintf(file->logfp, "Close took: (%f s)\n", close_times.elapsed); } /* end if */ -#endif /* H5_HAVE_GETTIMEOFDAY */ /* Dump the total number of seek/read/write operations */ if(file->fa.flags & H5FD_LOG_NUM_READ) @@ -817,11 +818,11 @@ H5FD_log_close(H5FD_t *_file) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_close() */ +} /* end H5FD__log_close() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_cmp + * Function: H5FD__log_cmp * * Purpose: Compares two files belonging to this driver using an * arbitrary (but consistent) ordering. @@ -836,13 +837,13 @@ done: *------------------------------------------------------------------------- */ static int -H5FD_log_cmp(const H5FD_t *_f1, const H5FD_t *_f2) +H5FD__log_cmp(const H5FD_t *_f1, const H5FD_t *_f2) { const H5FD_log_t *f1 = (const H5FD_log_t *)_f1; const H5FD_log_t *f2 = (const H5FD_log_t *)_f2; int ret_value = 0; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR #ifdef H5_HAVE_WIN32_API if(f1->dwVolumeSerialNumber < f2->dwVolumeSerialNumber) HGOTO_DONE(-1) @@ -873,11 +874,11 @@ H5FD_log_cmp(const H5FD_t *_f1, const H5FD_t *_f2) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_cmp() */ +} /* end H5FD__log_cmp() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_query + * Function: H5FD__log_query * * Purpose: Set the flags that this VFL driver is capable of supporting. * (listed in H5FDpublic.h) @@ -890,11 +891,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_query(const H5FD_t *_file, unsigned long *flags /* out */) +H5FD__log_query(const H5FD_t *_file, unsigned long *flags /* out */) { const H5FD_log_t *file = (const H5FD_log_t *)_file; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* Set the VFL feature flags that this driver supports */ if(flags) { @@ -913,11 +914,11 @@ H5FD_log_query(const H5FD_t *_file, unsigned long *flags /* out */) } /* end if */ FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5FD_log_query() */ +} /* end H5FD__log_query() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_alloc + * Function: H5FD__log_alloc * * Purpose: Allocate file memory. * @@ -930,13 +931,13 @@ H5FD_log_query(const H5FD_t *_file, unsigned long *flags /* out */) *------------------------------------------------------------------------- */ static haddr_t -H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hsize_t size) +H5FD__log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hsize_t size) { H5FD_log_t *file = (H5FD_log_t *)_file; haddr_t addr; haddr_t ret_value = HADDR_UNDEF; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* Compute the address for the block to allocate */ addr = file->eoa; @@ -960,7 +961,7 @@ H5FD_log_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hsi ret_value = addr; FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_alloc() */ +} /* end H5FD__log_alloc() */ /*------------------------------------------------------------------------- @@ -1001,7 +1002,7 @@ H5FD__log_free(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, /*------------------------------------------------------------------------- - * Function: H5FD_log_get_eoa + * Function: H5FD__log_get_eoa * * Purpose: Gets the end-of-address marker for the file. The EOA marker * is the first address past the last byte allocated in the @@ -1016,18 +1017,18 @@ H5FD__log_free(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, *------------------------------------------------------------------------- */ static haddr_t -H5FD_log_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +H5FD__log_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { const H5FD_log_t *file = (const H5FD_log_t *)_file; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR FUNC_LEAVE_NOAPI(file->eoa) -} /* end H5FD_log_get_eoa() */ +} /* end H5FD__log_get_eoa() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_set_eoa + * Function: H5FD__log_set_eoa * * Purpose: Set the end-of-address marker for the file. This function is * called shortly after an existing HDF5 file is opened in order @@ -1041,11 +1042,11 @@ H5FD_log_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) *------------------------------------------------------------------------- */ static herr_t -H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) +H5FD__log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) { H5FD_log_t *file = (H5FD_log_t *)_file; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR if(file->fa.flags != 0) { /* Check for increasing file size */ @@ -1084,11 +1085,11 @@ H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) file->eoa = addr; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5FD_log_set_eoa() */ +} /* end H5FD__log_set_eoa() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_get_eof + * Function: H5FD__log_get_eof * * Purpose: Returns the end-of-file marker, which is the greater of * either the filesystem end-of-file or the HDF5 end-of-address @@ -1105,18 +1106,18 @@ H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) *------------------------------------------------------------------------- */ static haddr_t -H5FD_log_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +H5FD__log_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) { const H5FD_log_t *file = (const H5FD_log_t *)_file; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR FUNC_LEAVE_NOAPI(file->eof) -} /* end H5FD_log_get_eof() */ +} /* end H5FD__log_get_eof() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_get_handle + * Function: H5FD__log_get_handle * * Purpose: Returns the file handle of LOG file driver. * @@ -1128,12 +1129,12 @@ H5FD_log_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) *------------------------------------------------------------------------- */ static herr_t -H5FD_log_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void **file_handle) +H5FD__log_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void **file_handle) { H5FD_log_t *file = (H5FD_log_t *)_file; herr_t ret_value = SUCCEED; - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC if(!file_handle) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid") @@ -1142,11 +1143,11 @@ H5FD_log_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void **file_handle done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_get_handle() */ +} /* end H5FD__log_get_handle() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_read + * Function: H5FD__log_read * * Purpose: Reads SIZE bytes of data from FILE beginning at address ADDR * into buffer BUF according to data transfer properties in @@ -1162,19 +1163,22 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, +H5FD__log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, size_t size, void *buf/*out*/) { H5FD_log_t *file = (H5FD_log_t *)_file; size_t orig_size = size; /* Save the original size for later */ haddr_t orig_addr = addr; -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval timeval_start, timeval_stop; -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_t read_timer; /* Timer for read operation */ + H5_timevals_t read_times; /* Elapsed time for read operation */ +#ifndef H5_HAVE_PREADWRITE + H5_timer_t seek_timer; /* Timer for seek operation */ + H5_timevals_t seek_times; /* Elapsed time for seek operation */ +#endif /* H5_HAVE_PREADWRITE */ HDoff_t offset = (HDoff_t)addr; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(file && file->pub.cls); HDassert(buf); @@ -1201,59 +1205,56 @@ H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hadd #ifndef H5_HAVE_PREADWRITE /* Seek to the correct location (if we don't have pread) */ if(addr != file->pos || OP_READ != file->op) { -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_SEEK) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Start timer for seek() call */ + if(file->fa.flags & H5FD_LOG_TIME_SEEK) { + H5_timer_init(&seek_timer); + H5_timer_start(&seek_timer); + } /* end if */ + if(HDlseek(file->fd, (HDoff_t)addr, SEEK_SET) < 0) HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") -#ifdef H5_HAVE_GETTIMEOFDAY + + /* Stop timer for seek() call */ if(file->fa.flags & H5FD_LOG_TIME_SEEK) - HDgettimeofday(&timeval_stop, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_stop(&seek_timer); - /* Log information about the seek */ + /* Add to the number of seeks, when tracking that */ if(file->fa.flags & H5FD_LOG_NUM_SEEK) file->total_seek_ops++; + + /* Add to the total seek time, when tracking that */ + if(file->fa.flags & H5FD_LOG_TIME_SEEK) { + H5_timer_get_times(seek_timer, &seek_times); + file->total_seek_time += seek_times.elapsed; + } /* end if */ + + /* Emit log string if we're tracking individual seek events. */ if(file->fa.flags & H5FD_LOG_LOC_SEEK) { HDfprintf(file->logfp, "Seek: From %10a To %10a", file->pos, addr); -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_SEEK) { - struct timeval timeval_diff; - double time_diff; - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(timeval_diff.tv_usec < 0) { - timeval_diff.tv_usec += 1000000; - timeval_diff.tv_sec--; - } /* end if */ - time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); - /* Add to total seek time */ - file->total_seek_time += time_diff; - } /* end if */ - else - HDfprintf(file->logfp, "\n"); -#else /* H5_HAVE_GETTIMEOFDAY */ - HDfprintf(file->logfp, "\n"); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Add the seek time, if we're tracking that. + * Note that the seek time is NOT emitted for when just H5FD_LOG_TIME_SEEK + * is set. + */ + if(file->fa.flags & H5FD_LOG_TIME_SEEK) + HDfprintf(file->logfp, " (%fs @ %f)\n", seek_times.elapsed, seek_timer.initial.elapsed); + else + HDfprintf(file->logfp, "\n"); } /* end if */ } /* end if */ #endif /* H5_HAVE_PREADWRITE */ + /* Start timer for read operation */ + if(file->fa.flags & H5FD_LOG_TIME_READ) { + H5_timer_init(&read_timer); + H5_timer_start(&read_timer); + } /* end if */ + /* * Read data, being careful of interrupted system calls, partial results, * and the end of the file. */ -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_READ) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ while(size > 0) { - h5_posix_io_t bytes_in = 0; /* # of bytes to read */ h5_posix_io_ret_t bytes_read = -1; /* # of bytes actually read */ @@ -1301,14 +1302,22 @@ H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hadd buf = (char *)buf + bytes_read; } /* end while */ -#ifdef H5_HAVE_GETTIMEOFDAY + + /* Stop timer for read operation */ if(file->fa.flags & H5FD_LOG_TIME_READ) - HDgettimeofday(&timeval_stop, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_stop(&read_timer); - /* Log information about the read */ + /* Add to the number of reads, when tracking that */ if(file->fa.flags & H5FD_LOG_NUM_READ) file->total_read_ops++; + + /* Add to the total read time, when tracking that */ + if(file->fa.flags & H5FD_LOG_TIME_READ) { + H5_timer_get_times(read_timer, &read_times); + file->total_read_time += read_times.elapsed; + } /* end if */ + + /* Log information about the read */ if(file->fa.flags & H5FD_LOG_LOC_READ) { HDfprintf(file->logfp, "%10a-%10a (%10Zu bytes) (%s) Read", orig_addr, (orig_addr + orig_size) - 1, orig_size, flavors[type]); @@ -1318,30 +1327,14 @@ H5FD_log_read(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, hadd HDassert(type == H5FD_MEM_DEFAULT || type == (H5FD_mem_t)file->flavor[(orig_addr + orig_size) - 1] || (H5FD_mem_t)file->flavor[(orig_addr + orig_size) - 1] == H5FD_MEM_DEFAULT); } /* end if */ - -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_READ) { - struct timeval timeval_diff; - double time_diff; - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(timeval_diff.tv_usec < 0) { - timeval_diff.tv_usec += 1000000; - timeval_diff.tv_sec--; - } /* end if */ - time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); - - /* Add to total read time */ - file->total_read_time += time_diff; - } /* end if */ + /* Add the read time, if we're tracking that. + * Note that the read time is NOT emitted for when just H5FD_LOG_TIME_READ + * is set. + */ + if(file->fa.flags & H5FD_LOG_TIME_READ) + HDfprintf(file->logfp, " (%fs @ %f)\n", read_times.elapsed, read_timer.initial.elapsed); else HDfprintf(file->logfp, "\n"); -#else /* H5_HAVE_GETTIMEOFDAY */ - HDfprintf(file->logfp, "\n"); -#endif /* H5_HAVE_GETTIMEOFDAY */ } /* end if */ /* Update current position */ @@ -1356,11 +1349,11 @@ done: } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_read() */ +} /* end H5FD__log_read() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_write + * Function: H5FD__log_write * * Purpose: Writes SIZE bytes of data to FILE beginning at address ADDR * from buffer BUF according to data transfer properties in @@ -1374,19 +1367,22 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, +H5FD__log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, size_t size, const void *buf) { H5FD_log_t *file = (H5FD_log_t *)_file; size_t orig_size = size; /* Save the original size for later */ haddr_t orig_addr = addr; -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval timeval_start, timeval_stop; -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_t write_timer; /* Timer for write operation */ + H5_timevals_t write_times; /* Elapsed time for write operation */ +#ifndef H5_HAVE_PREADWRITE + H5_timer_t seek_timer; /* Timer for seek operation */ + H5_timevals_t seek_times; /* Elapsed time for seek operation */ +#endif /* H5_HAVE_PREADWRITE */ HDoff_t offset = (HDoff_t)addr; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(file && file->pub.cls); HDassert(size > 0); @@ -1418,59 +1414,56 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, had #ifndef H5_HAVE_PREADWRITE /* Seek to the correct location (if we don't have pwrite) */ if(addr != file->pos || OP_WRITE != file->op) { -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_SEEK) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Start timer for seek() call */ + if(file->fa.flags & H5FD_LOG_TIME_SEEK) { + H5_timer_init(&seek_timer); + H5_timer_start(&seek_timer); + } /* end if */ + if(HDlseek(file->fd, (HDoff_t)addr, SEEK_SET) < 0) HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to seek to proper position") -#ifdef H5_HAVE_GETTIMEOFDAY + + /* Stop timer for seek() call */ if(file->fa.flags & H5FD_LOG_TIME_SEEK) - HDgettimeofday(&timeval_stop, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_stop(&seek_timer); - /* Log information about the seek */ + /* Add to the number of seeks, when tracking that */ if(file->fa.flags & H5FD_LOG_NUM_SEEK) file->total_seek_ops++; + + /* Add to the total seek time, when tracking that */ + if(file->fa.flags & H5FD_LOG_TIME_SEEK) { + H5_timer_get_times(seek_timer, &seek_times); + file->total_seek_time += seek_times.elapsed; + } /* end if */ + + /* Emit log string if we're tracking individual seek events. */ if(file->fa.flags & H5FD_LOG_LOC_SEEK) { HDfprintf(file->logfp, "Seek: From %10a To %10a", file->pos, addr); -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_SEEK) { - struct timeval timeval_diff; - double time_diff; - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(timeval_diff.tv_usec < 0) { - timeval_diff.tv_usec += 1000000; - timeval_diff.tv_sec--; - } /* end if */ - time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); - /* Add to total seek time */ - file->total_seek_time += time_diff; - } /* end if */ - else - HDfprintf(file->logfp, "\n"); -#else /* H5_HAVE_GETTIMEOFDAY */ - HDfprintf(file->logfp, "\n"); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Add the seek time, if we're tracking that. + * Note that the seek time is NOT emitted for when just H5FD_LOG_TIME_SEEK + * is set. + */ + if(file->fa.flags & H5FD_LOG_TIME_SEEK) + HDfprintf(file->logfp, " (%fs @ %f)\n", seek_times.elapsed, seek_timer.initial.elapsed); + else + HDfprintf(file->logfp, "\n"); } /* end if */ } /* end if */ #endif /* H5_HAVE_PREADWRITE */ + /* Start timer for write operation */ + if(file->fa.flags&H5FD_LOG_TIME_WRITE) { + H5_timer_init(&write_timer); + H5_timer_start(&write_timer); + } /* end if */ + /* * Write the data, being careful of interrupted system calls and partial * results */ -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags&H5FD_LOG_TIME_WRITE) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ while(size > 0) { - h5_posix_io_t bytes_in = 0; /* # of bytes to write */ h5_posix_io_ret_t bytes_wrote = -1; /* # of bytes written */ @@ -1511,14 +1504,22 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, had addr += (haddr_t)bytes_wrote; buf = (const char *)buf + bytes_wrote; } /* end while */ -#ifdef H5_HAVE_GETTIMEOFDAY + + /* Stop timer for write operation */ if(file->fa.flags & H5FD_LOG_TIME_WRITE) - HDgettimeofday(&timeval_stop, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_stop(&write_timer); - /* Log information about the write */ + /* Add to the number of writes, when tracking that */ if(file->fa.flags & H5FD_LOG_NUM_WRITE) file->total_write_ops++; + + /* Add to the total write time, when tracking that */ + if(file->fa.flags & H5FD_LOG_TIME_WRITE) { + H5_timer_get_times(write_timer, &write_times); + file->total_write_time += write_times.elapsed; + } /* end if */ + + /* Log information about the write */ if(file->fa.flags & H5FD_LOG_LOC_WRITE) { HDfprintf(file->logfp, "%10a-%10a (%10Zu bytes) (%s) Written", orig_addr, (orig_addr + orig_size) - 1, orig_size, flavors[type]); @@ -1530,29 +1531,14 @@ H5FD_log_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, had } /* end if */ } /* end if */ -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_WRITE) { - struct timeval timeval_diff; - double time_diff; - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(timeval_diff.tv_usec < 0) { - timeval_diff.tv_usec += 1000000; - timeval_diff.tv_sec--; - } /* end if */ - time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); - - /* Add to total write time */ - file->total_write_time += time_diff; - } /* end if */ - else - HDfprintf(file->logfp, "\n"); -#else /* H5_HAVE_GETTIMEOFDAY */ - HDfprintf(file->logfp, "\n"); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Add the write time, if we're tracking that. + * Note that the write time is NOT emitted for when just H5FD_LOG_TIME_WRITE + * is set. + */ + if(file->fa.flags & H5FD_LOG_TIME_WRITE) + HDfprintf(file->logfp, " (%fs @ %f)\n", write_times.elapsed, write_timer.initial.elapsed); + else + HDfprintf(file->logfp, "\n"); } /* end if */ /* Update current position and eof */ @@ -1569,11 +1555,11 @@ done: } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_write() */ +} /* end H5FD__log_write() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_truncate + * Function: H5FD__log_truncate * * Purpose: Makes sure that the true file size is the same (or larger) * than the end-of-address. @@ -1586,34 +1572,33 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing) +H5FD__log_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing) { H5FD_log_t *file = (H5FD_log_t *)_file; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(file); /* Extend the file to make sure it's large enough */ if(!H5F_addr_eq(file->eoa, file->eof)) { -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval timeval_start, timeval_stop; -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_t trunc_timer; /* Timer for truncate operation */ + H5_timevals_t trunc_times; /* Elapsed time for truncate operation */ + + /* Start timer for truncate operation */ + if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) { + H5_timer_init(&trunc_timer); + H5_timer_start(&trunc_timer); + } /* end if */ + #ifdef H5_HAVE_WIN32_API +{ LARGE_INTEGER li; /* 64-bit (union) integer for SetFilePointer() call */ DWORD dwPtrLow; /* Low-order pointer bits from SetFilePointer() * Only used as an error code here. */ - DWORD dwError; /* DWORD error code from GetLastError() */ - BOOL bError; /* Boolean error flag */ -#endif /* H5_HAVE_WIN32_API */ -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) - HDgettimeofday(&timeval_start, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ -#ifdef H5_HAVE_WIN32_API /* Windows uses this odd QuadPart union for 32/64-bit portability */ li.QuadPart = (__int64)file->eoa; @@ -1624,51 +1609,48 @@ H5FD_log_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_U */ dwPtrLow = SetFilePointer(file->hFile, li.LowPart, &li.HighPart, FILE_BEGIN); if(INVALID_SET_FILE_POINTER == dwPtrLow) { + DWORD dwError; /* DWORD error code from GetLastError() */ + dwError = GetLastError(); if(dwError != NO_ERROR ) HGOTO_ERROR(H5E_FILE, H5E_FILEOPEN, FAIL, "unable to set file pointer") - } + } /* end if */ - bError = SetEndOfFile(file->hFile); - if(0 == bError) + if(0 == SetEndOfFile(file->hFile)) HGOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly") +} #else /* H5_HAVE_WIN32_API */ + /* Truncate/extend the file */ if(-1 == HDftruncate(file->fd, (HDoff_t)file->eoa)) HSYS_GOTO_ERROR(H5E_IO, H5E_SEEKERROR, FAIL, "unable to extend file properly") #endif /* H5_HAVE_WIN32_API */ -#ifdef H5_HAVE_GETTIMEOFDAY + + /* Stop timer for truncate operation */ if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) - HDgettimeofday(&timeval_stop, NULL); -#endif /* H5_HAVE_GETTIMEOFDAY */ + H5_timer_stop(&trunc_timer); - /* Log information about the truncate */ + /* Add to the number of truncates, when tracking that */ if(file->fa.flags & H5FD_LOG_NUM_TRUNCATE) file->total_truncate_ops++; + + /* Add to the total truncate time, when tracking that */ + if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) { + H5_timer_get_times(trunc_timer, &trunc_times); + file->total_truncate_time += trunc_times.elapsed; + } /* end if */ + + /* Emit log string if we're tracking individual truncate events. */ if(file->fa.flags & H5FD_LOG_TRUNCATE) { HDfprintf(file->logfp, "Truncate: To %10a", file->eoa); -#ifdef H5_HAVE_GETTIMEOFDAY - if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) { - struct timeval timeval_diff; - double time_diff; - - /* Calculate the elapsed gettimeofday time */ - timeval_diff.tv_usec = timeval_stop.tv_usec - timeval_start.tv_usec; - timeval_diff.tv_sec = timeval_stop.tv_sec - timeval_start.tv_sec; - if(timeval_diff.tv_usec < 0) { - timeval_diff.tv_usec += 1000000; - timeval_diff.tv_sec--; - } /* end if */ - time_diff = (double)timeval_diff.tv_sec + ((double)timeval_diff.tv_usec / (double)1000000.0f); - HDfprintf(file->logfp, " (%fs @ %.6lu.%.6llu)\n", time_diff, (unsigned long long)timeval_start.tv_sec, (unsigned long long)timeval_start.tv_usec); - /* Add to total truncate time */ - file->total_truncate_time += time_diff; - } /* end if */ - else - HDfprintf(file->logfp, "\n"); -#else /* H5_HAVE_GETTIMEOFDAY */ - HDfprintf(file->logfp, "\n"); -#endif /* H5_HAVE_GETTIMEOFDAY */ + /* Add the truncate time, if we're tracking that. + * Note that the truncate time is NOT emitted for when just H5FD_LOG_TIME_TRUNCATE + * is set. + */ + if(file->fa.flags & H5FD_LOG_TIME_TRUNCATE) + HDfprintf(file->logfp, " (%fs @ %f)\n", trunc_times.elapsed, trunc_timer.initial.elapsed); + else + HDfprintf(file->logfp, "\n"); } /* end if */ /* Update the eof value */ @@ -1681,11 +1663,11 @@ H5FD_log_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_U done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_truncate() */ +} /* end H5FD__log_truncate() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_lock + * Function: H5FD__log_lock * * Purpose: Place a lock on the file * @@ -1697,13 +1679,13 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_lock(H5FD_t *_file, hbool_t rw) +H5FD__log_lock(H5FD_t *_file, hbool_t rw) { H5FD_log_t *file = (H5FD_log_t *)_file; /* VFD file struct */ int lock_flags; /* file locking flags */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC /* Sanity check */ HDassert(file); @@ -1713,19 +1695,23 @@ H5FD_log_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_lock() */ +} /* end H5FD__log_lock() */ /*------------------------------------------------------------------------- - * Function: H5FD_log_unlock + * Function: H5FD__log_unlock * * Purpose: Remove the existing lock on the file * @@ -1736,23 +1722,27 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FD_log_unlock(H5FD_t *_file) +H5FD__log_unlock(H5FD_t *_file) { H5FD_log_t *file = (H5FD_log_t *)_file; /* VFD file struct */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(file); if(HDflock(file->fd, LOCK_UN) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5FD_log_unlock() */ +} /* end H5FD__log_unlock() */ diff --git a/src/H5FDlog.h b/src/H5FDlog.h index a69bb18..db51f3d 100644 --- a/src/H5FDlog.h +++ b/src/H5FDlog.h @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Monday, April 17, 2000 * * Purpose: The public header file for the log driver. diff --git a/src/H5FDmirror.c b/src/H5FDmirror.c new file mode 100644 index 0000000..6bee4a7 --- /dev/null +++ b/src/H5FDmirror.c @@ -0,0 +1,2017 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Transmit write-only operations to a receiver/writer process on + * a remote host. + */ + +#include "H5FDdrvr_module.h" /* This source code file is part of the H5FD driver module */ + +#include "H5private.h" /* Generic Functions */ + +#ifdef H5_HAVE_MIRROR_VFD + +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDmirror.h" /* "Mirror" definitions */ +#include "H5FDmirror_priv.h" /* Private header for the mirror VFD */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Pprivate.h" /* Property lists */ + +/* The driver identification number, initialized at runtime */ +static hid_t H5FD_MIRROR_g = 0; + +/* Virtual file structure for a Mirror Driver */ +typedef struct H5FD_mirror_t { + H5FD_t pub; /* Public stuff, must be first */ + H5FD_mirror_fapl_t fa; /* Configuration structure */ + haddr_t eoa; /* End of allocated region */ + haddr_t eof; /* End of file; current file size */ + int sock_fd; /* Handle of socket to remote operator */ + H5FD_mirror_xmit_t xmit; /* Primary communication header */ + uint32_t xmit_i; /* Counter of transmission sent and rec'd */ +} H5FD_mirror_t; + +/* + * These macros check for overflow of various quantities. These macros + * assume that HDoff_t is signed and haddr_t and size_t are unsigned. + * + * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' + * is too large to be represented by the second argument + * of the file seek function. + * + * SIZE_OVERFLOW: Checks whether a buffer size of type `hsize_t' is too + * large to be represented by the `size_t' type. + * + * REGION_OVERFLOW: Checks whether an address and size pair describe data + * which can be addressed entirely by the second + * argument of the file seek function. + */ +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) + +#ifndef BSWAP_64 +#define BSWAP_64(X) \ + (uint64_t)( (((X) & 0x00000000000000FF) << 56) \ + | (((X) & 0x000000000000FF00) << 40) \ + | (((X) & 0x0000000000FF0000) << 24) \ + | (((X) & 0x00000000FF000000) << 8) \ + | (((X) & 0x000000FF00000000) >> 8) \ + | (((X) & 0x0000FF0000000000) >> 24) \ + | (((X) & 0x00FF000000000000) >> 40) \ + | (((X) & 0xFF00000000000000) >> 56)) +#endif /* BSWAP_64 */ + +/* Debugging flabs for verbose tracing -- nonzero to enable */ +#define MIRROR_DEBUG_OP_CALLS 0 +#define MIRROR_DEBUG_XMIT_BYTES 0 + +#if MIRROR_DEBUG_XMIT_BYTES +#define LOG_XMIT_BYTES(label, buf, len) do { \ + ssize_t bytes_written = 0; \ + const unsigned char *b = NULL; \ + \ + HDfprintf(stdout, "%s bytes:\n```\n", (label)); \ + \ + /* print whole lines */ \ + while ((len - bytes_written) >= 32) { \ + b = (const unsigned char *)(buf) + bytes_written; \ + HDfprintf(stdout, \ + "%04zX %02X%02X%02X%02X %02X%02X%02X%02X" \ + " %02X%02X%02X%02X %02X%02X%02X%02X" \ + " %02X%02X%02X%02X %02X%02X%02X%02X" \ + " %02X%02X%02X%02X %02X%02X%02X%02X\n", \ + bytes_written, \ + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], \ + b[8], b[9], b[10],b[11], b[12],b[13],b[14],b[15], \ + b[16],b[17],b[18],b[19], b[20],b[21],b[22],b[23], \ + b[24],b[25],b[26],b[27], b[28],b[29],b[30],b[31]); \ + bytes_written += 32; \ + } \ + \ + /* start partial line */ \ + if (len > bytes_written) { \ + HDfprintf(stdout, "%04zX ", bytes_written); \ + } \ + \ + /* partial line blocks */ \ + while ((len - bytes_written) >= 4) { \ + HDfprintf(stdout, " %02X%02X%02X%02X", \ + (buf)[bytes_written], (buf)[bytes_written+1], \ + (buf)[bytes_written+2], (buf)[bytes_written+3]); \ + bytes_written += 4; \ + } \ + \ + /* block separator before partial block */ \ + if (len > bytes_written) { \ + HDfprintf(stdout, " "); \ + } \ + \ + /* partial block individual bytes */ \ + while (len > bytes_written) { \ + HDfprintf(stdout, "%02X", (buf)[bytes_written++]); \ + } \ + \ + /* end partial line */ \ + HDfprintf(stdout, "\n"); \ + HDfprintf(stdout, "```\n"); \ + HDfflush(stdout); \ +} while (0) +#else +#define LOG_XMIT_BYTES(label, buf, len) /* no-op */ +#endif /* MIRROR_DEBUG_XMIT_BYTE */ + +#if MIRROR_DEBUG_OP_CALLS +#define LOG_OP_CALL(name) do { \ + HDprintf("called %s()\n", (name)); \ + HDfflush(stdout); \ +} while (0) +#else +#define LOG_OP_CALL(name) /* no-op */ +#endif /* MIRROR_DEBUG_OP_CALLS */ + +/* Prototypes */ +static herr_t H5FD__mirror_term(void); +static void *H5FD__mirror_fapl_get(H5FD_t *_file); +static void *H5FD__mirror_fapl_copy(const void *_old_fa); +static herr_t H5FD__mirror_fapl_free(void *_fa); +static haddr_t H5FD__mirror_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD__mirror_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD__mirror_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static H5FD_t *H5FD__mirror_open(const char *name, unsigned flags, + hid_t fapl_id, haddr_t maxaddr); +static herr_t H5FD__mirror_close(H5FD_t *_file); +static herr_t H5FD__mirror_query(const H5FD_t *_file, unsigned long *flags); +static herr_t H5FD__mirror_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, const void *buf); +static herr_t H5FD__mirror_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, void *buf); +static herr_t H5FD__mirror_truncate(H5FD_t *_file, hid_t dxpl_id, + hbool_t closing); +static herr_t H5FD__mirror_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD__mirror_unlock(H5FD_t *_file); + +static herr_t H5FD__mirror_verify_reply(H5FD_mirror_t *file); + +static const H5FD_class_t H5FD_mirror_g = { + "mirror", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD__mirror_term, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + 0, /* fapl_size */ + H5FD__mirror_fapl_get, /* fapl_get */ + H5FD__mirror_fapl_copy, /* fapl_copy */ + H5FD__mirror_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD__mirror_open, /* open */ + H5FD__mirror_close, /* close */ + NULL, /* cmp */ + H5FD__mirror_query, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD__mirror_get_eoa, /* get_eoa */ + H5FD__mirror_set_eoa, /* set_eoa */ + H5FD__mirror_get_eof, /* get_eof */ + NULL, /* get_handle */ + H5FD__mirror_read, /* read */ + H5FD__mirror_write, /* write */ + NULL, /* flush */ + H5FD__mirror_truncate, /* truncate */ + H5FD__mirror_lock, /* lock */ + H5FD__mirror_unlock, /* unlock */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +/* Declare a free list to manage the transmission buffers */ +H5FL_BLK_DEFINE_STATIC(xmit); + +/* Declare a free list to manage the H5FD_mirror_t struct */ +H5FL_DEFINE_STATIC(H5FD_mirror_t); + +/* Declare a free list to manage the H5FD_mirror_xmit_open_t struct */ +H5FL_DEFINE_STATIC(H5FD_mirror_xmit_open_t); + + +/*------------------------------------------------------------------------- + * Function: H5FD__init_package + * + * Purpose: Initializes any interface-specific data or routines. + * + * Return: Non-negative on success/Negative on failure + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__init_package(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + if(H5FD_mirror_init() < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize mirror VFD"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__init_package() */ + + +/* ------------------------------------------------------------------------- + * Function: H5FD_mirror_init + * + * Purpose: Initialize this driver by registering the driver with the + * library. + * + * Return: Success: The driver ID for the mirror driver. + * Failure: Negative + * ------------------------------------------------------------------------- + */ +hid_t +H5FD_mirror_init(void) +{ + hid_t ret_value = H5I_INVALID_HID; + + FUNC_ENTER_NOAPI(FAIL) + + LOG_OP_CALL(FUNC); + + if(H5I_VFL != H5I_get_type(H5FD_MIRROR_g)) + H5FD_MIRROR_g = H5FD_register(&H5FD_mirror_g, sizeof(H5FD_class_t), FALSE); + + ret_value = H5FD_MIRROR_g; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_mirror_init() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_term + * + * Purpose: Shut down the VFD + * + * Returns: SUCCEED (Can't fail) + * --------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_term(void) +{ + FUNC_ENTER_STATIC_NOERR + + /* Reset VFL ID */ + H5FD_MIRROR_g = 0; + + LOG_OP_CALL(FUNC); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD__mirror_term() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_decode_uint16 + * + * Purpose: Extract a 16-bit integer in "network" (Big-Endian) word order + * from the byte-buffer and return it with the local word order at + * the destination pointer. + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * Return: The number of bytes read from the buffer (2). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_decode_uint16(uint16_t *out, const unsigned char *_buf) +{ + uint16_t n = 0; + + LOG_OP_CALL(__func__); + + HDassert(_buf && out); + + HDmemcpy(&n, _buf, sizeof(n)); + *out = (uint16_t)HDntohs(n); + + return 2; /* number of bytes eaten */ +} /* end H5FD__mirror_xmit_decode_uint16() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_decode_uint32 + * + * Purpose: Extract a 32-bit integer in "network" (Big-Endian) word order + * from the byte-buffer and return it with the local word order at + * the destination pointer. + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * Return: The number of bytes read from the buffer (4). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_decode_uint32(uint32_t *out, const unsigned char *_buf) +{ + uint32_t n = 0; + + LOG_OP_CALL(__func__); + + HDassert(_buf && out); + + HDmemcpy(&n, _buf, sizeof(n)); + *out = (uint32_t)HDntohl(n); + + return 4; /* number of bytes eaten */ +} /* end H5FD__mirror_xmit_decode_uint32() */ + + +/* --------------------------------------------------------------------------- + * Function: is_host_little_endian + * + * Purpose: Determine whether the host machine is is little-endian. + * + * Store an intger with a known value, re-map the memory to a + * character array, and inspect the array's contents. + * + * Return: The number of bytes written to the buffer (8). + * + * Programmer: Jacob Smith + * 2020-03-05 + * --------------------------------------------------------------------------- + */ +static hbool_t +is_host_little_endian(void) +{ + union { + uint32_t u32; + uint8_t u8[4]; + } echeck; + echeck.u32 = 0xA1B2C3D4; + + if(echeck.u8[0] == 0xD4) + return TRUE; + else + return FALSE; +} /* end is_host_little_endian() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_decode_uint64 + * + * Purpose: Extract a 64-bit integer in "network" (Big-Endian) word order + * from the byte-buffer and return it with the local word order. + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * WARNING: Does not accommodate other forms of endianness, + * e.g. "middle-endian". + * + * Return: The number of bytes written to the buffer (8). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_decode_uint64(uint64_t *out, const unsigned char *_buf) +{ + uint64_t n = 0; + + LOG_OP_CALL(__func__); + + HDassert(_buf && out); + + HDmemcpy(&n, _buf, sizeof(n)); + if(TRUE == is_host_little_endian()) + *out = BSWAP_64(n); + else + *out = n; + + return 8; +} /* end H5FD__mirror_xmit_decode_uint64() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_decode_uint8 + * + * Purpose: Extract a 8-bit integer in "network" (Big-Endian) word order + * from the byte-buffer and return it with the local word order at + * the destination pointer. + * (yes, it's one byte). + * + * Return: The number of bytes read from the buffer (1). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_decode_uint8(uint8_t *out, const unsigned char *_buf) +{ + LOG_OP_CALL(__func__); + + HDassert(_buf && out); + + HDmemcpy(out, _buf, sizeof(uint8_t)); + + return 1; /* number of bytes eaten */ +} /* end H5FD__mirror_xmit_decode_uint8() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_encode_uint16 + * + * Purpose: Encode a 16-bit integer in "network" (Big-Endian) word order + * in place in the destination bytes-buffer. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer (2). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_encode_uint16(unsigned char *_dest, uint16_t v) +{ + uint16_t n = 0; + + LOG_OP_CALL(__func__); + + HDassert(_dest); + + n = (uint16_t)HDhtons(v); + HDmemcpy(_dest, &n, sizeof(n)); + + return 2; +} /* end H5FD__mirror_xmit_encode_uint16() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_encode_uint32 + * + * Purpose: Encode a 32-bit integer in "network" (Big-Endian) word order + * in place in the destination bytes-buffer. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer (4). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_encode_uint32(unsigned char *_dest, uint32_t v) +{ + uint32_t n = 0; + + LOG_OP_CALL(__func__); + + HDassert(_dest); + + n = (uint32_t)HDhtonl(v); + HDmemcpy(_dest, &n, sizeof(n)); + + return 4; +} /* end H5FD__mirror_xmit_encode_uint32() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_encode_uint64 + * + * Purpose: Encode a 64-bit integer in "network" (Big-Endian) word order + * in place in the destination bytes-buffer. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer (8). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_encode_uint64(unsigned char *_dest, uint64_t v) +{ + uint64_t n = v; + + LOG_OP_CALL(__func__); + + HDassert(_dest); + + if(TRUE == is_host_little_endian()) + n = BSWAP_64(v); + HDmemcpy(_dest, &n, sizeof(n)); + + return 8; +} /* H5FD__mirror_xmit_encode_uint64() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD__mirror_xmit_encode_uint8 + * + * Purpose: Encode a 8-bit integer in "network" (Big-Endian) word order + * in place in the destination bytes-buffer. + * (yes, it's one byte). + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes read from the buffer (1). + * --------------------------------------------------------------------------- + */ +size_t +H5FD__mirror_xmit_encode_uint8(unsigned char *dest, uint8_t v) +{ + LOG_OP_CALL(__func__); + + HDassert(dest); + + HDmemcpy(dest, &v, sizeof(v)); + + return 1; +} /* end H5FD__mirror_xmit_encode_uint8() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_header + * + * Purpose: Extract a mirror_xmit_t "header" from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_xmit() before use. + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * Return: The number of bytes consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_header(H5FD_mirror_xmit_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + + LOG_OP_CALL(__func__); + + HDassert(out && buf); + + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->magic), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint8(&(out->version), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->session_token), + &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->xmit_count), + &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint8(&(out->op), &buf[n_eaten]); + HDassert(n_eaten == H5FD_MIRROR_XMIT_HEADER_SIZE); + + return n_eaten; +} /* end H5FD_mirror_xmit_decode_header() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_lock + * + * Purpose: Extract a mirror_xmit_lock_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_lock() before use. + * + * Return: The number of bytes consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_lock(H5FD_mirror_xmit_lock_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + + LOG_OP_CALL(__func__); + + HDassert(out && buf); + + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->rw), &buf[n_eaten]); + HDassert(n_eaten == H5FD_MIRROR_XMIT_LOCK_SIZE); + + return n_eaten; +} /* end H5FD_mirror_xmit_decode_lock() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_open + * + * Purpose: Extract a mirror_xmit_open_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_open() before use. + * + * Return: The maximum number of bytes that this decoding operation might + * have consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_open(H5FD_mirror_xmit_open_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + + LOG_OP_CALL(__func__); + + HDassert(out && buf); + + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->flags), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->maxaddr), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->size_t_blob), + &buf[n_eaten]); + HDassert((H5FD_MIRROR_XMIT_OPEN_SIZE - H5FD_MIRROR_XMIT_FILEPATH_MAX) + == n_eaten); + HDstrncpy(out->filename, (const char *)&buf[n_eaten], + H5FD_MIRROR_XMIT_FILEPATH_MAX - 1); + out->filename[H5FD_MIRROR_XMIT_FILEPATH_MAX - 1] = 0; /* force final NULL */ + + return H5FD_MIRROR_XMIT_OPEN_SIZE; +} /* end H5FD_mirror_xmit_decode_open() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_reply + * + * Purpose: Extract a mirror_xmit_reply_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_reply() before use. + * + * Return: The maximum number of bytes that this decoding operation might + * have consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_reply(H5FD_mirror_xmit_reply_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + + LOG_OP_CALL(__func__); + + HDassert(out && buf); + + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint32(&(out->status), &buf[n_eaten]); + HDassert((H5FD_MIRROR_XMIT_REPLY_SIZE - H5FD_MIRROR_STATUS_MESSAGE_MAX) + == n_eaten); + HDstrncpy(out->message, (const char *)&buf[n_eaten], + H5FD_MIRROR_STATUS_MESSAGE_MAX - 1); + out->message[H5FD_MIRROR_STATUS_MESSAGE_MAX - 1] = 0; /* force NULL term */ + + return H5FD_MIRROR_XMIT_REPLY_SIZE; +} /* end H5FD_mirror_xmit_decode_reply() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_set_eoa + * + * Purpose: Extract a mirror_xmit_eoa_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_set_eoa() before use. + * + * Return: The number of bytes consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_set_eoa(H5FD_mirror_xmit_eoa_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + + LOG_OP_CALL(__func__); + + HDassert(out && buf); + + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint8(&(out->type), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->eoa_addr), &buf[n_eaten]); + HDassert(n_eaten == H5FD_MIRROR_XMIT_EOA_SIZE); + + return n_eaten; +} /* end H5FD_mirror_xmit_decode_set_eoa() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_decode_write + * + * Purpose: Extract a mirror_xmit_write_t from the bytes-buffer. + * + * Fields will be lifted from the buffer and stored in the + * target structure, using in the correct location (different + * systems may insert different padding between components) and + * word order (Big- vs Little-Endian). + * + * The programmer must ensure that the received buffer holds + * at least the expected size of data. + * + * The resulting structure should be sanity-checked with + * H5FD_mirror_xmit_is_write() before use. + * + * Return: The number of bytes consumed from the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_decode_write(H5FD_mirror_xmit_write_t *out, + const unsigned char *buf) +{ + size_t n_eaten = 0; + + LOG_OP_CALL(__func__); + + HDassert(out && buf); + + n_eaten += H5FD_mirror_xmit_decode_header(&(out->pub), buf); + n_eaten += H5FD__mirror_xmit_decode_uint8(&(out->type), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->offset), &buf[n_eaten]); + n_eaten += H5FD__mirror_xmit_decode_uint64(&(out->size), &buf[n_eaten]); + HDassert(n_eaten == H5FD_MIRROR_XMIT_WRITE_SIZE); + + return n_eaten; +} /* end H5FD_mirror_xmit_decode_write() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_header + * + * Purpose: Encode a mirror_xmit_t "header" to the bytes-buffer. + * + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_header(unsigned char *dest, + const H5FD_mirror_xmit_t *x) +{ + size_t n_writ = 0; + + LOG_OP_CALL(__func__); + + HDassert(dest && x); + + n_writ += H5FD__mirror_xmit_encode_uint32((dest+n_writ), x->magic); + n_writ += H5FD__mirror_xmit_encode_uint8((dest+n_writ), x->version); + n_writ += H5FD__mirror_xmit_encode_uint32((dest+n_writ), x->session_token); + n_writ += H5FD__mirror_xmit_encode_uint32((dest+n_writ), x->xmit_count); + n_writ += H5FD__mirror_xmit_encode_uint8((dest+n_writ), x->op); + HDassert(n_writ == H5FD_MIRROR_XMIT_HEADER_SIZE); + + return n_writ; +} /* end H5FD_mirror_xmit_encode_header() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_lock + * + * Purpose: Encode a mirror_xmit_lock_t to the bytes-buffer. + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_lock(unsigned char *dest, + const H5FD_mirror_xmit_lock_t *x) +{ + size_t n_writ = 0; + + LOG_OP_CALL(__func__); + + HDassert(dest && x); + + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->rw); + HDassert(n_writ == H5FD_MIRROR_XMIT_LOCK_SIZE); + + return n_writ; +} /* end H5FD_mirror_xmit_encode_lock() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_open + * + * Purpose: Encode a mirror_xmit_open_t to the bytes-buffer. + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The maximum number of bytes that this decoding operation might + * have written into the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_open(unsigned char *dest, + const H5FD_mirror_xmit_open_t *x) +{ + size_t n_writ = 0; + + LOG_OP_CALL(__func__); + + HDassert(dest && x); + + /* clear entire structure, but especially its filepath string area */ + HDmemset(dest, 0, H5FD_MIRROR_XMIT_OPEN_SIZE); + + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint32(&dest[n_writ], x->flags); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->maxaddr); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->size_t_blob); + HDassert((H5FD_MIRROR_XMIT_OPEN_SIZE - H5FD_MIRROR_XMIT_FILEPATH_MAX) + == n_writ); + HDstrncpy((char *)&dest[n_writ], x->filename, H5FD_MIRROR_XMIT_FILEPATH_MAX); + + return H5FD_MIRROR_XMIT_OPEN_SIZE; +} /* end H5FD_mirror_xmit_encode_open() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_reply + * + * Purpose: Encode a mirror_xmit_reply_t to the bytes-buffer. + * + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The maximum number of bytes that this decoding operation might + * have written into the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_reply(unsigned char *dest, + const H5FD_mirror_xmit_reply_t *x) +{ + size_t n_writ = 0; + + LOG_OP_CALL(__func__); + + HDassert(dest && x); + + /* clear entire structure, but especially its message string area */ + HDmemset(dest, 0, H5FD_MIRROR_XMIT_REPLY_SIZE); + + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint32(&dest[n_writ], x->status); + HDassert((H5FD_MIRROR_XMIT_REPLY_SIZE - H5FD_MIRROR_STATUS_MESSAGE_MAX) + == n_writ); + HDstrncpy((char *)&dest[n_writ], x->message, H5FD_MIRROR_STATUS_MESSAGE_MAX); + + return H5FD_MIRROR_XMIT_REPLY_SIZE; +} /* end H5FD_mirror_xmit_encode_reply() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_set_eoa + * + * Purpose: Encode a mirror_xmit_eoa_t to the bytes-buffer. + * + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_set_eoa(unsigned char *dest, + const H5FD_mirror_xmit_eoa_t *x) +{ + size_t n_writ = 0; + + LOG_OP_CALL(__func__); + + HDassert(dest && x); + + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint8(&dest[n_writ], x->type); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->eoa_addr); + HDassert(n_writ == H5FD_MIRROR_XMIT_EOA_SIZE); + + return n_writ; +} /* end H5FD_mirror_xmit_encode_set_eoa() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_encode_write + * + * Purpose: Encode a mirror_xmit_write_t to the bytes-buffer. + * + * Fields will be packed into the buffer in a predictable manner, + * any numbers stored in "network" (Big-Endian) word order. + * + * The programmer must ensure that the destination buffer is + * large enough to hold the expected data. + * + * Return: The number of bytes written to the buffer. + * --------------------------------------------------------------------------- + */ +size_t +H5FD_mirror_xmit_encode_write(unsigned char *dest, + const H5FD_mirror_xmit_write_t *x) +{ + size_t n_writ = 0; + + LOG_OP_CALL(__func__); + + HDassert(dest && x); + + n_writ += H5FD_mirror_xmit_encode_header(dest, + (const H5FD_mirror_xmit_t *)&(x->pub)); + n_writ += H5FD__mirror_xmit_encode_uint8(&dest[n_writ], x->type); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->offset); + n_writ += H5FD__mirror_xmit_encode_uint64(&dest[n_writ], x->size); + HDassert(n_writ == H5FD_MIRROR_XMIT_WRITE_SIZE); + + return n_writ; +} /* end H5FD_mirror_xmit_encode_write() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_close + * + * Purpose: Verify that a mirror_xmit_t is a valid CLOSE xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_close(const H5FD_mirror_xmit_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((TRUE == H5FD_mirror_xmit_is_xmit(xmit)) && (H5FD_MIRROR_OP_CLOSE == xmit->op)) + return TRUE; + + return FALSE; +} /* end H5FD_mirror_xmit_is_close() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_lock + * + * Purpose: Verify that a mirror_xmit_lock_t is a valid LOCK xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_lock(const H5FD_mirror_xmit_lock_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && (H5FD_MIRROR_OP_LOCK == xmit->pub.op)) + return TRUE; + + return FALSE; +} /* end H5FD_mirror_xmit_is_lock() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_open + * + * Purpose: Verify that a mirror_xmit_open_t is a valid OPEN xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_open(const H5FD_mirror_xmit_open_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && (H5FD_MIRROR_OP_OPEN == xmit->pub.op)) + + return TRUE; + + return FALSE; +} /* end H5FD_mirror_xmit_is_open() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_eoa + * + * Purpose: Verify that a mirror_xmit_eoa_t is a valid SET-EOA xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_set_eoa(const H5FD_mirror_xmit_eoa_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && (H5FD_MIRROR_OP_SET_EOA == xmit->pub.op)) + return TRUE; + + return FALSE; +} /* end H5FD_mirror_xmit_is_eoa() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_reply + * + * Purpose: Verify that a mirror_xmit_reply_t is a valid REPLY xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_reply(const H5FD_mirror_xmit_reply_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && (H5FD_MIRROR_OP_REPLY == xmit->pub.op)) + return TRUE; + + return FALSE; +} /* end H5FD_mirror_xmit_is_reply() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_write + * + * Purpose: Verify that a mirror_xmit_write_t is a valid WRITE xmit. + * + * Checks header validity and op code. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_write(const H5FD_mirror_xmit_write_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((TRUE == H5FD_mirror_xmit_is_xmit(&(xmit->pub))) && (H5FD_MIRROR_OP_WRITE == xmit->pub.op)) + return TRUE; + + return FALSE; +} /* end H5FD_mirror_xmit_is_write() */ + + +/* --------------------------------------------------------------------------- + * Function: H5FD_mirror_xmit_is_xmit + * + * Purpose: Verify that a mirror_xmit_t is well-formed. + * + * Checks magic number and structure version. + * + * Return: TRUE if valid; else FALSE. + * --------------------------------------------------------------------------- + */ +H5_ATTR_PURE hbool_t +H5FD_mirror_xmit_is_xmit(const H5FD_mirror_xmit_t *xmit) +{ + LOG_OP_CALL(__func__); + + HDassert(xmit); + + if((H5FD_MIRROR_XMIT_MAGIC != xmit->magic) || (H5FD_MIRROR_XMIT_CURR_VERSION != xmit->version)) + return FALSE; + + return TRUE; +} /* end H5FD_mirror_xmit_is_xmit() */ + + +/* ---------------------------------------------------------------------------- + * Function: H5FD__mirror_verify_reply + * + * Purpose: Wait for and read reply data from remote processes. + * Sanity-check that a reply is well-formed and valid. + * If all checks pass, inspect the reply contents and handle + * reported error, if not an OK reply. + * + * Return: SUCCEED if ok, else FAIL. + * ---------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_verify_reply(H5FD_mirror_t *file) +{ + unsigned char *xmit_buf = NULL; + struct H5FD_mirror_xmit_reply_t reply; + ssize_t read_ret = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + HDassert(file && file->sock_fd); + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + read_ret = HDread(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_REPLY_SIZE); + if(read_ret < 0) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to read reply"); + if(read_ret != H5FD_MIRROR_XMIT_REPLY_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unexpected read size"); + + LOG_XMIT_BYTES("reply", xmit_buf, read_ret); + + if(H5FD_mirror_xmit_decode_reply(&reply, xmit_buf) != H5FD_MIRROR_XMIT_REPLY_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unable to decode reply xmit"); + + if(H5FD_mirror_xmit_is_reply(&reply) != TRUE) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "xmit op code was not REPLY"); + + if(reply.pub.session_token != file->xmit.session_token) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "wrong session"); + if(reply.pub.xmit_count != (file->xmit_i)++) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "xmit out of sync"); + if(reply.status != H5FD_MIRROR_STATUS_OK) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "%s", (const char *)(reply.message)); + +done: + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__mirror_verify_reply() */ + + +/* ------------------------------------------------------------------------- + * Function: H5FD__mirror_fapl_get + * + * Purpose: Get the file access propety list which could be used to create + * an identical file. + * + * Return: Success: pointer to the new file access property list value. + * Failure: NULL + * ------------------------------------------------------------------------- + */ +static void * +H5FD__mirror_fapl_get(H5FD_t *_file) +{ + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + H5FD_mirror_fapl_t *fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + fa = (H5FD_mirror_fapl_t *)H5MM_calloc(sizeof(H5FD_mirror_fapl_t)); + if(NULL == fa) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "calloc failed"); + + HDmemcpy(fa, &(file->fa), sizeof(H5FD_mirror_fapl_t)); + + ret_value = fa; + +done: + if(ret_value == NULL) + if(fa != NULL) + H5MM_xfree(fa); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_fapl_get() */ + + +/* ------------------------------------------------------------------------- + * Function: H5FD__mirror_fapl_copy + * + * Purpose: Copies the mirror vfd-specific file access properties. + * + * Return: Success: Pointer to a new property list + * Failure: NULL + * ------------------------------------------------------------------------- + */ +static void * +H5FD__mirror_fapl_copy(const void *_old_fa) +{ + const H5FD_mirror_fapl_t *old_fa = (const H5FD_mirror_fapl_t *)_old_fa; + H5FD_mirror_fapl_t *new_fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + new_fa = (H5FD_mirror_fapl_t *)H5MM_malloc(sizeof(H5FD_mirror_fapl_t)); + if(new_fa == NULL) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "memory allocation failed"); + + HDmemcpy(new_fa, old_fa, sizeof(H5FD_mirror_fapl_t)); + ret_value = new_fa; + +done: + if(ret_value == NULL) + if(new_fa != NULL) + H5MM_xfree(new_fa); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_fapl_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_fapl_free + * + * Purpose: Frees the mirror VFD-specific file access properties. + * + * Return: SUCCEED (cannot fail) + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_fapl_free(void *_fa) +{ + H5FD_mirror_fapl_t *fa = (H5FD_mirror_fapl_t*)_fa; + + FUNC_ENTER_STATIC_NOERR + + LOG_OP_CALL(FUNC); + + /* sanity check */ + HDassert(fa != NULL); + HDassert(fa->magic == H5FD_MIRROR_FAPL_MAGIC); + + fa->magic += 1; /* invalidate */ + H5MM_xfree(fa); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD__mirror_fapl_free() */ + + +/* ------------------------------------------------------------------------- + * Function: H5Pget_fapl_mirror + * + * Purpose: Get the configuration information for this fapl. + * Data is memcopied into the fa_out pointer. + * + * Return: SUCCEED/FAIL + * ------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa_out) +{ + const H5FD_mirror_fapl_t *fa = NULL; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa_out); + + LOG_OP_CALL(FUNC); + + if(NULL == fa_out) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL"); + + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if(NULL == plist) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list"); + if(H5P_peek_driver(plist) != H5FD_MIRROR) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver"); + + fa = (const H5FD_mirror_fapl_t *)H5P_peek_driver_info(plist); + if(NULL == fa) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info"); + + HDassert(fa->magic == H5FD_MIRROR_FAPL_MAGIC); /* sanity check */ + + HDmemcpy(fa_out, fa, sizeof(H5FD_mirror_fapl_t)); + +done: + FUNC_LEAVE_API(ret_value); +} /* end H5Pget_fapl_mirror() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_mirror + * + * Purpose: Modify the file access property list to use the mirror + * driver (H5FD_MIRROR) defined in this source file. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa) +{ + H5P_genplist_t *plist = NULL; + herr_t ret_value = FAIL; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa); + + LOG_OP_CALL(FUNC); + + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if(NULL == plist) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list"); + if(NULL == fa) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null fapl_t pointer"); + if(H5FD_MIRROR_FAPL_MAGIC != fa->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid fapl_t magic"); + if(H5FD_MIRROR_CURR_FAPL_T_VERSION != fa->version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unknown fapl_t version"); + + ret_value = H5P_set_driver(plist, H5FD_MIRROR, (const void *)fa); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_fapl_mirror() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_open + * + * Purpose: Create and/or opens a file as an HDF5 file. + * + * Initiate connection with remote Server/Writer. + * If successful, the remote file is open. + * + * Return: Success: A pointer to a new file data structure. The + * public fields will be initialized by the + * caller, which is always H5FD_open(). + * Failure: NULL + *------------------------------------------------------------------------- + */ +static H5FD_t * +H5FD__mirror_open(const char *name, + unsigned flags, + hid_t fapl_id, + haddr_t maxaddr) +{ + int live_socket = -1; + struct sockaddr_in target_addr; + socklen_t addr_size; + unsigned char *xmit_buf = NULL; + H5FD_mirror_fapl_t fa; + H5FD_mirror_t *file = NULL; + H5FD_mirror_xmit_open_t *open_xmit = NULL; + H5FD_t *ret_value = NULL; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + /* --------------- */ + /* Check arguments */ + /* --------------- */ + + if(!name || !*name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name"); + if(HDstrlen(name) >= H5FD_MIRROR_XMIT_FILEPATH_MAX) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "filename is too long"); + if(0 == maxaddr || HADDR_UNDEF == maxaddr) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr"); + if(ADDR_OVERFLOW(maxaddr)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr"); + + if(H5Pget_fapl_mirror(fapl_id, &fa) == FAIL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't get config info"); + if(H5FD_MIRROR_FAPL_MAGIC != fa.magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid fapl magic"); + if(H5FD_MIRROR_CURR_FAPL_T_VERSION != fa.version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid fapl version"); + + /* --------------------- */ + /* Handshake with remote */ + /* --------------------- */ + + live_socket = HDsocket(AF_INET, SOCK_STREAM, 0); + if(live_socket < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't create socket"); + + target_addr.sin_family = AF_INET; + target_addr.sin_port = HDhtons((uint16_t)fa.handshake_port); + target_addr.sin_addr.s_addr = HDinet_addr(fa.remote_ip); + HDmemset(target_addr.sin_zero, '\0', sizeof target_addr.sin_zero); + + addr_size = sizeof(target_addr); + if(HDconnect(live_socket, (struct sockaddr *)&target_addr, addr_size) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't connect to remote server"); + + /* ------------- */ + /* Open the file */ + /* ------------- */ + + file = (H5FD_mirror_t *)H5FL_CALLOC(H5FD_mirror_t); + if(NULL == file) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate file struct"); + + file->sock_fd = live_socket; + file->xmit_i = 0; + + file->xmit.magic = H5FD_MIRROR_XMIT_MAGIC; + file->xmit.version = H5FD_MIRROR_XMIT_CURR_VERSION; + file->xmit.xmit_count = file->xmit_i++; + file->xmit.session_token = (uint32_t)(0x01020304 ^ file->sock_fd); /* TODO: hashing? */ + /* int --> uint32_t may truncate on some systems... shouldn't matter? */ + + open_xmit = (H5FD_mirror_xmit_open_t *)H5FL_CALLOC(H5FD_mirror_xmit_open_t); + if(NULL == open_xmit) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate open_xmit struct"); + + file->xmit.op = H5FD_MIRROR_OP_OPEN; + open_xmit->pub = file->xmit; + open_xmit->flags = (uint32_t)flags; + open_xmit->maxaddr = (uint64_t)maxaddr; + open_xmit->size_t_blob = (uint64_t)((size_t)(-1)); + HDsnprintf(open_xmit->filename, H5FD_MIRROR_XMIT_FILEPATH_MAX-1, "%s", name); + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate xmit buffer"); + + if(H5FD_mirror_xmit_encode_open(xmit_buf, open_xmit) != H5FD_MIRROR_XMIT_OPEN_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, NULL, "unable to encode open"); + + LOG_XMIT_BYTES("open", xmit_buf, H5FD_MIRROR_XMIT_OPEN_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_OPEN_SIZE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, NULL, "unable to transmit open"); + + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "invalid reply"); + + ret_value = (H5FD_t *)file; + +done: + if(NULL == ret_value) { + if(file) + file = H5FL_FREE(H5FD_mirror_t, file); + if(live_socket >= 0 && HDclose(live_socket) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "can't close socket"); + } + + if(open_xmit) + open_xmit = H5FL_FREE(H5FD_mirror_xmit_open_t, open_xmit); + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_open() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_close + * + * Purpose: Closes the HDF5 file. + * + * Tries to send a CLOSE op to the remote Writer and expects + * a valid reply, then closes the socket. + * In error, attempts to send a deliberately invalid xmit to the + * Writer to get it to close/abort, then attempts to close the + * socket. + * + * Return: Success: SUCCEED + * Failure: FAIL, file possibly not closed but resources freed. + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_close(H5FD_t *_file) +{ + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + unsigned char *xmit_buf = NULL; + int xmit_encoded = 0; /* monitor point of failure */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + HDassert(file->sock_fd >= 0); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_CLOSE; + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + if(H5FD_mirror_xmit_encode_header(xmit_buf, &(file->xmit)) != H5FD_MIRROR_XMIT_HEADER_SIZE) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to encode close"); + xmit_encoded = 1; + + LOG_XMIT_BYTES("close", xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to transmit close"); + + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + + if(HDclose(file->sock_fd) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "can't close socket"); + +done: + if(ret_value == FAIL) { + if(xmit_encoded == 0) { + /* Encode failed; send GOODBYE to force writer halt. + * We can ignore any response from the writer, if we receive + * any reply at all. + */ + if(HDwrite(file->sock_fd, "GOODBYE", HDstrlen("GOODBYE")) < 0) { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unable to transmit close"); + if(HDclose(file->sock_fd) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "can't close socket"); + file->sock_fd = -1; /* invalidate for later */ + } /* end if problem writing goodbye; go down hard */ + else + if(HDshutdown(file->sock_fd, SHUT_WR) < 0) + HDONE_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't shutdown socket write: %s", HDstrerror(errno)); + } /* end if xmit encode failed */ + + if(file->sock_fd >= 0) + if(HDclose(file->sock_fd) < 0) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "can't close socket"); + } /* end if error */ + + file = H5FL_FREE(H5FD_mirror_t, file); /* always release resources */ + + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_query + * + * Purpose: Get the driver feature flags implemented by the driver. + * + * Return: SUCCEED (non-negative) (can't fail) + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags) +{ + FUNC_ENTER_STATIC_NOERR; + + LOG_OP_CALL(FUNC); + + /* Notice: the Mirror VFD Writer currently uses only the Sec2 driver as + * the underying driver -- as such, the Mirror VFD implementation copies + * the Sec2 feature flags as its own. + * + * File pointer is always NULL/unused -- the H5FD_FEAT_IGNORE_DRVRINFO flag + * is never included. + * -- JOS 2020-01-13 + */ + if(flags) + *flags = H5FD_FEAT_AGGREGATE_METADATA \ + | H5FD_FEAT_ACCUMULATE_METADATA \ + | H5FD_FEAT_DATA_SIEVE \ + | H5FD_FEAT_AGGREGATE_SMALLDATA \ + | H5FD_FEAT_POSIX_COMPAT_HANDLE \ + | H5FD_FEAT_SUPPORTS_SWMR_IO \ + | H5FD_FEAT_DEFAULT_VFD_COMPATIBLE; + + FUNC_LEAVE_NOAPI(SUCCEED); +} /* end H5FD__mirror_query() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_get_eoa + * + * Purpose: Gets the end-of-address marker for the file. The EOA marker + * is the first address past the last byte allocated in the + * format address space. + * + * Required to register the driver. + * + * Return: The end-of-address marker. + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD__mirror_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_mirror_t *file = (const H5FD_mirror_t *)_file; + + FUNC_ENTER_STATIC_NOERR + + LOG_OP_CALL(FUNC); + + HDassert(file); + + FUNC_LEAVE_NOAPI(file->eoa) +} /* end H5FD__mirror_get_eoa() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_set_eoa + * + * Purpose: Set the end-of-address marker for the file. This function is + * called shortly after an existing HDF5 file is opened in order + * to tell the driver where the end of the HDF5 data is located. + * + * Return: SUCCEED / FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) +{ + H5FD_mirror_xmit_eoa_t xmit_eoa; + unsigned char *xmit_buf = NULL; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + HDassert(file); + + file->eoa = addr; /* local copy */ + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_SET_EOA; + + xmit_eoa.pub = file->xmit; + xmit_eoa.type = (uint8_t)type; + xmit_eoa.eoa_addr = (uint64_t)addr; + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + if(H5FD_mirror_xmit_encode_set_eoa(xmit_buf, &xmit_eoa) != H5FD_MIRROR_XMIT_EOA_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode set-eoa"); + + LOG_XMIT_BYTES("set-eoa", xmit_buf, H5FD_MIRROR_XMIT_EOA_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_EOA_SIZE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit set-eoa"); + + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + +done: + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_set_eoa() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_get_eof + * + * Purpose: Returns the end-of-file marker, which is the greater of + * either the filesystem end-of-file or the HDF5 end-of-address + * markers. + * + * Required to register the driver. + * + * Return: End of file address, the first address past the end of the + * "file", either the filesystem file or the HDF5 file. + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD__mirror_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_mirror_t *file = (const H5FD_mirror_t *)_file; + + FUNC_ENTER_STATIC_NOERR + + LOG_OP_CALL(FUNC); + + HDassert(file); + + FUNC_LEAVE_NOAPI(file->eof) +} /* end H5FD__mirror_get_eof() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_read + * + * Purpose: Required to register the driver, but if called, MUST fail. + * + * Return: FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_read(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED fapl_id, haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, void H5_ATTR_UNUSED *buf) +{ + FUNC_ENTER_STATIC_NOERR + + LOG_OP_CALL(FUNC); + + FUNC_LEAVE_NOAPI(FAIL) +} /* end H5FD__mirror_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_write + * + * Purpose: Writes SIZE bytes of data to FILE beginning at address ADDR + * from buffer BUF according to data transfer properties in + * DXPL_ID. + * + * Send metadata regarding the write (location, size) to the + * remote Writer, then separately transmits the data. + * Both transmission expect an OK reply from the Writer. + * This two-exchange approach incurs significant overhead, + * but is a simple and modular approach. + * Start optimizations here. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_write(H5FD_t *_file, H5FD_mem_t type, hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t addr, size_t size, const void *buf) +{ + H5FD_mirror_xmit_write_t xmit_write; + unsigned char *xmit_buf = NULL; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + HDassert(file); + HDassert(buf); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_WRITE; + + xmit_write.pub = file->xmit; + xmit_write.size = (uint64_t)size; + xmit_write.offset = (uint64_t)addr; + xmit_write.type = (uint8_t)type; + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + /* Notify Writer of incoming data to write. */ + if(H5FD_mirror_xmit_encode_write(xmit_buf, &xmit_write) != H5FD_MIRROR_XMIT_WRITE_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode write"); + + LOG_XMIT_BYTES("write", xmit_buf, H5FD_MIRROR_XMIT_WRITE_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_WRITE_SIZE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit write"); + + /* Check that our write xmission was received */ + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + + /* Send the data to be written */ + if(HDwrite(file->sock_fd, buf, size) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit data"); + + /* Writer should reply that it got the data and is still okay/ready */ + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + +done: + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_truncate + * + * Purpose: Makes sure that the true file size is the same (or larger) + * than the end-of-address. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_truncate(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, + hbool_t H5_ATTR_UNUSED closing) +{ + unsigned char *xmit_buf = NULL; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_TRUNCATE; + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + if(H5FD_mirror_xmit_encode_header(xmit_buf, &(file->xmit)) != H5FD_MIRROR_XMIT_HEADER_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode truncate"); + + LOG_XMIT_BYTES("truncate", xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit truncate"); + + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + +done: + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__mirror_truncate() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_lock + * + * Purpose: To place an advisory lock on a file. + * The lock type to apply depends on the parameter "rw": + * TRUE--opens for write: an exclusive lock + * FALSE--opens for read: a shared lock + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_lock(H5FD_t *_file, hbool_t rw) +{ + H5FD_mirror_xmit_lock_t xmit_lock; + unsigned char *xmit_buf = NULL; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_LOCK; + + xmit_lock.pub = file->xmit; + xmit_lock.rw = (uint64_t)rw; + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + if(H5FD_mirror_xmit_encode_lock(xmit_buf, &xmit_lock) != H5FD_MIRROR_XMIT_LOCK_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode lock"); + + LOG_XMIT_BYTES("lock", xmit_buf, H5FD_MIRROR_XMIT_LOCK_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_LOCK_SIZE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit lock"); + + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + +done: + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__mirror_lock */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__mirror_unlock + * + * Purpose: Remove the existing lock on the file. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__mirror_unlock(H5FD_t *_file) +{ + unsigned char *xmit_buf = NULL; + H5FD_mirror_t *file = (H5FD_mirror_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + LOG_OP_CALL(FUNC); + + file->xmit.xmit_count = (file->xmit_i)++; + file->xmit.op = H5FD_MIRROR_OP_UNLOCK; + + xmit_buf = H5FL_BLK_MALLOC(xmit, H5FD_MIRROR_XMIT_BUFFER_MAX); + if(NULL == xmit_buf) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate xmit buffer"); + + if(H5FD_mirror_xmit_encode_header(xmit_buf, &(file->xmit)) != H5FD_MIRROR_XMIT_HEADER_SIZE) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to encode unlock"); + + LOG_XMIT_BYTES("unlock", xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE); + + if(HDwrite(file->sock_fd, xmit_buf, H5FD_MIRROR_XMIT_HEADER_SIZE) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "unable to transmit unlock"); + + if(H5FD__mirror_verify_reply(file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid reply"); + +done: + if(xmit_buf) + xmit_buf = H5FL_BLK_FREE(xmit, xmit_buf); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__mirror_unlock */ + +#endif /* H5_HAVE_MIRROR_VFD */ + diff --git a/src/H5FDmirror.h b/src/H5FDmirror.h new file mode 100644 index 0000000..7d15c1b --- /dev/null +++ b/src/H5FDmirror.h @@ -0,0 +1,79 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Public, shared definitions for Mirror VFD & remote Writer. + */ + +#ifndef H5FDmirror_H +#define H5FDmirror_H + +#ifdef H5_HAVE_MIRROR_VFD + +#define H5FD_MIRROR (H5FD_mirror_init()) + +#ifdef __cplusplus +extern "C" { +#endif + +/* ============================================================================ + * Mirror VFD use and operation. + * ============================================================================ + */ + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_fapl_t + * + * Used to pass configuraiton information to the Mirror VFD. + * Populate components as appropriate and pass structure pointer to + * `H5Pset_fapl_mirror()`. + * + * `magic` (uint32_t) + * Semi-unique number to sanity-check pointers to this structure type. + * MUST equal H5FD_MIRROR_FAPL_MAGIC to be considered valid. + * + * `version` (uint32_t) + * Indicates expected components of the structure. + * + * `handshake_port (int) + * Port number to expect to reach the "Mirror Server" on the remote host. + * + * `remote_ip` (char[]) + * IP address string of "Mirror Server" remote host. + * --------------------------------------------------------------------------- + */ +#define H5FD_MIRROR_FAPL_MAGIC 0xF8DD514C +#define H5FD_MIRROR_CURR_FAPL_T_VERSION 1 +#define H5FD_MIRROR_MAX_IP_LEN 32 +typedef struct H5FD_mirror_fapl_t { + uint32_t magic; + uint32_t version; + int handshake_port; + char remote_ip[H5FD_MIRROR_MAX_IP_LEN + 1]; +} H5FD_mirror_fapl_t; + +H5_DLL hid_t H5FD_mirror_init(void); +H5_DLL herr_t H5Pget_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa_out); +H5_DLL herr_t H5Pset_fapl_mirror(hid_t fapl_id, H5FD_mirror_fapl_t *fa); + +#ifdef __cplusplus +} +#endif + +#else /* H5_HAVE_MIRROR_VFD */ + +#define H5FD_MIRROR (H5I_INAVLID_HID) + +#endif /* H5_HAVE_MIRROR_VFD */ + +#endif /* H5FDmirror_H */ + diff --git a/src/H5FDmirror_priv.h b/src/H5FDmirror_priv.h new file mode 100644 index 0000000..dc15441 --- /dev/null +++ b/src/H5FDmirror_priv.h @@ -0,0 +1,323 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Public, shared definitions for Mirror VFD & remote Writer. + */ + +#ifndef H5FDmirror_priv_H +#define H5FDmirror_priv_H + +#ifdef H5_HAVE_MIRROR_VFD + +#ifdef __cplusplus +extern "C" { +#endif + +/* = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = + * IPC - Mirror VFD and Remote Worker application. + * = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = + */ + +/* The maximum allowed size for a receiving buffer when accepting bytes to + * write. Writes larger than this size are performed by multiple accept-write + * steps by the Writer. */ +#define H5FD_MIRROR_DATA_BUFFER_MAX H5_GB /* 1 Gigabyte */ + +#define H5FD_MIRROR_XMIT_CURR_VERSION 1 +#define H5FD_MIRROR_XMIT_MAGIC 0x87F8005B + +#define H5FD_MIRROR_OP_OPEN 1 +#define H5FD_MIRROR_OP_CLOSE 2 +#define H5FD_MIRROR_OP_WRITE 3 +#define H5FD_MIRROR_OP_TRUNCATE 4 +#define H5FD_MIRROR_OP_REPLY 5 +#define H5FD_MIRROR_OP_SET_EOA 6 +#define H5FD_MIRROR_OP_LOCK 7 +#define H5FD_MIRROR_OP_UNLOCK 8 + +#define H5FD_MIRROR_STATUS_OK 0 +#define H5FD_MIRROR_STATUS_ERROR 1 +#define H5FD_MIRROR_STATUS_MESSAGE_MAX 256 /* Dedicated error message size */ + +/* Maximum length of a path/filename string, including the NULL-terminator. + * Must not be smaller than H5FD_SPLITTER_PATH_MAX. */ +#define H5FD_MIRROR_XMIT_FILEPATH_MAX 4097 + +/* Define the exact sizes of the various xmit blobs as sent over the wire. + * This is used to minimize the number of bytes transmitted as well as to + * sanity-check received bytes. + * Any modifications to the xmit structures and/or the encode/decode functions + * must be reflected here. + * */ +#define H5FD_MIRROR_XMIT_HEADER_SIZE 14 +#define H5FD_MIRROR_XMIT_EOA_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 9) +#define H5FD_MIRROR_XMIT_LOCK_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 8) +#define H5FD_MIRROR_XMIT_OPEN_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 20 + H5FD_MIRROR_XMIT_FILEPATH_MAX) +#define H5FD_MIRROR_XMIT_REPLY_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 4 + H5FD_MIRROR_STATUS_MESSAGE_MAX) +#define H5FD_MIRROR_XMIT_WRITE_SIZE (H5FD_MIRROR_XMIT_HEADER_SIZE + 17) + +/* Maximum length of any xmit. */ +#define H5FD_MIRROR_XMIT_BUFFER_MAX MAX2( MAX3(H5FD_MIRROR_XMIT_HEADER_SIZE, \ + H5FD_MIRROR_XMIT_EOA_SIZE, \ + H5FD_MIRROR_XMIT_LOCK_SIZE), \ + MAX3(H5FD_MIRROR_XMIT_OPEN_SIZE, \ + H5FD_MIRROR_XMIT_REPLY_SIZE, \ + H5FD_MIRROR_XMIT_WRITE_SIZE) ) \ + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_t + * + * Common structure 'header' for all mirror VFD/worker IPC. + * Must be the first component of a derived operation xmit structure, + * such as file-open or write command. + * + * `magic` (uint32_t) + * A "unique" number identifying the structure and endianness of + * transmitting maching. + * Must be set to H5FD_MIRROR_XMIT_MAGIC native to the VFD "sender". + * + * `version` (uint8_t) + * Number used to identify the structure membership. + * Allows sane modifications to this structure in the future. + * Must be set to H5FD_MIRROR_XMIT_CURR_VERSION. + * + * `session_token` (uint32_t) + * A "unique" number identifying the session between VFD sender and + * remote receiver/worker/writer. Exists to help sanity-check. + * + * `xmit_count` (uint32_t) + * Which transmission this is since the session began. + * Used to sanity-check transmission errors. + * First xmit (file-open) must be 0. + * + * `op` (uint8_t) + * Number identifying which operation to perform. + * Corresponds with the extended structure outside of this xmit header. + * Possible values are all defined H5FD_MIRROR_OP_* constants. + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_t { + uint32_t magic; + uint8_t version; + uint32_t session_token; + uint32_t xmit_count; + uint8_t op; +} H5FD_mirror_xmit_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_eoa_t + * + * Structure containing eoa-set information from VFD sender. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `type` (uint8_t) + * System-independent alias for H5F[D]_mem_t. + * Specifies datatype to be written. + * + * `eoa_addr` (uint64_t) + * New address for eoa. + * (Natively 'haddr_t', always a 64-bit field) + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_eoa_t { + H5FD_mirror_xmit_t pub; + uint8_t type; + uint64_t eoa_addr; +} H5FD_mirror_xmit_eoa_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_lock_t + * + * Structure containing eoa-set information from VFD sender. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `rw` (uint64_t) + * The Read/Write mode flag passed into H5FDlock(). + * (Natively `hbool_t`, an 'int') TODO: native int may be 64-bit? + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_lock_t { + H5FD_mirror_xmit_t pub; + uint64_t rw; +} H5FD_mirror_xmit_lock_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_open_t + * + * Structure containing file-open information from the VFD sender. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `flags` (uint32_t) + * VFL-layer file-open flags passed directly to H5FDopen(). + * (Natively 'unsigned [int]') TODO: native int may be 64-bit? + * + * `maxaddr` (uint64_t) + * VFL-layer maximum allowed address space for the file to open passed + * directly to H5FDopen(). + * (Natively 'haddr_t', always a 64-bit field) + * + * `size_t_blob` (uint64_t) + * A number indicating how large a size_t is on the sending system. + * Must be set to (uint64_t)((size_t)(-1)) + * (maximum possible value of size_t, cast to uint64_t). + * The receiving system inspects this value -- if the local (remote) + * size_t is smaller than that of the Sender, issues a warning. + * Not an error, as: + * 1. It is assumed that underlying file systems/drivers have become + * smart enough to handle file sizes that otherwise might be + * constrained. + * 2. The Mirror Writer ingests bytes to write multiple 'slices' if the + * size is greater than H5FD_MIRROR_DATA_BUFFER_MAX, regardless of + * any size_t storage size disparity. + * + * `filename` (char[]) + * String giving the filename and path of file to open. + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_open_t { + H5FD_mirror_xmit_t pub; + uint32_t flags; + uint64_t maxaddr; + uint64_t size_t_blob; + char filename[H5FD_MIRROR_XMIT_FILEPATH_MAX]; +} H5FD_mirror_xmit_open_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_reply_t + * + * Structure used by the remote receiver/worker/writer to respond to + * a command from the VFD sender. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `status` (uint32_t) + * Number indicating whether the command was successful or if an + * occured. + * Allowed values are H5FD_MIRROR_STATUS_OK and + * H5FD_MIRROR_STATUS_ERROR. + * + * `message` (char[]) + * Error message. Populated if and only if there was a problem. + * It is possible that a message may reach the end of the alloted + * space without a NULL terminator -- the onus is on the programmer to + * handle this situation. + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_reply_t { + H5FD_mirror_xmit_t pub; + uint32_t status; + char message[H5FD_MIRROR_STATUS_MESSAGE_MAX]; +} H5FD_mirror_xmit_reply_t; + +/* --------------------------------------------------------------------------- + * Structure: H5FD_mirror_xmit_write_t + * + * Structure containing data-write information from VFD sender. + * + * The data to be written is transmitted in subsequent, packets + * and may be broken up into more than one transmission buffer. + * The VFD sender and remote receiver/worker/writer must coordinate + * the receipt of data. + * + * `pub` (H5FD_mirror_xmit_t) + * Common transmission header, containing session information. + * Must be first. + * + * `type` (uint8_t) + * Specifies datatype to be written. + * (Natively 'H5FD_mem_t', an enumerated type in H5Fpublic.h) + * + * `offset` (uint64_t) + * Start location of write in file. + * (Natively 'haddr_t', always a 64-bit field) + * + * `size` (uint64_t) + * Size of the data to be written, in bytes. + * (Natively 'size_t', accommodate the largest possible as 64-bits) + * + * --------------------------------------------------------------------------- + */ +typedef struct H5FD_mirror_xmit_write_t { + H5FD_mirror_xmit_t pub; + uint8_t type; + uint64_t offset; + uint64_t size; +} H5FD_mirror_xmit_write_t; + + + +/* Encode/decode routines are required to "pack" the xmit data into a known + * byte format for transmission over the wire. + * + * All component numbers must be stored in "network" word order (Big-Endian). + * + * All components must be packed in the order given in the structure definition. + * + * All components must be packed with zero padding between. + */ + +H5_DLL size_t H5FD__mirror_xmit_decode_uint16(uint16_t *out, const unsigned char *buf); +H5_DLL size_t H5FD__mirror_xmit_decode_uint32(uint32_t *out, const unsigned char *buf); +H5_DLL size_t H5FD__mirror_xmit_decode_uint64(uint64_t *out, const unsigned char *buf); +H5_DLL size_t H5FD__mirror_xmit_decode_uint8(uint8_t *out, const unsigned char *buf); +H5_DLL size_t H5FD__mirror_xmit_encode_uint16(unsigned char *dest, uint16_t v); +H5_DLL size_t H5FD__mirror_xmit_encode_uint32(unsigned char *dest, uint32_t v); +H5_DLL size_t H5FD__mirror_xmit_encode_uint64(unsigned char *dest, uint64_t v); +H5_DLL size_t H5FD__mirror_xmit_encode_uint8(unsigned char *dest, uint8_t v); + +H5_DLL size_t H5FD_mirror_xmit_decode_header(H5FD_mirror_xmit_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_lock(H5FD_mirror_xmit_lock_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_open(H5FD_mirror_xmit_open_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_reply(H5FD_mirror_xmit_reply_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_set_eoa(H5FD_mirror_xmit_eoa_t *out, const unsigned char *buf); +H5_DLL size_t H5FD_mirror_xmit_decode_write(H5FD_mirror_xmit_write_t *out, const unsigned char *buf); + +H5_DLL size_t H5FD_mirror_xmit_encode_header(unsigned char *dest, const H5FD_mirror_xmit_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_lock(unsigned char *dest, const H5FD_mirror_xmit_lock_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_open(unsigned char *dest, const H5FD_mirror_xmit_open_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_reply(unsigned char *dest, const H5FD_mirror_xmit_reply_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_set_eoa(unsigned char *dest, const H5FD_mirror_xmit_eoa_t *x); +H5_DLL size_t H5FD_mirror_xmit_encode_write(unsigned char *dest, const H5FD_mirror_xmit_write_t *x); + +H5_DLL hbool_t H5FD_mirror_xmit_is_close(const H5FD_mirror_xmit_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_lock(const H5FD_mirror_xmit_lock_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_open(const H5FD_mirror_xmit_open_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_reply(const H5FD_mirror_xmit_reply_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_set_eoa(const H5FD_mirror_xmit_eoa_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_write(const H5FD_mirror_xmit_write_t *xmit); +H5_DLL hbool_t H5FD_mirror_xmit_is_xmit(const H5FD_mirror_xmit_t *xmit); + +#ifdef __cplusplus +} +#endif + +#endif /* H5_HAVE_MIRROR_VFD */ + +#endif /* H5FDmirror_priv_H */ + diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index 72f4da5..f80cac5 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -1856,7 +1856,7 @@ H5FD_multi_lock(H5FD_t *_file, hbool_t rw) } /* end if */ if(nerrors) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "error locking member files", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTLOCKFILE, "error locking member files", -1) return 0; } /* H5FD_multi_lock() */ @@ -1893,7 +1893,7 @@ H5FD_multi_unlock(H5FD_t *_file) } END_MEMBERS; if(nerrors) - H5Epush_ret(func, H5E_ERR_CLS, H5E_INTERNAL, H5E_BADVALUE, "error unlocking member files", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTUNLOCKFILE, "error unlocking member files", -1) return 0; } /* H5FD_multi_unlock() */ diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index 54ca8f7..61bf212 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -255,6 +255,8 @@ typedef enum H5F_mem_t H5FD_mem_t; * that creates a file which is compatible with the default VFD. * Generally, this means that the VFD creates a single file that follows * the canonical HDF5 file format. + * Regarding the Splitter VFD specifically, only drivers with this flag + * enabled may be used as the Write-Only (W/O) channel driver. */ #define H5FD_FEAT_DEFAULT_VFD_COMPATIBLE 0x00008000 diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c index ba4750e..ce11c05 100644 --- a/src/H5FDsec2.c +++ b/src/H5FDsec2.c @@ -39,6 +39,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_SEC2_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = FAIL; + /* The description of a file belonging to this driver. The 'eoa' and 'eof' * determine the amount of hdf5 address space in use and the high-water mark * of the file (the current size of the underlying filesystem file). The @@ -57,6 +60,7 @@ typedef struct H5FD_sec2_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ H5FD_file_op_t op; /* last operation */ + hbool_t ignore_disabled_file_locks; char filename[H5FD_MAX_FILENAME_LEN]; /* Copy of file name from open operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -190,10 +194,20 @@ H5FL_DEFINE_STATIC(H5FD_sec2_t); static herr_t H5FD__init_package(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ herr_t ret_value = SUCCEED; FUNC_ENTER_STATIC + /* Check the use disabled file locks environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !HDstrcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = TRUE; /* Override: Ignore disabled locks */ + else if(lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = FALSE; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = FAIL; /* Environment variable not set, or not set correctly */ + if(H5FD_sec2_init() < 0) HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize sec2 VFD") @@ -316,6 +330,7 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) struct _BY_HANDLE_FILE_INFORMATION fileinfo; #endif h5_stat_t sb; + H5P_genplist_t *plist; /* Property list pointer */ H5FD_t *ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -373,17 +388,26 @@ H5FD_sec2_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) file->inode = sb.st_ino; #endif /* H5_HAVE_WIN32_API */ + /* Get the FAPL */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") + + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != FAIL) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + /* Use the value in the property list */ + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &file->ignore_disabled_file_locks) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, NULL, "can't get ignore disabled file locks property") + } + /* Retain a copy of the name used to open the file, for possible error reporting */ HDstrncpy(file->filename, name, sizeof(file->filename)); file->filename[sizeof(file->filename) - 1] = '\0'; /* Check for non-default FAPL */ if(H5P_FILE_ACCESS_DEFAULT != fapl_id) { - H5P_genplist_t *plist; /* Property list pointer */ - - /* Get the FAPL */ - if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) - HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, NULL, "not a file access property list") /* This step is for h5repart tool only. If user wants to change file driver from * family to one that uses single files (sec2, etc.) while using h5repart, this @@ -521,6 +545,12 @@ H5FD_sec2_query(const H5FD_t *_file, unsigned long *flags /* out */) FUNC_ENTER_NOAPI_NOINIT_NOERR /* Set the VFL feature flags that this driver supports */ + /* Notice: the Mirror VFD Writer currently uses only the Sec2 driver as + * the underying driver -- as such, the Mirror VFD implementation copies + * these feature flags as its own. Any modifications made here must be + * reflected in H5FDmirror.c + * -- JOS 2020-01-13 + */ if(flags) { *flags = 0; *flags |= H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */ @@ -955,11 +985,15 @@ H5FD_sec2_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if(HDflock(file->fd, lock_flags | LOCK_NB) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to lock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock file") + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -988,11 +1022,15 @@ H5FD_sec2_unlock(H5FD_t *_file) HDassert(file); if(HDflock(file->fd, LOCK_UN) < 0) { - if(ENOSYS == errno) - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)") + if(file->ignore_disabled_file_locks && ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; + } else - HSYS_GOTO_ERROR(H5E_FILE, H5E_BADFILE, FAIL, "unable to unlock file") - } /* end if */ + HSYS_GOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5FDsplitter.c b/src/H5FDsplitter.c new file mode 100644 index 0000000..d0ed250d --- /dev/null +++ b/src/H5FDsplitter.c @@ -0,0 +1,1346 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: The Splitter VFD implements a file driver which relays all the + * VFD calls to an underlying VFD, and send all the write calls to + * another underlying VFD. Maintains two files simultaneously. + */ + +/* This source code file is part of the H5FD driver module */ +#include "H5FDdrvr_module.h" + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDsplitter.h" /* Splitter file driver */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Pprivate.h" /* Property lists */ + +/* The driver identification number, initialized at runtime */ +static hid_t H5FD_SPLITTER_g = 0; + +/* Driver-specific file access properties */ +typedef struct H5FD_splitter_fapl_t { + hid_t rw_fapl_id; /* fapl for the R/W channel */ + hid_t wo_fapl_id; /* fapl for the W/O channel */ + char wo_path[H5FD_SPLITTER_PATH_MAX + 1]; /* file name for the W/O channel */ + char log_file_path[H5FD_SPLITTER_PATH_MAX + 1]; /* file to record errors reported by the W/O channel */ + hbool_t ignore_wo_errs; /* TRUE to ignore errors on the W/O channel */ +} H5FD_splitter_fapl_t; + +/* The information of this splitter */ +typedef struct H5FD_splitter_t { + H5FD_t pub; /* public stuff, must be first */ + unsigned version; /* version of the H5FD_splitter_vfd_config_t structure used */ + H5FD_splitter_fapl_t fa; /* driver-specific file access properties */ + H5FD_t *rw_file; /* pointer of R/W channel */ + H5FD_t *wo_file; /* pointer of W/O channel */ + FILE *logfp; /* Log file pointer */ +} H5FD_splitter_t; + +/* + * These macros check for overflow of various quantities. These macros + * assume that HDoff_t is signed and haddr_t and size_t are unsigned. + * + * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' + * is too large to be represented by the second argument + * of the file seek function. + * + * SIZE_OVERFLOW: Checks whether a buffer size of type `hsize_t' is too + * large to be represented by the `size_t' type. + * + * REGION_OVERFLOW: Checks whether an address and size pair describe data + * which can be addressed entirely by the second + * argument of the file seek function. + */ +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) +#define SIZE_OVERFLOW(Z) ((Z) & ~(hsize_t)MAXADDR) +#define REGION_OVERFLOW(A,Z) (ADDR_OVERFLOW(A) || SIZE_OVERFLOW(Z) || \ + HADDR_UNDEF==(A)+(Z) || \ + (HDoff_t)((A)+(Z))<(HDoff_t)(A)) + +/* This macro provides a wrapper for shared fail-log-ignore behavior + * for errors arising in the splitter's W/O channel. + * Logs an error entry in a log file, if the file exists. + * If not set to ignore errors, registers an error with the library. + */ +#define H5FD_SPLITTER_WO_ERROR(file, funcname, errmajor, errminor, ret, mesg) \ +{ \ + H5FD__splitter_log_error((file), (funcname), (mesg)); \ + if(FALSE == (file)->fa.ignore_wo_errs) \ + HGOTO_ERROR((errmajor), (errminor), (ret), (mesg)) \ +} + +#define H5FD_SPLITTER_DEBUG_OP_CALLS 0 /* debugging print toggle; 0 disables */ + +#if H5FD_SPLITTER_DEBUG_OP_CALLS +#define H5FD_SPLITTER_LOG_CALL(name) do { \ + HDprintf("called %s()\n", (name)); \ + HDfflush(stdout); \ +} while (0) +#else +#define H5FD_SPLITTER_LOG_CALL(name) /* no-op */ +#endif /* H5FD_SPLITTER_DEBUG_OP_CALLS */ + +/* Private functions */ + +/* Print error messages from W/O channel to log file */ +static herr_t H5FD__splitter_log_error(const H5FD_splitter_t *file, const char *atfunc, const char *msg); +static int H5FD__copy_plist(hid_t fapl_id, hid_t *id_out_ptr); + +/* Prototypes */ +static herr_t H5FD__splitter_term(void); +static hsize_t H5FD__splitter_sb_size(H5FD_t *_file); +static herr_t H5FD__splitter_sb_encode(H5FD_t *_file, char *name/*out*/, unsigned char *buf/*out*/); +static herr_t H5FD__splitter_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf); +static void *H5FD__splitter_fapl_get(H5FD_t *_file); +static void *H5FD__splitter_fapl_copy(const void *_old_fa); +static herr_t H5FD__splitter_fapl_free(void *_fapl); +static H5FD_t *H5FD__splitter_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr); +static herr_t H5FD__splitter_close(H5FD_t *_file); +static int H5FD__splitter_cmp(const H5FD_t *_f1, const H5FD_t *_f2); +static herr_t H5FD__splitter_query(const H5FD_t *_file, unsigned long *flags /* out */); +static herr_t H5FD__splitter_get_type_map(const H5FD_t *_file, H5FD_mem_t *type_map); +static haddr_t H5FD__splitter_alloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size); +static herr_t H5FD__splitter_free(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size); +static haddr_t H5FD__splitter_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type); +static herr_t H5FD__splitter_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr); +static haddr_t H5FD__splitter_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type); +static herr_t H5FD__splitter_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void** file_handle); +static herr_t H5FD__splitter_read(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, void *buf); +static herr_t H5FD__splitter_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, const void *buf); +static herr_t H5FD__splitter_flush(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD__splitter_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD__splitter_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD__splitter_unlock(H5FD_t *_file); + +static const H5FD_class_t H5FD_splitter_g = { + "splitter", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD__splitter_term, /* terminate */ + H5FD__splitter_sb_size, /* sb_size */ + H5FD__splitter_sb_encode, /* sb_encode */ + H5FD__splitter_sb_decode, /* sb_decode */ + sizeof(H5FD_splitter_fapl_t), /* fapl_size */ + H5FD__splitter_fapl_get, /* fapl_get */ + H5FD__splitter_fapl_copy, /* fapl_copy */ + H5FD__splitter_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD__splitter_open, /* open */ + H5FD__splitter_close, /* close */ + H5FD__splitter_cmp, /* cmp */ + H5FD__splitter_query, /* query */ + H5FD__splitter_get_type_map, /* get_type_map */ + H5FD__splitter_alloc, /* alloc */ + H5FD__splitter_free, /* free */ + H5FD__splitter_get_eoa, /* get_eoa */ + H5FD__splitter_set_eoa, /* set_eoa */ + H5FD__splitter_get_eof, /* get_eof */ + H5FD__splitter_get_handle, /* get_handle */ + H5FD__splitter_read, /* read */ + H5FD__splitter_write, /* write */ + H5FD__splitter_flush, /* flush */ + H5FD__splitter_truncate, /* truncate */ + H5FD__splitter_lock, /* lock */ + H5FD__splitter_unlock, /* unlock */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +/* Declare a free list to manage the H5FD_splitter_t struct */ +H5FL_DEFINE_STATIC(H5FD_splitter_t); + +/* Declare a free list to manage the H5FD_splitter_fapl_t struct */ +H5FL_DEFINE_STATIC(H5FD_splitter_fapl_t); + + +/*------------------------------------------------------------------------- + * Function: H5FD__init_package + * + * Purpose: Initializes any interface-specific data or routines. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__init_package(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + if(H5FD_splitter_init() < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize splitter VFD") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD__init_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_splitter_init + * + * Purpose: Initialize the splitter driver by registering it with the + * library. + * + * Return: Success: The driver ID for the splitter driver. + * Failure: Negative + *------------------------------------------------------------------------- + */ +hid_t +H5FD_splitter_init(void) +{ + hid_t ret_value = H5I_INVALID_HID; + + FUNC_ENTER_NOAPI(FAIL) + + H5FD_SPLITTER_LOG_CALL(FUNC); + + if(H5I_VFL != H5I_get_type(H5FD_SPLITTER_g)) + H5FD_SPLITTER_g = H5FDregister(&H5FD_splitter_g); + + ret_value = H5FD_SPLITTER_g; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_splitter_init() */ + + +/*--------------------------------------------------------------------------- + * Function: H5FD__splitter_term + * + * Purpose: Shut down the splitter VFD. + * + * Returns: SUCCEED (Can't fail) + *--------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_term(void) +{ + FUNC_ENTER_STATIC_NOERR + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Reset VFL ID */ + H5FD_SPLITTER_g = 0; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD__splitter_term() */ + + + /*------------------------------------------------------------------------- + * Function: H5FD__copy_plist + * + * Purpose: Sanity-wrapped H5P_copy_plist() for each channel. + * Utility function for operation in multiple locations. + * + * Return: 0 on success, -1 on error. + *------------------------------------------------------------------------- + */ +static int +H5FD__copy_plist(hid_t fapl_id, + hid_t *id_out_ptr) +{ + int ret_value = 0; + H5P_genplist_t *plist_ptr = NULL; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + HDassert(id_out_ptr != NULL); + + if(FALSE == H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, -1, "not a file access property list"); + + plist_ptr = (H5P_genplist_t *)H5I_object(fapl_id); + if(NULL == plist_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, -1, "unable to get property list"); + + *id_out_ptr = H5P_copy_plist(plist_ptr, FALSE); + if(H5I_INVALID_HID == *id_out_ptr) + HGOTO_ERROR(H5E_VFL, H5E_BADTYPE, -1, "unable to copy file access property list"); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__copy_plist() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_splitter + * + * Purpose: Sets the file access property list to use the + * splitter driver. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *vfd_config) +{ + H5FD_splitter_fapl_t *info = NULL; + H5P_genplist_t *plist_ptr = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*Dr", fapl_id, vfd_config); + + H5FD_SPLITTER_LOG_CALL(FUNC); + + if(H5FD_SPLITTER_MAGIC != vfd_config->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid configuration (magic number mismatch)") + if(H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION != vfd_config->version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid config (version number mismatch)") + if(NULL == (plist_ptr = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid property list") + + /* Make sure that the W/O channel supports write-only capability. + * Some drivers (e.g. family or multi) do revision of the superblock + * in-memory, causing problems in that channel. + * Uses the feature flag H5FD_FEAT_DEFAULT_VFD_COMPATIBLE as the + * determining attribute. + */ + if(H5P_DEFAULT != vfd_config->wo_fapl_id) { + H5FD_class_t *wo_driver = NULL; + H5FD_driver_prop_t wo_driver_prop; + H5P_genplist_t *wo_plist_ptr = NULL; + unsigned long wo_driver_flags = 0; + + wo_plist_ptr = (H5P_genplist_t *)H5I_object(vfd_config->wo_fapl_id); + if(NULL == wo_plist_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if(H5P_peek(wo_plist_ptr, H5F_ACS_FILE_DRV_NAME, &wo_driver_prop) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get driver ID & info") + wo_driver = (H5FD_class_t *)H5I_object(wo_driver_prop.driver_id); + if(NULL == wo_driver) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "invalid driver ID in file access property list") + if(H5FD_driver_query(wo_driver, &wo_driver_flags) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't query VFD flags") + if(0 == (H5FD_FEAT_DEFAULT_VFD_COMPATIBLE & wo_driver_flags)) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "unsuitable W/O driver") + } /* end if W/O VFD is non-default */ + + info = H5FL_CALLOC(H5FD_splitter_fapl_t); + if(NULL == info) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, FAIL, "unable to allocate file access property list struct") + + info->ignore_wo_errs = vfd_config->ignore_wo_errs; + HDstrncpy(info->wo_path, vfd_config->wo_path, H5FD_SPLITTER_PATH_MAX); + HDstrncpy(info->log_file_path, vfd_config->log_file_path, H5FD_SPLITTER_PATH_MAX); + info->rw_fapl_id = H5P_FILE_ACCESS_DEFAULT; /* pre-set value */ + info->wo_fapl_id = H5P_FILE_ACCESS_DEFAULT; /* pre-set value */ + + /* Set non-default channel FAPL IDs in splitter configuration info */ + if(H5P_DEFAULT != vfd_config->rw_fapl_id) { + if(FALSE == H5P_isa_class(vfd_config->rw_fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") + info->rw_fapl_id = vfd_config->rw_fapl_id; + } + if(H5P_DEFAULT != vfd_config->wo_fapl_id) { + if(FALSE == H5P_isa_class(vfd_config->wo_fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") + info->wo_fapl_id = vfd_config->wo_fapl_id; + } + + ret_value = H5P_set_driver(plist_ptr, H5FD_SPLITTER, info); + +done: + if(info) + info = H5FL_FREE(H5FD_splitter_fapl_t, info); + + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_fapl_splitter() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_fapl_splitter + * + * Purpose: Returns information about the splitter file access property + * list through the structure config_out. + * + * Will fail if config_out is received without pre-set valid + * magic and version information. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_out) +{ + const H5FD_splitter_fapl_t *fapl_ptr = NULL; + H5P_genplist_t *plist_ptr = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*Dr", fapl_id, config_out); + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if(config_out == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "config_out pointer is null") + if(H5FD_SPLITTER_MAGIC != config_out->magic) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "info-out pointer invalid (magic number mismatch)") + if(H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION != config_out->version) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "info-out pointer invalid (version unsafe)") + + /* Pre-set out FAPL IDs with intent to replace these values */ + config_out->rw_fapl_id = H5I_INVALID_HID; + config_out->wo_fapl_id = H5I_INVALID_HID; + + /* Check and get the splitter fapl */ + if(NULL == (plist_ptr = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + if(H5FD_SPLITTER != H5P_peek_driver(plist_ptr)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + if(NULL == (fapl_ptr = (const H5FD_splitter_fapl_t *)H5P_peek_driver_info(plist_ptr))) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "unable to get specific-driver info") + + HDstrncpy(config_out->wo_path, fapl_ptr->wo_path, H5FD_SPLITTER_PATH_MAX); + HDstrncpy(config_out->log_file_path, fapl_ptr->log_file_path, H5FD_SPLITTER_PATH_MAX); + config_out->ignore_wo_errs = fapl_ptr->ignore_wo_errs; + + /* Copy R/W and W/O FAPLs */ + if(H5FD__copy_plist(fapl_ptr->rw_fapl_id, &(config_out->rw_fapl_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't copy R/W FAPL"); + if(H5FD__copy_plist(fapl_ptr->wo_fapl_id, &(config_out->wo_fapl_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "can't copy W/O FAPL"); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_fapl_splitter() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_flush + * + * Purpose: Flushes all data to disk for both channels. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_flush(H5FD_t *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t closing) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Public API for dxpl "context" */ + if(H5FDflush(file->rw_file, dxpl_id, closing) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTFLUSH, FAIL, "unable to flush R/W file") + if(H5FDflush(file->wo_file, dxpl_id, closing) < 0) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTFLUSH, FAIL, "unable to flush W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_flush() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_read + * + * Purpose: Reads SIZE bytes of data from the R/W channel, beginning at + * address ADDR into buffer BUF according to data transfer + * properties in DXPL_ID. + * + * Return: Success: SUCCEED + * The read result is written into the BUF buffer + * which should be allocated by the caller. + * Failure: FAIL + * The contents of BUF are undefined. + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, haddr_t addr, size_t size, void *buf) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + HDassert(file && file->pub.cls); + HDassert(buf); + + /* Check for overflow conditions */ + if(!H5F_addr_defined(addr)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "addr undefined, addr = %llu", (unsigned long long)addr) + if(REGION_OVERFLOW(addr, size)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "addr overflow, addr = %llu", (unsigned long long)addr) + + /* Only read from R/W channel */ + /* Public API for dxpl "context" */ + if(H5FDread(file->rw_file, type, dxpl_id, addr, size, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "Reading from R/W channel failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_write + * + * Purpose: Writes SIZE bytes of data to R/W and W/O channels, beginning + * at address ADDR from buffer BUF according to data transfer + * properties in DXPL_ID. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_write(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, + haddr_t addr, size_t size, const void *buf) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + H5P_genplist_t *plist_ptr = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + if(NULL == (plist_ptr = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") + + /* Write to each file */ + /* Public API for dxpl "context" */ + if(H5FDwrite(file->rw_file, type, dxpl_id, addr, size, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "R/W file write failed") + if(H5FDwrite(file->wo_file, type, dxpl_id, addr, size, buf) < 0) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_WRITEERROR, FAIL, "unable to write W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_fapl_get + * + * Purpose: Returns a file access property list which indicates how the + * specified file is being accessed. The return list could be + * used to access another file the same way. + * + * Return: Success: Ptr to new file access property list with all + * members copied from the file struct. + * Failure: NULL + *------------------------------------------------------------------------- + */ +static void * +H5FD__splitter_fapl_get(H5FD_t *_file) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + void *ret_value = NULL; + + FUNC_ENTER_STATIC_NOERR + + H5FD_SPLITTER_LOG_CALL(FUNC); + + ret_value = H5FD__splitter_fapl_copy(&(file->fa)); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_fapl_get() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_fapl_copy + * + * Purpose: Copies the file access properties. + * + * Return: Success: Pointer to a new property list info structure. + * Failure: NULL + *------------------------------------------------------------------------- + */ +static void * +H5FD__splitter_fapl_copy(const void *_old_fa) +{ + const H5FD_splitter_fapl_t *old_fa_ptr = (const H5FD_splitter_fapl_t *)_old_fa; + H5FD_splitter_fapl_t *new_fa_ptr = NULL; + void *ret_value = NULL; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + HDassert(old_fa_ptr); + + new_fa_ptr = H5FL_CALLOC(H5FD_splitter_fapl_t); + if(NULL == new_fa_ptr) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate log file FAPL") + + HDmemcpy(new_fa_ptr, old_fa_ptr, sizeof(H5FD_splitter_fapl_t)); + HDstrncpy(new_fa_ptr->wo_path, old_fa_ptr->wo_path, H5FD_SPLITTER_PATH_MAX); + HDstrncpy(new_fa_ptr->log_file_path, old_fa_ptr->log_file_path, H5FD_SPLITTER_PATH_MAX); + + /* Copy R/W and W/O FAPLs */ + if(H5FD__copy_plist(old_fa_ptr->rw_fapl_id, &(new_fa_ptr->rw_fapl_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy R/W FAPL"); + if(H5FD__copy_plist(old_fa_ptr->wo_fapl_id, &(new_fa_ptr->wo_fapl_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy W/O FAPL"); + + ret_value = (void *)new_fa_ptr; + +done: + if(NULL == ret_value) + if(new_fa_ptr) + new_fa_ptr = H5FL_FREE(H5FD_splitter_fapl_t, new_fa_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_fapl_copy() */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD__splitter_fapl_free + * + * Purpose: Releases the file access lists + * + * Return: SUCCEED/FAIL + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_fapl_free(void *_fapl) +{ + H5FD_splitter_fapl_t *fapl = (H5FD_splitter_fapl_t*)_fapl; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(fapl); + + if(H5I_dec_ref(fapl->rw_fapl_id) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "can't close R/W FAPL ID") + if(H5I_dec_ref(fapl->wo_fapl_id) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "can't close W/O FAPL ID") + + /* Free the property list */ + fapl = H5FL_FREE(H5FD_splitter_fapl_t, fapl); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_fapl_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_open + * + * Purpose: Create and/or opens a file as an HDF5 file. + * + * Return: Success: A pointer to a new file data structure. The + * public fields will be initialized by the + * caller, which is always H5FD_open(). + * Failure: NULL + *------------------------------------------------------------------------- + */ +static H5FD_t * +H5FD__splitter_open(const char *name, unsigned flags, hid_t splitter_fapl_id, haddr_t maxaddr) +{ + H5FD_splitter_t *file_ptr = NULL; /* Splitter VFD info */ + const H5FD_splitter_fapl_t *fapl_ptr = NULL; /* Driver-specific property list */ + H5P_genplist_t *plist_ptr = NULL; + H5FD_t *ret_value = NULL; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + if(!name || !*name) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name") + if(0 == maxaddr || HADDR_UNDEF == maxaddr) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr") + if(ADDR_OVERFLOW(maxaddr)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr") + if((H5P_FILE_ACCESS_DEFAULT == splitter_fapl_id) || + (H5FD_SPLITTER != H5Pget_driver(splitter_fapl_id)) ) + /* presupposes that H5P_FILE_ACCESS_DEFAULT is not a splitter */ + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "driver is not splitter") + + file_ptr = (H5FD_splitter_t *)H5FL_CALLOC(H5FD_splitter_t); + if(NULL == file_ptr) + HGOTO_ERROR(H5E_VFL, H5E_CANTALLOC, NULL, "unable to allocate file struct") + file_ptr->fa.rw_fapl_id = H5I_INVALID_HID; + file_ptr->fa.wo_fapl_id = H5I_INVALID_HID; + + /* Get the driver-specific file access properties */ + plist_ptr = (H5P_genplist_t *)H5I_object(splitter_fapl_id); + if(NULL == plist_ptr) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") + fapl_ptr = (const H5FD_splitter_fapl_t *)H5P_peek_driver_info(plist_ptr); + if(NULL == fapl_ptr) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "unable to get VFL driver info") + + /* Copy simpler info */ + HDstrncpy(file_ptr->fa.wo_path, fapl_ptr->wo_path, H5FD_SPLITTER_PATH_MAX); + HDstrncpy(file_ptr->fa.log_file_path, fapl_ptr->log_file_path, H5FD_SPLITTER_PATH_MAX); + file_ptr->fa.ignore_wo_errs = fapl_ptr->ignore_wo_errs; + + /* Copy R/W and W/O channel FAPLs. */ + if(H5FD__copy_plist(fapl_ptr->rw_fapl_id, &(file_ptr->fa.rw_fapl_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy R/W FAPL"); + if(H5FD__copy_plist(fapl_ptr->wo_fapl_id, &(file_ptr->fa.wo_fapl_id)) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "can't copy W/O FAPL"); + + /* Prepare log file if necessary. + * If application wants to ignore the errors from W/O channel and + * provided a name for the log file, then open it + */ + if(!file_ptr->logfp) { + if(file_ptr->fa.log_file_path[0] != '\0') { + file_ptr->logfp = HDfopen(file_ptr->fa.log_file_path, "w"); + if(file_ptr->logfp == NULL) + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open log file") + } /* end if logfile path given */ + } /* end if logfile pointer/handle does not exist */ + + file_ptr->rw_file = H5FD_open(name, flags, fapl_ptr->rw_fapl_id, HADDR_UNDEF); + if(!file_ptr->rw_file) + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open R/W file") + + file_ptr->wo_file = H5FD_open(fapl_ptr->wo_path, flags, fapl_ptr->wo_fapl_id, HADDR_UNDEF); + if(!file_ptr->wo_file) + H5FD_SPLITTER_WO_ERROR(file_ptr, FUNC, H5E_VFL, H5E_CANTOPENFILE, NULL, "unable to open W/O file") + + ret_value = (H5FD_t*)file_ptr; + +done: + if(NULL == ret_value) { + if(file_ptr) { + if(H5I_INVALID_HID != file_ptr->fa.rw_fapl_id) + H5I_dec_ref(file_ptr->fa.rw_fapl_id); + if(H5I_INVALID_HID != file_ptr->fa.wo_fapl_id) + H5I_dec_ref(file_ptr->fa.wo_fapl_id); + if(file_ptr->rw_file) + H5FD_close(file_ptr->rw_file); + if(file_ptr->wo_file) + H5FD_close(file_ptr->wo_file); + if(file_ptr->logfp) + HDfclose(file_ptr->logfp); + H5FL_FREE(H5FD_splitter_t, file_ptr); + } + } /* end if error */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_open() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_close + * + * Purpose: Closes files on both read-write and write-only channels. + * + * Return: Success: SUCCEED + * Failure: FAIL, file not closed. + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_close(H5FD_t *_file) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + + if(H5I_dec_ref(file->fa.rw_fapl_id) < 0) + HGOTO_ERROR(H5E_VFL, H5E_ARGS, FAIL, "can't close R/W FAPL") + if(H5I_dec_ref(file->fa.wo_fapl_id) < 0) + HGOTO_ERROR(H5E_VFL, H5E_ARGS, FAIL, "can't close W/O FAPL") + + if(file->rw_file) + if(H5FD_close(file->rw_file) == FAIL) + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close R/W file") + if(file->wo_file) + if(H5FD_close(file->wo_file) == FAIL) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close W/O file") + + if(file->logfp) { + HDfclose(file->logfp); + file->logfp = NULL; + } + + /* Release the file info */ + file = H5FL_FREE(H5FD_splitter_t, file); + file = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_close() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_get_eoa + * + * Purpose: Returns the end-of-address marker for the file. The EOA + * marker is the first address past the last byte allocated in + * the format address space. + * + * Return: Success: The end-of-address-marker + * + * Failure: HADDR_UNDEF + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD__splitter_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_splitter_t *file = (const H5FD_splitter_t *)_file; + haddr_t ret_value = HADDR_UNDEF; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if((ret_value = H5FD_get_eoa(file->rw_file, type)) == HADDR_UNDEF) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, HADDR_UNDEF, "unable to get eoa") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_get_eoa */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_set_eoa + * + * Purpose: Set the end-of-address marker for the file. This function is + * called shortly after an existing HDF5 file is opened in order + * to tell the driver where the end of the HDF5 data is located. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC) + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + HDassert(file->wo_file); + + if(H5FD_set_eoa(file->rw_file, type, addr) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTSET, FAIL, "H5FDset_eoa failed for R/W file") + + if(H5FD_set_eoa(file->wo_file, type, addr) < 0) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTSET, FAIL, "unable to set EOA for W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_set_eoa() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_get_eof + * + * Purpose: Returns the end-of-address marker for the file. The EOA + * marker is the first address past the last byte allocated in + * the format address space. + * + * Return: Success: The end-of-address-marker + * + * Failure: HADDR_UNDEF + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD__splitter_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_splitter_t *file = (const H5FD_splitter_t *)_file; + haddr_t ret_value = HADDR_UNDEF; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if(HADDR_UNDEF == (ret_value = H5FD_get_eof(file->rw_file, type))) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, HADDR_UNDEF, "unable to get eof") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_get_eof */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_truncate + * + * Purpose: Notify driver to truncate the file back to the allocated size. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + HDassert(file); + HDassert(file->rw_file); + HDassert(file->wo_file); + + if(H5FDtruncate(file->rw_file, dxpl_id, closing) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "unable to truncate R/W file") + + if(H5FDtruncate(file->wo_file, dxpl_id, closing) < 0) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTUPDATE, FAIL, "unable to truncate W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_truncate */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_sb_size + * + * Purpose: Obtains the number of bytes required to store the driver file + * access data in the HDF5 superblock. + * + * Return: Success: Number of bytes required. + * + * Failure: 0 if an error occurs or if the driver has no + * data to store in the superblock. + * + * NOTE: no public API for H5FD_sb_size, it needs to be added + *------------------------------------------------------------------------- + */ +static hsize_t +H5FD__splitter_sb_size(H5FD_t *_file) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + hsize_t ret_value = 0; + + FUNC_ENTER_STATIC_NOERR + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if(file->rw_file) + ret_value = H5FD_sb_size(file->rw_file); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_sb_size */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_sb_encode + * + * Purpose: Encode driver-specific data into the output arguments. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_sb_encode(H5FD_t *_file, char *name/*out*/, unsigned char *buf/*out*/) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if(file->rw_file && H5FD_sb_encode(file->rw_file, name, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTENCODE, FAIL, "unable to encode the superblock in R/W file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_sb_encode */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_sb_decode + * + * Purpose: Decodes the driver information block. + * + * Return: SUCCEED/FAIL + * + * NOTE: no public API for H5FD_sb_size, need to add + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_sb_decode(H5FD_t *_file, const char *name, const unsigned char *buf) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Sanity check */ + HDassert(file); + HDassert(file->rw_file); + + if(H5FD_sb_load(file->rw_file, name, buf) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTDECODE, FAIL, "unable to decode the superblock in R/W file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_sb_decode */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_cmp + * + * Purpose: Compare the keys of two files. + * + * Return: Success: A value like strcmp() + * Failure: Must never fail + *------------------------------------------------------------------------- + */ +static int +H5FD__splitter_cmp(const H5FD_t *_f1, const H5FD_t *_f2) +{ + const H5FD_splitter_t *f1 = (const H5FD_splitter_t *)_f1; + const H5FD_splitter_t *f2 = (const H5FD_splitter_t *)_f2; + herr_t ret_value = 0; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + H5FD_SPLITTER_LOG_CALL(FUNC); + + HDassert(f1); + HDassert(f2); + + ret_value = H5FD_cmp(f1->rw_file, f2->rw_file); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_cmp */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD__splitter_get_handle + * + * Purpose: Returns a pointer to the file handle of low-level virtual + * file driver. + * + * Return: SUCCEED/FAIL + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, + void **file_handle) +{ + H5FD_splitter_t *file = (H5FD_splitter_t*)_file; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + HDassert(file_handle); + + /* Only do for R/W channel */ + if(H5FD_get_vfd_handle(file->rw_file, file->fa.rw_fapl_id, file_handle) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to get handle of R/W file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_get_handle */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD__splitter_lock + * + * Purpose: Sets a file lock. + * + * Return: SUCCEED/FAIL + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_lock(H5FD_t *_file, hbool_t rw) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; /* VFD file struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + HDassert(file); + HDassert(file->rw_file); + + /* Place the lock on each file */ + if(H5FD_lock(file->rw_file, rw) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock R/W file") + + if(file->wo_file != NULL) + if(H5FD_lock(file->wo_file, rw) < 0) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTLOCKFILE, FAIL, "unable to lock W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_lock */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD__splitter_unlock + * + * Purpose: Removes a file lock. + * + * Return: SUCCEED/FAIL + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_unlock(H5FD_t *_file) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; /* VFD file struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + + /* Remove the lock on each file */ + if(H5FD_unlock(file->rw_file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock R/W file") + + if(file->wo_file != NULL) + if(H5FD_unlock(file->wo_file) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTUNLOCKFILE, FAIL, "unable to unlock W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_unlock */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_query + * + * Purpose: Set the flags that this VFL driver is capable of supporting. + * (listed in H5FDpublic.h) + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_query(const H5FD_t *_file, unsigned long *flags /* out */) +{ + const H5FD_splitter_t *file = (const H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + if(file) { + HDassert(file); + HDassert(file->rw_file); + + if(H5FDquery(file->rw_file, flags) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTLOCK, FAIL, "unable to query R/W file"); + } + else { + /* There is no file. Because this is a pure passthrough VFD, + * it has no features of its own. + */ + if(flags) + *flags = 0; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_query() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_alloc + * + * Purpose: Allocate file memory. + * + * Return: Address of allocated space (HADDR_UNDEF if error). + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD__splitter_alloc(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; /* VFD file struct */ + haddr_t ret_value = HADDR_UNDEF; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + + /* Allocate memory for each file, only return the return value for R/W file. */ + if((ret_value = H5FDalloc(file->rw_file, type, dxpl_id, size)) == HADDR_UNDEF) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "unable to allocate for R/W file") + + if(H5FDalloc(file->wo_file, type, dxpl_id, size) == HADDR_UNDEF) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "unable to alloc for W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_alloc() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_get_type_map + * + * Purpose: Retrieve the memory type mapping for this file + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_get_type_map(const H5FD_t *_file, H5FD_mem_t *type_map) +{ + const H5FD_splitter_t *file = (const H5FD_splitter_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + + /* Retrieve memory type mapping for R/W channel only */ + if(H5FD_get_fs_type_map(file->rw_file, type_map) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTGET, FAIL, "unable to allocate for R/W file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_get_type_map() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_free + * + * Purpose: Free the resources for the splitter VFD. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_free(H5FD_t *_file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size) +{ + H5FD_splitter_t *file = (H5FD_splitter_t *)_file; /* VFD file struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(file); + HDassert(file->rw_file); + + if(H5FDfree(file->rw_file, type, dxpl_id, addr, size) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTFREE, FAIL, "unable to free for R/W file") + + if(H5FDfree(file->wo_file, type, dxpl_id, addr, size) < 0) + H5FD_SPLITTER_WO_ERROR(file, FUNC, H5E_VFL, H5E_CANTINIT, FAIL, "unable to free for W/O file") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_free() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD__splitter_log_error + * + * Purpose: Log an error from the W/O channel appropriately. + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5FD__splitter_log_error(const H5FD_splitter_t *file, const char *atfunc, const char *msg) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC_NOERR + + H5FD_SPLITTER_LOG_CALL(FUNC); + + /* Check arguments */ + HDassert(file); + HDassert(atfunc && *atfunc); + HDassert(msg && *msg); + + if(file->logfp != NULL) { + size_t size; + char *s; + + size = HDstrlen(atfunc) + HDstrlen(msg) + 3; /* ':', ' ', '\n' */ + s = (char *)HDmalloc(sizeof(char) * (size + 1)); + if(NULL == s) + ret_value = FAIL; + else if(size < (size_t)HDsnprintf(s, size+1, "%s: %s\n", atfunc, msg)) + ret_value = FAIL; + else if(size != HDfwrite(s, 1, size, file->logfp)) + ret_value = FAIL; + HDfree(s); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD__splitter_log_error() */ + diff --git a/src/H5FDsplitter.h b/src/H5FDsplitter.h new file mode 100644 index 0000000..5a5ef29 --- /dev/null +++ b/src/H5FDsplitter.h @@ -0,0 +1,99 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: The public header file for the "splitter" driver. + */ + +#ifndef H5FDsplitter_H +#define H5FDsplitter_H + +#define H5FD_SPLITTER (H5FD_splitter_init()) + +/* The version of the H5FD_splitter_vfd_config_t structure used */ +#define H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION 1 + +/* Maximum length of a filename/path string in the Write-Only channel, + * including the NULL-terminator. + */ +#define H5FD_SPLITTER_PATH_MAX 4096 + +/* Semi-unique constant used to help identify structure pointers */ +#define H5FD_SPLITTER_MAGIC 0x2B916880 + +/* ---------------------------------------------------------------------------- + * Structure: H5FD_spliiter_vfd_config_t + * + * One-stop shopping for configuring a Splitter VFD (rather than many + * paramaters passed into H5Pset/get functions). + * + * magic (int32_t) + * Semi-unique number, used to sanity-check that a given pointer is + * likely (or not) to be this structure type. MUST be first. + * If magic is not H5FD_SPLITTER_MAGIC, the structure (and/or pointer to) + * must be considered invalid. + * + * version (unsigned int) + * Version number of this structure -- informs component membership. + * If not H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION, the structure (and/or + * pointer to) must be considered invalid. + * + * rw_fapl_id (hid_t) + * Library-given identification number of the Read/Write channel driver + * File Access Property List. + * The driver must support read/write access. + * Must be set to H5P_DEFAULT or a valid FAPL ID. + * + * wo_fapl_id (hid_t) + * Library-given identification number of the Read/Write channel driver + * File Access Property List. + * The driver feature flags must include H5FD_FEAT_DEFAULT_VFD_COMPAITBLE. + * Must be set to H5P_DEFAULT or a valid FAPL ID. + * + * wo_file_path (char[H5FD_SPLITTER_PATH_MAX + 1]) + * String buffer for the Write-Only channel target file. + * Must be null-terminated, cannot be empty. + * + * log_file_path (char[H5FD_SPLITTER_PATH_MAX + 1]) + * String buffer for the Splitter VFD logging output. + * Must be null-terminated. + * If null, no logfile is created. + * + * ignore_wo_errors (hbool_t) + * Toggle flag for how judiciously to respond to errors on the Write-Only + * channel. + * + * ---------------------------------------------------------------------------- + */ +typedef struct H5FD_splitter_vfd_config_t { + int32_t magic; + unsigned int version; + hid_t rw_fapl_id; + hid_t wo_fapl_id; + char wo_path[H5FD_SPLITTER_PATH_MAX + 1]; + char log_file_path[H5FD_SPLITTER_PATH_MAX + 1]; + hbool_t ignore_wo_errs; +} H5FD_splitter_vfd_config_t; + +#ifdef __cplusplus +extern "C" { +#endif +H5_DLL hid_t H5FD_splitter_init(void); +H5_DLL herr_t H5Pset_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_ptr); +H5_DLL herr_t H5Pget_fapl_splitter(hid_t fapl_id, H5FD_splitter_vfd_config_t *config_ptr); + +#ifdef __cplusplus +} +#endif + +#endif + diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c index d29a1b4..a769e86 100644 --- a/src/H5FDstdio.c +++ b/src/H5FDstdio.c @@ -52,6 +52,9 @@ /* The driver identification number, initialized at runtime */ static hid_t H5FD_STDIO_g = 0; +/* Whether to ignore file locks when disabled (env var value) */ +static htri_t ignore_disabled_file_locks_s = -1; + /* The maximum number of bytes which can be written in a single I/O operation */ static size_t H5_STDIO_MAX_IO_BYTES_g = (size_t)-1; @@ -82,6 +85,7 @@ typedef struct H5FD_stdio_t { haddr_t eof; /* end of file; current file size */ haddr_t pos; /* current file I/O position */ unsigned write_access; /* Flag to indicate the file was opened with write access */ + hbool_t ignore_disabled_file_locks; H5FD_stdio_file_op op; /* last operation */ #ifndef H5_HAVE_WIN32_API /* On most systems the combination of device and i-node number uniquely @@ -231,11 +235,23 @@ static const H5FD_class_t H5FD_stdio_g = { hid_t H5FD_stdio_init(void) { + char *lock_env_var = NULL; /* Environment variable pointer */ + /* Clear the error stack */ H5Eclear2(H5E_DEFAULT); - if (H5I_VFL!=H5Iget_type(H5FD_STDIO_g)) + /* Check the use disabled file locks environment variable */ + lock_env_var = getenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && !strcmp(lock_env_var, "BEST_EFFORT")) + ignore_disabled_file_locks_s = 1; /* Override: Ignore disabled locks */ + else if(lock_env_var && (!strcmp(lock_env_var, "TRUE") || !strcmp(lock_env_var, "1"))) + ignore_disabled_file_locks_s = 0; /* Override: Don't ignore disabled locks */ + else + ignore_disabled_file_locks_s = -1; /* Environment variable not set, or not set correctly */ + + if (H5I_VFL != H5Iget_type(H5FD_STDIO_g)) H5FD_STDIO_g = H5FDregister(&H5FD_stdio_g); + return H5FD_STDIO_g; } /* end H5FD_stdio_init() */ @@ -318,7 +334,7 @@ H5Pset_fapl_stdio(hid_t fapl_id) *------------------------------------------------------------------------- */ static H5FD_t * -H5FD_stdio_open( const char *name, unsigned flags, hid_t /*UNUSED*/ fapl_id, +H5FD_stdio_open( const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) { FILE *f = NULL; @@ -396,6 +412,21 @@ H5FD_stdio_open( const char *name, unsigned flags, hid_t /*UNUSED*/ fapl_id, file->eof = (haddr_t)x; } + /* Check the file locking flags in the fapl */ + if(ignore_disabled_file_locks_s != -1) + /* The environment variable was set, so use that preferentially */ + file->ignore_disabled_file_locks = ignore_disabled_file_locks_s; + else { + hbool_t unused; + + /* Use the value in the property list */ + if(H5Pget_file_locking(fapl_id, &unused, &file->ignore_disabled_file_locks) < 0) { + free(file); + fclose(f); + H5Epush_ret(func, H5E_ERR_CLS, H5E_FILE, H5E_CANTGET, "unable to get use disabled file locks property", NULL); + } + } + /* Get the file descriptor (needed for truncate and some Windows information) */ #ifdef H5_HAVE_WIN32_API file->fd = _fileno(file->fp); @@ -1104,10 +1135,13 @@ H5FD_stdio_lock(H5FD_t *_file, hbool_t rw) /* Place a non-blocking lock on the file */ if(flock(file->fd, lock_flags | LOCK_NB) < 0) { - if(ENOSYS == errno) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)", -1) + if(file->ignore_disabled_file_locks && ENOSYS == errno) + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; else - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file lock failed", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTLOCKFILE, "file lock failed", -1) } /* end if */ /* Flush the stream */ @@ -1152,10 +1186,13 @@ H5FD_stdio_unlock(H5FD_t *_file) /* Place a non-blocking lock on the file */ if(flock(file->fd, LOCK_UN) < 0) { - if(ENOSYS == errno) - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file locking disabled on this file system (use HDF5_USE_FILE_LOCKING environment variable to override)", -1) + if(file->ignore_disabled_file_locks && ENOSYS == errno) + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. + */ + errno = 0; else - H5Epush_ret(func, H5E_ERR_CLS, H5E_IO, H5E_FCNTL, "file unlock failed", -1) + H5Epush_ret(func, H5E_ERR_CLS, H5E_VFL, H5E_CANTUNLOCKFILE, "file unlock failed", -1) } /* end if */ #endif /* H5_HAVE_FLOCK */ diff --git a/src/H5Fint.c b/src/H5Fint.c index eb023d6..29bbd45 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -77,6 +77,7 @@ static herr_t H5F__get_objects(const H5F_t *f, unsigned types, size_t max_index, static int H5F__get_objects_cb(void *obj_ptr, hid_t obj_id, void *key); static herr_t H5F__build_name(const char *prefix, const char *file_name, char **full_name/*out*/); static char *H5F__getenv_prefix_name(char **env_prefix/*in,out*/); +static herr_t H5F__check_if_using_file_locks(H5P_genplist_t *fapl, hbool_t *use_file_locking); static herr_t H5F__build_actual_name(const H5F_t *f, const H5P_genplist_t *fapl, const char *name, char ** /*out*/ actual_name); static herr_t H5F__flush_phase1(H5F_t *f); static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); @@ -86,6 +87,15 @@ static herr_t H5F__flush_phase2(H5F_t *f, hbool_t closing); /* Package Variables */ /*********************/ +/* Package initialization variable */ +hbool_t H5_PKG_INIT_VAR = FALSE; + +/* Based on the value of the HDF5_USE_FILE_LOCKING environment variable. + * TRUE/FALSE have obvious meanings. FAIL means the environment variable was + * not set, so the code should ignore it and use the fapl value instead. + */ +htri_t use_locks_env_g = FAIL; + /*****************************/ /* Library Private Variables */ @@ -102,6 +112,123 @@ H5FL_DEFINE(H5F_t); /* Declare a free list to manage the H5F_shared_t struct */ H5FL_DEFINE(H5F_shared_t); +/* File ID class */ +static const H5I_class_t H5I_FILE_CLS[1] = {{ + H5I_FILE, /* ID class value */ + 0, /* Class flags */ + 0, /* # of reserved IDs for class */ + (H5I_free_t)H5F__close_cb /* Callback routine for closing objects of this class */ +}}; + + + +/*-------------------------------------------------------------------------- +NAME + H5F__init_package -- Initialize interface-specific information +USAGE + herr_t H5F__init_package() +RETURNS + Non-negative on success/Negative on failure +DESCRIPTION + Initializes any interface-specific data or routines. + +--------------------------------------------------------------------------*/ +herr_t +H5F__init_package(void) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* + * Initialize the atom group for the file IDs. + */ + if(H5I_register_type(H5I_FILE_CLS) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to initialize interface") + + /* Check the file locking environment variable */ + if(H5F__parse_file_lock_env_var(&use_locks_env_g) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to parse file locking environment variable") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F__init_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_term_package + * + * Purpose: Terminate this interface: free all memory and reset global + * variables to their initial values. Release all ID groups + * associated with this interface. + * + * Return: Success: Positive if anything was done that might + * have affected other interfaces; + * zero otherwise. + * + * Failure: Never fails + * + *------------------------------------------------------------------------- + */ +int +H5F_term_package(void) +{ + int n = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if(H5_PKG_INIT_VAR) { + if(H5I_nmembers(H5I_FILE) > 0) { + (void)H5I_clear_type(H5I_FILE, FALSE, FALSE); + n++; /*H5I*/ + } /* end if */ + else { + /* Make certain we've cleaned up all the shared file objects */ + H5F_sfile_assert_num(0); + + /* Destroy the file object id group */ + n += (H5I_dec_type_ref(H5I_FILE) > 0); + + /* Mark closed */ + if(0 == n) + H5_PKG_INIT_VAR = FALSE; + } /* end else */ + } /* end if */ + + FUNC_LEAVE_NOAPI(n) +} /* end H5F_term_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__parse_file_lock_env_var + * + * Purpose: Parses the HDF5_USE_FILE_LOCKING environment variable. + * + * NOTE: This is done in a separate function so we can call it from + * the test code. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__parse_file_lock_env_var(htri_t *use_locks) +{ + char *lock_env_var = NULL; /* Environment variable pointer */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check the file locking environment variable */ + lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); + if(lock_env_var && (!HDstrcmp(lock_env_var, "FALSE") || !HDstrcmp(lock_env_var, "0"))) + *use_locks = FALSE; /* Override: Never use locks */ + else if(lock_env_var && (!HDstrcmp(lock_env_var, "TRUE") || !HDstrcmp(lock_env_var, "BEST_EFFORT") || !HDstrcmp(lock_env_var, "1"))) + *use_locks = TRUE; /* Override: Always use locks */ + else + *use_locks = FAIL; /* Environment variable not set, or not set correctly */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5F__parse_file_lock_env_var() */ /*------------------------------------------------------------------------- @@ -1334,6 +1461,52 @@ H5F__dest(H5F_t *f, hbool_t flush) /*------------------------------------------------------------------------- + * Function: H5F__check_if_using_file_locks + * + * Purpose: Determines if this file will use file locks. + * + * There are three ways that file locking can be controlled: + * + * 1) The configure/cmake option that sets the H5_USE_FILE_LOCKING + * symbol (which is used as the default fapl value). + * + * 2) The H5Pset_file_locking() API call, which will override + * the configuration default. + * + * 3) The HDF5_USE_FILE_LOCKING environment variable, which overrides + * everything above. + * + * The main reason to disable file locking is to prevent errors on file + * systems where locking is not supported or has been disabled (as is + * often the case in parallel file systems). + * + * Return: SUCCEED/FAIL + *------------------------------------------------------------------------- + */ +static herr_t +H5F__check_if_using_file_locks(H5P_genplist_t *fapl, hbool_t *use_file_locking) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Make sure the out parameter has a value */ + *use_file_locking = TRUE; + + /* Check the fapl property */ + if(H5P_get(fapl, H5F_ACS_USE_FILE_LOCKING_NAME, use_file_locking) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get use file locking flag") + + /* Check the environment variable */ + if(use_locks_env_g != FAIL) + *use_file_locking = (use_locks_env_g == TRUE) ? TRUE : FALSE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__check_if_using_file_locks() */ + + +/*------------------------------------------------------------------------- * Function: H5F_open * * Purpose: Opens (or creates) a file. This function understands the @@ -1422,8 +1595,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) hbool_t set_flag = FALSE; /*set the status_flags in the superblock */ hbool_t clear = FALSE; /*clear the status_flags */ hbool_t evict_on_close; /* evict on close value from plist */ - char *lock_env_var = NULL;/*env var pointer */ - hbool_t use_file_locking; /*read from env var */ + hbool_t use_file_locking = TRUE; /* Using file locks? */ hbool_t ci_load = FALSE; /* whether MDC ci load requested */ hbool_t ci_write = FALSE; /* whether MDC CI write requested */ H5F_t *ret_value = NULL; /*actual return value */ @@ -1441,15 +1613,13 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if(NULL == (drvr = H5FD_get_class(fapl_id))) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to retrieve VFL class") - /* Check the environment variable that determines if we care - * about file locking. File locking should be used unless explicitly - * disabled. - */ - lock_env_var = HDgetenv("HDF5_USE_FILE_LOCKING"); - if(lock_env_var && !HDstrcmp(lock_env_var, "FALSE")) - use_file_locking = FALSE; - else - use_file_locking = TRUE; + /* Get the file access property list, for future queries */ + if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") + + /* Check if we are using file locking */ + if (H5F__check_if_using_file_locks(a_plist, &use_file_locking) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file locking flag") /* * Opening a file is a two step process. First we try to open the @@ -1540,8 +1710,8 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) if(H5FD_lock(lf, (hbool_t)((flags & H5F_ACC_RDWR) ? TRUE : FALSE)) < 0) { /* Locking failed - Closing will remove the lock */ if(H5FD_close(lf) < 0) - HDONE_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to close low-level file info") - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to lock the file") + HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, NULL, "unable to close low-level file info") + HGOTO_ERROR(H5E_FILE, H5E_CANTLOCKFILE, NULL, "unable to lock the file") } /* end if */ /* Create the 'top' file structure */ @@ -1573,10 +1743,6 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) shared = file->shared; lf = shared->lf; - /* Get the file access property list, for future queries */ - if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") - /* Check if page buffering is enabled */ if(H5P_get(a_plist, H5F_ACS_PAGE_BUFFER_SIZE_NAME, &page_buf_size) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get page buffer size") @@ -1715,7 +1881,7 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id) /* Remove the file lock for SWMR_WRITE */ if(use_file_locking && (H5F_INTENT(file) & H5F_ACC_SWMR_WRITE)) { if(H5FD_unlock(file->shared->lf) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to unlock the file") + HGOTO_ERROR(H5E_FILE, H5E_CANTUNLOCKFILE, NULL, "unable to unlock the file") } /* end if */ } /* end if */ else { /* H5F_ACC_RDONLY: check consistency of status_flags */ diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index e4892ed..6e3f3d4 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -389,6 +389,10 @@ H5FL_EXTERN(H5F_t); /* Declare a free list to manage the H5F_shared_t struct */ H5FL_EXTERN(H5F_shared_t); +/* Whether or not to use file locking (based on the environment variable) + * FAIL means ignore the environment variable. + */ +H5_DLLVAR htri_t use_locks_env_g; /******************************/ /* Package Private Prototypes */ @@ -407,6 +411,7 @@ H5_DLL herr_t H5F__start_swmr_write(H5F_t *f); H5_DLL herr_t H5F__close(hid_t file_id); H5_DLL herr_t H5F__close_cb(H5F_t *f); H5_DLL herr_t H5F__set_libver_bounds(H5F_t *f, H5F_libver_t low, H5F_libver_t high); +H5_DLL herr_t H5F__parse_file_lock_env_var(htri_t *use_locks); /* File mount related routines */ H5_DLL herr_t H5F__mount(H5G_loc_t *loc, const char *name, H5F_t *child, hid_t plist_id); @@ -466,6 +471,7 @@ H5_DLL herr_t H5F__get_sohm_mesg_count_test(hid_t fid, unsigned type_id, size_t H5_DLL herr_t H5F__check_cached_stab_test(hid_t file_id); H5_DLL herr_t H5F__get_maxaddr_test(hid_t file_id, haddr_t *maxaddr); H5_DLL herr_t H5F__get_sbe_addr_test(hid_t file_id, haddr_t *sbe_addr); +H5_DLL herr_t H5F__reparse_file_lock_variable_test(void); #endif /* H5F_TESTING */ #endif /* _H5Fpkg_H */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 57318a7..fd161e2 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -516,6 +516,8 @@ typedef struct H5F_t H5F_t; #define H5F_ACS_PAGE_BUFFER_SIZE_NAME "page_buffer_size" /* the maximum size for the page buffer cache */ #define H5F_ACS_PAGE_BUFFER_MIN_META_PERC_NAME "page_buffer_min_meta_perc" /* the min metadata percentage for the page buffer cache */ #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_NAME "page_buffer_min_raw_perc" /* the min raw data percentage for the page buffer cache */ +#define H5F_ACS_USE_FILE_LOCKING_NAME "use_file_locking" /* whether or not we use file locks for SWMR control and to prevent multiple writers */ +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME "ignore_disabled_file_locks" /* whether or not we ignore "locks disabled" errors */ /* ======================== File Mount properties ====================*/ #define H5F_MNT_SYM_LOCAL_NAME "local" /* Whether absolute symlinks local to file. */ diff --git a/src/H5Ftest.c b/src/H5Ftest.c index df9c933..e9d697a 100644 --- a/src/H5Ftest.c +++ b/src/H5Ftest.c @@ -232,3 +232,36 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__get_sbe_addr_test() */ + +/*------------------------------------------------------------------------- + * Function: H5F__reparse_file_lock_variable_test + * + * Purpose: Re-parse the file locking environment variable. + * + * Since getenv(3) is fairly expensive, we only parse it once, + * when the library opens. This test function is used to + * re-parse the environment variable after we've changed it + * with setnev(3). + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Summer 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F__reparse_file_lock_variable_test(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check the file locking environment variable */ + if(H5F__parse_file_lock_env_var(&use_locks_env_g) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "unable to parse file locking environment variable") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__reparse_file_lock_variable_test() */ + diff --git a/src/H5Gpublic.h b/src/H5Gpublic.h index 170b74d..3f74d45 100644 --- a/src/H5Gpublic.h +++ b/src/H5Gpublic.h @@ -15,7 +15,7 @@ * * Created: H5Gpublic.h * Jul 11 1997 - * Robb Matzke <matzke@llnl.gov> + * Robb Matzke * * Purpose: Public declarations for the H5G package * diff --git a/src/H5Opublic.h b/src/H5Opublic.h index dcbf85a..2ef0c0d 100644 --- a/src/H5Opublic.h +++ b/src/H5Opublic.h @@ -15,7 +15,7 @@ * * Created: H5Opublic.h * Aug 5 1997 - * Robb Matzke <matzke@llnl.gov> + * Robb Matzke * * Purpose: Public declarations for the H5O (object header) * package. diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index b2a1a30..208c8dd 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -259,6 +259,29 @@ #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF 0 #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_ENC H5P__encode_unsigned #define H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEC H5P__decode_unsigned +/* Definition for using file locking or not. The default is set + * via the configure step. + */ +#define H5F_ACS_USE_FILE_LOCKING_SIZE sizeof(hbool_t) +#if defined H5_USE_FILE_LOCKING && H5_USE_FILE_LOCKING +#define H5F_ACS_USE_FILE_LOCKING_DEF TRUE +#else +#define H5F_ACS_USE_FILE_LOCKING_DEF FALSE +#endif +#define H5F_ACS_USE_FILE_LOCKING_ENC H5P__encode_hbool_t +#define H5F_ACS_USE_FILE_LOCKING_DEC H5P__decode_hbool_t +/* Definition for whether we ignore file locking errors when we can + * tell that file locking has been disabled on the file system. + * The default is set via the configure step. + */ +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_SIZE sizeof(hbool_t) +#if defined H5_IGNORE_DISABLED_FILE_LOCKS && H5_IGNORE_DISABLED_FILE_LOCKS +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF TRUE +#else +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF FALSE +#endif +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC H5P__encode_hbool_t +#define H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC H5P__decode_hbool_t /******************/ @@ -398,6 +421,8 @@ static const H5AC_cache_image_config_t H5F_def_mdc_initCacheImageCfg_g = H5F_ACS static const size_t H5F_def_page_buf_size_g = H5F_ACS_PAGE_BUFFER_SIZE_DEF; /* Default page buffer size */ static const unsigned H5F_def_page_buf_min_meta_perc_g = H5F_ACS_PAGE_BUFFER_MIN_META_PERC_DEF; /* Default page buffer minimum metadata size */ static const unsigned H5F_def_page_buf_min_raw_perc_g = H5F_ACS_PAGE_BUFFER_MIN_RAW_PERC_DEF; /* Default page buffer mininum raw data size */ +static const hbool_t H5F_def_use_file_locking_g = H5F_ACS_USE_FILE_LOCKING_DEF; /* Default use file locking flag */ +static const hbool_t H5F_def_ignore_disabled_file_locks_g = H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEF; /* Default ignore disabled file locks flag */ /*------------------------------------------------------------------------- @@ -647,6 +672,16 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the use file locking flag */ + if(H5P__register_real(pclass, H5F_ACS_USE_FILE_LOCKING_NAME, H5F_ACS_USE_FILE_LOCKING_SIZE, &H5F_def_use_file_locking_g, + NULL, NULL, NULL, H5F_ACS_USE_FILE_LOCKING_ENC, H5F_ACS_USE_FILE_LOCKING_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the ignore disabled file locks flag */ + if(H5P__register_real(pclass, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_SIZE, &H5F_def_ignore_disabled_file_locks_g, + NULL, NULL, NULL, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_ENC, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_DEC, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__facc_reg_prop() */ @@ -4586,6 +4621,98 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_evict_on_close() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_file_locking + * + * Purpose: Sets the file locking property values. + * + * Overrides the default file locking flag setting that was + * set when the library was configured. + * + * Can be overridden by the HDF5_USE_FILE_LOCKING environment + * variable. + * + * File locking is used when creating/opening a file to prevent + * problematic file accesses. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ibb", fapl_id, use_file_locking, ignore_when_disabled); + + /* Make sure this is a fapl */ + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not a file access plist") + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set values */ + if(H5P_set(plist, H5F_ACS_USE_FILE_LOCKING_NAME, &use_file_locking) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set use file locking property") + if(H5P_set(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, &ignore_when_disabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set ignore disabled file locks property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_file_locking() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_file_locking + * + * Purpose: Gets the file locking property values. + * + * File locking is used when creating/opening a file to prevent + * problematic file accesses. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Spring 2020 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_file_locking(hid_t fapl_id, hbool_t *use_file_locking, hbool_t *ignore_when_disabled) +{ + H5P_genplist_t *plist; /* property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "i*b*b", fapl_id, use_file_locking, ignore_when_disabled); + + /* Make sure this is a fapl */ + if(TRUE != H5P_isa_class(fapl_id, H5P_FILE_ACCESS)) + HGOTO_ERROR(H5E_PLIST, H5E_CANTREGISTER, FAIL, "property list is not an access plist") + + /* Get the plist structure */ + if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get values */ + if(H5P_get(plist, H5F_ACS_USE_FILE_LOCKING_NAME, use_file_locking) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get use file locking property") + if(H5P_get(plist, H5F_ACS_IGNORE_DISABLED_FILE_LOCKS_NAME, ignore_when_disabled) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get ignore disabled file locks property") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_file_locking() */ + #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 5d6dea8..dbd0724 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -359,6 +359,8 @@ H5_DLL herr_t H5Pset_mdc_log_options(hid_t plist_id, hbool_t is_enabled, const c H5_DLL herr_t H5Pget_mdc_log_options(hid_t plist_id, hbool_t *is_enabled, char *location, size_t *location_size, hbool_t *start_on_access); H5_DLL herr_t H5Pset_evict_on_close(hid_t fapl_id, hbool_t evict_on_close); H5_DLL herr_t H5Pget_evict_on_close(hid_t fapl_id, hbool_t *evict_on_close); +H5_DLL herr_t H5Pset_file_locking(hid_t fapl_id, hbool_t use_file_locking, hbool_t ignore_when_disabled); +H5_DLL herr_t H5Pget_file_locking(hid_t fapl_id, hbool_t *use_file_locking, hbool_t *ignore_when_disabled); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5Pset_all_coll_metadata_ops(hid_t plist_id, hbool_t is_collective); H5_DLL herr_t H5Pget_all_coll_metadata_ops(hid_t plist_id, hbool_t *is_collective); @@ -1815,12 +1815,6 @@ done: * Programmer: Robb Matzke * Friday, January 9, 1998 * - * Modifications: - * - * Robb Matzke, 1 Jun 1998 - * It is illegal to lock a named datatype since we must allow named - * types to be closed (to release file resources) but locking a type - * prevents that. *------------------------------------------------------------------------- */ herr_t @@ -1893,9 +1887,6 @@ done: * Programmer: Robb Matzke * Monday, December 8, 1997 * - * Modifications: - * Broke out from H5Tget_class - QAK - 6/4/99 - * *------------------------------------------------------------------------- */ H5T_class_t @@ -2882,10 +2873,6 @@ done: * Programmer: Raymond Lu * July 14, 2004 * - * Modification:Raymond Lu - * 17 February 2011 - * I changed the value for the APP_REF parameter of H5I_register - * from FALSE to TRUE. *------------------------------------------------------------------------- */ hid_t @@ -3046,10 +3033,6 @@ done: * Programmer: Robb Matzke * Friday, December 5, 1997 * - * Modifications: - * Raymond Lu - * 19 May 2011 - * We support fixed size or variable-length string now. *------------------------------------------------------------------------- */ H5T_t * @@ -5055,11 +5038,6 @@ H5T_path_noop(const H5T_path_t *p) * Programmer: Raymond Lu * 8 June 2007 * - * Modifications: Neil Fortner - * 19 September 2008 - * Changed return value to H5T_subset_info_t - * (to allow it to return copy_size) - * *------------------------------------------------------------------------- */ H5T_subset_info_t * @@ -5154,15 +5132,18 @@ H5T_convert(H5T_path_t *tpath, hid_t src_id, hid_t dst_id, size_t nelmts, size_t buf_stride, size_t bkg_stride, void *buf, void *bkg) { #ifdef H5T_DEBUG - H5_timer_t timer; + H5_timer_t timer; /* Timer for conversion */ #endif herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) #ifdef H5T_DEBUG - if(H5DEBUG(T)) - H5_timer_begin(&timer); + if(H5DEBUG(T)) { + /* Initialize and start timer */ + H5_timer_init(&timer); + H5_timer_start(&timer); + } /* end if */ #endif /* Call the appropriate conversion callback */ @@ -5176,10 +5157,16 @@ H5T_convert(H5T_path_t *tpath, hid_t src_id, hid_t dst_id, size_t nelmts, HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCONVERT, FAIL, "datatype conversion failed") #ifdef H5T_DEBUG if(H5DEBUG(T)) { - H5_timer_end(&(tpath->stats.timer), &timer); + /* Stop timer */ + H5_timer_stop(&timer); + + /* Record elapsed timer info */ + H5_timer_get_times(timer, &tpath->stats.times); + + /* Increment # of calls and # of elements converted */ tpath->stats.ncalls++; tpath->stats.nelmts += nelmts; - } + } /* end if */ #endif done: diff --git a/src/H5Tdbg.c b/src/H5Tdbg.c index eb648f3..6188138 100644 --- a/src/H5Tdbg.c +++ b/src/H5Tdbg.c @@ -13,11 +13,11 @@ /*------------------------------------------------------------------------- * - * Created: H5Tdbg.c - * Jul 19 2007 - * Quincey Koziol <koziol@hdfgroup.org> + * Created: H5Tdbg.c + * Jul 19 2007 + * Quincey Koziol * - * Purpose: Dump debugging information about a datatype + * Purpose: Dump debugging information about a datatype * *------------------------------------------------------------------------- */ @@ -95,25 +95,24 @@ herr_t H5T__print_stats(H5T_path_t H5_ATTR_UNUSED * path, int H5_ATTR_UNUSED * nprint/*in,out*/) { -#ifdef H5T_DEBUG - hsize_t nbytes; - char bandwidth[32]; -#endif - FUNC_ENTER_PACKAGE_NOERR #ifdef H5T_DEBUG - if (H5DEBUG(T) && path->stats.ncalls > 0) { - if (nprint && 0 == (*nprint)++) { - HDfprintf(H5DEBUG(T), "H5T: type conversion statistics:\n"); - HDfprintf(H5DEBUG(T), " %-16s %10s %10s %8s %8s %8s %10s\n", - "Conversion", "Elmts", "Calls", "User", - "System", "Elapsed", "Bandwidth"); - HDfprintf(H5DEBUG(T), " %-16s %10s %10s %8s %8s %8s %10s\n", - "----------", "-----", "-----", "----", - "------", "-------", "---------"); - } - if (path->src && path->dst) + if(H5DEBUG(T) && path->stats.ncalls > 0) { + hsize_t nbytes; + char bandwidth[32]; + + if(nprint && 0 == (*nprint)++) { + HDfprintf(H5DEBUG(T), "H5T: type conversion statistics:\n"); + HDfprintf(H5DEBUG(T), " %-16s %10s %10s %8s %8s %8s %10s\n", + "Conversion", "Elmts", "Calls", "User", + "System", "Elapsed", "Bandwidth"); + HDfprintf(H5DEBUG(T), " %-16s %10s %10s %8s %8s %8s %10s\n", + "----------", "-----", "-----", "----", + "------", "-------", "---------"); + } /* end if */ + + if(path->src && path->dst) nbytes = MAX(H5T_get_size(path->src), H5T_get_size(path->dst)); else if (path->src) nbytes = H5T_get_size(path->src); @@ -121,18 +120,20 @@ H5T__print_stats(H5T_path_t H5_ATTR_UNUSED * path, int H5_ATTR_UNUSED * nprint/* nbytes = H5T_get_size(path->dst); else nbytes = 0; - nbytes *= path->stats.nelmts; - H5_bandwidth(bandwidth, (double)nbytes, path->stats.timer.etime); - HDfprintf(H5DEBUG(T), " %-16s %10Hd %10d %8.2f %8.2f %8.2f %10s\n", - path->name, - path->stats.nelmts, - path->stats.ncalls, - path->stats.timer.utime, - path->stats.timer.stime, - path->stats.timer.etime, - bandwidth); - } + + nbytes *= path->stats.nelmts; + H5_bandwidth(bandwidth, (double)nbytes, path->stats.times.elapsed); + HDfprintf(H5DEBUG(T), " %-16s %10Hd %10d %8T %8T %8T %10s\n", + path->name, + path->stats.nelmts, + path->stats.ncalls, + path->stats.times.user, + path->stats.times.system, + path->stats.times.elapsed, + bandwidth); + } /* end if */ #endif + FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5T__print_stats() */ @@ -405,18 +406,18 @@ H5T_debug(const H5T_t *dt, FILE *stream) } /* end else */ } else if (H5T_ENUM == dt->shared->type) { - size_t base_size; + size_t base_size; /* Enumeration data type */ HDfprintf(stream, " "); H5T_debug(dt->shared->parent, stream); base_size = dt->shared->parent->shared->size; for (i = 0; i < dt->shared->u.enumer.nmembs; i++) { - size_t k; + size_t k; HDfprintf(stream, "\n\"%s\" = 0x", dt->shared->u.enumer.name[i]); for (k = 0; k < base_size; k++) - HDfprintf(stream, "%02lx", (unsigned long)(dt->shared->u.enumer.value + (i * base_size) + k)); + HDfprintf(stream, "%02p", ((uint8_t *)dt->shared->u.enumer.value + (i * base_size) + k)); } /* end for */ HDfprintf(stream, "\n"); } diff --git a/src/H5Tpkg.h b/src/H5Tpkg.h index 05879ff..b8d2435 100644 --- a/src/H5Tpkg.h +++ b/src/H5Tpkg.h @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Monday, December 8, 1997 * * Purpose: This file contains declarations which are visible only within @@ -146,7 +146,7 @@ struct H5T_stats_t { unsigned ncalls; /*num calls to conversion function */ hsize_t nelmts; /*total data points converted */ - H5_timer_t timer; /*total time for conversion */ + H5_timevals_t times; /*total time for conversion */ }; /* Library internal datatype conversion functions are... */ @@ -12,13 +12,14 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Friday, October 10, 1997 */ #include "H5private.h" #include "H5Eprivate.h" +#include "H5MMprivate.h" /* Memory management */ #include "H5Oprivate.h" #include "H5VMprivate.h" @@ -33,21 +34,21 @@ typedef struct H5VM_memcpy_ud_t { /* Local prototypes */ static void -H5VM_stride_optimize1(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, +H5VM__stride_optimize1(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, const hsize_t *size, hsize_t *stride1); static void -H5VM_stride_optimize2(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, +H5VM__stride_optimize2(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, const hsize_t *size, hsize_t *stride1, hsize_t *stride2); #ifdef LATER static void -H5VM_stride_copy2(hsize_t nelmts, hsize_t elmt_size, +H5VM__stride_copy2(hsize_t nelmts, hsize_t elmt_size, unsigned dst_n, const hsize_t *dst_size, const ssize_t *dst_stride, void *_dst, unsigned src_n, const hsize_t *src_size, const ssize_t *src_stride, const void *_src); #endif /* LATER */ /*------------------------------------------------------------------------- - * Function: H5VM_stride_optimize1 + * Function: H5VM__stride_optimize1 * * Purpose: Given a stride vector which references elements of the * specified size, optimize the dimensionality, the stride @@ -62,15 +63,13 @@ H5VM_stride_copy2(hsize_t nelmts, hsize_t elmt_size, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static void -H5VM_stride_optimize1(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, +H5VM__stride_optimize1(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, const hsize_t *size, hsize_t *stride1) { - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* * This has to be true because if we optimize the dimensionality down to @@ -93,7 +92,7 @@ H5VM_stride_optimize1(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, /*------------------------------------------------------------------------- - * Function: H5VM_stride_optimize2 + * Function: H5VM__stride_optimize2 * * Purpose: Given two stride vectors which reference elements of the * specified size, optimize the dimensionality, the stride @@ -108,18 +107,13 @@ H5VM_stride_optimize1(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * Unrolled loops for common cases - * Quincey Koziol - * ?, ? ?, 2001? - * *------------------------------------------------------------------------- */ static void -H5VM_stride_optimize2(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, +H5VM__stride_optimize2(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, const hsize_t *size, hsize_t *stride1, hsize_t *stride2) { - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR /* * This has to be true because if we optimize the dimensionality down to @@ -246,11 +240,6 @@ H5VM_stride_optimize2(unsigned *np/*in,out*/, hsize_t *elmt_size/*in,out*/, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * Unrolled loops for common cases - * Quincey Koziol - * ?, ? ?, 2001? - * *------------------------------------------------------------------------- */ hsize_t @@ -349,8 +338,6 @@ H5VM_hyper_stride(unsigned n, const hsize_t *size, * Programmer: Robb Matzke * Friday, October 17, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ htri_t @@ -400,8 +387,6 @@ done: * Programmer: Robb Matzke * Friday, October 10, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t @@ -438,7 +423,7 @@ H5VM_hyper_fill(unsigned n, const hsize_t *_size, /* Compute an optimal destination stride vector */ dst_start = H5VM_hyper_stride(n, size, total_size, offset, dst_stride); - H5VM_stride_optimize1(&n, &elmt_size, size, dst_stride); + H5VM__stride_optimize1(&n, &elmt_size, size, dst_stride); /* Copy */ ret_value = H5VM_stride_fill(n, elmt_size, size, dst_stride, dst+dst_start, @@ -475,11 +460,6 @@ H5VM_hyper_fill(unsigned n, const hsize_t *_size, * Programmer: Robb Matzke * Friday, October 10, 1997 * - * Modifications: - * Unrolled loops for common cases - * Quincey Koziol - * ?, ? ?, 2001? - * *------------------------------------------------------------------------- */ herr_t @@ -623,7 +603,7 @@ H5VM_hyper_copy(unsigned n, const hsize_t *_size, #endif /* NO_INLINED_CODE */ /* Optimize the strides as a pair */ - H5VM_stride_optimize2(&n, &elmt_size, size, dst_stride, src_stride); + H5VM__stride_optimize2(&n, &elmt_size, size, dst_stride, src_stride); /* Perform the copy in terms of stride */ ret_value = H5VM_stride_copy(n, elmt_size, size, @@ -644,8 +624,6 @@ H5VM_hyper_copy(unsigned n, const hsize_t *_size, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t @@ -704,8 +682,6 @@ H5VM_stride_fill(unsigned n, hsize_t elmt_size, const hsize_t *size, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t @@ -773,8 +749,6 @@ H5VM_stride_copy(unsigned n, hsize_t elmt_size, const hsize_t *size, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t @@ -827,7 +801,7 @@ H5VM_stride_copy_s(unsigned n, hsize_t elmt_size, const hsize_t *size, #ifdef LATER /*------------------------------------------------------------------------- - * Function: H5VM_stride_copy2 + * Function: H5VM__stride_copy2 * * Purpose: Similar to H5VM_stride_copy() except the source and * destination each have their own dimensionality and size and @@ -839,12 +813,10 @@ H5VM_stride_copy_s(unsigned n, hsize_t elmt_size, const hsize_t *size, * Programmer: Robb Matzke * Saturday, October 11, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static void -H5VM_stride_copy2(hsize_t nelmts, hsize_t elmt_size, +H5VM__stride_copy2(hsize_t nelmts, hsize_t elmt_size, /* destination */ unsigned dst_n, const hsize_t *dst_size, @@ -864,7 +836,7 @@ H5VM_stride_copy2(hsize_t nelmts, hsize_t elmt_size, int j; /* Local index variable */ hbool_t carry; - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR HDassert(elmt_size < SIZET_MAX); HDassert(dst_n>0); @@ -917,8 +889,6 @@ H5VM_stride_copy2(hsize_t nelmts, hsize_t elmt_size, * Programmer: Quincey Koziol * Thursday, June 18, 1998 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t @@ -974,8 +944,6 @@ H5VM_array_fill(void *_dst, const void *src, size_t size, size_t count) * Programmer: Quincey Koziol * Monday, April 28, 2003 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t @@ -1055,8 +1023,6 @@ H5VM_array_offset_pre(unsigned n, const hsize_t *acc, const hsize_t *offset) * Programmer: Quincey Koziol * Tuesday, June 22, 1999 * - * Modifications: - * *------------------------------------------------------------------------- */ hsize_t @@ -1139,8 +1105,6 @@ H5VM_array_calc_pre(hsize_t offset, unsigned n, const hsize_t *down, * Programmer: Quincey Koziol * Wednesday, April 16, 2003 * - * Modifications: - * *------------------------------------------------------------------------- */ herr_t diff --git a/src/H5VMprivate.h b/src/H5VMprivate.h index 26f59e2..5c975f6 100644 --- a/src/H5VMprivate.h +++ b/src/H5VMprivate.h @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Friday, October 10, 1997 */ #ifndef H5VMprivate_H @@ -28,16 +28,16 @@ typedef herr_t (*H5VM_opvv_func_t)(hsize_t dst_off, hsize_t src_off, size_t len, void *udata); /* Vector comparison functions like Fortran66 comparison operators */ -#define H5VM_vector_eq_s(N,V1,V2) (H5VM_vector_cmp_s (N, V1, V2)==0) -#define H5VM_vector_lt_s(N,V1,V2) (H5VM_vector_cmp_s (N, V1, V2)<0) -#define H5VM_vector_gt_s(N,V1,V2) (H5VM_vector_cmp_s (N, V1, V2)>0) -#define H5VM_vector_le_s(N,V1,V2) (H5VM_vector_cmp_s (N, V1, V2)<=0) -#define H5VM_vector_ge_s(N,V1,V2) (H5VM_vector_cmp_s (N, V1, V2)>=0) -#define H5VM_vector_eq_u(N,V1,V2) (H5VM_vector_cmp_u (N, V1, V2)==0) -#define H5VM_vector_lt_u(N,V1,V2) (H5VM_vector_cmp_u (N, V1, V2)<0) -#define H5VM_vector_gt_u(N,V1,V2) (H5VM_vector_cmp_u (N, V1, V2)>0) -#define H5VM_vector_le_u(N,V1,V2) (H5VM_vector_cmp_u (N, V1, V2)<=0) -#define H5VM_vector_ge_u(N,V1,V2) (H5VM_vector_cmp_u (N, V1, V2)>=0) +#define H5VM_vector_eq_s(N,V1,V2) (H5VM_vector_cmp_s(N, V1, V2) == 0) +#define H5VM_vector_lt_s(N,V1,V2) (H5VM_vector_cmp_s(N, V1, V2) < 0) +#define H5VM_vector_gt_s(N,V1,V2) (H5VM_vector_cmp_s(N, V1, V2) > 0) +#define H5VM_vector_le_s(N,V1,V2) (H5VM_vector_cmp_s(N, V1, V2) <= 0) +#define H5VM_vector_ge_s(N,V1,V2) (H5VM_vector_cmp_s(N, V1, V2) >= 0) +#define H5VM_vector_eq_u(N,V1,V2) (H5VM_vector_cmp_u(N, V1, V2) == 0) +#define H5VM_vector_lt_u(N,V1,V2) (H5VM_vector_cmp_u(N, V1, V2) < 0) +#define H5VM_vector_gt_u(N,V1,V2) (H5VM_vector_cmp_u(N, V1, V2) > 0) +#define H5VM_vector_le_u(N,V1,V2) (H5VM_vector_cmp_u(N, V1, V2) <= 0) +#define H5VM_vector_ge_u(N,V1,V2) (H5VM_vector_cmp_u(N, V1, V2) >= 0) /* Other functions */ #define H5VM_vector_cpy(N,DST,SRC) { \ @@ -146,6 +146,10 @@ H5_DLL ssize_t H5VM_memcpyvv(void *_dst, * elements in an array and array dimensions are always of type * size_t. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Success: Product of elements * * Failure: 1 if N is zero @@ -153,8 +157,6 @@ H5_DLL ssize_t H5VM_memcpyvv(void *_dst, * Programmer: Robb Matzke * Friday, October 10, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static H5_INLINE hsize_t H5_ATTR_UNUSED @@ -177,6 +179,10 @@ done: * * Purpose: Determines if all elements of a vector are zero. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Success: TRUE if all elements are zero, * FALSE otherwise * @@ -185,8 +191,6 @@ done: * Programmer: Robb Matzke * Friday, October 10, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static H5_INLINE htri_t H5_ATTR_UNUSED @@ -212,6 +216,10 @@ done: * * Purpose: Determines if all elements of a vector are zero. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Success: TRUE if all elements are zero, * FALSE otherwise * @@ -220,8 +228,6 @@ done: * Programmer: Robb Matzke * Friday, October 10, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static H5_INLINE htri_t H5_ATTR_UNUSED @@ -248,6 +254,10 @@ done: * Purpose: Compares two vectors of the same size and determines if V1 is * lexicographically less than, equal, or greater than V2. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Success: -1 if V1 is less than V2 * 0 if they are equal * 1 if V1 is greater than V2 @@ -257,12 +267,10 @@ done: * Programmer: Robb Matzke * Friday, October 10, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static H5_INLINE int H5_ATTR_UNUSED -H5VM_vector_cmp_u (unsigned n, const hsize_t *v1, const hsize_t *v2) +H5VM_vector_cmp_u(unsigned n, const hsize_t *v1, const hsize_t *v2) { int ret_value=0; /* Return value */ @@ -290,6 +298,10 @@ done: * Purpose: Compares two vectors of the same size and determines if V1 is * lexicographically less than, equal, or greater than V2. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Success: -1 if V1 is less than V2 * 0 if they are equal * 1 if V1 is greater than V2 @@ -299,12 +311,10 @@ done: * Programmer: Robb Matzke * Wednesday, April 8, 1998 * - * Modifications: - * *------------------------------------------------------------------------- */ static H5_INLINE int H5_ATTR_UNUSED -H5VM_vector_cmp_s (unsigned n, const hssize_t *v1, const hssize_t *v2) +H5VM_vector_cmp_s(unsigned n, const hssize_t *v1, const hssize_t *v2) { int ret_value=0; /* Return value */ @@ -331,13 +341,15 @@ done: * * Purpose: Increments V1 by V2 * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: void * * Programmer: Robb Matzke * Monday, October 13, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ static H5_INLINE void H5_ATTR_UNUSED @@ -380,6 +392,10 @@ static const unsigned char LogTable256[] = * The version on the web-site is for 32-bit quantities and this * version has been extended for 64-bit quantities. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: log2(n) (always - no failure condition) * * Programmer: Quincey Koziol @@ -428,6 +444,10 @@ static const unsigned MultiplyDeBruijnBitPosition[32] = * This is from the "Bit Twiddling Hacks" at: * http://graphics.stanford.edu/~seander/bithacks.html#IntegerLogDeBruijn * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: log2(n) (always - no failure condition) * * Programmer: Quincey Koziol @@ -450,6 +470,10 @@ H5VM_log2_of2(uint32_t n) * * Purpose: Round up a number to the next power of 2 * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Return the number which is a power of 2 * * Programmer: Vailin Choi; Nov 2014 @@ -478,6 +502,10 @@ H5VM_power2up(hsize_t n) * Purpose: Determine the # of bytes needed to encode values within a * range from 0 to a given limit * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: Number of bytes needed * * Programmer: Quincey Koziol @@ -507,6 +535,10 @@ static const unsigned char H5VM_bit_clear_g[8] = {0x7F, 0xBF, 0xDF, 0xEF, 0xF7, * to bit offset 7 in the first byte's low-bit position, then to * bit offset 8 in the second byte's high-bit position, etc. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: TRUE/FALSE * * Programmer: Quincey Koziol @@ -534,6 +566,10 @@ H5VM_bit_get(const unsigned char *buf, size_t offset) * to bit offset 7 in the first byte's low-bit position, then to * bit offset 8 in the second byte's high-bit position, etc. * + * Note: Although this routine is 'static' in this file, that's intended + * only as an optimization and the naming (with a single underscore) + * reflects its inclusion in a "private" header file. + * * Return: None * * Programmer: Quincey Koziol @@ -15,7 +15,7 @@ * * Created: H5WB.c * Jun 26 2007 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Implements the "wrapped buffer" code for wrapping * an existing [staticly sized] buffer, in order to @@ -97,7 +97,6 @@ H5FL_BLK_DEFINE_STATIC(extra_buf); * NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Jun 26 2007 * *------------------------------------------------------------------------- @@ -151,7 +150,6 @@ done: * NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Jun 26 2007 * *------------------------------------------------------------------------- @@ -219,7 +217,6 @@ done: * NULL on failure * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Jun 26 2007 * *------------------------------------------------------------------------- @@ -257,7 +254,6 @@ done: * Return: SUCCEED/FAIL * * Programmer: Quincey Koziol - * koziol@hdfgroup.org * Jun 26 2007 * *------------------------------------------------------------------------- diff --git a/src/H5WBprivate.h b/src/H5WBprivate.h index 4460808..3ec9261 100644 --- a/src/H5WBprivate.h +++ b/src/H5WBprivate.h @@ -15,7 +15,7 @@ * * Created: H5WBprivate.h * Jun 26 2007 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Private header for library accessible wrapped buffer routines. * @@ -35,10 +35,10 @@ #ifdef H5Z_DEBUG typedef struct H5Z_stats_t { struct { - hsize_t total; /* total number of bytes processed */ - hsize_t errors; /* bytes of total attributable to errors */ - H5_timer_t timer; /* execution time including errors */ - } stats[2]; /* 0=output, 1=input */ + hsize_t total; /* total number of bytes processed */ + hsize_t errors; /* bytes of total attributable to errors */ + H5_timevals_t times; /* execution time including errors */ + } stats[2]; /* 0 = output, 1 = input */ } H5Z_stats_t; #endif /* H5Z_DEBUG */ @@ -68,7 +68,7 @@ static H5Z_stats_t *H5Z_stat_table_g = NULL; #endif /* H5Z_DEBUG */ /* Local functions */ -static int H5Z_find_idx(H5Z_filter_t id); +static int H5Z__find_idx(H5Z_filter_t id); static int H5Z__check_unregister_dset_cb(void *obj_ptr, hid_t obj_id, void *key); static int H5Z__check_unregister_group_cb(void *obj_ptr, hid_t obj_id, void *key); static int H5Z__flush_file_cb(void *obj_ptr, hid_t obj_id, void *key); @@ -132,9 +132,9 @@ H5Z_term_package(void) if(H5_PKG_INIT_VAR) { #ifdef H5Z_DEBUG - char comment[16], bandwidth[32]; - int dir, nprint = 0; - size_t i; + char comment[16], bandwidth[32]; + int dir, nprint = 0; + size_t i; if(H5DEBUG(Z)) { for(i = 0; i < H5Z_table_used_g; i++) { @@ -167,25 +167,26 @@ H5Z_term_package(void) */ H5_bandwidth(bandwidth, (double)(H5Z_stat_table_g[i].stats[dir].total), - H5Z_stat_table_g[i].stats[dir].timer.etime); + H5Z_stat_table_g[i].stats[dir].times.elapsed); /* Print the statistics */ - HDfprintf(H5DEBUG(Z), - " %s%-15s %10Hd %10Hd %8.2f %8.2f %8.2f " - "%10s\n", dir?"<":">", comment, + HDfprintf(H5DEBUG(Z), " %s%-15s %10Hd %10Hd %8T %8T %8T %10s\n", + (dir ? "<" : ">"), comment, H5Z_stat_table_g[i].stats[dir].total, H5Z_stat_table_g[i].stats[dir].errors, - H5Z_stat_table_g[i].stats[dir].timer.utime, - H5Z_stat_table_g[i].stats[dir].timer.stime, - H5Z_stat_table_g[i].stats[dir].timer.etime, + H5Z_stat_table_g[i].stats[dir].times.user, + H5Z_stat_table_g[i].stats[dir].times.system, + H5Z_stat_table_g[i].stats[dir].times.elapsed, bandwidth); } /* end for */ } /* end for */ } /* end if */ #endif /* H5Z_DEBUG */ + /* Free the table of filters */ - if (H5Z_table_g) { + if(H5Z_table_g) { H5Z_table_g = (H5Z_class2_t *)H5MM_xfree(H5Z_table_g); + #ifdef H5Z_DEBUG H5Z_stat_table_g = (H5Z_stats_t *)H5MM_xfree(H5Z_stat_table_g); #endif /* H5Z_DEBUG */ @@ -378,8 +379,8 @@ done: * Purpose: Same as the public version except this one allows filters * to be unset for predefined method numbers <H5Z_FILTER_RESERVED * - * Return: Non-negative on success - * Negative on failure + * Return: SUCCEED/FAIL + * *------------------------------------------------------------------------- */ herr_t @@ -700,8 +701,8 @@ H5Z_filter_avail(H5Z_filter_t id) HGOTO_DONE(TRUE) key.id = (int)id; - if (NULL != (filter_info = (const H5Z_class2_t *)H5PL_load(H5PL_TYPE_FILTER, key))) { - if (H5Z_register(filter_info) < 0) + if(NULL != (filter_info = (const H5Z_class2_t *)H5PL_load(H5PL_TYPE_FILTER, key))) { + if(H5Z_register(filter_info) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to register loaded filter") HGOTO_DONE(TRUE) } /* end if */ @@ -712,7 +713,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5Z_prelude_callback + * Function: H5Z__prelude_callback * * Purpose: Makes a dataset creation "prelude" callback for the "can_apply" * or "set_local" routines. @@ -725,14 +726,14 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5Z_prelude_callback(const H5O_pline_t *pline, hid_t dcpl_id, hid_t type_id, +H5Z__prelude_callback(const H5O_pline_t *pline, hid_t dcpl_id, hid_t type_id, hid_t space_id, H5Z_prelude_type_t prelude_type) { H5Z_class2_t *fclass; /* Individual filter information */ size_t u; /* Local index variable */ htri_t ret_value = TRUE; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(pline->nused > 0); @@ -790,11 +791,11 @@ H5Z_prelude_callback(const H5O_pline_t *pline, hid_t dcpl_id, hid_t type_id, done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5Z_prelude_callback() */ +} /* end H5Z__prelude_callback() */ /*------------------------------------------------------------------------- - * Function: H5Z_prepare_prelude_callback_dcpl + * Function: H5Z__prepare_prelude_callback_dcpl * * Purpose: Prepares to make a dataset creation "prelude" callback * for the "can_apply" or "set_local" routines. @@ -807,13 +808,13 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5Z_prepare_prelude_callback_dcpl(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_type) +H5Z__prepare_prelude_callback_dcpl(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_type) { hid_t space_id = -1; /* ID for dataspace describing chunk */ H5O_layout_t *dcpl_layout = NULL; /* Dataset's layout information */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_STATIC HDassert(H5I_GENPROP_LST == H5I_get_type(dcpl_id)); HDassert(H5I_DATATYPE == H5I_get_type(type_id)); @@ -861,7 +862,7 @@ H5Z_prepare_prelude_callback_dcpl(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type } /* Make the callbacks */ - if (H5Z_prelude_callback(&dcpl_pline, dcpl_id, type_id, space_id, prelude_type) < 0) + if (H5Z__prelude_callback(&dcpl_pline, dcpl_id, type_id, space_id, prelude_type) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "unable to apply filter") } } @@ -875,7 +876,7 @@ done: dcpl_layout = (H5O_layout_t *)H5MM_xfree(dcpl_layout); FUNC_LEAVE_NOAPI(ret_value) -} /* end H5Z_prepare_prelude_callback_dcpl() */ +} /* end H5Z__prepare_prelude_callback_dcpl() */ /*------------------------------------------------------------------------- @@ -901,7 +902,7 @@ H5Z_can_apply(hid_t dcpl_id, hid_t type_id) FUNC_ENTER_NOAPI(FAIL) /* Make "can apply" callbacks for filters in pipeline */ - if (H5Z_prepare_prelude_callback_dcpl(dcpl_id, type_id, H5Z_PRELUDE_CAN_APPLY) < 0) + if (H5Z__prepare_prelude_callback_dcpl(dcpl_id, type_id, H5Z_PRELUDE_CAN_APPLY) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "unable to apply filter") done: @@ -932,7 +933,7 @@ H5Z_set_local(hid_t dcpl_id, hid_t type_id) FUNC_ENTER_NOAPI(FAIL) /* Make "set local" callbacks for filters in pipeline */ - if (H5Z_prepare_prelude_callback_dcpl(dcpl_id, type_id, H5Z_PRELUDE_SET_LOCAL) < 0) + if (H5Z__prepare_prelude_callback_dcpl(dcpl_id, type_id, H5Z_PRELUDE_SET_LOCAL) < 0) HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not set") done: @@ -961,7 +962,7 @@ H5Z_can_apply_direct(const H5O_pline_t *pline) HDassert(pline->nused > 0); /* Make "can apply" callbacks for filters in pipeline */ - if (H5Z_prelude_callback(pline, (hid_t)-1, (hid_t)-1, (hid_t)-1, H5Z_PRELUDE_CAN_APPLY) < 0) + if (H5Z__prelude_callback(pline, (hid_t)-1, (hid_t)-1, (hid_t)-1, H5Z_PRELUDE_CAN_APPLY) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "unable to apply filter") done: @@ -994,7 +995,7 @@ H5Z_set_local_direct(const H5O_pline_t *pline) HDassert(pline->nused > 0); /* Make "set local" callbacks for filters in pipeline */ - if (H5Z_prelude_callback(pline, (hid_t)-1, (hid_t)-1, (hid_t)-1, H5Z_PRELUDE_SET_LOCAL) < 0) + if (H5Z__prelude_callback(pline, (hid_t)-1, (hid_t)-1, (hid_t)-1, H5Z_PRELUDE_SET_LOCAL) < 0) HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not set") done: @@ -1167,7 +1168,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5Z_find_idx + * Function: H5Z__find_idx * * Purpose: Given a filter ID return the offset in the global array * that holds all the registered filters. @@ -1177,20 +1178,20 @@ done: *------------------------------------------------------------------------- */ static int -H5Z_find_idx(H5Z_filter_t id) +H5Z__find_idx(H5Z_filter_t id) { size_t i; /* Local index variable */ int ret_value = FAIL; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_STATIC_NOERR - for (i = 0; i < H5Z_table_used_g; i++) - if (H5Z_table_g[i].id == id) + for(i = 0; i < H5Z_table_used_g; i++) + if(H5Z_table_g[i].id == id) HGOTO_DONE((int)i) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5Z_find_idx() */ +} /* end H5Z__find_idx() */ /*------------------------------------------------------------------------- @@ -1212,7 +1213,7 @@ H5Z_find(H5Z_filter_t id) FUNC_ENTER_NOAPI(NULL) /* Get the index in the global table */ - if ((idx = H5Z_find_idx(id)) < 0) + if ((idx = H5Z__find_idx(id)) < 0) HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, NULL, "required filter %d is not registered", id) /* Set return value */ @@ -1248,20 +1249,22 @@ done: */ herr_t H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, - unsigned *filter_mask/*in,out*/, H5Z_EDC_t edc_read, - H5Z_cb_t cb_struct, size_t *nbytes/*in,out*/, - size_t *buf_size/*in,out*/, void **buf/*in,out*/) + unsigned *filter_mask/*in,out*/, H5Z_EDC_t edc_read, H5Z_cb_t cb_struct, + size_t *nbytes/*in,out*/, size_t *buf_size/*in,out*/, void **buf/*in,out*/) { - size_t i, idx, new_nbytes; - int fclass_idx; /* Index of filter class in global table */ - H5Z_class2_t *fclass=NULL; /* Filter class pointer */ + size_t idx; + size_t new_nbytes; + int fclass_idx; /* Index of filter class in global table */ + H5Z_class2_t *fclass = NULL; /* Filter class pointer */ #ifdef H5Z_DEBUG - H5Z_stats_t *fstats=NULL; /* Filter stats pointer */ - H5_timer_t timer; + H5Z_stats_t *fstats = NULL; /* Filter stats pointer */ + H5_timer_t timer; /* Timer for filter operations */ + H5_timevals_t times; /* Elapsed time for each operation */ #endif - unsigned failed = 0; - unsigned tmp_flags; - herr_t ret_value = SUCCEED; /* Return value */ + unsigned failed = 0; + unsigned tmp_flags; + size_t i; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -1272,11 +1275,13 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, HDassert(buf && *buf); HDassert(!pline || pline->nused < H5Z_MAX_NFILTERS); - if (pline && (flags & H5Z_FLAG_REVERSE)) { /* Read */ - for (i = pline->nused; i > 0; --i) { - idx = i-1; - - if (*filter_mask & ((unsigned)1 << idx)) { +#ifdef H5Z_DEBUG + H5_timer_init(&timer); +#endif + if(pline && (flags & H5Z_FLAG_REVERSE)) { /* Read */ + for(i = pline->nused; i > 0; --i) { + idx = i - 1; + if(*filter_mask & ((unsigned)1 << idx)) { failed |= (unsigned)1 << idx; continue; /* filter excluded */ } @@ -1285,10 +1290,10 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, * indicate no plugin through HDF5_PRELOAD_PLUG (using the symbol "::"), * try to load it dynamically and register it. Otherwise, return failure */ - if ((fclass_idx = H5Z_find_idx(pline->filter[idx].id)) < 0) { - hbool_t issue_error = FALSE; + if ((fclass_idx = H5Z__find_idx(pline->filter[idx].id)) < 0) { H5PL_key_t key; const H5Z_class2_t *filter_info; + hbool_t issue_error = FALSE; /* Try loading the filter */ key.id = (int)(pline->filter[idx].id); @@ -1298,7 +1303,7 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to register filter") /* Search in the table of registered filters again to find the dynamic filter just loaded and registered */ - if ((fclass_idx = H5Z_find_idx(pline->filter[idx].id)) < 0) + if((fclass_idx = H5Z__find_idx(pline->filter[idx].id)) < 0) issue_error = TRUE; } else @@ -1316,76 +1321,96 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, } /* end if */ fclass = &H5Z_table_g[fclass_idx]; + #ifdef H5Z_DEBUG fstats = &H5Z_stat_table_g[fclass_idx]; - H5_timer_begin (&timer); + H5_timer_start(&timer); #endif + tmp_flags = flags | (pline->filter[idx].flags); - tmp_flags |= (edc_read== H5Z_DISABLE_EDC) ? H5Z_FLAG_SKIP_EDC : 0; - new_nbytes = (fclass->filter)(tmp_flags, pline->filter[idx].cd_nelmts, - pline->filter[idx].cd_values, *nbytes, buf_size, buf); + tmp_flags |= (edc_read == H5Z_DISABLE_EDC) ? H5Z_FLAG_SKIP_EDC : 0; + new_nbytes = (fclass->filter)(tmp_flags, + pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, + *nbytes, buf_size, buf); #ifdef H5Z_DEBUG - H5_timer_end (&(fstats->stats[1].timer), &timer); + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + fstats->stats[1].times.elapsed += times.elapsed; + fstats->stats[1].times.system += times.system; + fstats->stats[1].times.user += times.user; + fstats->stats[1].total += MAX(*nbytes, new_nbytes); - if (0 == new_nbytes) + if(0 == new_nbytes) fstats->stats[1].errors += *nbytes; #endif - if (0 == new_nbytes) { - if ((cb_struct.func && (H5Z_CB_FAIL == cb_struct.func(pline->filter[idx].id, *buf, *buf_size, cb_struct.op_data))) || !cb_struct.func) + if(0 == new_nbytes) { + if((cb_struct.func + && (H5Z_CB_FAIL == cb_struct.func(pline->filter[idx].id, *buf, *buf_size, cb_struct.op_data))) + || !cb_struct.func) HGOTO_ERROR(H5E_PLINE, H5E_READERROR, FAIL, "filter returned failure during read") *nbytes = *buf_size; failed |= (unsigned)1 << idx; - H5E_clear_stack (NULL); + H5E_clear_stack(NULL); } else *nbytes = new_nbytes; } } - else if (pline) { /* Write */ - for (idx = 0; idx < pline->nused; idx++) { - if (*filter_mask & ((unsigned)1 << idx)) { + else if(pline) { /* Write */ + for(idx = 0; idx < pline->nused; idx++) { + if(*filter_mask & ((unsigned)1 << idx)) { failed |= (unsigned)1 << idx; - continue; /*filter excluded*/ + continue; /* filter excluded */ } - if ((fclass_idx = H5Z_find_idx(pline->filter[idx].id)) < 0) { + if((fclass_idx = H5Z__find_idx(pline->filter[idx].id)) < 0) { /* Check if filter is optional -- If it isn't, then error */ - if ((pline->filter[idx].flags & H5Z_FLAG_OPTIONAL) == 0) + if((pline->filter[idx].flags & H5Z_FLAG_OPTIONAL) == 0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "required filter is not registered") - failed |= (unsigned)1 << idx; - H5E_clear_stack (NULL); - continue; /*filter excluded*/ - } + H5E_clear_stack(NULL); + continue; /* filter excluded */ + } /* end if */ + fclass = &H5Z_table_g[fclass_idx]; + #ifdef H5Z_DEBUG fstats = &H5Z_stat_table_g[fclass_idx]; - H5_timer_begin (&timer); + H5_timer_start(&timer); #endif - new_nbytes = (fclass->filter)(flags | (pline->filter[idx].flags), pline->filter[idx].cd_nelmts, - pline->filter[idx].cd_values, *nbytes, buf_size, buf); + + new_nbytes = (fclass->filter)(flags | (pline->filter[idx].flags), + pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, + *nbytes, buf_size, buf); + #ifdef H5Z_DEBUG - H5_timer_end (&(fstats->stats[0].timer), &timer); + H5_timer_stop(&timer); + H5_timer_get_times(timer, ×); + fstats->stats[0].times.elapsed += times.elapsed; + fstats->stats[0].times.system += times.system; + fstats->stats[0].times.user += times.user; + fstats->stats[0].total += MAX(*nbytes, new_nbytes); - if (0 == new_nbytes) + if(0 == new_nbytes) fstats->stats[0].errors += *nbytes; #endif - if (0 == new_nbytes) { - if (0 == (pline->filter[idx].flags & H5Z_FLAG_OPTIONAL)) { - if ((cb_struct.func && (H5Z_CB_FAIL == cb_struct.func (pline->filter[idx].id, *buf, *nbytes, cb_struct.op_data))) || !cb_struct.func) + + if(0 == new_nbytes) { + if(0 == (pline->filter[idx].flags & H5Z_FLAG_OPTIONAL)) { + if((cb_struct.func && (H5Z_CB_FAIL == cb_struct.func(pline->filter[idx].id, *buf, *nbytes, cb_struct.op_data))) + || !cb_struct.func) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "filter returned failure") *nbytes = *buf_size; } - failed |= (unsigned)1 << idx; - H5E_clear_stack (NULL); + H5E_clear_stack(NULL); } else *nbytes = new_nbytes; - } + } /* end for */ } *filter_mask = failed; diff --git a/src/H5Zdeflate.c b/src/H5Zdeflate.c index 6d7b87e..d46108f 100644 --- a/src/H5Zdeflate.c +++ b/src/H5Zdeflate.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Friday, August 27, 1999 */ @@ -34,26 +34,26 @@ #endif /* Local function prototypes */ -static size_t H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, +static size_t H5Z__filter_deflate (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ const H5Z_class2_t H5Z_DEFLATE[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_DEFLATE, /* Filter id number */ - 1, /* encoder_present flag (set to true) */ - 1, /* decoder_present flag (set to true) */ - "deflate", /* Filter name for debugging */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "deflate", /* Filter name for debugging */ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ - H5Z_filter_deflate, /* The actual filter function */ + H5Z__filter_deflate, /* The actual filter function */ }}; #define H5Z_DEFLATE_SIZE_ADJUST(s) (HDceil(((double)(s)) * (double)1.001f) + 12) /*------------------------------------------------------------------------- - * Function: H5Z_filter_deflate + * Function: H5Z__filter_deflate * * Purpose: Implement an I/O filter around the 'deflate' algorithm in * libz @@ -64,12 +64,10 @@ const H5Z_class2_t H5Z_DEFLATE[1] = {{ * Programmer: Robb Matzke * Thursday, April 16, 1998 * - * Modifications: - * *------------------------------------------------------------------------- */ static size_t -H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, +H5Z__filter_deflate(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) { @@ -77,7 +75,7 @@ H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, int status; /* Status from zlib operation */ size_t ret_value = 0; /* Return value */ - FUNC_ENTER_NOAPI(0) + FUNC_ENTER_STATIC /* Sanity check */ HDassert(*buf_size > 0); diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index a2a44fa..b37dcf3 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Robb Matzke <matzke@llnl.gov> +/* Programmer: Robb Matzke * Thursday, April 16, 1998 */ diff --git a/src/H5err.txt b/src/H5err.txt index 30c646f..f3cca72 100644 --- a/src/H5err.txt +++ b/src/H5err.txt @@ -133,6 +133,8 @@ MINOR, FILEACC, H5E_NOTHDF5, Not an HDF5 file MINOR, FILEACC, H5E_BADFILE, Bad file ID accessed MINOR, FILEACC, H5E_TRUNCATED, File has been truncated MINOR, FILEACC, H5E_MOUNT, File mount error +MINOR, FILEACC, H5E_CANTLOCKFILE, Unable to lock file +MINOR, FILEACC, H5E_CANTUNLOCKFILE, Unable to unlock file # Generic low-level file I/O errors MINOR, FILE, H5E_SEEKERROR, Seek failed diff --git a/src/H5private.h b/src/H5private.h index c0d39fc..7793e0b 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Robb Matzke <matzke@llnl.gov> +/* Programmer: Robb Matzke * Friday, October 30, 1998 * * Purpose: This file is included by all HDF5 library source files to @@ -362,6 +362,22 @@ #endif /* __cplusplus */ /* + * Networking headers used by the mirror VFD and related tests and utilities. + */ +#ifdef H5_HAVE_ARPA_INET_H +# include <arpa/inet.h> +#endif +#ifdef H5_HAVE_NETDB_H +# include <netdb.h> +#endif +#ifdef H5_HAVE_NETINET_IN_H +# include <netinet/in.h> +#endif +#ifdef H5_HAVE_SYS_SOCKET_H +# include <sys/socket.h> +#endif + +/* * Status return values for the `herr_t' type. * Since some unix/c routines use 0 and -1 (or more precisely, non-negative * vs. negative) as their return code, and some assumption had been made in @@ -556,28 +572,6 @@ #define H5_REQUEST_NULL NULL /* - * A macro to portably decrement enumerated types. - */ -#ifndef H5_DEC_ENUM -# define H5_DEC_ENUM(TYPE,VAR) (VAR)=((TYPE)((VAR)-1)) -#endif - -/* Double constant wrapper - * - * Quiets gcc warnings from -Wunsuffixed-float-constants. - * - * This is a really annoying warning since the standard specifies that - * constants of type double do NOT get a suffix so there's no way - * to specify a constant of type double. To quiet gcc, we specify floating - * point constants as type long double and cast to double. - * - * Note that this macro only needs to be used where using a double - * is important. For most code, suffixing constants with F will quiet the - * compiler and not produce erroneous code. - */ -#define H5_DOUBLE(S) ((double) S ## L) - -/* * Methods to compare the equality of floating-point values: * * 1. H5_XXX_ABS_EQUAL - check if the difference is smaller than the @@ -617,21 +611,37 @@ #define LOCK_UN 0x08 #endif /* H5_HAVE_FLOCK */ -/* - * Data types and functions for timing certain parts of the library. +/* Typedefs and functions for timing certain parts of the library. */ + +/* A set of elapsed/user/system times emitted as a time point by the + * platform-independent timers. */ typedef struct { - double utime; /*user time */ - double stime; /*system time */ - double etime; /*elapsed wall-clock time */ + double user; /* User time in seconds */ + double system; /* System time in seconds */ + double elapsed; /* Elapsed (wall clock) time in seconds */ +} H5_timevals_t; + +/* Timer structure for platform-independent timers */ +typedef struct { + H5_timevals_t initial; /* Current interval start time */ + H5_timevals_t final_interval; /* Last interval elapsed time */ + H5_timevals_t total; /* Total elapsed time for all intervals */ + hbool_t is_running; /* Whether timer is running */ } H5_timer_t; -H5_DLL void H5_timer_reset (H5_timer_t *timer); -H5_DLL void H5_timer_begin (H5_timer_t *timer); -H5_DLL void H5_timer_end (H5_timer_t *sum/*in,out*/, - H5_timer_t *timer/*in,out*/); +/* Returns library bandwidth as a pretty string */ H5_DLL void H5_bandwidth(char *buf/*out*/, double nbytes, double nseconds); + +/* Timer functionality */ H5_DLL time_t H5_now(void); +H5_DLL uint64_t H5_now_usec(void); +H5_DLL herr_t H5_timer_init(H5_timer_t *timer /*in,out*/); +H5_DLL herr_t H5_timer_start(H5_timer_t *timer /*in,out*/); +H5_DLL herr_t H5_timer_stop(H5_timer_t *timer /*in,out*/); +H5_DLL herr_t H5_timer_get_times(H5_timer_t timer, H5_timevals_t *times /*in,out*/); +H5_DLL herr_t H5_timer_get_total_times(H5_timer_t timer, H5_timevals_t *times /*in,out*/); +H5_DLL char *H5_timer_get_time_string(double seconds); /* Depth of object copy */ typedef enum { @@ -665,6 +675,9 @@ typedef struct { #ifndef HDabs #define HDabs(X) abs(X) #endif /* HDabs */ +#ifndef HDaccept + #define HDaccept(A,B,C) accept((A),(B),(C)) /* mirror VFD */ +#endif /* HDaccept */ #ifndef HDaccess #define HDaccess(F,M) access(F, M) #endif /* HDaccess */ @@ -711,6 +724,9 @@ typedef struct { #ifndef HDatoll #define HDatoll(S) atoll(S) #endif /* HDatol */ +#ifndef HDbind + #define HDbind(A,B,C) bind((A),(B),(C)) /* mirror VFD */ +#endif /* HDbind */ #ifndef HDbsearch #define HDbsearch(K,B,N,Z,F) bsearch(K,B,N,Z,F) #endif /* HDbsearch */ @@ -747,12 +763,18 @@ typedef struct { #ifndef HDclock #define HDclock() clock() #endif /* HDclock */ +#ifndef HDclock_gettime + #define HDclock_gettime(CID, TS) clock_gettime(CID, TS) +#endif /* HDclock_gettime */ #ifndef HDclose #define HDclose(F) close(F) #endif /* HDclose */ #ifndef HDclosedir #define HDclosedir(D) closedir(D) #endif /* HDclosedir */ +#ifndef HDconnect + #define HDconnect(A,B,C) connect((A),(B),(C)) /* mirror VFD */ +#endif /* HDconnect */ #ifndef HDcos #define HDcos(X) cos(X) #endif /* HDcos */ @@ -863,8 +885,8 @@ H5_DLL H5_ATTR_CONST int Nflock(int fd, int operation); #ifndef HDflock /* NOTE: flock(2) is not present on all POSIX systems. * If it is not present, we try a flock() equivalent based on - * fcntl(2), then fall back to a function that always fails if - * it is not present at all (Windows uses a separate Wflock() + * fcntl(2), then fall back to a function that always succeeds + * if it is not present at all (Windows uses a separate Wflock() * function). */ #if defined(H5_HAVE_FLOCK) @@ -953,7 +975,7 @@ typedef off_t h5_stat_size_t; #define H5_SIZEOF_H5_STAT_SIZE_T H5_SIZEOF_OFF_T #ifndef HDftell - #define HDftell(F) ftello(F) + #define HDftell(F) ftell(F) #endif /* HDftell */ #ifndef HDftruncate #define HDftruncate(F,L) ftruncate(F,L) @@ -997,6 +1019,9 @@ typedef off_t h5_stat_size_t; #ifndef HDgetgroups #define HDgetgroups(Z,G) getgroups(Z,G) #endif /* HDgetgroups */ +#ifndef HDgethostbyaddr + #define HDgethostbyaddr(A,B,C) gethostbyaddr((A),(B),(C)) /* mirror VFD */ +#endif /* HDgethostbyaddr */ #ifndef HDgethostname #define HDgethostname(N,L) gethostname(N,L) #endif /* HDgethostname */ @@ -1036,6 +1061,18 @@ typedef off_t h5_stat_size_t; #ifndef HDgmtime #define HDgmtime(T) gmtime(T) #endif /* HDgmtime */ +#ifndef HDhtonl + #define HDhtonl(X) htonl((X)) /* mirror VFD */ +#endif /* HDhtonl */ +#ifndef HDhtons + #define HDhtons(X) htons((X)) /* mirror VFD */ +#endif /* HDhtons */ +#ifndef HDinet_addr + #define HDinet_addr(C) inet_addr((C)) /* mirror VFD */ +#endif /* HDinet_addr */ +#ifndef HDinet_ntoa + #define HDinet_ntoa(C) inet_ntoa((C)) /* mirror VFD */ +#endif /* HDinet_ntoa */ #ifndef HDisalnum #define HDisalnum(C) isalnum((int)(C)) /*cast for solaris warning*/ #endif /* HDisalnum */ @@ -1090,6 +1127,9 @@ typedef off_t h5_stat_size_t; #ifndef HDlink #define HDlink(OLD,NEW) link(OLD,NEW) #endif /* HDlink */ +#ifndef HDlisten + #define HDlisten(A,B) listen((A),(B)) /* mirror VFD */ +#endif /* HDlisten */ #ifndef HDllround #define HDllround(V) llround(V) #endif /* HDround */ @@ -1171,6 +1211,12 @@ typedef off_t h5_stat_size_t; #ifndef HDnanosleep #define HDnanosleep(N, O) nanosleep(N, O) #endif /* HDnanosleep */ +#ifndef HDntohl + #define HDntohl(A) ntohl((A)) /* mirror VFD */ +#endif /* HDntohl */ +#ifndef HDntohs + #define HDntohs(A) ntohs((A)) /* mirror VFD */ +#endif /* HDntohs */ #ifndef HDopen #define HDopen(F,...) open(F,__VA_ARGS__) #endif /* HDopen */ @@ -1225,13 +1271,23 @@ typedef off_t h5_stat_size_t; #define HDrandom() HDrand() #endif /* HDrandom */ H5_DLL int HDrand(void); -#elif H5_HAVE_RANDOM + #ifndef HDsrandom + #define HDsrandom(S) HDsrand(S) + #endif /* HDsrandom */ + H5_DLL void HDsrand(unsigned int seed); +#elif defined(H5_HAVE_RANDOM) #ifndef HDrand #define HDrand() random() #endif /* HDrand */ #ifndef HDrandom #define HDrandom() random() #endif /* HDrandom */ + #ifndef HDsrand + #define HDsrand(S) srandom(S) + #endif /* HDsrand */ + #ifndef HDsrandom + #define HDsrandom(S) srandom(S) + #endif /* HDsrandom */ #else /* H5_HAVE_RANDOM */ #ifndef HDrand #define HDrand() rand() @@ -1239,6 +1295,12 @@ typedef off_t h5_stat_size_t; #ifndef HDrandom #define HDrandom() rand() #endif /* HDrandom */ + #ifndef HDsrand + #define HDsrand(S) srand(S) + #endif /* HDsrand */ + #ifndef HDsrandom + #define HDsrandom(S) srand(S) + #endif /* HDsrandom */ #endif /* H5_HAVE_RANDOM */ #ifndef HDread @@ -1302,12 +1364,21 @@ typedef off_t h5_stat_size_t; #ifndef HDsetsid #define HDsetsid() setsid() #endif /* HDsetsid */ +#ifndef HDsetsockopt + #define HDsetsockopt(A,B,C,D,E) setsockopt((A),(B),(C),(D),(E)) /* mirror VFD */ +#endif /* HDsetsockopt */ #ifndef HDsetuid #define HDsetuid(U) setuid(U) #endif /* HDsetuid */ #ifndef HDsetvbuf #define HDsetvbuf(F,S,M,Z) setvbuf(F,S,M,Z) #endif /* HDsetvbuf */ +#ifndef HDshutdown + #define HDshutdown(A, B) shutdown((A),(B)) /* mirror VFD */ +#endif /* HDshutdown */ +#ifndef HDsigaction + #define HDsigaction(S,A,O) sigaction((S),(A),(O)) +#endif /* HDsigaction */ #ifndef HDsigaddset #define HDsigaddset(S,N) sigaddset(S,N) #endif /* HDsigaddset */ @@ -1353,32 +1424,15 @@ typedef off_t h5_stat_size_t; #ifndef HDsnprintf #define HDsnprintf snprintf /*varargs*/ #endif /* HDsnprintf */ +#ifndef HDsocket + #define HDsocket(A,B,C) socket((A),(B),(C)) /* mirror VFD */ +#endif /* HDsocket */ #ifndef HDsprintf #define HDsprintf sprintf /*varargs*/ #endif /* HDsprintf */ #ifndef HDsqrt #define HDsqrt(X) sqrt(X) #endif /* HDsqrt */ -#ifdef H5_HAVE_RAND_R - H5_DLL void HDsrand(unsigned int seed); - #ifndef HDsrandom - #define HDsrandom(S) HDsrand(S) - #endif /* HDsrandom */ -#elif H5_HAVE_RANDOM - #ifndef HDsrand - #define HDsrand(S) srandom(S) - #endif /* HDsrand */ - #ifndef HDsrandom - #define HDsrandom(S) srandom(S) - #endif /* HDsrandom */ -#else /* H5_HAVE_RAND_R */ - #ifndef HDsrand - #define HDsrand(S) srand(S) - #endif /* HDsrand */ - #ifndef HDsrandom - #define HDsrandom(S) srand(S) - #endif /* HDsrandom */ -#endif /* H5_HAVE_RAND_R */ #ifndef HDsscanf #define HDsscanf(S,FMT,...) sscanf(S,FMT,__VA_ARGS__) #endif /* HDsscanf */ diff --git a/src/H5public.h b/src/H5public.h index 871c360..e451905 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -26,32 +26,32 @@ * it via H5public.h. The #ifndef _H5public_H guard above would * prevent repeated include. */ -#include "H5pubconf.h" /*from configure */ +#include "H5pubconf.h" /* From configure */ /* API Version macro wrapper definitions */ #include "H5version.h" #ifdef H5_HAVE_FEATURES_H -#include <features.h> /*for setting POSIX, BSD, etc. compatibility */ +#include <features.h> /* For setting POSIX, BSD, etc. compatibility */ #endif #ifdef H5_HAVE_SYS_TYPES_H #include <sys/types.h> #endif #ifdef H5_STDC_HEADERS -# include <limits.h> /*for H5T_NATIVE_CHAR defn in H5Tpublic.h */ -# include <stdarg.h> /*for variadic functions in H5VLpublic.h */ +# include <limits.h> /* For H5T_NATIVE_CHAR defn in H5Tpublic.h */ +# include <stdarg.h> /* For variadic functions in H5VLpublic.h */ #endif #ifndef __cplusplus # ifdef H5_HAVE_STDINT_H -# include <stdint.h> /*for C9x types */ +# include <stdint.h> /* For C9x types */ # endif #else # ifdef H5_HAVE_STDINT_H_CXX -# include <stdint.h> /*for C9x types when include from C++ */ +# include <stdint.h> /* For C9x types (when included from C++) */ # endif #endif #ifdef H5_HAVE_INTTYPES_H -# include <inttypes.h> /* For uint64_t on some platforms */ +# include <inttypes.h> /* C99/POSIX.1 header for uint64_t, PRIu64 */ #endif #ifdef H5_HAVE_STDDEF_H # include <stddef.h> @@ -61,7 +61,7 @@ # define MPICH_SKIP_MPICXX 1 # define OMPI_SKIP_MPICXX 1 # include <mpi.h> -#ifndef MPI_FILE_NULL /*MPIO may be defined in mpi.h already */ +#ifndef MPI_FILE_NULL /* MPIO may be defined in mpi.h already */ # include <mpio.h> #endif #endif @@ -94,15 +94,15 @@ extern "C" { #endif /* Version numbers */ -#define H5_VERS_MAJOR 1 /* For major interface/format changes */ -#define H5_VERS_MINOR 10 /* For minor interface/format changes */ -#define H5_VERS_RELEASE 7 /* For tweaks, bug-fixes, or development */ -#define H5_VERS_SUBRELEASE "1" /* For pre-releases like snap0 */ - /* Empty string for real releases. */ +#define H5_VERS_MAJOR 1 /* For major interface/format changes */ +#define H5_VERS_MINOR 10 /* For minor interface/format changes */ +#define H5_VERS_RELEASE 7 /* For tweaks, bug-fixes, or development */ +#define H5_VERS_SUBRELEASE "1" /* For pre-releases like snap0 */ + /* Empty string for real releases. */ #define H5_VERS_INFO "HDF5 library version: 1.10.7-1" /* Full version string */ -#define H5check() H5check_version(H5_VERS_MAJOR,H5_VERS_MINOR, \ - H5_VERS_RELEASE) +#define H5check() H5check_version(H5_VERS_MAJOR,H5_VERS_MINOR, \ + H5_VERS_RELEASE) /* macros for comparing the version */ #define H5_VERSION_GE(Maj,Min,Rel) \ @@ -122,8 +122,8 @@ extern "C" { * The negative failure value is most commonly -1, but don't bet on it. The * proper way to detect failure is something like: * - * if((dset = H5Dopen2(file, name)) < 0) - * fprintf(stderr, "unable to open the requested dataset\n"); + * if((dset = H5Dopen2(file, name)) < 0) + * fprintf(stderr, "unable to open the requested dataset\n"); */ typedef int herr_t; @@ -135,13 +135,13 @@ typedef int herr_t; * (false), positive (true), or negative (failure). The proper way to test * for truth from a htri_t function is: * - * if ((retval = H5Tcommitted(type))>0) { - * printf("data type is committed\n"); - * } else if (!retval) { - * printf("data type is not committed\n"); - * } else { - * printf("error determining whether data type is committed\n"); - * } + * if ((retval = H5Tcommitted(type)) > 0) { + * printf("data type is committed\n"); + * } else if (!retval) { + * printf("data type is not committed\n"); + * } else { + * printf("error determining whether data type is committed\n"); + * } */ #ifdef H5_HAVE_STDBOOL_H #include <stdbool.h> @@ -183,8 +183,8 @@ typedef long long ssize_t; */ #if H5_SIZEOF_LONG_LONG >= 8 H5_GCC_DIAG_OFF(long-long) -typedef unsigned long long hsize_t; -typedef signed long long hssize_t; +typedef unsigned long long hsize_t; +typedef signed long long hssize_t; H5_GCC_DIAG_ON(long-long) # define H5_SIZEOF_HSIZE_T H5_SIZEOF_LONG_LONG # define H5_SIZEOF_HSSIZE_T H5_SIZEOF_LONG_LONG @@ -229,7 +229,7 @@ H5_GCC_DIAG_ON(long-long) #else # error "nothing appropriate for H5_PRINTF_HADDR_FMT" #endif -#define HADDR_MAX (HADDR_UNDEF-1) +#define HADDR_MAX (HADDR_UNDEF-1) /* uint32_t type is used for creation order field for messages. It may be * defined in Posix.1g, otherwise it is defined here. @@ -297,7 +297,7 @@ typedef enum { H5_ITER_INC, /* Increasing order */ H5_ITER_DEC, /* Decreasing order */ H5_ITER_NATIVE, /* No particular order, whatever is fastest */ - H5_ITER_N /* Number of iteration orders */ + H5_ITER_N /* Number of iteration orders */ } H5_iter_order_t; /* Iteration callback values */ @@ -314,10 +314,10 @@ typedef enum { * links in groups/attributes on objects. */ typedef enum H5_index_t { - H5_INDEX_UNKNOWN = -1, /* Unknown index type */ - H5_INDEX_NAME, /* Index on names */ - H5_INDEX_CRT_ORDER, /* Index on creation order */ - H5_INDEX_N /* Number of indices defined */ + H5_INDEX_UNKNOWN = -1, /* Unknown index type */ + H5_INDEX_NAME, /* Index on names */ + H5_INDEX_CRT_ORDER, /* Index on creation order */ + H5_INDEX_N /* Number of indices defined */ } H5_index_t; /* @@ -353,9 +353,9 @@ H5_DLL herr_t H5get_free_list_sizes(size_t *reg_size, size_t *arr_size, size_t *blk_size, size_t *fac_size); H5_DLL herr_t H5get_alloc_stats(H5_alloc_stats_t *stats); H5_DLL herr_t H5get_libversion(unsigned *majnum, unsigned *minnum, - unsigned *relnum); + unsigned *relnum); H5_DLL herr_t H5check_version(unsigned majnum, unsigned minnum, - unsigned relnum); + unsigned relnum); H5_DLL herr_t H5is_library_threadsafe(hbool_t *is_ts); H5_DLL herr_t H5free_memory(void *mem); H5_DLL void *H5allocate_memory(size_t size, hbool_t clear); diff --git a/src/H5system.c b/src/H5system.c index a7ca3f8..e2747f4 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -13,11 +13,11 @@ /*------------------------------------------------------------------------- * - * Created: H5system.c - * Aug 21 2006 - * Quincey Koziol <koziol@hdfgroup.org> + * Created: H5system.c + * Aug 21 2006 + * Quincey Koziol * - * Purpose: System call wrapper implementations. + * Purpose: System call wrapper implementations. * *------------------------------------------------------------------------- */ @@ -30,10 +30,10 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fprivate.h" /* File access */ -#include "H5MMprivate.h" /* Memory management */ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5MMprivate.h" /* Memory management */ /****************/ @@ -377,7 +377,7 @@ HDfprintf(FILE *stream, const char *fmt, ...) if(fwidth) len += HDsnprintf(format_templ + len, (sizeof(format_templ) - (size_t)(len + 1)), "%d", fwidth); HDstrncat(format_templ, "s", (sizeof(format_templ) - (size_t)(len + 1))); - fprintf(stream, format_templ, "UNDEF"); + n = fprintf(stream, format_templ, "UNDEF"); } } break; @@ -407,11 +407,25 @@ HDfprintf(FILE *stream, const char *fmt, ...) htri_t tri_var = HDva_arg(ap, htri_t); if(tri_var > 0) - fprintf(stream, "TRUE"); + n = fprintf(stream, "TRUE"); else if(!tri_var) - fprintf(stream, "FALSE"); + n = fprintf(stream, "FALSE"); else - fprintf(stream, "FAIL(%d)", (int)tri_var); + n = fprintf(stream, "FAIL(%d)", (int)tri_var); + } + break; + + case 'T': /* Elapsed time, in seconds */ + { + double seconds = HDva_arg(ap, double); + char *time_string = H5_timer_get_time_string(seconds); + + if(time_string) { + n = fprintf(stream, format_templ, time_string); + HDfree(time_string); + } /* end if */ + else + n = fprintf(stream, format_templ, "(error)"); } break; @@ -474,8 +488,6 @@ H5_GCC_DIAG_ON(format-nonliteral) * Programmer: Robb Matzke * Thursday, April 9, 1998 * - * Modifications: - * *------------------------------------------------------------------------- */ #ifndef HDstrtoll @@ -647,14 +659,14 @@ Pflock(int fd, int operation) { * Purpose: Wrapper function for systems where no file locking is * available. * - * Return: Failure: -1 (always fails) + * Return: 0 (success) * *------------------------------------------------------------------------- */ int H5_ATTR_CONST Nflock(int H5_ATTR_UNUSED fd, int H5_ATTR_UNUSED operation) { - /* just fail */ - return -1; + /* just succeed */ + return 0; } /* end Nflock() */ @@ -833,11 +845,88 @@ Wsetenv(const char *name, const char *value, int overwrite) #pragma comment(lib, "advapi32.lib") #endif + +/*------------------------------------------------------------------------- + * Function: H5_get_win32_times + * + * Purpose: Gets the elapsed, system and user times on Windows platforms. + * All time values are in seconds. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_WIN32_API +int +H5_get_win32_times(H5_timevals_t *tvs /*in,out*/) +{ + static HANDLE process_handle; + ULARGE_INTEGER kernel_start; + ULARGE_INTEGER user_start; + FILETIME KernelTime; + FILETIME UserTime; + FILETIME CreationTime; + FILETIME ExitTime; + LARGE_INTEGER counts_start; + static LARGE_INTEGER counts_freq; + static hbool_t is_initialized = FALSE; + BOOL err; + + HDassert(tvs); + + if(!is_initialized) { + /* NOTE: This is just a pseudo handle and does not need to be closed. */ + process_handle = GetCurrentProcess(); + err = QueryPerformanceFrequency(&counts_freq); + if(0 == err) + return -1; + is_initialized = TRUE; + } /* end if */ + + /************************* + * System and user times * + *************************/ + + err = GetProcessTimes(process_handle, &CreationTime, &ExitTime, &KernelTime, + &UserTime); + if(0 == err) + return -1; + + /* The 1.0E7 factor seems strange but it's due to the clock + * ticking in 100 ns increments. + */ + kernel_start.HighPart = KernelTime.dwHighDateTime; + kernel_start.LowPart = KernelTime.dwLowDateTime; + tvs->system = (double)(kernel_start.QuadPart / 1.0E7F); + + user_start.HighPart = UserTime.dwHighDateTime; + user_start.LowPart = UserTime.dwLowDateTime; + tvs->user = (double)(user_start.QuadPart / 1.0E7F); + + /**************** + * Elapsed time * + ****************/ + + err = QueryPerformanceCounter(&counts_start); + if(0 == err) + return -1; + + tvs->elapsed = (double)(counts_start.QuadPart) / (double)counts_freq.QuadPart; + + return 0; +} /* end H5_get_win32_times() */ +#endif + #define WloginBuffer_count 256 static char Wlogin_buffer[WloginBuffer_count]; + char* -Wgetlogin() +Wgetlogin(void) { #ifdef H5_HAVE_WINSOCK2_H @@ -1342,38 +1431,6 @@ H5_nanosleep(uint64_t nanosec) FUNC_LEAVE_NOAPI_VOID } /* end H5_nanosleep() */ - -/*-------------------------------------------------------------------------- - * Function: H5_get_time - * - * Purpose: Get the current time, as the time of seconds after the UNIX epoch - * - * Return: SUCCEED/FAIL - * - * Programmer: Quincey Koziol - * October 05, 2016 - *-------------------------------------------------------------------------- - */ -double -H5_get_time(void) -{ -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval curr_time; -#endif /* H5_HAVE_GETTIMEOFDAY */ - double ret_value = (double)0.0f; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&curr_time, NULL); - - ret_value = (double)curr_time.tv_sec + ((double)curr_time.tv_usec / (double)1000000.0f); -#endif /* H5_HAVE_GETTIMEOFDAY */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5_get_time() */ - - #ifdef H5_HAVE_WIN32_API #define H5_WIN32_ENV_VAR_BUFFER_SIZE 32767 diff --git a/src/H5timer.c b/src/H5timer.c index 4b1ec06..61d6f0f 100644 --- a/src/H5timer.c +++ b/src/H5timer.c @@ -12,12 +12,11 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /*------------------------------------------------------------------------- - * * Created: H5timer.c * Aug 21 2006 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * - * Purpose: Internal 'timer' routines & support routines. + * Purpose: Internal, platform-independent 'timer' support routines. * *------------------------------------------------------------------------- */ @@ -32,20 +31,22 @@ /***********/ #include "H5private.h" /* Generic Functions */ -/* We need this for the struct rusage declaration */ -#if defined(H5_HAVE_GETRUSAGE) && defined(H5_HAVE_SYS_RESOURCE_H) -# include <sys/resource.h> -#endif - -#if defined(H5_HAVE_GETTIMEOFDAY) && defined(H5_HAVE_SYS_TIME_H) -#include <sys/time.h> -#endif - /****************/ /* Local Macros */ /****************/ +/* Size of a generated time string. + * Most time strings should be < 20 or so characters (max!) so this should be a + * safe size. Dynamically allocating the correct size would be painful. + */ +#define H5TIMER_TIME_STRING_LEN 1536 + +/* Conversion factors */ +#define H5_SEC_PER_DAY (double)(24.0F * 60.0F * 60.0F) +#define H5_SEC_PER_HOUR (double)(60.0F * 60.0F) +#define H5_SEC_PER_MIN (double)(60.0F) + /******************/ /* Local Typedefs */ @@ -76,104 +77,6 @@ /* Local Variables */ /*******************/ - -/*------------------------------------------------------------------------- - * Function: H5_timer_reset - * - * Purpose: Resets the timer struct to zero. Use this to reset a timer - * that's being used as an accumulator for summing times. - * - * Return: void - * - * Programmer: Robb Matzke - * Thursday, April 16, 1998 - * - *------------------------------------------------------------------------- - */ -void -H5_timer_reset (H5_timer_t *timer) -{ - HDassert(timer); - HDmemset(timer, 0, sizeof *timer); -} /* end H5_timer_reset() */ - - -/*------------------------------------------------------------------------- - * Function: H5_timer_begin - * - * Purpose: Initialize a timer to time something. - * - * Return: void - * - * Programmer: Robb Matzke - * Thursday, April 16, 1998 - * - *------------------------------------------------------------------------- - */ -void -H5_timer_begin (H5_timer_t *timer) -{ -#ifdef H5_HAVE_GETRUSAGE - struct rusage rusage; -#endif -#ifdef H5_HAVE_GETTIMEOFDAY - struct timeval etime; -#endif - - HDassert(timer); - -#ifdef H5_HAVE_GETRUSAGE - HDgetrusage (RUSAGE_SELF, &rusage); - timer->utime = (double)rusage.ru_utime.tv_sec + - ((double)rusage.ru_utime.tv_usec / (double)1e6F); - timer->stime = (double)rusage.ru_stime.tv_sec + - ((double)rusage.ru_stime.tv_usec / (double)1e6F); -#else - timer->utime = 0.0F; - timer->stime = 0.0F; -#endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday (&etime, NULL); - timer->etime = (double)etime.tv_sec + ((double)etime.tv_usec / (double)1e6F); -#else - timer->etime = 0.0F; -#endif -} /* end H5_timer_begin() */ - - -/*------------------------------------------------------------------------- - * Function: H5_timer_end - * - * Purpose: This function should be called at the end of a timed region. - * The SUM is an optional pointer which will accumulate times. - * TMS is the same struct that was passed to H5_timer_start(). - * On return, TMS will contain total times for the timed region. - * - * Return: void - * - * Programmer: Robb Matzke - * Thursday, April 16, 1998 - * - *------------------------------------------------------------------------- - */ -void -H5_timer_end (H5_timer_t *sum/*in,out*/, H5_timer_t *timer/*in,out*/) -{ - H5_timer_t now; - - HDassert(timer); - H5_timer_begin(&now); - - timer->utime = MAX((double)0.0F, now.utime - timer->utime); - timer->stime = MAX((double)0.0F, now.stime - timer->stime); - timer->etime = MAX((double)0.0F, now.etime - timer->etime); - - if (sum) { - sum->utime += timer->utime; - sum->stime += timer->stime; - sum->etime += timer->etime; - } -} /* end H5_timer_end() */ /*------------------------------------------------------------------------- @@ -209,32 +112,35 @@ H5_bandwidth(char *buf/*out*/, double nbytes, double nseconds) if(nseconds <= (double)0.0F) HDstrcpy(buf, " NaN"); else { - bw = nbytes/nseconds; + bw = nbytes / nseconds; if(H5_DBL_ABS_EQUAL(bw, (double)0.0F)) HDstrcpy(buf, "0.000 B/s"); else if(bw < (double)1.0F) HDsprintf(buf, "%10.4e", bw); else if(bw < (double)H5_KB) { HDsprintf(buf, "%05.4f", bw); - HDstrcpy(buf+5, " B/s"); + HDstrcpy(buf + 5, " B/s"); } else if(bw < (double)H5_MB) { HDsprintf(buf, "%05.4f", bw / (double)H5_KB); - HDstrcpy(buf+5, " kB/s"); + HDstrcpy(buf + 5, " kB/s"); } else if(bw < (double)H5_GB) { HDsprintf(buf, "%05.4f", bw / (double)H5_MB); - HDstrcpy(buf+5, " MB/s"); + HDstrcpy(buf + 5, " MB/s"); } else if(bw < (double)H5_TB) { HDsprintf(buf, "%05.4f", bw / (double)H5_GB); - HDstrcpy(buf+5, " GB/s"); + HDstrcpy(buf + 5, " GB/s"); } else if(bw < (double)H5_PB) { HDsprintf(buf, "%05.4f", bw / (double)H5_TB); - HDstrcpy(buf+5, " TB/s"); + HDstrcpy(buf + 5, " TB/s"); + } else if(bw < (double)H5_EB) { + HDsprintf(buf, "%05.4f", bw / (double)H5_PB); + HDstrcpy(buf + 5, " PB/s"); } else { HDsprintf(buf, "%10.4e", bw); if(HDstrlen(buf) > 10) HDsprintf(buf, "%10.3e", bw); - } - } + } /* end else-if */ + } /* end else */ } /* end H5_bandwidth() */ @@ -269,3 +175,488 @@ H5_now(void) return(now); } /* end H5_now() */ + +/*------------------------------------------------------------------------- + * Function: H5_now_usec + * + * Purpose: Retrieves the current time, as microseconds after the UNIX epoch. + * + * Return: # of microseconds from the epoch (can't fail) + * + * Programmer: Quincey Koziol + * Tuesday, November 28, 2006 + * + *------------------------------------------------------------------------- + */ +uint64_t +H5_now_usec(void) +{ + uint64_t now; /* Current time, in microseconds */ + +#if defined(H5_HAVE_CLOCK_GETTIME) + { + struct timespec ts; + + HDclock_gettime(CLOCK_MONOTONIC, &ts); + now = (uint64_t)(ts.tv_sec * (1000 * 1000)) + (uint64_t)(ts.tv_nsec / 1000); + } +#elif defined(H5_HAVE_GETTIMEOFDAY) + { + struct timeval now_tv; + + HDgettimeofday(&now_tv, NULL); + now = (uint64_t)(now_tv.tv_sec * (1000 * 1000)) + (uint64_t)now_tv.tv_usec; + } +#else /* H5_HAVE_GETTIMEOFDAY */ + now = (uint64_t)(HDtime(NULL) * (1000 * 1000)); +#endif /* H5_HAVE_GETTIMEOFDAY */ + + return(now); +} /* end H5_now_usec() */ + + +/*-------------------------------------------------------------------------- + * Function: H5_get_time + * + * Purpose: Get the current time, as the time of seconds after the UNIX epoch + * + * Return: Success: A non-negative time value + * Failure: -1.0 (in theory, can't currently fail) + * + * Programmer: Quincey Koziol + * October 05, 2016 + *-------------------------------------------------------------------------- + */ +double +H5_get_time(void) +{ + double ret_value = (double)0.0f; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if defined(H5_HAVE_CLOCK_GETTIME) + { + struct timespec ts; + + HDclock_gettime(CLOCK_MONOTONIC, &ts); + ret_value = (double)ts.tv_sec + ((double)ts.tv_nsec / (double)1000000000.0f); + } +#elif defined(H5_HAVE_GETTIMEOFDAY) + { + struct timeval now_tv; + + HDgettimeofday(&now_tv, NULL); + ret_value = (double)now_tv.tv_sec + ((double)now_tv.tv_usec / (double)1000000.0f); + } +#else + ret_value = (double)HDtime(NULL); +#endif + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_get_time() */ + + + +/*------------------------------------------------------------------------- + * Function: H5__timer_get_timevals + * + * Purpose: Internal platform-specific function to get time system, + * user and elapsed time values. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5__timer_get_timevals(H5_timevals_t *times /*in,out*/) +{ + /* Sanity check */ + HDassert(times); + + /* Windows call handles both system/user and elapsed times */ +#ifdef H5_HAVE_WIN32_API + if(H5_get_win32_times(times) < 0) { + times->elapsed = -1.0; + times->system = -1.0; + times->user = -1.0; + + return -1; + } /* end if */ +#else /* H5_HAVE_WIN32_API */ + + /************************* + * System and user times * + *************************/ +#if defined(H5_HAVE_GETRUSAGE) +{ + struct rusage res; + + if(HDgetrusage(RUSAGE_SELF, &res) < 0) + return -1; + times->system = (double)res.ru_stime.tv_sec + ((double)res.ru_stime.tv_usec / (double)1.0E6F); + times->user = (double)res.ru_utime.tv_sec + ((double)res.ru_utime.tv_usec / (double)1.0E6F); +} +#else + /* No suitable way to get system/user times */ + /* This is not an error condition, they just won't be available */ + times->system = -1.0; + times->user = -1.0; +#endif + + /**************** + * Elapsed time * + ****************/ + + times->elapsed = H5_get_time(); + +#endif /* H5_HAVE_WIN32_API */ + + return 0; +} /* end H5__timer_get_timevals() */ + + +/*------------------------------------------------------------------------- + * Function: H5_timer_init + * + * Purpose: Initialize a platform-independent timer. + * + * Timer usage is as follows: + * + * 1) Call H5_timer_init(), passing in a timer struct, to set + * up the timer. + * + * 2) Wrap any code you'd like to time with calls to + * H5_timer_start/stop(). For accurate timing, place these + * calls as close to the code of interest as possible. You + * can call start/stop multiple times on the same timer - + * when you do this, H5_timer_get_times() will return time + * values for the current/last session and + * H5_timer_get_total_times() will return the summed times + * of all sessions (see #3 and #4, below). + * + * 3) Use H5_timer_get_times() to get the current system, user + * and elapsed times from a running timer. If called on a + * stopped timer, this will return the time recorded at the + * stop point. + * + * 4) Call H5_timer_get_total_times() to get the total system, + * user and elapsed times recorded across multiple start/stop + * sessions. If called on a running timer, it will return the + * time recorded up to that point. On a stopped timer, it + * will return the time recorded at the stop point. + * + * NOTE: Obtaining a time point is not free! Keep in mind that + * the time functions make system calls and can have + * non-trivial overhead. If you call one of the get_time + * functions on a running timer, that overhead will be + * added to the reported times. + * + * 5) All times recorded will be in seconds. These can be + * converted into human-readable strings with the + * H5_timer_get_time_string() function. + * + * 6) A timer can be reset using by calling H5_timer_init() on + * it. This will set its state to 'stopped' and reset all + * accumulated times to zero. + * + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +herr_t +H5_timer_init(H5_timer_t *timer /*in,out*/) +{ + /* Sanity check */ + HDassert(timer); + + /* Initialize everything */ + HDmemset(timer, 0, sizeof(H5_timer_t)); + + return 0; +} /* end H5_timer_init() */ + + +/*------------------------------------------------------------------------- + * Function: H5_timer_start + * + * Purpose: Start tracking time in a platform-independent timer. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +herr_t +H5_timer_start(H5_timer_t *timer /*in,out*/) +{ + /* Sanity check */ + HDassert(timer); + + /* Start the timer + * This sets the "initial" times to the system-defined start times. + */ + if(H5__timer_get_timevals(&(timer->initial)) < 0) + return -1; + + timer->is_running = TRUE; + + return 0; +} /* end H5_timer_start() */ + + +/*------------------------------------------------------------------------- + * Function: H5_timer_stop + * + * Purpose: Stop tracking time in a platform-independent timer. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +herr_t +H5_timer_stop(H5_timer_t *timer /*in,out*/) +{ + /* Sanity check */ + HDassert(timer); + + /* Stop the timer */ + if(H5__timer_get_timevals(&(timer->final_interval)) < 0) + return -1; + + /* The "final" times are stored as intervals (final - initial) + * for more useful reporting to the user. + */ + timer->final_interval.elapsed = timer->final_interval.elapsed - timer->initial.elapsed; + timer->final_interval.system = timer->final_interval.system - timer->initial.system; + timer->final_interval.user = timer->final_interval.user - timer->initial.user; + + /* Add the intervals to the elapsed time */ + timer->total.elapsed += timer->final_interval.elapsed; + timer->total.system += timer->final_interval.system; + timer->total.user += timer->final_interval.user; + + timer->is_running = FALSE; + + return 0; +} /* end H5_timer_stop() */ + + +/*------------------------------------------------------------------------- + * Function: H5_timer_get_times + * + * Purpose: Get the system, user and elapsed times from a timer. These + * are the times since the timer was last started and will be + * 0.0 in a timer that has not been started since it was + * initialized. + * + * This function can be called either before or after + * H5_timer_stop() has been called. If it is called before the + * stop function, the timer will continue to run. + * + * The system and user times will be -1.0 if those times cannot + * be computed on a particular platform. The elapsed time will + * always be present. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +herr_t +H5_timer_get_times(H5_timer_t timer, H5_timevals_t *times /*in,out*/) +{ + /* Sanity check */ + HDassert(times); + + if(timer.is_running) { + H5_timevals_t now; + + /* Get the current times and report the current intervals without + * stopping the timer. + */ + if(H5__timer_get_timevals(&now) < 0) + return -1; + + times->elapsed = now.elapsed - timer.initial.elapsed; + times->system = now.system - timer.initial.system; + times->user = now.user - timer.initial.user; + } /* end if */ + else { + times->elapsed = timer.final_interval.elapsed; + times->system = timer.final_interval.system; + times->user = timer.final_interval.user; + } /* end else */ + + return 0; +} /* end H5_timer_get_times() */ + + +/*------------------------------------------------------------------------- + * Function: H5_timer_get_total_times + * + * Purpose: Get the TOTAL system, user and elapsed times recorded by + * the timer since its initialization. This is the sum of all + * times recorded while the timer was running. + * + * These will be 0.0 in a timer that has not been started + * since it was initialized. Calling H5_timer_init() on a + * timer will reset these values to 0.0. + * + * This function can be called either before or after + * H5_timer_stop() has been called. If it is called before the + * stop function, the timer will continue to run. + * + * The system and user times will be -1.0 if those times cannot + * be computed on a particular platform. The elapsed time will + * always be present. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +herr_t +H5_timer_get_total_times(H5_timer_t timer, H5_timevals_t *times /*in,out*/) +{ + /* Sanity check */ + HDassert(times); + + if(timer.is_running) { + H5_timevals_t now; + + /* Get the current times and report the current totals without + * stopping the timer. + */ + if(H5__timer_get_timevals(&now) < 0) + return -1; + + times->elapsed = timer.total.elapsed + (now.elapsed - timer.initial.elapsed); + times->system = timer.total.system + (now.system - timer.initial.system); + times->user = timer.total.user + (now.user - timer.initial.user); + } /* end if */ + else { + times->elapsed = timer.total.elapsed; + times->system = timer.total.system; + times->user = timer.total.user; + } /* end else */ + + return 0; +} /* end H5_timer_get_total_times() */ + + +/*------------------------------------------------------------------------- + * Function: H5_timer_get_time_string + * + * Purpose: Converts a time (in seconds) into a human-readable string + * suitable for log messages. + * + * Return: Success: The time string. + * + * The general format of the time string is: + * + * "N/A" time < 0 (invalid time) + * "%.f ns" time < 1 microsecond + * "%.1f us" time < 1 millisecond + * "%.1f ms" time < 1 second + * "%.2f s" time < 1 minute + * "%.f m %.f s" time < 1 hour + * "%.f h %.f m %.f s" longer times + * + * Failure: NULL + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +char * +H5_timer_get_time_string(double seconds) +{ + char *s; /* output string */ + + /* Used when the time is greater than 59 seconds */ + double days; + double hours; + double minutes; + double remainder_sec; + + /* Extract larger time units from count of seconds */ + if(seconds > (double)60.0F) { + /* Set initial # of seconds */ + remainder_sec = seconds; + + /* Extract days */ + days = HDfloor(remainder_sec / H5_SEC_PER_DAY); + remainder_sec -= (days * H5_SEC_PER_DAY); + + /* Extract hours */ + hours = HDfloor(remainder_sec / H5_SEC_PER_HOUR); + remainder_sec -= (hours * H5_SEC_PER_HOUR); + + /* Extract minutes */ + minutes = HDfloor(remainder_sec / H5_SEC_PER_MIN); + remainder_sec -= (minutes * H5_SEC_PER_MIN); + + /* The # of seconds left is in remainder_sec */ + } /* end if */ + + /* Allocate */ + if(NULL == (s = (char *)HDcalloc(H5TIMER_TIME_STRING_LEN, sizeof(char)))) + return NULL; + + /* Do we need a format string? Some people might like a certain + * number of milliseconds or s before spilling to the next highest + * time unit. Perhaps this could be passed as an integer. + * (name? round_up_size? ?) + */ + if(seconds < (double)0.0F) + HDsprintf(s, "N/A"); + else if(H5_DBL_ABS_EQUAL((double)0.0F, seconds)) + HDsprintf(s, "0.0 s"); + else if(seconds < (double)1.0E-6F) + /* t < 1 us, Print time in ns */ + HDsprintf(s, "%.f ns", seconds * (double)1.0E9F); + else if(seconds < (double)1.0E-3F) + /* t < 1 ms, Print time in us */ + HDsprintf(s, "%.1f us", seconds * (double)1.0E6F); + else if(seconds < (double)1.0F) + /* t < 1 s, Print time in ms */ + HDsprintf(s, "%.1f ms", seconds * (double)1.0E3F); + else if(seconds < H5_SEC_PER_MIN) + /* t < 1 m, Print time in s */ + HDsprintf(s, "%.2f s", seconds); + else if(seconds < H5_SEC_PER_HOUR) + /* t < 1 h, Print time in m and s */ + HDsprintf(s, "%.f m %.f s", minutes, remainder_sec); + else if(seconds < H5_SEC_PER_DAY) + /* t < 1 d, Print time in h, m and s */ + HDsprintf(s, "%.f h %.f m %.f s", hours, minutes, remainder_sec); + else + /* Print time in d, h, m and s */ + HDsprintf(s, "%.f d %.f h %.f m %.f s", days, hours, minutes, remainder_sec); + + return s; +} /* end H5_timer_get_time_string() */ + diff --git a/src/H5trace.c b/src/H5trace.c index 93ff681..cf14565 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -15,7 +15,7 @@ * * Created: H5trace.c * Aug 21 2006 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Internal code for tracing API calls * @@ -125,8 +125,11 @@ H5_trace(const double *returning, const char *func, const char *type, ...) hssize_t i; void *vp = NULL; FILE *out = H5_debug_g.trace; - H5_timer_t event_time; - static H5_timer_t first_time = {0.0F, 0.0F, 0.0F}; + static hbool_t is_first_invocation = TRUE; + H5_timer_t function_timer; + H5_timevals_t function_times; + static H5_timer_t running_timer; + H5_timevals_t running_times; static int current_depth = 0; static int last_call_depth = 0; @@ -152,13 +155,18 @@ H5_trace(const double *returning, const char *func, const char *type, ...) } /* end else */ } /* end if */ - /* Get time for event */ - if(H5_DBL_ABS_EQUAL(first_time.etime, (double)0.0f)) - H5_timer_begin(&first_time); - if(H5_debug_g.ttimes) - H5_timer_begin(&event_time); - else - HDmemset(&event_time, 0, sizeof event_time); + /* Get time for event if the trace times flag is set */ + if(is_first_invocation && H5_debug_g.ttimes) { + /* start the library-wide timer */ + is_first_invocation = FALSE; + H5_timer_init(&running_timer); + H5_timer_start(&running_timer); + } /* end if */ + if(H5_debug_g.ttimes) { + /* start the timer for this function */ + H5_timer_init(&function_timer); + H5_timer_start(&function_timer); + } /* end if */ /* Print the first part of the line. This is the indication of the * nesting depth followed by the function name and either start of @@ -174,7 +182,9 @@ H5_trace(const double *returning, const char *func, const char *type, ...) if(H5_debug_g.ttimes) { char tmp[320]; - HDsprintf(tmp, "%.6f", event_time.etime-first_time.etime); + H5_timer_get_times(function_timer, &function_times); + H5_timer_get_times(running_timer, &running_times); + HDsprintf(tmp, "%.6f", (function_times.elapsed - running_times.elapsed)); HDfprintf(out, " %*s ", (int)HDstrlen(tmp), ""); } /* end if */ for(i = 0; i < current_depth; i++) @@ -189,8 +199,11 @@ H5_trace(const double *returning, const char *func, const char *type, ...) else { if(current_depth>last_call_depth) HDfputs(" = <delayed>\n", out); - if(H5_debug_g.ttimes) - HDfprintf(out, "@%.6f ", event_time.etime - first_time.etime); + if(H5_debug_g.ttimes) { + H5_timer_get_times(function_timer, &function_times); + H5_timer_get_times(running_timer, &running_times); + HDfprintf(out, "@%.6f ", (function_times.elapsed - running_times.elapsed)); + } /* end if */ for(i = 0; i < current_depth; i++) HDfputc('+', out); HDfprintf(out, "%*s%s(", 2*current_depth, "", func); @@ -296,7 +309,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) case 'a': if(ptr) { if(vp) - HDfprintf (out, "0x%p", vp); + HDfprintf(out, "0x%p", vp); else HDfprintf(out, "NULL"); } /* end if */ @@ -749,7 +762,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) break; default: - HDfprintf (out, "BADTYPE(D%c)", type[1]); + HDfprintf(out, "BADTYPE(D%c)", type[1]); goto error; } /* end switch */ break; @@ -1095,7 +1108,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) case 's': if(ptr) { if(vp) - HDfprintf (out, "0x%p", vp); + HDfprintf(out, "0x%p", vp); else HDfprintf(out, "NULL"); } /* end if */ @@ -1172,7 +1185,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) break; default: - HDfprintf (out, "BADTYPE(H%c)", type[1]); + HDfprintf(out, "BADTYPE(H%c)", type[1]); goto error; } /* end switch */ break; @@ -1356,7 +1369,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) break; case H5I_NTYPES: - HDfprintf (out, "%ld (ntypes - error)", (long)obj); + HDfprintf(out, "%ld (ntypes - error)", (long)obj); break; default: @@ -1460,7 +1473,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) else { int is = HDva_arg(ap, int); - HDfprintf (out, "%d", is); + HDfprintf(out, "%d", is); asize[argno] = is; } /* end else */ break; @@ -1576,7 +1589,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) break; default: - HDfprintf (out, "BADTYPE(I%c)", type[1]); + HDfprintf(out, "BADTYPE(I%c)", type[1]); goto error; } /* end switch */ break; @@ -1586,7 +1599,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) case 'l': if(ptr) { if(vp) - HDfprintf (out, "0x%p", vp); + HDfprintf(out, "0x%p", vp); else HDfprintf(out, "NULL"); } /* end if */ @@ -1730,7 +1743,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) else { off_t offset = HDva_arg(ap, off_t); - HDfprintf (out, "%ld", (long)offset); + HDfprintf(out, "%ld", (long)offset); } /* end else */ break; @@ -2394,7 +2407,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) break; default: - HDfprintf (out, "BADTYPE(T%c)", type[1]); + HDfprintf(out, "BADTYPE(T%c)", type[1]); goto error; } /* end switch */ break; @@ -2410,9 +2423,9 @@ H5_trace(const double *returning, const char *func, const char *type, ...) htri_t tri_var = HDva_arg (ap, htri_t); if(tri_var>0) - HDfprintf (out, "TRUE"); + HDfprintf(out, "TRUE"); else if(!tri_var) - HDfprintf (out, "FALSE"); + HDfprintf(out, "FALSE"); else HDfprintf(out, "FAIL(%d)", (int)tri_var); } /* end else */ @@ -2469,7 +2482,7 @@ H5_trace(const double *returning, const char *func, const char *type, ...) break; default: - HDfprintf (out, "BADTYPE(U%c)", type[1]); + HDfprintf(out, "BADTYPE(U%c)", type[1]); goto error; } /* end switch */ break; @@ -2650,9 +2663,12 @@ H5_trace(const double *returning, const char *func, const char *type, ...) } /* end for */ /* Display event time for return */ - if(returning && H5_debug_g.ttimes) - HDfprintf(out, " @%.6f [dt=%.6f]", (event_time.etime - first_time.etime), - (event_time.etime - *returning)); + if(returning && H5_debug_g.ttimes) { + H5_timer_get_times(function_timer, &function_times); + H5_timer_get_times(running_timer, &running_times); + HDfprintf(out, " @%.6f [dt=%.6f]", (function_times.elapsed - running_times.elapsed), + (function_times.elapsed - *returning)); + } /* end if */ error: HDva_end(ap); @@ -2660,10 +2676,13 @@ error: HDfprintf(out, ";\n"); else { last_call_depth = current_depth++; - HDfprintf (out, ")"); + HDfprintf(out, ")"); } /* end else */ HDfflush(out); - return event_time.etime; + if(H5_debug_g.ttimes) + return function_times.elapsed; + else + return 0.0F; } /* end H5_trace() */ diff --git a/src/Makefile.am b/src/Makefile.am index 0ec6c41..05a5856 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -63,7 +63,8 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5FA.c H5FAcache.c H5FAdbg.c H5FAdblock.c H5FAdblkpage.c H5FAhdr.c \ H5FAint.c H5FAstat.c H5FAtest.c \ H5FD.c H5FDcore.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c \ - H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c H5FDtest.c \ + H5FDmulti.c H5FDsec2.c H5FDspace.c \ + H5FDsplitter.c H5FDstdio.c H5FDtest.c \ H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSint.c H5FSsection.c \ H5FSstat.c H5FStest.c \ H5G.c H5Gbtree2.c H5Gcache.c \ @@ -126,6 +127,11 @@ if DIRECT_VFD_CONDITIONAL libhdf5_la_SOURCES += H5FDdirect.c endif +# Only compile the mirror VFD if necessary +if MIRROR_VFD_CONDITIONAL + libhdf5_la_SOURCES += H5FDmirror.c +endif + # Only compile the read-only S3 VFD if necessary if ROS3_VFD_CONDITIONAL libhdf5_la_SOURCES += H5FDros3.c H5FDs3comms.c @@ -137,8 +143,8 @@ include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5vers H5Cpublic.h H5Dpublic.h \ H5Epubgen.h H5Epublic.h H5Fpublic.h \ H5FDpublic.h H5FDcore.h H5FDdirect.h H5FDfamily.h H5FDhdfs.h \ - H5FDlog.h H5FDmpi.h H5FDmpio.h H5FDmulti.h H5FDros3.h \ - H5FDsec2.h H5FDstdio.h H5FDwindows.h \ + H5FDlog.h H5FDmirror.h H5FDmpi.h H5FDmpio.h H5FDmulti.h H5FDros3.h \ + H5FDsec2.h H5FDsplitter.h H5FDstdio.h H5FDwindows.h \ H5Gpublic.h H5Ipublic.h H5Lpublic.h \ H5MMpublic.h H5Opublic.h H5Ppublic.h \ H5PLextern.h H5PLpublic.h \ @@ -44,10 +44,12 @@ #include "H5FDfamily.h" /* File families */ #include "H5FDhdfs.h" /* Hadoop HDFS */ #include "H5FDlog.h" /* sec2 driver with I/O logging (for debugging) */ +#include "H5FDmirror.h" /* Mirror VFD and IPC definitions */ #include "H5FDmpi.h" /* MPI-based file drivers */ #include "H5FDmulti.h" /* Usage-partitioned file family */ #include "H5FDros3.h" /* R/O S3 "file" I/O */ #include "H5FDsec2.h" /* POSIX unbuffered file I/O */ +#include "H5FDsplitter.h" /* Twin-channel (R/W & R/O) I/O passthrough */ #include "H5FDstdio.h" /* Standard C buffered I/O */ #ifdef H5_HAVE_WINDOWS #include "H5FDwindows.h" /* Win32 I/O */ diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in index 422ca0b..128e107 100644 --- a/src/libhdf5.settings.in +++ b/src/libhdf5.settings.in @@ -79,6 +79,7 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ I/O filters (external): @EXTERNAL_FILTERS@ MPE: @MPE@ Direct VFD: @DIRECT_VFD@ + Mirror VFD: @MIRROR_VFD@ (Read-Only) S3 VFD: @ROS3_VFD@ (Read-Only) HDFS VFD: @HAVE_LIBHDFS@ dmalloc: @HAVE_DMALLOC@ @@ -87,5 +88,6 @@ Parallel Filtered Dataset Writes: @PARALLEL_FILTERED_WRITES@ Using memory checker: @USINGMEMCHECKER@ Memory allocation sanity checks: @MEMORYALLOCSANITYCHECK@ Function stack tracing: @CODESTACK@ + Use file locking: @DESIRED_FILE_LOCKING@ Strict file format checks: @STRICT_FORMAT_CHECKS@ Optimization instrumentation: @INSTRUMENT_LIBRARY@ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 8e553d0..1911903 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -173,6 +173,11 @@ set (cache_image_SOURCES ${HDF5_TEST_SOURCE_DIR}/genall5.c ) +set(mirror_vfd_SOURCES + ${HDF5_TEST_SOURCE_DIR}/mirror_vfd.c + ${HDF5_TEST_SOURCE_DIR}/genall5.c +) + set (ttsafe_SOURCES ${HDF5_TEST_SOURCE_DIR}/ttsafe.c ${HDF5_TEST_SOURCE_DIR}/ttsafe_dcreate.c @@ -206,7 +211,7 @@ set (H5_TESTS page_buffer dtypes dsets - chunk_info + chunk_info # compression lib link cmpd_dset filter_fail extend @@ -235,6 +240,7 @@ set (H5_TESTS ros3 s3comms hdfs + mirror_vfd ntypes dangle dtransform @@ -250,6 +256,7 @@ set (H5_TESTS cork swmr thread_id # special link + timer ) macro (ADD_H5_EXE file) @@ -267,11 +274,13 @@ macro (ADD_H5_EXE file) endmacro () set (H5_TESTS_MULTIPLE + chunk_info direct_chunk testhdf5 cache_image ttsafe thread_id # special link + mirror_vfd ) # Only build single source tests here foreach (h5_test ${H5_TESTS}) @@ -284,6 +293,19 @@ endforeach () ### M U L T I P L E S O U R C E T E S T S ### ############################################################################## ######### Also special handling of link libs ############# +#-- Adding test for chunk_info +add_executable (chunk_info ${HDF5_TEST_SOURCE_DIR}/chunk_info.c) +target_compile_options(chunk_info PRIVATE "${HDF5_CMAKE_C_FLAGS}") +target_include_directories (chunk_info PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};${HDF5_TEST_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") +if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (chunk_info STATIC) + target_link_libraries (chunk_info PRIVATE ${HDF5_TEST_LIB_TARGET} ${LINK_COMP_LIBS}) +else () + TARGET_C_PROPERTIES (chunk_info SHARED) + target_link_libraries (chunk_info PRIVATE ${HDF5_TEST_LIBSH_TARGET} ${LINK_COMP_LIBS}) +endif () +set_target_properties (chunk_info PROPERTIES FOLDER test) + #-- Adding test for direct_chunk add_executable (direct_chunk ${HDF5_TEST_SOURCE_DIR}/direct_chunk.c) target_compile_options(direct_chunk PRIVATE "${HDF5_CMAKE_C_FLAGS}") @@ -362,6 +384,18 @@ else () endif () set_target_properties (thread_id PROPERTIES FOLDER test) +#-- Adding test for mirror_vfd +add_executable (mirror_vfd ${mirror_vfd_SOURCES}) +target_include_directories (mirror_vfd PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") +if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (mirror_vfd STATIC) + target_link_libraries (mirror_vfd PRIVATE ${HDF5_TEST_LIB_TARGET}) +else () + TARGET_C_PROPERTIES (mirror_vfd SHARED) + target_link_libraries (mirror_vfd PRIVATE ${HDF5_TEST_LIBSH_TARGET}) +endif () +set_target_properties (mirror_vfd PROPERTIES FOLDER test) + ############################################################################## ### A D D I T I O N A L T E S T S ### ############################################################################## diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index 800bfb6..4e14067 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -446,6 +446,10 @@ set (test_CLEANFILES vds_swmr_src_*.h5 tmp_vds_env/vds_src_2.h5 direct_chunk.h5 + splitter*.h5 + splitter.log + mirror_rw/* + mirror_wo/* ) # Remove any output file left over from previous test run diff --git a/test/Makefile.am b/test/Makefile.am index f60f55c..7e1cd4b 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -33,9 +33,11 @@ AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_builddir)/src # testvdsswmr.sh: vds_swmr* # testabort_fail.sh: filenotclosed.c and del_many_dense_attrs.c # test_filter_plugin.sh: filter_plugin.c +# test_mirror.sh: mirror_vfd ../utils/mirror_vfd/* # test_usecases.sh: use_append_chunk, use_append_mchunks, use_disable_mdc_flushes TEST_SCRIPT = testerror.sh testlibinfo.sh testcheck_version.sh testlinks_env.sh testexternal_env.sh \ - testswmr.sh testvds_env.sh testvdsswmr.sh testflushrefresh.sh test_usecases.sh testabort_fail.sh + testswmr.sh testvds_env.sh testvdsswmr.sh testflushrefresh.sh test_usecases.sh testabort_fail.sh \ + test_mirror.sh SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) \ external_env$(EXEEXT) filenotclosed$(EXEEXT) del_many_dense_attrs$(EXEEXT) \ flushrefresh$(EXEEXT) use_append_chunk$(EXEEXT) use_append_mchunks$(EXEEXT) use_disable_mdc_flushes$(EXEEXT) \ @@ -63,7 +65,7 @@ TEST_PROG= testhdf5 \ flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \ enc_dec_plist_cross_platform getname vfd ros3 s3comms hdfs ntypes \ dangle dtransform reserved cross_read freespace mf vds file_image \ - unregister cache_logging cork swmr thread_id + unregister cache_logging cork swmr thread_id timer # List programs to be built when testing here. # error_test and err_compat are built at the same time as the other tests, but executed by testerror.sh. @@ -78,16 +80,18 @@ TEST_PROG= testhdf5 \ # swmr_* files (besides swmr.c) are used by testswmr.sh. # vds_swmr_* files are used by testvdsswmr.sh # vds_env is used by testvds_env.sh +# mirror_vfd is used by test_mirror.sh # 'make check' doesn't run them directly, so they are not included in TEST_PROG. # Also build testmeta, which is used for timings test. It builds quickly, # and this lets automake keep all its test programs in one place. check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version \ testmeta accum_swmr_reader atomic_writer atomic_reader external_env \ links_env filenotclosed del_many_dense_attrs flushrefresh \ - use_append_chunk use_append_mchunks use_disable_mdc_flushes \ + use_append_chunk use_append_chunk_mirror use_append_mchunks use_disable_mdc_flushes \ swmr_generator swmr_start_write swmr_reader swmr_writer swmr_remove_reader \ swmr_remove_writer swmr_addrem_writer swmr_sparse_reader swmr_sparse_writer \ - swmr_check_compat_vfd vds_env vds_swmr_gen vds_swmr_reader vds_swmr_writer + swmr_check_compat_vfd vds_env vds_swmr_gen vds_swmr_reader vds_swmr_writer \ + mirror_vfd if HAVE_SHARED_CONDITIONAL check_PROGRAMS+= filter_plugin endif @@ -137,6 +141,7 @@ LDADD=libh5test.la $(LIBHDF5) ttsafe_SOURCES=ttsafe.c ttsafe_dcreate.c ttsafe_error.c ttsafe_cancel.c \ ttsafe_acreate.c ttsafe_attr_vlen.c cache_image_SOURCES=cache_image.c genall5.c +mirror_vfd_SOURCES=mirror_vfd.c genall5.c VFD_LIST = sec2 stdio core core_paged split multi family if DIRECT_VFD_CONDITIONAL @@ -201,7 +206,8 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offse flushrefresh_VERIFICATION_DONE atomic_data accum_swmr_big.h5 ohdr_swmr.h5 \ test_swmr*.h5 cache_logging.h5 cache_logging.out vds_swmr.h5 vds_swmr_src_*.h5 \ swmr[0-2].h5 swmr_writer.out swmr_writer.log.* swmr_reader.out.* swmr_reader.log.* \ - tbogus.h5.copy cache_image_test.h5 direct_chunk.h5 + tbogus.h5.copy cache_image_test.h5 direct_chunk.h5 \ + splitter*.h5 splitter.log mirror_rw mirror_ro # Sources for testhdf5 executable testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \ @@ -211,11 +217,13 @@ testhdf5_SOURCES=testhdf5.c tarray.c tattr.c tchecksum.c tconfig.c tfile.c \ # Sources for Use Cases use_append_chunk_SOURCES=use_append_chunk.c use_common.c +use_append_chunk_mirror_SOURCES=use_append_chunk_mirror.c use_common.c use_append_mchunks_SOURCES=use_append_mchunks.c use_common.c use_disable_mdc_flushes_SOURCES=use_disable_mdc_flushes.c # Temporary files. DISTCLEANFILES=testerror.sh testlibinfo.sh testcheck_version.sh testlinks_env.sh test_filter_plugin.sh \ - testexternal_env.sh testswmr.sh testvds_env.sh testvdsswmr.sh test_usecases.sh testflushrefresh.sh testabort_fail.sh + testexternal_env.sh testswmr.sh testvds_env.sh testvdsswmr.sh test_usecases.sh testflushrefresh.sh \ + testabort_fail.sh test_mirror.sh include $(top_srcdir)/config/conclude.am diff --git a/test/SWMR_UseCase_UG.txt b/test/SWMR_UseCase_UG.txt index 18d4927..1e3d1e6 100644 --- a/test/SWMR_UseCase_UG.txt +++ b/test/SWMR_UseCase_UG.txt @@ -6,8 +6,8 @@ case program and explain how to run them. 2.1. Author and Dates: - Version 2: By Albert Cheng (acheng@hdfgroup.org), 2013/06/18. - Version 1: By Albert Cheng (acheng@hdfgroup.org), 2013/06/01. + Version 2: By Albert Cheng, 2013/06/18. + Version 1: By Albert Cheng, 2013/06/01. %%%%Use Case 1.7%%%% diff --git a/test/app_ref.c b/test/app_ref.c index a4853fa..c1735fa 100644 --- a/test/app_ref.c +++ b/test/app_ref.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Neil Fortner <nfortne2@hdfgroup.org> + * Programmer: Neil Fortner * Thursday, August 14, 2008 * * Purpose: Tests closing the library after reference counts have been @@ -12,9 +12,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Wednesday, April 8, 1998 - * Modified: Albert Cheng <acheng@hdfgroup.org> + * Modified: Albert Cheng * September 11, 2010 */ /* @@ -803,10 +803,10 @@ main (int ac, char **av) /* Choose random # seed */ seed = (unsigned long)HDtime(NULL); -#ifdef QAK +#if 0 /* seed = (unsigned long)1155438845; */ HDfprintf(stderr, "Random # seed was: %lu\n", seed); -#endif /* QAK */ +#endif HDsrandom((unsigned)seed); /* run VFD-specific test */ diff --git a/test/bittests.c b/test/bittests.c index 046528a..6f236d6 100644 --- a/test/bittests.c +++ b/test/bittests.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Tuesday, June 16, 1998 * * Purpose: Tests functions in H5Tbit.c diff --git a/test/btree2.c b/test/btree2.c index fb18048..b95c595 100644 --- a/test/btree2.c +++ b/test/btree2.c @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> +/* Programmer: Quincey Koziol * Tuesday, February 1, 2005 */ #include "h5test.h" @@ -26,7 +26,7 @@ /* Other private headers that this test requires */ #include "H5CXprivate.h" /* API Contexts */ -#include "H5Iprivate.h" /* Virtual Object Layer */ +#include "H5Iprivate.h" /* Identifiers */ const char *FILENAME[] = { "btree2", diff --git a/test/cache.c b/test/cache.c index 49bb303..10e7bf6 100644 --- a/test/cache.c +++ b/test/cache.c @@ -28146,7 +28146,6 @@ check_auto_cache_resize_aux_fcns(unsigned paged) * Return: void * * Programmer: Mike McGreevy - * <mamcgree@hdfgroup.org> * 12/16/08 * * Modifications: diff --git a/test/cache_api.c b/test/cache_api.c index 1e1a2c9..d6ea7e2 100644 --- a/test/cache_api.c +++ b/test/cache_api.c @@ -1664,7 +1664,7 @@ init_invalid_configs(void) { configs[13].lower_hr_threshold = 1.00000001f; /* 14 -- increment too small */ - configs[14].increment = H5_DOUBLE(0.999999999999); + configs[14].increment = 0.999999999999; /* 15 -- invalid flash_incr_mode */ configs[15].flash_incr_mode = (enum H5C_cache_flash_incr_mode)-1; @@ -1697,7 +1697,7 @@ init_invalid_configs(void) { /* 23 -- decrement too big */ configs[23].decr_mode = H5C_decr__threshold; - configs[23].decrement = H5_DOUBLE(1.0000000001); + configs[23].decrement = 1.0000000001; /* 24 -- epochs_before_eviction too small */ configs[24].epochs_before_eviction = 0; @@ -1709,13 +1709,13 @@ init_invalid_configs(void) { configs[26].empty_reserve = -0.0000000001f; /* 27 -- empty_reserve too big */ - configs[27].empty_reserve = H5_DOUBLE(1.00000000001); + configs[27].empty_reserve = 1.00000000001; /* 28 -- upper_hr_threshold too small */ configs[28].upper_hr_threshold = -0.000000001f; /* 29 -- upper_hr_threshold too big */ - configs[29].upper_hr_threshold = H5_DOUBLE(1.00000001); + configs[29].upper_hr_threshold = 1.00000001; /* 30 -- upper_hr_threshold <= lower_hr_threshold */ configs[30].lower_hr_threshold = 0.9f; diff --git a/test/cache_tagging.c b/test/cache_tagging.c index 57b5d5c..aaf7d9a 100644 --- a/test/cache_tagging.c +++ b/test/cache_tagging.c @@ -439,7 +439,9 @@ check_file_creation_tags(hid_t fcpl_id, int type) { /* Variable Declarations */ hid_t fid = -1; /* File Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose test outout */ +#endif /* NDEBUG */ /* end debugging functions */ haddr_t root_tag = 0; haddr_t sbe_tag = 0; @@ -520,7 +522,9 @@ check_file_open_tags(hid_t fcpl, int type) { /* Variable Declarations */ hid_t fid = -1; /* File Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ haddr_t root_tag; /* Root Group Tag */ haddr_t sbe_tag; /* Sblock Extension Tag */ @@ -627,7 +631,9 @@ check_group_creation_tags(void) /* Variable Declarations */ hid_t fid = -1; /* File Identifier */ hid_t gid = -1; /* Group Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ haddr_t root_tag = HADDR_UNDEF; /* Root Group Tag */ haddr_t g_tag; /* Group Tag */ @@ -722,7 +728,9 @@ check_multi_group_creation_tags(void) /* Variable Declarations */ hid_t fid = -1; /* File Identifier */ hid_t gid = -1; /* Group Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ char gname[16]; /* group name buffer */ int i = 0; /* iterator */ hid_t fapl = -1; /* File access prop list */ @@ -850,7 +858,9 @@ check_link_iteration_tags(void) hid_t fid = -1; /* File Identifier */ hid_t sid = -1; /* Group Identifier */ hid_t did = -1; /* Group Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ int i = 0; /* iterator */ haddr_t root_tag = 0; /* Root Group Tag Value */ char dsetname[500]; /* Name of dataset */ @@ -964,7 +974,9 @@ check_dense_attribute_tags(void) hid_t sid = -1; /* Group Identifier */ hid_t did = -1; /* Group Identifier */ hid_t dcpl = -1; /* Group Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ int i = 0; /* iterator */ hid_t fapl = -1; /* File access property list */ haddr_t d_tag = 0; /* Dataset tag value */ @@ -1144,7 +1156,9 @@ check_group_open_tags(void) /* Variable Declarations */ hid_t fid = -1; /* File Identifier */ hid_t gid = -1; /* Group Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file output */ +#endif /* NDEBUG */ /* end debugging functions */ haddr_t root_tag = HADDR_UNDEF; haddr_t g_tag; @@ -1247,7 +1261,9 @@ check_attribute_creation_tags(hid_t fcpl, int type) hid_t aid = -1; /* Attribute Identifier */ hid_t gid = -1; /* Group Identifier */ hid_t sid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ haddr_t root_tag = 0; /* Root group tag */ haddr_t g_tag = 0; hsize_t dims1[2] = {DIMS, DIMS}; /* dimensions */ @@ -1376,7 +1392,9 @@ check_attribute_open_tags(hid_t fcpl, int type) hid_t aid = -1; /* Attribute Identifier */ hid_t gid = -1; /* Group Identifier */ hid_t sid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ haddr_t root_tag = 0; haddr_t g_tag = 0; hsize_t dims1[2] = {DIMS, DIMS}; /* dimensions */ @@ -1507,7 +1525,9 @@ check_attribute_rename_tags(hid_t fcpl, int type) hid_t gid = -1; /* Group Identifier */ hid_t aid = -1; /* Attribute Identifier */ hid_t sid = -1; /* Dataset Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ int *data = NULL; /* data buffer */ int i,j,k = 0; /* iterators */ haddr_t root_tag = 0; @@ -1686,7 +1706,9 @@ check_attribute_delete_tags(hid_t fcpl, int type) hid_t gid = -1; /* Group Identifier */ hid_t aid = -1; /* Attribute Identifier */ hid_t sid = -1; /* Dataset Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ int *data = NULL; /* data buffer */ int i,j,k = 0; /* iterators */ haddr_t root_tag = 0; @@ -1843,7 +1865,9 @@ check_dataset_creation_tags(hid_t fcpl, int type) hid_t fid = -1; /* File Identifier */ hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -1970,7 +1994,9 @@ check_dataset_creation_earlyalloc_tags(hid_t fcpl, int type) hid_t fid = -1; /* File Identifier */ hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -2101,7 +2127,9 @@ check_dataset_open_tags(void) hid_t fid = -1; /* File Identifier */ hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -2221,7 +2249,9 @@ check_dataset_write_tags(void) hid_t fid = -1; /* File Identifier */ hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -2357,7 +2387,9 @@ check_attribute_write_tags(hid_t fcpl, int type) hid_t gid = -1; /* Group Identifier */ hid_t aid = -1; /* Attribute Identifier */ hid_t sid = -1; /* Dataset Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ int *data = NULL; /* data buffer */ int i,j,k = 0; /* iterators */ haddr_t root_tag = 0; @@ -2504,7 +2536,9 @@ check_dataset_read_tags(void) hid_t fid = -1; /* File Identifier */ hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -2635,7 +2669,9 @@ check_dataset_size_retrieval(void) hid_t fid = -1; /* File Identifier */ hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -2768,7 +2804,9 @@ check_dataset_extend_tags(void) hid_t fid = -1; /* File Identifier */ hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -2899,7 +2937,9 @@ check_object_info_tags(void) /* Variable Declarations */ hid_t fid = -1; /* File Identifier */ hid_t gid = -1; /* Group Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file output */ +#endif /* NDEBUG */ /* end debugging functions */ haddr_t root_tag = HADDR_UNDEF; haddr_t g_tag; H5O_info_t oinfo; /* Object info struct */ @@ -3002,7 +3042,9 @@ check_object_copy_tags(void) /* Variable Declarations */ hid_t fid = -1; /* File Identifier */ hid_t gid = -1; /* Group Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file output */ +#endif /* NDEBUG */ /* end debugging functions */ haddr_t root_tag = HADDR_UNDEF; haddr_t g_tag; haddr_t copy_tag; @@ -3117,7 +3159,9 @@ check_link_removal_tags(hid_t fcpl, int type) hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ hid_t gid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -3270,7 +3314,9 @@ check_link_getname_tags(void) hid_t did = -1; /* Dataset Identifier */ hid_t sid = -1; /* Dataspace Identifier */ hid_t gid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ hid_t dcpl = -1; /* dataset creation pl */ hsize_t cdims[2] = {1,1}; /* chunk dimensions */ int fillval = 0; @@ -3413,7 +3459,9 @@ check_external_link_creation_tags(void) hid_t fid = -1; /* File Identifier */ hid_t fid2 = -1; /* File Identifier */ hid_t gid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ haddr_t root_tag = 0; /* Testing Macro */ @@ -3513,7 +3561,9 @@ check_external_link_open_tags(void) hid_t fid2 = -1; /* File Identifier */ hid_t gid = -1; /* Dataspace Identifier */ hid_t xid = -1; /* Dataspace Identifier */ +#ifndef NDEBUG int verbose = FALSE; /* verbose file outout */ +#endif /* NDEBUG */ /* end debugging functions */ haddr_t root_tag = 0; haddr_t root2_tag = 0; diff --git a/test/cmpd_dset.c b/test/cmpd_dset.c index ff3767c..c8ef1e4 100644 --- a/test/cmpd_dset.c +++ b/test/cmpd_dset.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Friday, January 23, 1998 */ diff --git a/test/cross_read.c b/test/cross_read.c index c16ddbc..dabd3ab 100644 --- a/test/cross_read.c +++ b/test/cross_read.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu <slu@ncsa.uiuc.edu> + * Programmer: Raymond Lu * Thursday, March 23, 2006 * * Purpose: Check if floating-point data created on big-endian and diff --git a/test/dangle.c b/test/dangle.c index 9f30f10..8300792 100644 --- a/test/dangle.c +++ b/test/dangle.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Tuesday, May 13, 2003 * * Purpose: Test dangling IDs diff --git a/test/del_many_dense_attrs.c b/test/del_many_dense_attrs.c index 4c1efae..6f48901 100644 --- a/test/del_many_dense_attrs.c +++ b/test/del_many_dense_attrs.c @@ -62,17 +62,17 @@ static void catch_signal(int H5_ATTR_UNUSED signo) int main(void) { - hid_t fid = -1; /* HDF5 File ID */ - hid_t gid = -1; /* Group ID */ - hid_t sid = -1; /* Dataspace ID */ - hid_t aid = -1; /* Attribute ID */ - hid_t tid = -1; /* Datatype ID */ - hid_t fapl = -1; /* File access property lists */ - hid_t gcpl = -1; /* Group creation property list */ - char aname[50]; /* Name of attribute */ - char *basename="attr"; /* Name prefix for attribute */ - char filename[100]; /* File name */ - int i; /* Local index variable */ + hid_t fid = -1; /* HDF5 File ID */ + hid_t gid = -1; /* Group ID */ + hid_t sid = -1; /* Dataspace ID */ + hid_t aid = -1; /* Attribute ID */ + hid_t tid = -1; /* Datatype ID */ + hid_t fapl = -1; /* File access property lists */ + hid_t gcpl = -1; /* Group creation property list */ + char aname[50]; /* Name of attribute */ + const char *basename="attr"; /* Name prefix for attribute */ + char filename[100]; /* File name */ + int i; /* Local index variable */ /* Testing setup */ h5_reset(); diff --git a/test/direct_chunk.c b/test/direct_chunk.c index 7b17043..2e6cc68 100644 --- a/test/direct_chunk.c +++ b/test/direct_chunk.c @@ -43,7 +43,7 @@ #define CHUNK_NX 4 #define CHUNK_NY 4 -#define DEFLATE_SIZE_ADJUST(s) (HDceil(((double)(s))*H5_DOUBLE(1.001))+H5_DOUBLE(12.0)) +#define DEFLATE_SIZE_ADJUST(s) (HDceil(((double)(s))*1.001)+12.0) /* Temporary filter IDs used for testing */ #define H5Z_FILTER_BOGUS1 305 @@ -661,20 +661,20 @@ filter_bogus1(unsigned int flags, size_t H5_ATTR_UNUSED cd_nelmts, size_t *buf_size, void **buf) { int *int_ptr=(int *)*buf; /* Pointer to the data values */ - ssize_t buf_left=(ssize_t)*buf_size; /* Amount of data buffer left to process */ + size_t buf_left=*buf_size; /* Amount of data buffer left to process */ if(flags & H5Z_FLAG_REVERSE) { /* read */ /* Substract the "add on" value to all the data values */ while(buf_left>0) { *int_ptr++ -= (int)ADD_ON; - buf_left -= (ssize_t)sizeof(int); + buf_left -= sizeof(int); } /* end while */ } /* end if */ else { /* write */ /* Add the "add on" value to all the data values */ while(buf_left>0) { *int_ptr++ += (int)ADD_ON; - buf_left -= (ssize_t)sizeof(int); + buf_left -= sizeof(int); } /* end while */ } /* end else */ @@ -698,20 +698,20 @@ filter_bogus2(unsigned int flags, size_t H5_ATTR_UNUSED cd_nelmts, size_t *buf_size, void **buf) { int *int_ptr=(int *)*buf; /* Pointer to the data values */ - ssize_t buf_left=(ssize_t)*buf_size; /* Amount of data buffer left to process */ + size_t buf_left=*buf_size; /* Amount of data buffer left to process */ if(flags & H5Z_FLAG_REVERSE) { /* read */ /* Substract the "add on" value to all the data values */ while(buf_left>0) { *int_ptr++ /= (int)FACTOR; - buf_left -= (ssize_t)sizeof(int); + buf_left -= sizeof(int); } /* end while */ } /* end if */ else { /* write */ /* Add the "add on" value to all the data values */ while(buf_left>0) { *int_ptr++ *= (int)FACTOR; - buf_left -= (ssize_t)sizeof(int); + buf_left -= sizeof(int); } /* end while */ } /* end else */ diff --git a/test/dsets.c b/test/dsets.c index f7e90a5..d72c65b 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Tuesday, December 9, 1997 * * Purpose: Tests the dataset interface (H5D) @@ -274,13 +274,6 @@ const char *FILENAME[] = { #define STORAGE_SIZE_CHUNK_DIM1 5 #define STORAGE_SIZE_CHUNK_DIM2 5 -/* Parameters for testing version bounds */ -#define VDS_FNAME1 "virtual_file1" -#define VDS_FNAME2 "virtual_file2" -#define SRC_FNAME "source_file" -#define SRC_DSET "src_dset" -#define V_DSET "v_dset" - /* Shared global arrays */ #define DSET_DIM1 100 #define DSET_DIM2 200 @@ -3176,18 +3169,18 @@ test_nbit_double(hid_t file) */ double orig_data[2][5] = { { - H5_DOUBLE(1.6081706885101836e+60), - H5_DOUBLE(-255.32099170994480), - H5_DOUBLE(1.2677579992621376e-61), - H5_DOUBLE(64568.289448797700), - H5_DOUBLE(-1.0619721778839084e-75) + (double)1.6081706885101836e+60L, + -255.32099170994480f, + (double)1.2677579992621376e-61L, + 64568.289448797700f, + (double)-1.0619721778839084e-75L }, { - H5_DOUBLE(2.1499497833454840e+56), - H5_DOUBLE(6.6562295504670740e-3), - H5_DOUBLE(-1.5747263393432150), - H5_DOUBLE(1.0711093225222612), - H5_DOUBLE(-9.8971679387636870e-1) + (double)2.1499497833454840e+56L, + 6.6562295504670740e-3f, + -1.5747263393432150f, + 1.0711093225222612f, + -9.8971679387636870e-1f }}; double new_data[2][5]; size_t precision, offset; @@ -13232,6 +13225,11 @@ error: * *------------------------------------------------------------------------- */ +#define VDS_FNAME1 "virtual_file1" +#define VDS_FNAME2 "virtual_file2" +#define SRC_FNAME "source_file" +#define SRC_DSET "src_dset" +#define V_DSET "v_dset" static herr_t test_versionbounds(void) { diff --git a/test/dt_arith.c b/test/dt_arith.c index 7e1adf5..9b89225 100644 --- a/test/dt_arith.c +++ b/test/dt_arith.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Tuesday, December 9, 1997 * * Purpose: Tests the data type interface (H5T) @@ -770,7 +770,7 @@ static int test_particular_fp_integer(void) /* Print errors */ if(dst_c != SCHAR_MAX) { - double x; + double x = 0.0; signed char y; if(0 == fails_this_test++) @@ -814,7 +814,7 @@ static int test_particular_fp_integer(void) /* Print errors */ if(dst_i != fill_value) { - float x; + float x = 0.0; int y; if(0 == fails_this_test++) @@ -2723,16 +2723,16 @@ my_isnan(dtype_t type, void *val) char s[256]; if (FLT_FLOAT==type) { - float x; + float x = 0.0; HDmemcpy(&x, val, sizeof(float)); retval = (x!=x); } else if (FLT_DOUBLE==type) { - double x; + double x = 0.0; HDmemcpy(&x, val, sizeof(double)); retval = (x!=x); #if H5_SIZEOF_LONG_DOUBLE!=H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE!=0 } else if (FLT_LDOUBLE==type) { - long double x; + long double x = 0.0; HDmemcpy(&x, val, sizeof(long double)); retval = (x!=x); #endif @@ -2746,18 +2746,18 @@ my_isnan(dtype_t type, void *val) */ if (!retval) { if (FLT_FLOAT==type) { - float x; + float x = 0.0; HDmemcpy(&x, val, sizeof(float)); HDsnprintf(s, sizeof(s), "%g", (double)x); } else if (FLT_DOUBLE==type) { - double x; + double x = 0.0; HDmemcpy(&x, val, sizeof(double)); HDsnprintf(s, sizeof(s), "%g", x); #if H5_SIZEOF_LONG_DOUBLE!=H5_SIZEOF_DOUBLE && H5_SIZEOF_LONG_DOUBLE!=0 } else if (FLT_LDOUBLE==type) { - long double x; + long double x = 0.0; HDmemcpy(&x, val, sizeof(long double)); HDsnprintf(s, sizeof(s), "%Lg", x); @@ -3197,7 +3197,7 @@ test_conv_flt_1 (const char *name, int run_test, hid_t src, hid_t dst) int check_expo[2]; if (FLT_FLOAT==dst_type) { - float x; + float x = 0.0; HDmemcpy(&x, &buf[j*dst_size], sizeof(float)); if (underflow && HDfabsf(x) <= FLT_MIN && HDfabsf(hw_f) <= FLT_MIN) @@ -3208,7 +3208,7 @@ test_conv_flt_1 (const char *name, int run_test, hid_t src, hid_t dst) check_mant[0] = HDfrexpf(x, check_expo+0); check_mant[1] = HDfrexpf(hw_f, check_expo+1); } else if (FLT_DOUBLE==dst_type) { - double x; + double x = 0.0; HDmemcpy(&x, &buf[j*dst_size], sizeof(double)); if (underflow && HDfabs(x) <= DBL_MIN && HDfabs(hw_d) <= DBL_MIN) @@ -3220,7 +3220,7 @@ test_conv_flt_1 (const char *name, int run_test, hid_t src, hid_t dst) check_mant[1] = HDfrexp(hw_d, check_expo+1); #if H5_SIZEOF_LONG_DOUBLE !=0 && (H5_SIZEOF_LONG_DOUBLE!=H5_SIZEOF_DOUBLE) } else { - long double x; + long double x = 0.0; HDmemcpy(&x, &buf[j*dst_size], sizeof(long double)); /* dst is largest float, no need to check underflow. */ check_mant[0] = (double)HDfrexpl(x, check_expo+0); @@ -3265,16 +3265,16 @@ test_conv_flt_1 (const char *name, int run_test, hid_t src, hid_t dst) HDprintf(" %02x", saved[j*src_size+ENDIAN(src_size,k,sendian)]); HDprintf("%*s", (int)(3*MAX(0, (ssize_t)dst_size-(ssize_t)src_size)), ""); if (FLT_FLOAT==src_type) { - float x; + float x = 0.0; HDmemcpy(&x, &saved[j*src_size], sizeof(float)); HDprintf(" %29.20e\n", (double)x); } else if (FLT_DOUBLE==src_type) { - double x; + double x = 0.0; HDmemcpy(&x, &saved[j*src_size], sizeof(double)); HDprintf(" %29.20e\n", x); #if H5_SIZEOF_LONG_DOUBLE!=H5_SIZEOF_DOUBLE } else { - long double x; + long double x = 0.0; HDmemcpy(&x, &saved[j*src_size], sizeof(long double)); HDfprintf(stdout," %29.20Le\n", x); #endif @@ -3285,16 +3285,16 @@ test_conv_flt_1 (const char *name, int run_test, hid_t src, hid_t dst) HDprintf(" %02x", buf[j*dst_size+ENDIAN(dst_size,k,dendian)]); HDprintf("%*s", (int)(3*MAX(0, (ssize_t)src_size-(ssize_t)dst_size)), ""); if (FLT_FLOAT==dst_type) { - float x; + float x = 0.0; HDmemcpy(&x, &buf[j*dst_size], sizeof(float)); HDprintf(" %29.20e\n", (double)x); } else if (FLT_DOUBLE==dst_type) { - double x; + double x = 0.0; HDmemcpy(&x, &buf[j*dst_size], sizeof(double)); HDprintf(" %29.20e\n", x); #if H5_SIZEOF_LONG_DOUBLE!=H5_SIZEOF_DOUBLE } else { - long double x; + long double x = 0.0; HDmemcpy(&x, &buf[j*dst_size], sizeof(long double)); HDfprintf(stdout," %29.20Le\n", x); #endif diff --git a/test/dtypes.c b/test/dtypes.c index c89688c..f0c4714 100644 --- a/test/dtypes.c +++ b/test/dtypes.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Tuesday, December 9, 1997 * * Purpose: Tests the datatype interface (H5T) @@ -7642,19 +7642,19 @@ test_versionbounds(void) ret = H5Tenum_insert(enum_type, "RED", &enum_val); if (ret < 0) TEST_ERROR - enum_val++; + enum_val = E1_GREEN; ret = H5Tenum_insert(enum_type, "GREEN", &enum_val); if (ret < 0) TEST_ERROR - enum_val++; + enum_val = E1_BLUE; ret = H5Tenum_insert(enum_type, "BLUE", &enum_val); if (ret < 0) TEST_ERROR - enum_val++; + enum_val = E1_ORANGE; ret = H5Tenum_insert(enum_type, "ORANGE", &enum_val); if (ret < 0) TEST_ERROR - enum_val++; + enum_val = E1_YELLOW; ret = H5Tenum_insert(enum_type, "YELLOW", &enum_val); if (ret < 0) TEST_ERROR diff --git a/test/earray.c b/test/earray.c index aef7e58..d75e125 100644 --- a/test/earray.c +++ b/test/earray.c @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Quincey Koziol <koziol@hdfgroup.org> +/* Programmer: Quincey Koziol * Tuesday, June 17, 2008 */ #include "h5test.h" diff --git a/test/enum.c b/test/enum.c index 109f7c3..1933ce1 100644 --- a/test/enum.c +++ b/test/enum.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Tuesday, December 22, 1998 */ #include "h5test.h" diff --git a/test/extend.c b/test/extend.c index 1e2b5b5..369ad32 100644 --- a/test/extend.c +++ b/test/extend.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Friday, January 30, 1998 * * Purpose: Tests extendible datasets. diff --git a/test/external.c b/test/external.c index d29fb6b..c98c228 100644 --- a/test/external.c +++ b/test/external.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Tuesday, March 3, 1998 * * Purpose: Tests datasets stored in external raw files. diff --git a/test/external_common.c b/test/external_common.c index cfd95a3..f0e7961 100644 --- a/test/external_common.c +++ b/test/external_common.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu <songyulu@hdfgroup.org> + * Programmer: Raymond Lu * April, 2019 * * Purpose: Private function for external.c and external_env.c diff --git a/test/external_common.h b/test/external_common.h index f02652b..3633ea3 100644 --- a/test/external_common.h +++ b/test/external_common.h @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu <songyulu@hdfgroup.org> + * Programmer: Raymond Lu * April, 2019 * * Purpose: Private function for external.c and external_env.c diff --git a/test/external_fname.h b/test/external_fname.h index c5111b6..f5aca6d 100644 --- a/test/external_fname.h +++ b/test/external_fname.h @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@lbl.gov> + * Programmer: Quincey Koziol * July, 2019 * * Purpose: Private declaration for external.c and external_env.c diff --git a/test/fheap.c b/test/fheap.c index 771081b..2e63149 100644 --- a/test/fheap.c +++ b/test/fheap.c @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> +/* Programmer: Quincey Koziol * Friday, February 24, 2006 */ #include "h5test.h" diff --git a/test/filter_fail.c b/test/filter_fail.c index e5187be..b3e5c8a 100644 --- a/test/filter_fail.c +++ b/test/filter_fail.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu <songyulu@hdfgroup.org> + * Programmer: Raymond Lu * 7 September 2010 * * Purpose: Make sure dataset, file, and library can close properly when a diff --git a/test/flush1.c b/test/flush1.c index 04f24f7..e01f4a5 100644 --- a/test/flush1.c +++ b/test/flush1.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Friday, October 23, 1998 * * Purpose: This is the first half of a two-part test that makes sure diff --git a/test/flush2.c b/test/flush2.c index 5fc716b..c3103d9 100644 --- a/test/flush2.c +++ b/test/flush2.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Friday, October 23, 1998 * * Purpose: This is the second half of a two-part test that makes sure diff --git a/test/gen_bad_compound.c b/test/gen_bad_compound.c index 292659c..bd857b4 100644 --- a/test/gen_bad_compound.c +++ b/test/gen_bad_compound.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * Programmer: Quincey Koziol * April 14, 2011 * * Purpose: This program is run to generate an HDF5 data file with objects diff --git a/test/gen_bad_ohdr.c b/test/gen_bad_ohdr.c index 36ba64a..0f85cfe 100644 --- a/test/gen_bad_ohdr.c +++ b/test/gen_bad_ohdr.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * Programmer: Quincey Koziol * Jan 5, 2008 * * Purpose: This program is run to generate an HDF5 data file with a diff --git a/test/gen_bogus.c b/test/gen_bogus.c index 1ab18a4..ab2620f 100644 --- a/test/gen_bogus.c +++ b/test/gen_bogus.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * Programmer: Quincey Koziol * Apr 17, 2007 * * Purpose: This program is run to generate an HDF5 data file with several diff --git a/test/gen_cross.c b/test/gen_cross.c index 105895d..2c1ff4d 100644 --- a/test/gen_cross.c +++ b/test/gen_cross.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu <slu@ncsa.uiuc.edu> + * Programmer: Raymond Lu * Thursday, March 23, 2006 * * This program writes floating-point data to the HDF5 file. It generates diff --git a/test/gen_filters.c b/test/gen_filters.c index 9764830..ace86ba 100644 --- a/test/gen_filters.c +++ b/test/gen_filters.c @@ -38,7 +38,7 @@ static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, * * Failure: -1 * - * Programmer: Pedro Vicente <pvn@ncsa.uiuc.edu> + * Programmer: Pedro Vicente * Thursday, March 25, 2004 * *------------------------------------------------------------------------- diff --git a/test/gen_mergemsg.c b/test/gen_mergemsg.c index f158d57..6633eb0 100644 --- a/test/gen_mergemsg.c +++ b/test/gen_mergemsg.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Friday, June 30, 2006 * * This program creates an object with fragmented object header messages diff --git a/test/gen_new_array.c b/test/gen_new_array.c index 27f162c..041b691 100644 --- a/test/gen_new_array.c +++ b/test/gen_new_array.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Thursday, November 09, 2000 * * Purpose: Create a two datasets, one with a compound datatypes with array diff --git a/test/gen_new_fill.c b/test/gen_new_fill.c index 5bdbf73..c012d8b 100644 --- a/test/gen_new_fill.c +++ b/test/gen_new_fill.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu <slu@ncsa.uiuc.edu> + * Programmer: Raymond Lu * Feb 27, 2002 * * Purpose: This program is run to generate a HDF5 data file with fill diff --git a/test/gen_new_group.c b/test/gen_new_group.c index 6924291..42a751f 100644 --- a/test/gen_new_group.c +++ b/test/gen_new_group.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Oct 24, 2005 * * Purpose: This program is run to generate an HDF5 data file with both diff --git a/test/gen_new_mtime.c b/test/gen_new_mtime.c index b44d567..90259c7 100644 --- a/test/gen_new_mtime.c +++ b/test/gen_new_mtime.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Friday, January 3, 2003 * * Purpose: Create a dataset, which should have the newer mtime information diff --git a/test/gen_new_super.c b/test/gen_new_super.c index d371f5f..36f2187 100644 --- a/test/gen_new_super.c +++ b/test/gen_new_super.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Tuesday, July 15, 2003 * * Purpose: Create a file which will have the newer superblock format. diff --git a/test/gen_old_array.c b/test/gen_old_array.c index 3fab657..402fd40 100644 --- a/test/gen_old_array.c +++ b/test/gen_old_array.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Thursday, November 09, 2000 * * Purpose: Create a two datasets with compound datatypes, one with no array diff --git a/test/gen_old_group.c b/test/gen_old_group.c index d109329..55dbde3 100644 --- a/test/gen_old_group.c +++ b/test/gen_old_group.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Oct 24, 2005 * * Purpose: This program is run to generate an HDF5 data file with an diff --git a/test/gen_old_layout.c b/test/gen_old_layout.c index 56c3e4e..0210786 100644 --- a/test/gen_old_layout.c +++ b/test/gen_old_layout.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Thursday, May 27, 2004 * * Purpose: Create two datasets (one for version 1 and one for version 2 of diff --git a/test/gen_old_mtime.c b/test/gen_old_mtime.c index cbe3bdc..e72c9b1 100644 --- a/test/gen_old_mtime.c +++ b/test/gen_old_mtime.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Friday, January 3, 2003 * * Purpose: Create a dataset, which should have the older mtime information diff --git a/test/gen_plist.c b/test/gen_plist.c index f82cc7b..e66b1d6 100644 --- a/test/gen_plist.c +++ b/test/gen_plist.c @@ -461,7 +461,7 @@ encode_plist(hid_t plist_id, int little_endian, int word_length, const char *fil herr_t ret = 0; void *temp_buf = NULL; size_t temp_size = 0; - ssize_t write_size; + ssize_t H5_ATTR_NDEBUG_UNUSED write_size; char filename[1024]; /* Generate filename */ diff --git a/test/gen_sizes_lheap.c b/test/gen_sizes_lheap.c index 81742df..0e62019 100644 --- a/test/gen_sizes_lheap.c +++ b/test/gen_sizes_lheap.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Neil Fortner <nfortne2@hdfgroup.org> + * Programmer: Neil Fortner * Thursday, July 15, 2010 * * Purpose: Creates a file with non-default sizes of lengths and addresses. diff --git a/test/gen_specmetaread.c b/test/gen_specmetaread.c index d06bd59..ca44f9a 100644 --- a/test/gen_specmetaread.c +++ b/test/gen_specmetaread.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@hdfgroup.org> + * Programmer: Quincey Koziol * Thursday, October 8, 2009 * * Purpose: Create a file with a dataset who's raw data immediately follows @@ -23,8 +23,8 @@ * the library on the trunk as of when this file is checked in. */ -#include "hdf5.h" -#include <assert.h> + +#include "h5test.h" #define FILENAME "specmetaread.h5" #define DIM 10 @@ -39,7 +39,7 @@ main(void) hsize_t dim[1] = {DIM}; unsigned data[DIM]; unsigned u; - herr_t ret; /* Generic return value */ + herr_t H5_ATTR_NDEBUG_UNUSED ret; /* Initialize the data */ for(u = 0; u < DIM; u++) diff --git a/test/gen_udlinks.c b/test/gen_udlinks.c index e48d0e8..456cb5c 100644 --- a/test/gen_udlinks.c +++ b/test/gen_udlinks.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: James Laird <jlaird@hdfgroup.org> + * Programmer: James Laird * Tuesday, June 6, 2006 * * This program creates HDF5 files with user-defined links. These files diff --git a/test/getname.c b/test/getname.c index 7933608..7233d17 100644 --- a/test/getname.c +++ b/test/getname.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Pedro Vicente <pvn@ncsa.uiuc.edu> + * Programmer: Pedro Vicente * April 12, 2002 * * Purpose: Tests the "ID to name" functionality diff --git a/test/gheap.c b/test/gheap.c index 32e2785..23fe0ba 100644 --- a/test/gheap.c +++ b/test/gheap.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Tuesday, March 31, 1998 * * Purpose: Tests the global heap. The global heap is the set of all diff --git a/test/h5test.c b/test/h5test.c index 2613ce1..4fa5102 100644 --- a/test/h5test.c +++ b/test/h5test.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Thursday, November 19, 1998 * * Purpose: Provides support functions for most of the hdf5 tests cases. @@ -1889,6 +1889,84 @@ h5_get_version_string(H5F_libver_t libver) } /* end of h5_get_version_string */ /*------------------------------------------------------------------------- + * Function: h5_compare_file_bytes() + * + * Purpose: Helper function to compare two files byte-for-byte. + * + * Return: Success: 0, if files are identical + * Failure: -1, if files differ + * + * Programmer: Binh-Minh Ribler + * October, 2018 + *------------------------------------------------------------------------- + */ +int +h5_compare_file_bytes(char *f1name, char *f2name) +{ + FILE *f1ptr = NULL; /* two file pointers */ + FILE *f2ptr = NULL; + off_t f1size = 0; /* size of the files */ + off_t f2size = 0; + char f1char = 0; /* one char from each file */ + char f2char = 0; + off_t ii = 0; + int ret_value = 0; /* for error handling */ + + /* Open files for reading */ + f1ptr = HDfopen(f1name, "rb"); + if (f1ptr == NULL) { + HDfprintf(stderr, "Unable to fopen() %s\n", f1name); + ret_value = -1; + goto done; + } + f2ptr = HDfopen(f2name, "rb"); + if (f2ptr == NULL) { + HDfprintf(stderr, "Unable to fopen() %s\n", f2name); + ret_value = -1; + goto done; + } + + /* Get the file sizes and verify that they equal */ + HDfseek(f1ptr , 0 , SEEK_END); + f1size = HDftell(f1ptr); + + HDfseek(f2ptr , 0 , SEEK_END); + f2size = HDftell(f2ptr); + + if (f1size != f2size) { + HDfprintf(stderr, "Files differ in size, %llu vs. %llu\n", f1size, f2size); + ret_value = -1; + goto done; + } + + /* Compare each byte and fail if a difference is found */ + HDrewind(f1ptr); + HDrewind(f2ptr); + for (ii = 0; ii < f1size; ii++) { + if(HDfread(&f1char, 1, 1, f1ptr) != 1) { + ret_value = -1; + goto done; + } + if(HDfread(&f2char, 1, 1, f2ptr) != 1) { + ret_value = -1; + goto done; + } + if (f1char != f2char) { + HDfprintf(stderr, "Mismatch @ 0x%llX: 0x%X != 0x%X\n", ii, f1char, f2char); + ret_value = -1; + goto done; + } + } + +done: + if (f1ptr) + HDfclose(f1ptr); + if (f2ptr) + HDfclose(f2ptr); + return ret_value; +} /* end h5_compare_file_bytes() */ + +/*------------------------------------------------------------------------- * Function: H5_get_srcdir_filename * * Purpose: Append the test file name to the srcdir path and return the whole string @@ -1941,3 +2019,131 @@ const char *H5_get_srcdir(void) return(NULL); } /* end H5_get_srcdir() */ +/*------------------------------------------------------------------------- + * Function: h5_duplicate_file_by_bytes + * + * Purpose: Duplicate a file byte-for-byte at filename/path 'orig' + * to a new (or replaced) file at 'dest'. + * + * Return: Success: 0, completed successfully + * Failure: -1 + * + * Programmer: Jake Smith + * 24 June 2020 + * + *------------------------------------------------------------------------- + */ +int +h5_duplicate_file_by_bytes(const char *orig, const char *dest) +{ + FILE *orig_ptr = NULL; + FILE *dest_ptr = NULL; + hsize_t fsize = 0; + hsize_t read_size = 0; + hsize_t max_buf = 0; + void *dup_buf = NULL; + int ret_value = 0; + + max_buf = 4096 * sizeof(char); + + orig_ptr = HDfopen(orig, "rb"); + if (NULL == orig_ptr) { + ret_value = -1; + goto done; + } + + HDfseek(orig_ptr , 0 , SEEK_END); + fsize = (hsize_t)HDftell(orig_ptr); + HDrewind(orig_ptr); + + dest_ptr = HDfopen(dest, "wb"); + if (NULL == dest_ptr) { + ret_value = -1; + goto done; + } + + read_size = MIN(fsize, max_buf); + dup_buf = HDmalloc(read_size); + if (NULL == dup_buf) { + ret_value = -1; + goto done; + } + + while (read_size > 0) { + if (HDfread(dup_buf, read_size, 1, orig_ptr) != 1) { + ret_value = -1; + goto done; + } + HDfwrite(dup_buf, read_size, 1, dest_ptr); + fsize -= read_size; + read_size = MIN(fsize, max_buf); + } + +done: + if (orig_ptr != NULL) + HDfclose(orig_ptr); + if (dest_ptr != NULL) + HDfclose(dest_ptr); + if (dup_buf != NULL) + HDfree(dup_buf); + return ret_value; +} /* end h5_duplicate_file_by_bytes() */ + +/*------------------------------------------------------------------------- + * Function: h5_check_if_file_locking_enabled + * + * Purpose: Checks if file locking is enabled on this file system. + * + * Return: SUCCEED/FAIL + * are_enabled will be FALSE if file locking is disabled on + * the file system of if there were errors. + * + *------------------------------------------------------------------------- + */ +herr_t +h5_check_if_file_locking_enabled(hbool_t *is_enabled) +{ + const char *filename = "locking_test_file"; + int pmode = O_RDWR | O_CREAT | O_TRUNC; + int fd = -1; + + *is_enabled = TRUE; + + if((fd = HDopen(filename, pmode, H5_POSIX_CREATE_MODE_RW)) < 0) + goto error; + + /* Test HDflock() to see if it works */ + if(HDflock(fd, LOCK_EX | LOCK_NB) < 0) { + if(ENOSYS == errno) { + /* When errno is set to ENOSYS, the file system does not support + * locking, so ignore it. This is most frequently used on + * Lustre. If we also want to check for disabled NFS locks + * we'll need to check for ENOLCK, too. That isn't done by + * default here since that could also represent an actual + * error condition. + */ + errno = 0; + *is_enabled = FALSE; + } + else + goto error; + } + if(HDflock(fd, LOCK_UN) < 0) + goto error; + + if(HDclose(fd) < 0) + goto error; + if(HDremove(filename) < 0) + goto error; + + return SUCCEED; + +error: + *is_enabled = FALSE; + if (fd > -1) { + HDclose(fd); + HDremove(filename); + } + return FAIL; +} /* end h5_check_if_file_locking_enabled() */ + diff --git a/test/h5test.h b/test/h5test.h index 0d937e1..93cdf96 100644 --- a/test/h5test.h +++ b/test/h5test.h @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Friday, November 20, 1998 * * Purpose: Test support stuff. @@ -206,6 +206,9 @@ H5TEST_DLL int h5_make_local_copy(const char *origfilename, const char *local_co H5TEST_DLL herr_t h5_verify_cached_stabs(const char *base_name[], hid_t fapl); H5TEST_DLL H5FD_class_t *h5_get_dummy_vfd_class(void); H5TEST_DLL const char *h5_get_version_string(H5F_libver_t libver); +H5TEST_DLL int h5_compare_file_bytes(char *fname1, char *fname2); +H5TEST_DLL int h5_duplicate_file_by_bytes(const char *orig, const char *dest); +H5TEST_DLL herr_t h5_check_if_file_locking_enabled(hbool_t *are_enabled); /* Functions that will replace components of a FAPL */ H5TEST_DLL herr_t h5_get_vfd_fapl(hid_t fapl_id); diff --git a/test/hyperslab.c b/test/hyperslab.c index e702023..8168eed 100644 --- a/test/hyperslab.c +++ b/test/hyperslab.c @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Robb Matzke <matzke@llnl.gov> +/* Programmer: Robb Matzke * Friday, October 10, 1997 * * Purpose: Hyperslab operations are rather complex, so this file diff --git a/test/istore.c b/test/istore.c index c8fe866..e80f260 100644 --- a/test/istore.c +++ b/test/istore.c @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Robb Matzke <matzke@llnl.gov> +/* Programmer: Robb Matzke * Wednesday, October 15, 1997 * * Purpose: Tests various aspects of indexed raw data storage. diff --git a/test/lheap.c b/test/lheap.c index 4b40740..54c77a7 100644 --- a/test/lheap.c +++ b/test/lheap.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Tuesday, November 24, 1998 * * Purpose: Test local heaps used by symbol tables (groups). diff --git a/test/mirror_vfd.c b/test/mirror_vfd.c new file mode 100644 index 0000000..35c3f90 --- /dev/null +++ b/test/mirror_vfd.c @@ -0,0 +1,2763 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Test the Mirror VFD functionality. + */ + +/* WARNING: The use of realpath() is probably system-dependent, as are + * other things here such as the socket calls. + * Notable to realpath() in particular is the use of "PATH_MAX", which + * apparently has some major potential issues if paths are abused. + * http://insanecoding.blogspot.com/2007/11/pathmax-simply-isnt.html + * so BE CAREFUL about the paths we throw around? + */ + +#include "h5test.h" +#include "cache_common.h" +#include "genall5.h" + +#ifdef H5_HAVE_MIRROR_VFD + +#include "H5FDmirror_priv.h" /* Private header for the mirror VFD */ + +/* For future consideration, IP address and port number might be + * environment variables? + */ +#define SERVER_IP_ADDRESS "127.0.0.1" + +/* Primary listening port on server. */ +#define SERVER_HANDSHAKE_PORT 3000 + +#define DATABUFFER_SIZE 128 +#define DSET_NAME_LEN 16 + +/* Parameters for the "large chunked dataset" writing */ +#define MAX_DSET_COUNT 255 +#define DSET_DIM 32 +#define CHUNK_DIM 8 + +#define CONCURRENT_COUNT 3 /* Number of files in concurrent test */ + +/* Macro: LOGPRINT() + * Prints logging and debugging messages to the output stream based + * on the level of verbosity. + * 0 : no logging + * 1 : errors only + * 2 : details + * 3 : all + */ +#define DEFAULT_VERBOSITY 1 +static unsigned int g_verbosity = DEFAULT_VERBOSITY; + +/* Macro for selective debug printing / logging */ +#define LOGPRINT(lvl, ...) \ +do { \ + if ((lvl) <= g_verbosity) { \ + fprintf(g_log_stream, __VA_ARGS__); \ + fflush(g_log_stream); \ + } \ +} while (0) + +#define MIRROR_RW_DIR "mirror_rw/" +#define MIRROR_WO_DIR "mirror_wo/" + +/* String buffer for error messages */ +#define MIRR_MESG_SIZE 128 +static char mesg[MIRR_MESG_SIZE + 1]; + +/* Convenience structure for passing file names via helper functions. + */ +struct mirrortest_filenames { + char rw[H5FD_SPLITTER_PATH_MAX+1]; + char wo[H5FD_SPLITTER_PATH_MAX+1]; + char log[H5FD_SPLITTER_PATH_MAX+1]; +}; + +static FILE *g_log_stream = NULL; /* initialized at runtime */ + +static herr_t _verify_datasets(unsigned min_dset, unsigned max_dset, + hid_t *filespace_id, hid_t *dataset_id, hid_t memspace_id); + +static herr_t _create_chunking_ids(hid_t file_id, unsigned min_dset, + unsigned max_dset, hsize_t *chunk_dims, hsize_t *dset_dims, + hid_t *dataspace_ids, hid_t *filespace_ids, hid_t *dataset_ids, + hid_t *memspace_id); + +static herr_t _close_chunking_ids(unsigned min_dset, unsigned max_dset, + hid_t *dataspace_ids, hid_t *filespace_ids, hid_t *dataset_ids, + hid_t *memspace_id); + +static herr_t _populate_filepath(const char *dirname, const char *_basename, + hid_t fapl_id, char *path_out, hbool_t h5suffix); + +static hid_t create_mirroring_split_fapl(const char *_basename, + struct mirrortest_filenames *names); + +static void mybzero(void *dest, size_t size); + + +/* ---------------------------------------------------------------------------- + * Function: mybzero + * + * Purpose: Have bzero simplicity and abstraction in (possible) absence of + * it being available. + * + * Programmer: Jacob Smith + * 2020-03-30 + * ---------------------------------------------------------------------------- + */ +static void +mybzero(void *dest, size_t size) +{ + size_t i = 0; + char *s = NULL; + HDassert(dest != NULL); + s = (char *)dest; + for (i = 0; i < size; i++) { + *(s+i) = 0; + } +} /* end mybzero() */ + + +/* ---------------------------------------------------------------------------- + * Function: _populate_filepath + * + * Purpose: Given a directory name and a base name, concatenate the two and + * run h5fixname() to get the "actual" path to the intented target. + * `h5suffix' should be FALSE to keep the base name unaltered; + * TRUE will append the '.h5' h5suffix to the basename... + * FALSE -> h5fixname_no_suffix(), TRUE -> h5fixname() + * <h5fixname_prefix> / <dirname> / <_basename> <h5prefix?> + * + * Programmer: Jacob Smith + * 2019-08-16 + * ---------------------------------------------------------------------------- + */ +static herr_t +_populate_filepath(const char *dirname, const char *_basename, hid_t fapl_id, + char *path_out, hbool_t h5suffix) +{ + char _path[H5FD_SPLITTER_PATH_MAX]; + + if ((_basename == NULL) || + (*_basename == 0) || + (dirname == NULL) || + (*dirname == 0) || + (path_out == NULL)) + { + TEST_ERROR; + } + + if (HDsnprintf(_path, H5FD_SPLITTER_PATH_MAX, "%s%s%s", dirname, + (dirname[strlen(dirname)] == '/') ? "" : "/", /* slash iff needed */ + _basename) + > H5FD_SPLITTER_PATH_MAX) + { + TEST_ERROR; + } + + if (h5suffix == TRUE) { + if (h5_fixname(_path, fapl_id, path_out, + H5FD_SPLITTER_PATH_MAX) + == NULL) + { + TEST_ERROR; + } + } + else { + if (h5_fixname_no_suffix(_path, fapl_id, path_out, + H5FD_SPLITTER_PATH_MAX) + == NULL) + { + TEST_ERROR; + } + } + + return SUCCEED; + +error: + return FAIL; +} /* end _populate_filepath() */ + + +/* --------------------------------------------------------------------------- + * Function: build_paths + * + * Purpose: Convenience function to create the three file paths used in + * most mirror tests. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2019-08-16 + * --------------------------------------------------------------------------- + */ +static herr_t +build_paths( + const char *_basename, + H5FD_splitter_vfd_config_t *splitter_config, + struct mirrortest_filenames *names) +{ + char baselogname[H5FD_SPLITTER_PATH_MAX]; + + if (_populate_filepath(MIRROR_RW_DIR, _basename, + splitter_config->rw_fapl_id, names->rw, TRUE) + == FAIL) + { + TEST_ERROR; + } + + if (_populate_filepath(MIRROR_WO_DIR, _basename, + splitter_config->wo_fapl_id, names->wo, TRUE) + == FAIL) + { + TEST_ERROR; + } + + if (_basename == NULL || *_basename == 0) + return FAIL; + if (HDsnprintf(baselogname, H5FD_SPLITTER_PATH_MAX, "%s_err.log", + _basename) + > H5FD_SPLITTER_PATH_MAX) + { + TEST_ERROR; + } + + if (_populate_filepath(MIRROR_WO_DIR, baselogname, + splitter_config->wo_fapl_id, names->log, FALSE) + == FAIL) + { + TEST_ERROR; + } + + return SUCCEED; + +error: + return FAIL; +} /* end build_paths() */ + + +/* --------------------------------------------------------------------------- + * Function: test_fapl_configuration + * + * Purpose: Test FAPL configuration and examination. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2019-03-12 + * --------------------------------------------------------------------------- + */ +static int +test_fapl_configuration(void) +{ + hid_t fapl_id; + H5FD_mirror_fapl_t mirror_conf = { + H5FD_MIRROR_FAPL_MAGIC, /* magic */ + H5FD_MIRROR_CURR_FAPL_T_VERSION, /* version */ + SERVER_HANDSHAKE_PORT, /* handhake_port */ + SERVER_IP_ADDRESS, /* remote_ip "IP address" */ + }; + H5FD_mirror_fapl_t fa_out = {0, 0, 0, ""}; + + TESTING("Mirror fapl configuration (set/get)"); + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) { + TEST_ERROR; + } + + if (H5Pset_fapl_mirror(fapl_id, &mirror_conf) == FAIL) { + TEST_ERROR; + } + + if (H5Pget_fapl_mirror(fapl_id, &fa_out) == FAIL) { + TEST_ERROR; + } + if (H5FD_MIRROR_FAPL_MAGIC != fa_out.magic) { + TEST_ERROR; + } + if (H5FD_MIRROR_CURR_FAPL_T_VERSION != fa_out.version) { + TEST_ERROR; + } + if (SERVER_HANDSHAKE_PORT != fa_out.handshake_port) { + TEST_ERROR; + } + if (HDstrncmp(SERVER_IP_ADDRESS, (const char *)fa_out.remote_ip, + H5FD_MIRROR_MAX_IP_LEN)) + { + TEST_ERROR; + } + + if (H5Pclose(fapl_id) == FAIL) { + TEST_ERROR; + } + + PASSED(); + return 0; + +error: + if (H5I_INVALID_HID != fapl_id) { + (void)H5Pclose(fapl_id); + } + return -1; +} /* end test_fapl_configuration() */ + + + +#define PRINT_BUFFER_DIFF(act, exp, len) do { \ + size_t _x = 0; \ + while ((act)[_x] == (exp)[_x]) { \ + _x++; \ + } \ + if (_x != (len)) { \ + size_t _y = 0; \ + HDprintf("First bytes differ at %zu\n", _x); \ + HDprintf("exp "); \ + for (_y = _x; _y < (len); _y++) { \ + HDprintf("%02X", (unsigned char)(exp)[_y]); \ + } \ + HDprintf("\nact "); \ + for (_y = _x; _y < (len); _y++) { \ + HDprintf("%02X", (unsigned char)(act)[_y]); \ + } \ + HDprintf("\n"); \ + } \ +} while (0); /* end PRINT_BUFFER_DIFF */ + + +/* --------------------------------------------------------------------------- + * Function: test_xmit_encode_decode + * + * Purpose: Test byte-encoding operations for network transport. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2020-02-02 + * --------------------------------------------------------------------------- + */ +static int +test_xmit_encode_decode(void) +{ + H5FD_mirror_xmit_t xmit_mock; /* re-used header in various xmit tests */ + + TESTING("Mirror encode/decode of xmit elements"); + + /* Set bogus values matching expected; encoding doesn't care + * Use sequential values to easily generate the expected buffer with a + * for loop. + */ + xmit_mock.magic = 0x00010203; + xmit_mock.version = 0x04; + xmit_mock.session_token = 0x05060708; + xmit_mock.xmit_count = 0x090A0B0C; + xmit_mock.op = 0x0D; + + /* Test uint8_t encode/decode + */ + do { + unsigned char buf[8]; + unsigned char expected[8]; + const uint8_t v = 200; + unsigned char out = 0; + + /* Start of buffer uint8_t + */ + mybzero(buf, 8); + mybzero(expected, 8); + expected[0] = 200; + out = 0; + if (H5FD__mirror_xmit_encode_uint8(buf, v) != 1) { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, 8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, 8); + TEST_ERROR; + } + if (H5FD__mirror_xmit_decode_uint8(&out, buf) != 1) { + TEST_ERROR; + } + if (v != out) { + TEST_ERROR; + } + + /* Middle of buffer uint8_t + */ + mybzero(buf, 8); + mybzero(expected, 8); + expected[3] = v; + out = 0; + if (H5FD__mirror_xmit_encode_uint8((buf+3), v) != 1) { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, 8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, 8); + TEST_ERROR; + } + if (H5FD__mirror_xmit_decode_uint8(&out, (buf+3)) != 1) { + TEST_ERROR; + } + if (v != out) { + TEST_ERROR; + } + + /* End of buffer uint8_t + */ + mybzero(buf, 8); + mybzero(expected, 8); + expected[7] = v; + out = 0; + if (H5FD__mirror_xmit_encode_uint8((buf+7), v) != 1) { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, 8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, 8); + TEST_ERROR; + } + if (H5FD__mirror_xmit_decode_uint8(&out, (buf+7)) != 1) { + TEST_ERROR; + } + if (v != out) { + TEST_ERROR; + } + + } while (0); /* end uint8_t en/decode */ + + /* Test uint16_t encode/decode + */ + do { + unsigned char buf[8]; + unsigned char expected[8]; + const uint16_t v = 0x8F02; + uint16_t out = 0; + + /* Start of buffer uint16_t + */ + mybzero(buf, 8); + mybzero(expected, 8); + expected[0] = 0x8F; + expected[1] = 0x02; + out = 0; + if (H5FD__mirror_xmit_encode_uint16(buf, v) != 2) { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, 8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, 8); + TEST_ERROR; + } + if (H5FD__mirror_xmit_decode_uint16(&out, buf) != 2) { + TEST_ERROR; + } + if (out != v) { + TEST_ERROR; + } + + /* Middle of buffer uint16_t + */ + mybzero(buf, 8); + mybzero(expected, 8); + expected[3] = 0x8F; + expected[4] = 0x02; + out = 0; + if (H5FD__mirror_xmit_encode_uint16((buf+3), v) != 2) { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, 8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, 8); + TEST_ERROR; + } + if (H5FD__mirror_xmit_decode_uint16(&out, (buf+3)) != 2) { + TEST_ERROR; + } + if (out != v) { + TEST_ERROR; + } + /* slice */ + if (H5FD__mirror_xmit_decode_uint16(&out, (buf+4)) != 2) { + TEST_ERROR; + } + if (out != 0x0200) { + TEST_ERROR; + } + + /* End of buffer uint16_t + */ + mybzero(buf, 8); + mybzero(expected, 8); + expected[6] = 0x8F; + expected[7] = 0x02; + out = 0; + if (H5FD__mirror_xmit_encode_uint16((buf+6), v) != 2) { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, 8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, 8); + TEST_ERROR; + } + if (H5FD__mirror_xmit_decode_uint16(&out, (buf+6)) != 2) { + TEST_ERROR; + } + if (out != v) { + TEST_ERROR; + } + + } while (0); /* end uint16_t en/decode */ + + /* Test uint32_t encode/decode + */ + do { + unsigned char buf[8]; + unsigned char expected[8]; + const uint32_t v = 0x8F020048; + uint32_t out = 0; + + /* Start of buffer uint32_t + */ + mybzero(buf, 8); + mybzero(expected, 8); + expected[0] = 0x8F; + expected[1] = 0x02; + expected[2] = 0x00; + expected[3] = 0x48; + out = 0; + if (H5FD__mirror_xmit_encode_uint32(buf, v) != 4) { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, 8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, 8); + TEST_ERROR; + } + if (H5FD__mirror_xmit_decode_uint32(&out, buf) != 4) { + TEST_ERROR; + } + if (out != v) { + TEST_ERROR; + } + + /* Middle of buffer uint32_t + */ + mybzero(buf, 8); + mybzero(expected, 8); + expected[3] = 0x8F; + expected[4] = 0x02; + expected[5] = 0x00; + expected[6] = 0x48; + out = 0; + if (H5FD__mirror_xmit_encode_uint32((buf+3), v) != 4) { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, 8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, 8); + TEST_ERROR; + } + if (H5FD__mirror_xmit_decode_uint32(&out, (buf+3)) != 4) { + TEST_ERROR; + } + if (out != v) { + TEST_ERROR; + } + /* slice */ + if (H5FD__mirror_xmit_decode_uint32(&out, (buf+4)) != 4) { + TEST_ERROR; + } + if (out != 0x02004800) { + TEST_ERROR; + } + + /* End of buffer uint32_t + */ + mybzero(buf, 8); + mybzero(expected, 8); + expected[4] = 0x8F; + expected[5] = 0x02; + expected[6] = 0x00; + expected[7] = 0x48; + out = 0; + if (H5FD__mirror_xmit_encode_uint32((buf+4), v) != 4) { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, 8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, 8); + TEST_ERROR; + } + if (H5FD__mirror_xmit_decode_uint32(&out, (buf+4)) != 4) { + TEST_ERROR; + } + if (out != v) { + TEST_ERROR; + } + + } while (0); /* end uint32_t en/decode */ + + /* Test uint64_t encode/decode + */ + do { + unsigned char buf[16]; + unsigned char expected[16]; + const uint64_t v = 0x90DCBE17939CE4BB; + uint64_t out = 0; + + /* Start of buffer uint64_t + */ + mybzero(buf, 16); + mybzero(expected, 16); + expected[0] = 0x90; + expected[1] = 0xDC; + expected[2] = 0xBE; + expected[3] = 0x17; + expected[4] = 0x93; + expected[5] = 0x9C; + expected[6] = 0xE4; + expected[7] = 0xBB; + out = 0; + if (H5FD__mirror_xmit_encode_uint64(buf, v) != 8) { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, 16) != 0) { + PRINT_BUFFER_DIFF(buf, expected, 16); + TEST_ERROR; + } + if (H5FD__mirror_xmit_decode_uint64(&out, buf) != 8) { + TEST_ERROR; + } + if (out != v) { + TEST_ERROR; + } + + /* Middle of buffer uint64_t + */ + mybzero(buf, 16); + mybzero(expected, 16); + expected[3] = 0x90; + expected[4] = 0xDC; + expected[5] = 0xBE; + expected[6] = 0x17; + expected[7] = 0x93; + expected[8] = 0x9C; + expected[9] = 0xE4; + expected[10] = 0xBB; + out = 0; + if (H5FD__mirror_xmit_encode_uint64((buf+3), v) != 8) { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, 16) != 0) { + PRINT_BUFFER_DIFF(buf, expected, 16); + TEST_ERROR; + } + if (H5FD__mirror_xmit_decode_uint64(&out, (buf+3)) != 8) { + TEST_ERROR; + } + if (out != v) { + TEST_ERROR; + } + /* slice */ + if (H5FD__mirror_xmit_decode_uint64(&out, (buf+6)) != 8) { + TEST_ERROR; + } + if (out != 0x17939CE4BB000000) { + TEST_ERROR; + } + + /* End of buffer uint64_t + */ + mybzero(buf, 16); + mybzero(expected, 16); + expected[8] = 0x90; + expected[9] = 0xDC; + expected[10] = 0xBE; + expected[11] = 0x17; + expected[12] = 0x93; + expected[13] = 0x9C; + expected[14] = 0xE4; + expected[15] = 0xBB; + out = 0; + if (H5FD__mirror_xmit_encode_uint64((buf+8), v) != 8) { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, 16) != 0) { + PRINT_BUFFER_DIFF(buf, expected, 16); + TEST_ERROR; + } + if (H5FD__mirror_xmit_decode_uint64(&out, (buf+8)) != 8) { + TEST_ERROR; + } + if (out != v) { + TEST_ERROR; + } + + } while (0); /* end uint64_t en/decode */ + + /* Test xmit header structure encode/decode + * Write bogus but easily verifiable data to inside a buffer, and compare. + * Then decode the buffer and compare the structure contents. + * Then repeat from a different offset in the buffer and compare. + */ + do { + unsigned char buf[H5FD_MIRROR_XMIT_HEADER_SIZE+8]; + unsigned char expected[H5FD_MIRROR_XMIT_HEADER_SIZE+8]; + H5FD_mirror_xmit_t xmit_out; + size_t i = 0; + + /* sanity check */ + if (14 != H5FD_MIRROR_XMIT_HEADER_SIZE) { + FAIL_PUTS_ERROR("Header size definition does not match test\n"); + } + + /* Populate the expected buffer; expect end padding of 0xFF + */ + HDmemset(expected, 0xFF, H5FD_MIRROR_XMIT_HEADER_SIZE+8); + for (i=0; i < H5FD_MIRROR_XMIT_HEADER_SIZE; i++) { + expected[i+2] = (unsigned char)i; + } + + /* Encode, and compare buffer contents + * Initial buffer is filled with 0xFF to match expected padding + */ + HDmemset(buf, 0xFF, H5FD_MIRROR_XMIT_HEADER_SIZE+8); + if (H5FD_mirror_xmit_encode_header((buf+2), &xmit_mock) + != H5FD_MIRROR_XMIT_HEADER_SIZE) + { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, H5FD_MIRROR_XMIT_HEADER_SIZE+8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, H5FD_MIRROR_XMIT_HEADER_SIZE+8); + TEST_ERROR; + } + + /* Decode from buffer + */ + if (H5FD_mirror_xmit_decode_header(&xmit_out, (buf+2)) + != H5FD_MIRROR_XMIT_HEADER_SIZE) + { + TEST_ERROR; + } + if (xmit_out.magic != xmit_mock.magic) TEST_ERROR; + if (xmit_out.version != xmit_mock.version) TEST_ERROR; + if (xmit_out.session_token != xmit_mock.session_token) TEST_ERROR; + if (xmit_out.xmit_count != xmit_mock.xmit_count) TEST_ERROR; + if (xmit_out.op != xmit_mock.op) TEST_ERROR; + + /* Decode from different offset in buffer + * Observe changes when ingesting the padding + */ + if (H5FD_mirror_xmit_decode_header(&xmit_out, (buf)) + != H5FD_MIRROR_XMIT_HEADER_SIZE) + { + TEST_ERROR; + } + if (xmit_out.magic != 0xFFFF0001) TEST_ERROR; + if (xmit_out.version != 0x02) TEST_ERROR; + if (xmit_out.session_token != 0x03040506) TEST_ERROR; + if (xmit_out.xmit_count != 0x0708090A) TEST_ERROR; + if (xmit_out.op != 0x0B) TEST_ERROR; + + } while (0); /* end xmit header en/decode */ + + /* Test xmit set-eoa structure encode/decode + * Write bogus but easily verifiable data to inside a buffer, and compare. + * Then decode the buffer and compare the structure contents. + * Then repeat from a different offset in the buffer and compare. + */ + do { + unsigned char buf[H5FD_MIRROR_XMIT_EOA_SIZE+8]; + unsigned char expected[H5FD_MIRROR_XMIT_EOA_SIZE+8]; + H5FD_mirror_xmit_eoa_t xmit_in; + H5FD_mirror_xmit_eoa_t xmit_out; + size_t i = 0; + + /* sanity check */ + if ((14+9) != H5FD_MIRROR_XMIT_EOA_SIZE) { + FAIL_PUTS_ERROR("Header size definition does not match test\n"); + } + if (xmit_mock.op != 0x0D) { + FAIL_PUTS_ERROR("shared header structure is not in expected state"); + } + + /* Populate the expected buffer; expect end padding of 0xFF + */ + HDmemset(expected, 0xFF, H5FD_MIRROR_XMIT_EOA_SIZE+8); + for (i=0; i < H5FD_MIRROR_XMIT_EOA_SIZE; i++) { + expected[i+2] = (unsigned char)i; + } + + /* Set xmit_in + */ + xmit_in.pub = xmit_mock; /* shared/common */ + xmit_in.type = 0x0E; + xmit_in.eoa_addr = 0x0F10111213141516; + + /* Encode, and compare buffer contents + * Initial buffer is filled with 0xFF to match expected padding + */ + HDmemset(buf, 0xFF, H5FD_MIRROR_XMIT_EOA_SIZE+8); + if (H5FD_mirror_xmit_encode_set_eoa((buf+2), &xmit_in) + != H5FD_MIRROR_XMIT_EOA_SIZE) + { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, H5FD_MIRROR_XMIT_EOA_SIZE+8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, H5FD_MIRROR_XMIT_EOA_SIZE+8); + TEST_ERROR; + } + + /* Decode from buffer + */ + if (H5FD_mirror_xmit_decode_set_eoa(&xmit_out, (buf+2)) + != H5FD_MIRROR_XMIT_EOA_SIZE) + { + TEST_ERROR; + } + if (xmit_out.pub.magic != xmit_mock.magic) TEST_ERROR; + if (xmit_out.pub.version != xmit_mock.version) TEST_ERROR; + if (xmit_out.pub.session_token != xmit_mock.session_token) TEST_ERROR; + if (xmit_out.pub.xmit_count != xmit_mock.xmit_count) TEST_ERROR; + if (xmit_out.pub.op != xmit_mock.op) TEST_ERROR; + if (xmit_out.type != 0x0E) TEST_ERROR; + if (xmit_out.eoa_addr != 0x0F10111213141516) TEST_ERROR; + + /* Decode from different offset in buffer + * Observe changes when ingesting the padding + */ + if (H5FD_mirror_xmit_decode_set_eoa(&xmit_out, (buf)) + != H5FD_MIRROR_XMIT_EOA_SIZE) + { + TEST_ERROR; + } + if (xmit_out.pub.magic != 0xFFFF0001) TEST_ERROR; + if (xmit_out.pub.version != 0x02) TEST_ERROR; + if (xmit_out.pub.session_token != 0x03040506) TEST_ERROR; + if (xmit_out.pub.xmit_count != 0x0708090A) TEST_ERROR; + if (xmit_out.pub.op != 0x0B) TEST_ERROR; + if (xmit_out.type != 0x0C) TEST_ERROR; + if (xmit_out.eoa_addr != 0x0D0E0F1011121314) TEST_ERROR; + + } while (0); /* end xmit set-eoa en/decode */ + + /* Test xmit lock structure encode/decode + * Write bogus but easily verifiable data to inside a buffer, and compare. + * Then decode the buffer and compare the structure contents. + * Then repeat from a different offset in the buffer and compare. + */ + do { + unsigned char buf[H5FD_MIRROR_XMIT_LOCK_SIZE+8]; + unsigned char expected[H5FD_MIRROR_XMIT_LOCK_SIZE+8]; + H5FD_mirror_xmit_lock_t xmit_in; + H5FD_mirror_xmit_lock_t xmit_out; + size_t i = 0; + + /* sanity check */ + if ((14+8) != H5FD_MIRROR_XMIT_LOCK_SIZE) { + FAIL_PUTS_ERROR("Header size definition does not match test\n"); + } + if (xmit_mock.op != 0x0D) { + FAIL_PUTS_ERROR("shared header structure is not in expected state"); + } + + /* Populate the expected buffer; expect end padding of 0xFF + */ + HDmemset(expected, 0xFF, H5FD_MIRROR_XMIT_LOCK_SIZE+8); + for (i=0; i < H5FD_MIRROR_XMIT_LOCK_SIZE; i++) { + expected[i+2] = (unsigned char)i; + } + + /* Set xmit_in + */ + xmit_in.pub = xmit_mock; /* shared/common */ + xmit_in.rw = 0x0E0F101112131415; + + /* Encode, and compare buffer contents + * Initial buffer is filled with 0xFF to match expected padding + */ + HDmemset(buf, 0xFF, H5FD_MIRROR_XMIT_LOCK_SIZE+8); + if (H5FD_mirror_xmit_encode_lock((buf+2), &xmit_in) + != H5FD_MIRROR_XMIT_LOCK_SIZE) + { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, H5FD_MIRROR_XMIT_LOCK_SIZE+8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, H5FD_MIRROR_XMIT_LOCK_SIZE+8); + TEST_ERROR; + } + + /* Decode from buffer + */ + if (H5FD_mirror_xmit_decode_lock(&xmit_out, (buf+2)) + != H5FD_MIRROR_XMIT_LOCK_SIZE) + { + TEST_ERROR; + } + if (xmit_out.pub.magic != xmit_mock.magic) TEST_ERROR; + if (xmit_out.pub.version != xmit_mock.version) TEST_ERROR; + if (xmit_out.pub.session_token != xmit_mock.session_token) TEST_ERROR; + if (xmit_out.pub.xmit_count != xmit_mock.xmit_count) TEST_ERROR; + if (xmit_out.pub.op != xmit_mock.op) TEST_ERROR; + if (xmit_out.rw != 0x0E0F101112131415) TEST_ERROR; + + /* Decode from different offset in buffer + * Observe changes when ingesting the padding + */ + if (H5FD_mirror_xmit_decode_lock(&xmit_out, (buf)) + != H5FD_MIRROR_XMIT_LOCK_SIZE) + { + TEST_ERROR; + } + if (xmit_out.pub.magic != 0xFFFF0001) TEST_ERROR; + if (xmit_out.pub.version != 0x02) TEST_ERROR; + if (xmit_out.pub.session_token != 0x03040506) TEST_ERROR; + if (xmit_out.pub.xmit_count != 0x0708090A) TEST_ERROR; + if (xmit_out.pub.op != 0x0B) TEST_ERROR; + if (xmit_out.rw != 0x0C0D0E0F10111213) TEST_ERROR; + + } while (0); /* end xmit lock en/decode */ + + /* Test xmit open structure encode/decode + * Write bogus but easily verifiable data to inside a buffer, and compare. + * Then decode the buffer and compare the structure contents. + * Then repeat from a different offset in the buffer and compare. + * + * Verifies that the first zero character in the filepath will end the + * string, with all following bytes in the encoded buffer being zeroed. + */ + do { + unsigned char buf[H5FD_MIRROR_XMIT_OPEN_SIZE+8]; + unsigned char expected[H5FD_MIRROR_XMIT_OPEN_SIZE+8]; + H5FD_mirror_xmit_open_t xmit_in; + H5FD_mirror_xmit_open_t xmit_out; + size_t i = 0; + + /* sanity check */ + if ((14+20+4097) != H5FD_MIRROR_XMIT_OPEN_SIZE) { + FAIL_PUTS_ERROR("Header size definition does not match test\n"); + } + if (xmit_mock.op != 0x0D) { + FAIL_PUTS_ERROR("shared header structure is not in expected state"); + } + + /* Populate the expected buffer; expect end padding of 0xFF + */ + HDmemset(expected, 0xFF, H5FD_MIRROR_XMIT_OPEN_SIZE+8); + for (i=0; i < H5FD_MIRROR_XMIT_OPEN_SIZE; i++) { + /* 0x100 is "zero" in a byte, so encode will treat it as a NULL- + * terminator in the filepath string. Expect all zeroes following. + */ + expected[i+2] = (i > 0xFF) ? 0 : (unsigned char)i; + } + + /* Set xmit_in + */ + xmit_in.pub = xmit_mock; /* shared/common */ + xmit_in.flags = 0x0E0F1011; + xmit_in.maxaddr = 0x1213141516171819; + xmit_in.size_t_blob = 0x1A1B1C1D1E1F2021; + for (i=0x22; i < H5FD_MIRROR_XMIT_FILEPATH_MAX+0x22; i++) { + /* nonzero values repeat after 0x100, but will not be encoded */ + xmit_in.filename[i-0x22] = (char)(i % 0x100); + } + xmit_in.filename[H5FD_MIRROR_XMIT_FILEPATH_MAX-1] = 0; + + /* Encode, and compare buffer contents + * Initial buffer is filled with 0xFF to match expected padding + */ + HDmemset(buf, 0xFF, H5FD_MIRROR_XMIT_OPEN_SIZE+8); + if (H5FD_mirror_xmit_encode_open((buf+2), &xmit_in) + != H5FD_MIRROR_XMIT_OPEN_SIZE) + { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, H5FD_MIRROR_XMIT_OPEN_SIZE+8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, H5FD_MIRROR_XMIT_OPEN_SIZE+8); + TEST_ERROR; + } + + /* Decode from buffer + */ + if (H5FD_mirror_xmit_decode_open(&xmit_out, (buf+2)) + != H5FD_MIRROR_XMIT_OPEN_SIZE) + { + TEST_ERROR; + } + if (xmit_out.pub.magic != xmit_mock.magic) TEST_ERROR; + if (xmit_out.pub.version != xmit_mock.version) TEST_ERROR; + if (xmit_out.pub.session_token != xmit_mock.session_token) TEST_ERROR; + if (xmit_out.pub.xmit_count != xmit_mock.xmit_count) TEST_ERROR; + if (xmit_out.pub.op != xmit_mock.op) TEST_ERROR; + if (xmit_out.flags != xmit_in.flags) TEST_ERROR; + if (xmit_out.maxaddr != xmit_in.maxaddr) TEST_ERROR; + if (xmit_out.size_t_blob != xmit_in.size_t_blob) TEST_ERROR; + if (HDstrncmp(xmit_out.filename, xmit_in.filename, + H5FD_MIRROR_XMIT_FILEPATH_MAX) + != 0) + { + PRINT_BUFFER_DIFF(xmit_out.filename, xmit_in.filename, + H5FD_MIRROR_XMIT_FILEPATH_MAX); + TEST_ERROR; + } + + /* Decode from different offset in buffer + * Observe changes when ingesting the padding + */ + if (H5FD_mirror_xmit_decode_open(&xmit_out, (buf)) + != H5FD_MIRROR_XMIT_OPEN_SIZE) + { + TEST_ERROR; + } + if (xmit_out.pub.magic != 0xFFFF0001) TEST_ERROR; + if (xmit_out.pub.version != 0x02) TEST_ERROR; + if (xmit_out.pub.session_token != 0x03040506) TEST_ERROR; + if (xmit_out.pub.xmit_count != 0x0708090A) TEST_ERROR; + if (xmit_out.pub.op != 0x0B) TEST_ERROR; + if (xmit_out.flags != 0x0C0D0E0F) TEST_ERROR; + if (xmit_out.maxaddr != 0x1011121314151617) TEST_ERROR; + if (xmit_out.size_t_blob != 0x18191A1B1C1D1E1F) TEST_ERROR; + /* update expected "filepath" in structure */ + for (i=0x20; i < H5FD_MIRROR_XMIT_FILEPATH_MAX+0x20; i++) { + xmit_in.filename[i-0x20] = (i > 0xFF) ? 0 : (char)i; + } + if (HDstrncmp(xmit_out.filename, xmit_in.filename, + H5FD_MIRROR_XMIT_FILEPATH_MAX) + != 0) + { + PRINT_BUFFER_DIFF(xmit_out.filename, xmit_in.filename, + H5FD_MIRROR_XMIT_FILEPATH_MAX); + TEST_ERROR; + } + + } while (0); /* end xmit open en/decode */ + + /* Test xmit reply structure encode/decode + * Write bogus but easily verifiable data to inside a buffer, and compare. + * Then decode the buffer and compare the structure contents. + * Then repeat from a different offset in the buffer and compare. + * + * Verifies that the first zero character in the filepath will end the + * string, with all following bytes in the encoded buffer being zeroed. + */ + do { + unsigned char buf[H5FD_MIRROR_XMIT_REPLY_SIZE+8]; + unsigned char expected[H5FD_MIRROR_XMIT_REPLY_SIZE+8]; + H5FD_mirror_xmit_reply_t xmit_in; + H5FD_mirror_xmit_reply_t xmit_out; + size_t i = 0; + + /* sanity check */ + if ((14+4+256) != H5FD_MIRROR_XMIT_REPLY_SIZE) { + FAIL_PUTS_ERROR("Header size definition does not match test\n"); + } + if (xmit_mock.op != 0x0D) { + FAIL_PUTS_ERROR("shared header structure is not in expected state"); + } + + /* Populate the expected buffer; expect end padding of 0xFF + */ + HDmemset(expected, 0xFF, H5FD_MIRROR_XMIT_REPLY_SIZE+8); + for (i=0; i < H5FD_MIRROR_XMIT_REPLY_SIZE; i++) { + /* 0x100 is "zero" in a byte, so encode will treat it as a NULL- + * terminator in the filepath string. Expect all zeroes following. + */ + expected[i+2] = (i > 0xFF) ? 0 : (unsigned char)i; + } + + /* Set xmit_in + */ + xmit_in.pub = xmit_mock; /* shared/common */ + xmit_in.status = 0x0E0F1011; + for (i=0x12; i < H5FD_MIRROR_STATUS_MESSAGE_MAX+0x12; i++) { + /* nonzero values repeat after 0x100, but will not be encoded */ + xmit_in.message[i-0x12] = (char)(i % 0x100); + } + xmit_in.message[H5FD_MIRROR_STATUS_MESSAGE_MAX-1] = 0; + + /* Encode, and compare buffer contents + * Initial buffer is filled with 0xFF to match expected padding + */ + HDmemset(buf, 0xFF, H5FD_MIRROR_XMIT_REPLY_SIZE+8); + if (H5FD_mirror_xmit_encode_reply((buf+2), &xmit_in) + != H5FD_MIRROR_XMIT_REPLY_SIZE) + { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, H5FD_MIRROR_XMIT_REPLY_SIZE+8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, H5FD_MIRROR_XMIT_REPLY_SIZE+8); + TEST_ERROR; + } + + /* Decode from buffer + */ + if (H5FD_mirror_xmit_decode_reply(&xmit_out, (buf+2)) + != H5FD_MIRROR_XMIT_REPLY_SIZE) + { + TEST_ERROR; + } + if (xmit_out.pub.magic != xmit_mock.magic) TEST_ERROR; + if (xmit_out.pub.version != xmit_mock.version) TEST_ERROR; + if (xmit_out.pub.session_token != xmit_mock.session_token) TEST_ERROR; + if (xmit_out.pub.xmit_count != xmit_mock.xmit_count) TEST_ERROR; + if (xmit_out.pub.op != xmit_mock.op) TEST_ERROR; + if (xmit_out.status != xmit_in.status) TEST_ERROR; + if (HDstrncmp(xmit_out.message, xmit_in.message, + H5FD_MIRROR_STATUS_MESSAGE_MAX) + != 0) + { + PRINT_BUFFER_DIFF(xmit_out.message, xmit_in.message, + H5FD_MIRROR_STATUS_MESSAGE_MAX); + TEST_ERROR; + } + + /* Decode from different offset in buffer + * Observe changes when ingesting the padding + */ + if (H5FD_mirror_xmit_decode_reply(&xmit_out, (buf)) + != H5FD_MIRROR_XMIT_REPLY_SIZE) + { + TEST_ERROR; + } + if (xmit_out.pub.magic != 0xFFFF0001) TEST_ERROR; + if (xmit_out.pub.version != 0x02) TEST_ERROR; + if (xmit_out.pub.session_token != 0x03040506) TEST_ERROR; + if (xmit_out.pub.xmit_count != 0x0708090A) TEST_ERROR; + if (xmit_out.pub.op != 0x0B) TEST_ERROR; + if (xmit_out.status != 0x0C0D0E0F) TEST_ERROR; + /* update expected "message" in structure */ + for (i=0x10; i < H5FD_MIRROR_STATUS_MESSAGE_MAX+0x10; i++) { + xmit_in.message[i-0x10] = (i > 0xFF) ? 0 : (char)i; + } + if (HDstrncmp(xmit_out.message, xmit_in.message, + H5FD_MIRROR_STATUS_MESSAGE_MAX) + != 0) + { + PRINT_BUFFER_DIFF(xmit_out.message, xmit_in.message, + H5FD_MIRROR_STATUS_MESSAGE_MAX); + TEST_ERROR; + } + + } while (0); /* end xmit reply en/decode */ + + /* Test xmit write structure encode/decode + * Write bogus but easily verifiable data to inside a buffer, and compare. + * Then decode the buffer and compare the structure contents. + * Then repeat from a different offset in the buffer and compare. + */ + do { + unsigned char buf[H5FD_MIRROR_XMIT_WRITE_SIZE+8]; + unsigned char expected[H5FD_MIRROR_XMIT_WRITE_SIZE+8]; + H5FD_mirror_xmit_write_t xmit_in; + H5FD_mirror_xmit_write_t xmit_out; + size_t i = 0; + + /* sanity check */ + if ((14+17) != H5FD_MIRROR_XMIT_WRITE_SIZE) { + FAIL_PUTS_ERROR("Header size definition does not match test\n"); + } + if (xmit_mock.op != 0x0D) { + FAIL_PUTS_ERROR("shared header structure is not in expected state"); + } + + /* Populate the expected buffer; expect end padding of 0xFF + */ + HDmemset(expected, 0xFF, H5FD_MIRROR_XMIT_WRITE_SIZE+8); + for (i=0; i < H5FD_MIRROR_XMIT_WRITE_SIZE; i++) { + expected[i+2] = (unsigned char)i; + } + + /* Set xmit_in + */ + xmit_in.pub = xmit_mock; /* shared/common */ + xmit_in.type = 0x0E; + xmit_in.offset = 0x0F10111213141516; + xmit_in.size = 0x1718191A1B1C1D1E; + + /* Encode, and compare buffer contents + * Initial buffer is filled with 0xFF to match expected padding + */ + HDmemset(buf, 0xFF, H5FD_MIRROR_XMIT_WRITE_SIZE+8); + if (H5FD_mirror_xmit_encode_write((buf+2), &xmit_in) + != H5FD_MIRROR_XMIT_WRITE_SIZE) + { + TEST_ERROR; + } + if (HDmemcmp(buf, expected, H5FD_MIRROR_XMIT_WRITE_SIZE+8) != 0) { + PRINT_BUFFER_DIFF(buf, expected, H5FD_MIRROR_XMIT_WRITE_SIZE+8); + TEST_ERROR; + } + + /* Decode from buffer + */ + if (H5FD_mirror_xmit_decode_write(&xmit_out, (buf+2)) + != H5FD_MIRROR_XMIT_WRITE_SIZE) + { + TEST_ERROR; + } + if (xmit_out.pub.magic != xmit_mock.magic) TEST_ERROR; + if (xmit_out.pub.version != xmit_mock.version) TEST_ERROR; + if (xmit_out.pub.session_token != xmit_mock.session_token) TEST_ERROR; + if (xmit_out.pub.xmit_count != xmit_mock.xmit_count) TEST_ERROR; + if (xmit_out.pub.op != xmit_mock.op) TEST_ERROR; + if (xmit_out.type != 0x0E) TEST_ERROR; + if (xmit_out.offset != 0x0F10111213141516) TEST_ERROR; + if (xmit_out.size != 0x1718191A1B1C1D1E) TEST_ERROR; + + /* Decode from different offset in buffer + * Observe changes when ingesting the padding + */ + if (H5FD_mirror_xmit_decode_write(&xmit_out, (buf)) + != H5FD_MIRROR_XMIT_WRITE_SIZE) + { + TEST_ERROR; + } + if (xmit_out.pub.magic != 0xFFFF0001) TEST_ERROR; + if (xmit_out.pub.version != 0x02) TEST_ERROR; + if (xmit_out.pub.session_token != 0x03040506) TEST_ERROR; + if (xmit_out.pub.xmit_count != 0x0708090A) TEST_ERROR; + if (xmit_out.pub.op != 0x0B) TEST_ERROR; + if (xmit_out.type != 0x0C) TEST_ERROR; + if (xmit_out.offset != 0x0D0E0F1011121314) TEST_ERROR; + if (xmit_out.size != 0x15161718191A1B1C) TEST_ERROR; + + } while (0); /* end xmit write en/decode */ + + PASSED(); + return 0; + +error: + return -1; +} /* end test_xmit_encode_decode */ + + +/* --------------------------------------------------------------------------- + * Function: create_mirroring_split_fapl + * + * Purpose: Create and populate a mirroring FAPL ID. + * Creates target files with the given base name -- ideally the + * test name -- and creates mirroring/split FAPL set to use the + * global mirroring info and a sec2 R/W channel driver. + * + * TODO: receive target IP from caller? + * + * Return: Success: HID of the top-level (splitter) FAPL, a non-negative + * value. + * Failure: H5I_INVALID_HID, a negative value. + * + * Programmer: Jacob Smith + * 2019 + * --------------------------------------------------------------------------- + */ +static hid_t +create_mirroring_split_fapl(const char *_basename, + struct mirrortest_filenames *names) +{ + H5FD_splitter_vfd_config_t splitter_config; + H5FD_mirror_fapl_t mirror_conf; + hid_t ret_value = H5I_INVALID_HID; + + if (_basename == NULL || *_basename == '\0') { + TEST_ERROR; + } + + splitter_config.magic = H5FD_SPLITTER_MAGIC; + splitter_config.version = H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION; + splitter_config.ignore_wo_errs = FALSE; + + /* Create Splitter R/W channel driver (sec2) + */ + splitter_config.rw_fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == splitter_config.rw_fapl_id) { + TEST_ERROR; + } + if (H5Pset_fapl_sec2(splitter_config.rw_fapl_id) == FAIL) { + TEST_ERROR; + } + + /* Create Splitter W/O channel driver (mirror) + */ + mirror_conf.magic = H5FD_MIRROR_FAPL_MAGIC; + mirror_conf.version = H5FD_MIRROR_CURR_FAPL_T_VERSION; + mirror_conf.handshake_port = SERVER_HANDSHAKE_PORT; + if (HDstrncpy(mirror_conf.remote_ip, SERVER_IP_ADDRESS, + H5FD_MIRROR_MAX_IP_LEN) + == NULL) + { + TEST_ERROR; + } + splitter_config.wo_fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == splitter_config.wo_fapl_id) { + TEST_ERROR; + } + if (H5Pset_fapl_mirror(splitter_config.wo_fapl_id, &mirror_conf) == FAIL) { + TEST_ERROR; + } + + /* Build r/w, w/o, and log file paths + */ + if (build_paths(_basename, &splitter_config, names) < 0) { + TEST_ERROR; + } + + /* Set file paths for w/o and logfile + */ + if (HDstrncpy(splitter_config.wo_path, (const char *)names->wo, + H5FD_SPLITTER_PATH_MAX) + == NULL) + { + TEST_ERROR; + } + if (HDstrncpy(splitter_config.log_file_path, (const char *)names->log, + H5FD_SPLITTER_PATH_MAX) + == NULL) + { + TEST_ERROR; + } + + /* Create Splitter FAPL + */ + ret_value = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == ret_value) { + TEST_ERROR; + } + if (H5Pset_fapl_splitter(ret_value, &splitter_config) == FAIL) { + TEST_ERROR; + } + + /* Close FAPLs created for child channels + */ + if (H5Pclose(splitter_config.rw_fapl_id) < 0) { + TEST_ERROR; + } + splitter_config.rw_fapl_id = H5I_INVALID_HID; + if (H5Pclose(splitter_config.wo_fapl_id) < 0) { + TEST_ERROR; + } + splitter_config.wo_fapl_id = H5I_INVALID_HID; + + return ret_value; + +error: + if (splitter_config.wo_fapl_id >= 0) { + (void)H5Pclose(splitter_config.wo_fapl_id); + } + if (splitter_config.rw_fapl_id >= 0) { + (void)H5Pclose(splitter_config.rw_fapl_id); + } + if (ret_value >= 0) { + (void)H5Pclose(ret_value); + } + return H5I_INVALID_HID; +} /* end create_mirroring_split_fapl() */ + + +/* --------------------------------------------------------------------------- + * Function: test_create_and_close + * + * Purpose: Test/demonstrate a do-nothing file open and close. + * + * Verifying file existence and contents is part of other tests. + * + * TODO: receive target IP from caller? + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2019-12-17 + * --------------------------------------------------------------------------- + */ +static int +test_create_and_close(void) +{ + struct mirrortest_filenames names; + hid_t file_id = H5I_INVALID_HID; + hid_t fapl_id = H5P_DEFAULT; + + TESTING("File creation and immediate close"); + + /* Create FAPL for Splitter[sec2|mirror] + */ + fapl_id = create_mirroring_split_fapl("basic_create", &names); + if (H5I_INVALID_HID == fapl_id) { + TEST_ERROR; + } + + /* -------------------- */ + /* TEST: Create and Close */ + + file_id = H5Fcreate(names.rw, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + if (H5I_INVALID_HID == file_id) { + TEST_ERROR; + } + + /* -------------------- */ + /* Standard cleanup */ + + if (H5Fclose(file_id) == FAIL) { + TEST_ERROR; + } + if (fapl_id != H5P_DEFAULT && fapl_id >= 0) { + if (H5Pclose(fapl_id) == FAIL) { + TEST_ERROR; + } + } + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY{ + (void)H5Fclose(file_id); + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + return -1; +} /* end test_create_and_close() */ + + +/* ---------------------------------------------------------------------------- + * Function: create_datasets + * + * Purpose: Given a file ID and least and greateset dataset indices, create + * populated chunked datasets in the target file from min_dset to + * (and including) max_dset. + * Uses #defined constants to determine chunk and dataset sizes + * and values. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2019-08-14 + * ---------------------------------------------------------------------------- + */ +static herr_t +create_datasets(hid_t file_id, + unsigned min_dset, + unsigned max_dset) +{ + hid_t dataspace_ids[MAX_DSET_COUNT + 1]; + hid_t dataset_ids[MAX_DSET_COUNT + 1]; + hid_t filespace_ids[MAX_DSET_COUNT + 1]; + int data_chunk[CHUNK_DIM][CHUNK_DIM]; + unsigned int i, j, k, l, m; + hsize_t offset[2]; + hid_t memspace_id = H5I_INVALID_HID; + hsize_t a_size[2] = {CHUNK_DIM, CHUNK_DIM}; + hsize_t chunk_dims[2] = {CHUNK_DIM, CHUNK_DIM}; + hsize_t dset_dims[2] = {DSET_DIM, DSET_DIM}; + + HDassert(file_id >= 0); + HDassert(min_dset <= max_dset); + HDassert(max_dset <= MAX_DSET_COUNT); + + LOGPRINT(2, "create_dataset()\n"); + + /* --------------------------------- + * "Clear" ID arrays + */ + + for (i = 0; i < MAX_DSET_COUNT; i++) { + LOGPRINT(3, "clearing IDs [%d]\n", i); + dataspace_ids[i] = H5I_INVALID_HID; + dataset_ids[i] = H5I_INVALID_HID; + filespace_ids[i] = H5I_INVALID_HID; + } + + /* --------------------------------- + * Generate dataspace, dataset, and 'filespace' IDs + */ + + if (_create_chunking_ids(file_id, min_dset, max_dset, chunk_dims, + dset_dims, dataspace_ids, filespace_ids, dataset_ids, &memspace_id) + == FAIL) + { + TEST_ERROR; + } + + /* --------------------------------- + * Initialize (write) all datasets in a "round robin"... + * for a given chunk 'location', write chunk data to each dataset. + */ + + for (i = 0; i < DSET_DIM; i += CHUNK_DIM) + { + LOGPRINT(3, "i: %d\n", i); + for (j = 0; j < DSET_DIM; j += CHUNK_DIM) + { + LOGPRINT(3, " j: %d\n", j); + for (m = min_dset; m <= max_dset; m++) + { + LOGPRINT(3, " m: %d\n", m); + for (k = 0; k < CHUNK_DIM; k++) + { + for (l = 0; l < CHUNK_DIM; l++) + { + data_chunk[k][l] = (int)((DSET_DIM * DSET_DIM * m) + + (DSET_DIM * (i + k)) + j + l); + LOGPRINT(3, " data_chunk[%d][%d]: %d\n", + k, l, data_chunk[k][l]); + } + } + + /* select on disk hyperslab */ + offset[0] = (hsize_t)i; + offset[1] = (hsize_t)j; + LOGPRINT(3, " H5Sselect_hyperslab()\n"); + if (H5Sselect_hyperslab(filespace_ids[m], H5S_SELECT_SET, + offset, NULL, a_size, NULL) + < 0) + { + TEST_ERROR; + } + + LOGPRINT(3, " H5Dwrite()\n"); + if (H5Dwrite(dataset_ids[m], H5T_NATIVE_INT, memspace_id, + filespace_ids[m], H5P_DEFAULT, data_chunk) + < 0) + { + TEST_ERROR; + } + + } + } + } + + /* --------------------------------- + * Read and verify data from datasets + */ + + if (_verify_datasets(min_dset, max_dset, filespace_ids, dataset_ids, + memspace_id) + == FAIL) + { + TEST_ERROR; + } + + /* --------------------------------- + * Cleanup + */ + + if (_close_chunking_ids(min_dset, max_dset, dataspace_ids, filespace_ids, + dataset_ids, &memspace_id) + == FAIL) + { + TEST_ERROR; + } + + return SUCCEED; + +error: + (void)_close_chunking_ids(min_dset, max_dset, dataspace_ids, + filespace_ids, dataset_ids, &memspace_id); + LOGPRINT(1, "create_datasets() FAILED\n"); + return FAIL; +} /* end create_datasets() */ + + +/* ---------------------------------------------------------------------------- + * Function: _create_chunking_ids + * + * Purpose: Create new IDs to be used with the associated file. + * + * Return: SUCCEED/FAIL + * + * Programer: Jacob Smith + * 2019 + * ---------------------------------------------------------------------------- + */ +static herr_t +_create_chunking_ids(hid_t file_id, + unsigned min_dset, + unsigned max_dset, + hsize_t *chunk_dims, + hsize_t *dset_dims, + hid_t *dataspace_ids, + hid_t *filespace_ids, + hid_t *dataset_ids, + hid_t *memspace_id) +{ + char dset_name[DSET_NAME_LEN + 1]; + unsigned m = 0; + hid_t dcpl_id = H5I_INVALID_HID; + + LOGPRINT(2, "_create_chunking_ids()\n"); + + /* -------------------- + * Create chunking DCPL + */ + + dcpl_id = H5Pcreate(H5P_DATASET_CREATE); + if (dcpl_id < 0) { + TEST_ERROR; + } + if (H5Pset_chunk(dcpl_id, 2, chunk_dims) == FAIL) { + TEST_ERROR; + } + + /* -------------------- + * Create dataspace IDs + */ + + for (m = min_dset; m <= max_dset; m++) { + dataspace_ids[m] = H5Screate_simple(2, dset_dims, NULL); + if (dataspace_ids[m] < 0) { + HDsnprintf(mesg, MIRR_MESG_SIZE, + "unable to create dataspace ID %d\n", m); + FAIL_PUTS_ERROR(mesg); + } + } + + /* -------------------- + * Create dataset IDs + */ + + for (m = min_dset; m <= max_dset; m++) { + if (HDsnprintf(dset_name, DSET_NAME_LEN, "/dset%03d", m) + > DSET_NAME_LEN) + { + HDsnprintf(mesg, MIRR_MESG_SIZE, + "unable to compose dset name %d\n", m); + FAIL_PUTS_ERROR(mesg); + } + + dataset_ids[m] = H5Dcreate(file_id, dset_name, H5T_STD_I32BE, + dataspace_ids[m], H5P_DEFAULT, dcpl_id, H5P_DEFAULT); + if (dataset_ids[m] < 0) { + HDsnprintf(mesg, MIRR_MESG_SIZE, + "unable to create dset ID %d\n", m); + FAIL_PUTS_ERROR(mesg); + } + } + + /* -------------------- + * Get file space IDs + */ + + for (m = min_dset; m <= max_dset; m++) { + filespace_ids[m] = H5Dget_space(dataset_ids[m]); + if (filespace_ids[m] < 0) { + HDsnprintf(mesg, MIRR_MESG_SIZE, + "unable to create filespace ID %d\n", m); + FAIL_PUTS_ERROR(mesg); + } + } + + /* -------------------- + * Create mem space to be used to read and write chunks + */ + + *memspace_id = H5Screate_simple(2, chunk_dims, NULL); + if (*memspace_id < 0) { + TEST_ERROR; + } + + /* -------------------- + * Clean up the DCPL, even if there were errors before + */ + + if (dcpl_id != H5P_DEFAULT && dcpl_id != H5I_INVALID_HID) { + if (H5Pclose(dcpl_id) == FAIL) { + TEST_ERROR; + } + } + + return SUCCEED; + +error: + if (dcpl_id != H5P_DEFAULT && dcpl_id != H5I_INVALID_HID) { + (void)H5Pclose(dcpl_id); + } + LOGPRINT(1, "_create_chunking_ids() FAILED\n"); + return FAIL; +} /* end _create_chunking_ids() */ + + +/* ---------------------------------------------------------------------------- + * Function: _open_chunking_ids + * + * Purpose: Open/access IDs from the given file. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2019 + * ---------------------------------------------------------------------------- + */ +static herr_t +_open_chunking_ids( + hid_t file_id, + unsigned min_dset, + unsigned max_dset, + hsize_t *chunk_dims, + hid_t *filespace_ids, + hid_t *dataset_ids, + hid_t *memspace_id) +{ + char dset_name[DSET_NAME_LEN+1]; + unsigned m = 0; + + LOGPRINT(2, "_open_chunking_ids()\n"); + + /* -------------------- + * Open dataset IDs + */ + + for (m = min_dset; m <= max_dset; m++) { + if (HDsnprintf(dset_name, DSET_NAME_LEN, "/dset%03d", m) + > DSET_NAME_LEN) + { + HDsnprintf(mesg, MIRR_MESG_SIZE, + "unable to compose dset name %d\n", m); + FAIL_PUTS_ERROR(mesg); + } + + dataset_ids[m] = H5Dopen2(file_id, dset_name, H5P_DEFAULT); + if (dataset_ids[m] < 0) { + HDsnprintf(mesg, MIRR_MESG_SIZE, "unable to open dset ID %d\n", m); + FAIL_PUTS_ERROR(mesg); + } + } + + /* -------------------- + * Open filespace IDs + */ + + for (m = min_dset; m <= max_dset; m++) { + filespace_ids[m] = H5Dget_space(dataset_ids[m]); + if (filespace_ids[m] < 0) { + HDsnprintf(mesg, MIRR_MESG_SIZE, + "unable to get filespace ID %d\n", m); + FAIL_PUTS_ERROR(mesg); + } + } + + /* -------------------- + * Create mem space to be used to read and write chunks + */ + + *memspace_id = H5Screate_simple(2, chunk_dims, NULL); + if (*memspace_id < 0) { + TEST_ERROR; + } + + return SUCCEED; + +error: + LOGPRINT(1, "_open_chunking_ids() FAILED\n"); + return FAIL; +} /* end _open_chunking_ids() */ + + +/* --------------------------------------------------------------------------- + * Function: _close_chunking_ids + * + * Purpose: Close IDs that were created or opened. + * Pass NULL into `dataspace_ids` when closing items opened with + * _open_chunking_ids(). (as opposed to created IDs) + * + * Return: SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2019 + * --------------------------------------------------------------------------- + */ +static herr_t +_close_chunking_ids(unsigned min_dset, + unsigned max_dset, + hid_t *dataspace_ids, + hid_t *filespace_ids, + hid_t *dataset_ids, + hid_t *memspace_id) +{ + unsigned m; + + LOGPRINT(2, "_close_chunking_ids()\n"); + + for (m = min_dset; m <= max_dset; m++) { + LOGPRINT(3, "closing ids[%d]\n", m); + if (dataspace_ids) { + if (H5Sclose(dataspace_ids[m]) < 0) { + HDsnprintf(mesg, MIRR_MESG_SIZE, + "unable to close dataspace_id[%d]\n", m); + FAIL_PUTS_ERROR(mesg); + } + } + if (H5Dclose(dataset_ids[m]) < 0) { + HDsnprintf(mesg, MIRR_MESG_SIZE, + "unable to close dataset_id[%d]\n", m); + FAIL_PUTS_ERROR(mesg); + } + if (H5Sclose(filespace_ids[m]) < 0) { + HDsnprintf(mesg, MIRR_MESG_SIZE, + "unable to close filespace_id[%d]\n", m); + FAIL_PUTS_ERROR(mesg); + } + } + + if ( (*memspace_id != H5I_INVALID_HID) && + (H5Sclose(*memspace_id) < 0) ) + { + TEST_ERROR; + } + + return SUCCEED; + +error: + LOGPRINT(1, "_close_chunking_ids() FAILED\n"); + return FAIL; +} /* end _close_chunking_ids() */ + + +/* --------------------------------------------------------------------------- + * Function: _verify_datasets + * + * Purpose: Check that each chunk's contents are as expected, as pertaining + * to create_datasets(). + * + * Return: SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2019 + * --------------------------------------------------------------------------- + */ +static herr_t +_verify_datasets(unsigned min_dset, + unsigned max_dset, + hid_t *filespace_ids, + hid_t *dataset_ids, + hid_t memspace_id) +{ + unsigned i, j, k, l, m; + int data_chunk[CHUNK_DIM][CHUNK_DIM]; + hsize_t offset[2]; + hsize_t a_size[2] = {CHUNK_DIM, CHUNK_DIM}; + + LOGPRINT(2, "_verify_datasets()\n"); + + for (i = 0; i < DSET_DIM; i += CHUNK_DIM) + { + LOGPRINT(3, "i: %d\n", i); + for (j = 0; j < DSET_DIM; j += CHUNK_DIM) + { + LOGPRINT(3, " j: %d\n", j); + for (m = min_dset; m <= max_dset; m++) + { + LOGPRINT(3, " m: %d\n", m); + + /* select on disk hyperslab */ + offset[0] = (hsize_t)i; + offset[1] = (hsize_t)j; + if (H5Sselect_hyperslab(filespace_ids[m], H5S_SELECT_SET, + offset, NULL, a_size, NULL) + < 0) + { + TEST_ERROR; + } + + if (H5Dread(dataset_ids[m], H5T_NATIVE_INT, memspace_id, + filespace_ids[m], H5P_DEFAULT, data_chunk) + < 0) + { + HDsnprintf(mesg, MIRR_MESG_SIZE, + " H5Dread() [%d][%d][%d]\n", + i, j, m); + FAIL_PUTS_ERROR(mesg); + } + + for (k = 0; k < CHUNK_DIM; k++) { + for (l = 0; l < CHUNK_DIM; l++) { + if ((unsigned)data_chunk[k][l] + != + ((DSET_DIM * DSET_DIM * m) + + (DSET_DIM * (i + k)) + j + l)) + { + HDsnprintf(mesg, MIRR_MESG_SIZE, + " MISMATCH [%d][%d][%d][%d][%d]\n", + i, j, m, k, l); + FAIL_PUTS_ERROR(mesg); + } + } + } + + } + } + } + + return SUCCEED; + +error: + LOGPRINT(1, "_verify_datasets() FAILED\n"); + return FAIL; +} /* end _verify_datasets() */ + + +/* --------------------------------------------------------------------------- + * Function: verify_datasets + * + * Purpose: Inspect the datasets in the file created by create_datasets(). + * Wrapper for _verify_datasets() -- this function sets up and + * tears down accessor information. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2019 + * --------------------------------------------------------------------------- + */ +static herr_t +verify_datasets(hid_t file_id, + unsigned min_dset, + unsigned max_dset) +{ + hid_t dataset_ids[MAX_DSET_COUNT + 1]; + hid_t filespace_ids[MAX_DSET_COUNT + 1]; + unsigned i; + hid_t memspace_id = H5I_INVALID_HID; + hsize_t chunk_dims[2] = {CHUNK_DIM, CHUNK_DIM}; + + HDassert(file_id >= 0); + HDassert(min_dset <= max_dset); + HDassert(max_dset <= MAX_DSET_COUNT); + + LOGPRINT(2, "verify_datasets()\n"); + + /* --------------------------------- + * "Clear" ID arrays + */ + + for (i = 0; i < MAX_DSET_COUNT; i++) { + LOGPRINT(3, "clearing IDs [%d]\n", i); + dataset_ids[i] = H5I_INVALID_HID; + filespace_ids[i] = H5I_INVALID_HID; + } + + /* --------------------------------- + * Generate dataspace, dataset, and 'filespace' IDs + */ + + if (_open_chunking_ids(file_id, min_dset, max_dset, chunk_dims, + filespace_ids, dataset_ids, &memspace_id) + == FAIL) + { + TEST_ERROR; + } + + /* --------------------------------- + * Read and verify data from datasets + */ + + if (_verify_datasets(min_dset, max_dset, filespace_ids, dataset_ids, + memspace_id) + == FAIL) + { + TEST_ERROR; + } + + /* --------------------------------- + * Cleanup + */ + + if (_close_chunking_ids(min_dset, max_dset, NULL, filespace_ids, + dataset_ids, &memspace_id) + == FAIL) + { + TEST_ERROR; + } + + return SUCCEED; + +error: + LOGPRINT(1, "verify_datasets() FAILED\n"); + (void)_close_chunking_ids(min_dset, max_dset, NULL, filespace_ids, + dataset_ids, &memspace_id); + return FAIL; + +} /* end verify_datasets() */ + + +/* --------------------------------------------------------------------------- + * Function: test_basic_dataset_write + * + * Purpose: Create and close files; repoen files and write a dataset, + * close; compare files. + * + * TODO: receive target IP from caller? + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2019 + * --------------------------------------------------------------------------- + */ +static int +test_basic_dataset_write(void) +{ + struct mirrortest_filenames names; + hid_t file_id = H5I_INVALID_HID; + hid_t fapl_id = H5P_DEFAULT; + hid_t dset_id = H5I_INVALID_HID; + hid_t dspace_id = H5I_INVALID_HID; + hid_t dtype_id = H5T_NATIVE_INT; + hsize_t dims[2] = { DATABUFFER_SIZE, DATABUFFER_SIZE }; + int *buf = NULL; + int i = 0; + int j = 0; + + TESTING("Mirror open and dataset writing"); + + /* Create FAPL for Splitter[sec2|mirror] + */ + fapl_id = create_mirroring_split_fapl("basic_write", &names); + if (H5I_INVALID_HID == fapl_id) { + TEST_ERROR; + } + + /* Prepare data to be written + */ + buf = (int *)HDmalloc(DATABUFFER_SIZE * DATABUFFER_SIZE * sizeof(int)); + if (NULL == buf) { + TEST_ERROR; + } + for (i = 0; i < DATABUFFER_SIZE; i++) { + for (j = 0; j < DATABUFFER_SIZE; j++) { + int k = i * DATABUFFER_SIZE + j; + buf[k] = k; + } + } + + /* -------------------- */ + /* TEST: Create and Close */ + + file_id = H5Fcreate(names.rw, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + if (H5I_INVALID_HID == file_id) { + TEST_ERROR; + } + if (H5Fclose(file_id) == FAIL) { + TEST_ERROR; + } + file_id = H5I_INVALID_HID; + + + /* -------------------- */ + /* TEST: Repoen and Write */ + + file_id = H5Fopen(names.rw, H5F_ACC_RDWR, fapl_id); + if (H5I_INVALID_HID == file_id) { + TEST_ERROR; + } + + dspace_id = H5Screate_simple(2, dims, NULL); + if (H5I_INVALID_HID == dspace_id) { + TEST_ERROR; + } + dset_id = H5Dcreate2(file_id, "dataset", dtype_id, dspace_id, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); + if (H5I_INVALID_HID == dset_id) { + TEST_ERROR; + } + + if (H5Dwrite(dset_id, dtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) + == FAIL) + { + TEST_ERROR; + } + + /* -------------------- */ + /* Standard cleanup */ + + HDfree(buf); + buf = NULL; + if (H5Dclose(dset_id) == FAIL) { + TEST_ERROR; + } + if (H5Sclose(dspace_id) == FAIL) { + TEST_ERROR; + } + if (H5Fclose(file_id) == FAIL) { + TEST_ERROR; + } + if (fapl_id != H5P_DEFAULT && fapl_id > 0) { + if (H5Pclose(fapl_id) == FAIL) { + TEST_ERROR; + } + } + + /* -------------------- */ + /* TEST: Verify that the R/W and W/O files are identical */ + + if (h5_compare_file_bytes(names.rw, names.wo) < 0) { + TEST_ERROR; + } + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY{ + (void)H5Fclose(file_id); + if (buf) { + HDfree(buf); + } + (void)H5Dclose(dset_id); + (void)H5Sclose(dspace_id); + if (fapl_id != H5P_DEFAULT && fapl_id > 0) { + (void)H5Pclose(fapl_id); + } + } H5E_END_TRY; + return -1; +} /* end test_basic_dataset_write() */ + + +/* --------------------------------------------------------------------------- + * Function: test_chunked_dataset_write + * + * Purpose: Create and close files; repoen files and write a dataset, + * close; compare files. + * + * TODO: receive target IP from caller? + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2019 + * --------------------------------------------------------------------------- + */ +static int +test_chunked_dataset_write(void) +{ + struct mirrortest_filenames names; + hid_t file_id = H5I_INVALID_HID; + hid_t fapl_id = H5P_DEFAULT; + + TESTING("Mirror open and dataset writing (chunked)"); + + /* Create FAPL for Splitter[sec2|mirror] + */ + fapl_id = create_mirroring_split_fapl("chunked_write", &names); + if (H5I_INVALID_HID == fapl_id) { + TEST_ERROR; + } + + /* -------------------- */ + /* TEST: Create and Close */ + + file_id = H5Fcreate(names.rw, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + if (H5I_INVALID_HID == file_id) { + TEST_ERROR; + } + if (H5Fclose(file_id) == FAIL) { + TEST_ERROR; + } + file_id = H5I_INVALID_HID; + + /* -------------------- */ + /* TEST: Reopen and Write */ + + file_id = H5Fopen(names.rw, H5F_ACC_RDWR, fapl_id); + if (H5I_INVALID_HID == file_id) { + TEST_ERROR; + } + + /* Write datasets to file + */ + if (create_datasets(file_id, 0, MAX_DSET_COUNT) == FAIL) { + TEST_ERROR; + } + + /* Close to 'flush to disk', and reopen file + */ + if (H5Fclose(file_id) == FAIL) { + TEST_ERROR; + } + file_id = H5I_INVALID_HID; + + /* Reopen file + */ + file_id = H5Fopen(names.rw, H5F_ACC_RDWR, fapl_id); + if (H5I_INVALID_HID == file_id) { + TEST_ERROR; + } + + /* Verify written data integrity + */ + if (verify_datasets(file_id, 0, MAX_DSET_COUNT) == FAIL) { + TEST_ERROR; + } + + /* -------------------- */ + /* Standard cleanup */ + + if (H5Fclose(file_id) == FAIL) { + TEST_ERROR; + } + file_id = H5I_INVALID_HID; + if (fapl_id != H5P_DEFAULT && fapl_id > 0) { + if (H5Pclose(fapl_id) == FAIL) { + TEST_ERROR; + } + fapl_id = H5I_INVALID_HID; + } + + /* -------------------- */ + /* TEST: Verify that the R/W and W/O files are identical */ + + if (h5_compare_file_bytes(names.rw, names.wo) < 0) { + TEST_ERROR; + } + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + (void)H5Fclose(file_id); + if (fapl_id != H5P_DEFAULT && fapl_id > 0) { + (void)H5Pclose(fapl_id); + } + } H5E_END_TRY; + return -1; +} /* end test_chunked_dataset_write() */ + + +/* --------------------------------------------------------------------------- + * Function: test_on_disk_zoo + * + * Purpose: Verify that the mirror can handle the passing of all the + * various on-disk data structures over the wire, as implemented + * in genall5.c:create_zoo(). + * + * TODO: receive target IP from caller? + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2019 + * --------------------------------------------------------------------------- + */ +static int +test_on_disk_zoo(void) +{ + const char grp_name[] = "/only"; + struct mirrortest_filenames names; + hid_t file_id = H5I_INVALID_HID; + hid_t grp_id = H5I_INVALID_HID; + hid_t fapl_id = H5P_DEFAULT; + + TESTING("'Zoo' of on-disk structures"); + + /* Create FAPL for Splitter[sec2|mirror] + */ + fapl_id = create_mirroring_split_fapl("zoo", &names); + if (H5I_INVALID_HID == fapl_id) { + TEST_ERROR; + } + + /* -------------------- */ + /* TEST: Create file */ + file_id = H5Fcreate(names.rw, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + if (H5I_INVALID_HID == file_id) { + TEST_ERROR; + } + + grp_id = H5Gcreate2(file_id, grp_name, H5P_DEFAULT, H5P_DEFAULT, + H5P_DEFAULT); + if (grp_id == H5I_INVALID_HID) { + TEST_ERROR; + } + + /* Create datasets in file, close (flush) and reopen, validate. + * Use of ( pass ) a conceit required for using create_ and validate_zoo() + * from cache_common and/or genall5. + */ + + if ( pass ) { + create_zoo(file_id, grp_name, 0); + } + if ( pass ) { + if (H5Fclose(file_id) == FAIL) { + TEST_ERROR; + } + file_id = H5Fopen(names.rw, H5F_ACC_RDWR, fapl_id); + if (H5I_INVALID_HID == file_id) { + TEST_ERROR; + } + } + if ( pass ) { + validate_zoo(file_id, grp_name, 0); /* sanity-check */ + } + if ( !pass ) { + HDprintf(failure_mssg); + TEST_ERROR; + } + + /* -------------------- */ + /* Standard cleanup */ + + if (fapl_id != H5P_DEFAULT && fapl_id >= 0) { + if (H5Pclose(fapl_id) == FAIL) { + TEST_ERROR; + } + } + if (H5Gclose(grp_id) == FAIL) { + TEST_ERROR; + } + if (H5Fclose(file_id) == FAIL) { + TEST_ERROR; + } + + /* -------------------- */ + /* TEST: Verify that the R/W and W/O files are identical */ + + if (h5_compare_file_bytes(names.rw, names.wo) < 0) { + TEST_ERROR; + } + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + (void)H5Fclose(file_id); + (void)H5Gclose(grp_id); + if (fapl_id != H5P_DEFAULT && fapl_id > 0) { + (void)H5Pclose(fapl_id); + } + } H5E_END_TRY; + return -1; +} /* end test_on_disk_zoo() */ + + +/* --------------------------------------------------------------------------- + * Function: test_vanishing_datasets + * + * Purpose: Verify behavior when writing to a file where data is deleted. + * + * Each dataset is populated with the value of its suffix + * (dset5 is all fives). + * + * Opens 0..15 create one new dataset each, '/dset[i]'. + * Opens 3..18 delete '/dset[1-3]' + * + * Should end with no data in file. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2019 + * --------------------------------------------------------------------------- + */ +static int +test_vanishing_datasets(void) +{ + struct mirrortest_filenames names; + hid_t file_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; + hid_t dspace_id = H5I_INVALID_HID; + hid_t mirror_fapl_id = H5I_INVALID_HID; + hsize_t dims[2] = {DATABUFFER_SIZE, DATABUFFER_SIZE}; + uint32_t buf[DATABUFFER_SIZE][DATABUFFER_SIZE]; /* consider malloc? */ + H5G_info_t group_info; + unsigned int i, j, k; + const unsigned int max_loops = 20; + const unsigned int max_at_one_time = 3; + + TESTING("Vanishing Datasets"); + + /* -------------------- */ + /* Set up recurrent data (FAPL, dataspace) */ + + /* Create FAPL for Splitter[sec2|mirror] + */ + fapl_id = create_mirroring_split_fapl("vanishing", &names); + if (H5I_INVALID_HID == fapl_id) { + TEST_ERROR; + } + + dspace_id = H5Screate_simple(2, dims, NULL); + if (dspace_id < 0) { + TEST_ERROR; + } + + /* create file */ + file_id = H5Fcreate(names.rw, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + if (H5I_INVALID_HID == file_id) { + TEST_ERROR; + } + + for (i=0; i < max_loops; i++) { + char namebuf[DSET_NAME_LEN + 1]; + + /* deleting datasets */ + if (i >= max_at_one_time) { + if (HDsnprintf(namebuf, DSET_NAME_LEN, "/dset%02d", + (i - max_at_one_time) ) + > DSET_NAME_LEN) + { + TEST_ERROR; + } + if (H5Ldelete(file_id, namebuf, H5P_DEFAULT) < 0) { + TEST_ERROR; + } + } /* end if deleting a dataset */ + + /* writing datasets */ + if (i < (max_loops - max_at_one_time)) { + if (HDsnprintf(namebuf, DSET_NAME_LEN, "/dset%02d", i) + > DSET_NAME_LEN) + { + TEST_ERROR; + } + dset_id = H5Dcreate2(file_id, namebuf, H5T_STD_U32LE, dspace_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (H5I_INVALID_HID == dset_id) { + TEST_ERROR; + } + + for (j=0; j < DATABUFFER_SIZE; j++) { + for (k=0; k < DATABUFFER_SIZE; k++) { + buf[j][k] = (uint32_t)i; + } + } + + if (H5Dwrite(dset_id, H5T_STD_U32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, + buf) + < 0) + { + TEST_ERROR; + } + + if (H5Dclose(dset_id) < 0) { + TEST_ERROR; + } + dset_id = H5I_INVALID_HID; + } /* end if writing a dataset */ + + } /* end for dataset create-destroy cycles */ + + if (H5Fclose(file_id) < 0) { + TEST_ERROR; + } + file_id = H5I_INVALID_HID; + + /* verify there are no datasets in file */ + file_id = H5Fopen(names.rw, H5F_ACC_RDONLY, H5P_DEFAULT); + if (file_id < 0) { + TEST_ERROR; + } + if (H5Gget_info(file_id, &group_info) < 0) { + TEST_ERROR; + } + if (group_info.nlinks > 0) { + HDfprintf(stderr, "links in rw file: %d\n", group_info.nlinks); + HDfflush(stderr); + TEST_ERROR; + } + if (H5Fclose(file_id) < 0) { + TEST_ERROR; + } + file_id = H5Fopen(names.wo, H5F_ACC_RDONLY, H5P_DEFAULT); + if (file_id < 0) { + TEST_ERROR; + } + if (H5Gget_info(file_id, &group_info) < 0) { + TEST_ERROR; + } + if (group_info.nlinks > 0) { + HDfprintf(stderr, "links in wo file: %d\n", group_info.nlinks); + HDfflush(stderr); + TEST_ERROR; + } + if (H5Fclose(file_id) < 0) { + TEST_ERROR; + } + file_id = H5I_INVALID_HID; + + if (h5_compare_file_bytes(names.rw, names.wo) < 0) + TEST_ERROR; + + /* -------------------- */ + /* Teardown */ + + if (H5Sclose(dspace_id) < 0) { + TEST_ERROR; + } + if (H5Pclose(fapl_id) < 0) { + TEST_ERROR; + } + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + if (mirror_fapl_id != H5I_INVALID_HID) { + H5Pclose(mirror_fapl_id); + } + if (fapl_id != H5I_INVALID_HID) { + H5Pclose(fapl_id); + } + if (file_id != H5I_INVALID_HID) { + H5Fclose(file_id); + } + if (dset_id != H5I_INVALID_HID) { + H5Dclose(dset_id); + } + if (dspace_id != H5I_INVALID_HID) { + H5Sclose(dspace_id); + } + } H5E_END_TRY; + return -1; +} /* test_vanishing_datasets() */ + + +/* --------------------------------------------------------------------------- + * Function: test_concurrent_access + * + * Purpose: Verify that more than one file may be opened at a time. + * + * TODO: receive target IP from caller? + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2020-03-09 + * --------------------------------------------------------------------------- + */ +static int +test_concurrent_access(void) +{ + struct file_bundle { + struct mirrortest_filenames names; + hid_t dset_id; + hid_t fapl_id; + hid_t file_id; + } bundle[CONCURRENT_COUNT]; + hid_t dspace_id = H5I_INVALID_HID; + hid_t dtype_id = H5T_NATIVE_INT; + hsize_t dims[2] = { DATABUFFER_SIZE, DATABUFFER_SIZE }; + int *buf = NULL; + int i = 0; + int j = 0; + + TESTING("Concurrent opened mirrored files"); + + /* blank bundle */ + for (i = 0; i < CONCURRENT_COUNT; i++) { + bundle[i].dset_id = H5I_INVALID_HID; + bundle[i].fapl_id = H5I_INVALID_HID; + bundle[i].file_id = H5I_INVALID_HID; + *bundle[i].names.rw = '\0'; + *bundle[i].names.wo = '\0'; + *bundle[i].names.log = '\0'; + } + + /* Create FAPL for Splitter[sec2|mirror] + */ + for (i = 0; i < CONCURRENT_COUNT; i++) { + char _name[16] = ""; + hid_t _fapl_id = H5I_INVALID_HID; + HDsnprintf(_name, 15, "concurrent%d", i); + _fapl_id = create_mirroring_split_fapl(_name, &bundle[i].names); + if (H5I_INVALID_HID == _fapl_id) { + TEST_ERROR; + } + bundle[i].fapl_id = _fapl_id; + } + + /* Prepare data to be written + */ + buf = (int *)HDmalloc(DATABUFFER_SIZE * DATABUFFER_SIZE * sizeof(int)); + if (NULL == buf) { + TEST_ERROR; + } + for (i = 0; i < DATABUFFER_SIZE; i++) { + for (j = 0; j < DATABUFFER_SIZE; j++) { + int k = i * DATABUFFER_SIZE + j; + buf[k] = k; + } + } + + /* Prepare generic dataspace + */ + dspace_id = H5Screate_simple(2, dims, NULL); + if (H5I_INVALID_HID == dspace_id) { + TEST_ERROR; + } + + /* -------------------- */ + /* TEST: Create file and open elements */ + + for (i = 0; i < CONCURRENT_COUNT; i++) { + hid_t _file_id = H5I_INVALID_HID; + hid_t _dset_id = H5I_INVALID_HID; + + _file_id = H5Fcreate(bundle[i].names.rw, H5F_ACC_TRUNC, H5P_DEFAULT, + bundle[i].fapl_id); + if (H5I_INVALID_HID == _file_id) { + TEST_ERROR; + } + + bundle[i].file_id = _file_id; + + _dset_id = H5Dcreate2(_file_id, "dataset", dtype_id, dspace_id, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (H5I_INVALID_HID == _dset_id) { + TEST_ERROR; + } + bundle[i].dset_id = _dset_id; + } + + /* -------------------- */ + /* TEST: Write to files */ + + for (i = 0; i < CONCURRENT_COUNT; i++) { + if (H5Dwrite(bundle[i].dset_id, dtype_id, H5S_ALL, H5S_ALL, + H5P_DEFAULT, buf) + == FAIL) + { + TEST_ERROR; + } + } + + /* -------------------- */ + /* TEST: Close elements */ + + for (i = 0; i < CONCURRENT_COUNT; i++) { + if (H5Dclose(bundle[i].dset_id) == FAIL) { + TEST_ERROR; + } + if (H5Fclose(bundle[i].file_id) == FAIL) { + TEST_ERROR; + } + if (H5Pclose(bundle[i].fapl_id) == FAIL) { + TEST_ERROR; + } + } + + /* -------------------- */ + /* Standard cleanup */ + + HDfree(buf); + buf = NULL; + if (H5Sclose(dspace_id) == FAIL) { + TEST_ERROR; + } + + /* -------------------- */ + /* TEST: Verify that the R/W and W/O files are identical */ + + for (i = 0; i < CONCURRENT_COUNT; i++) { + if (h5_compare_file_bytes(bundle[i].names.rw, bundle[i].names.wo) < 0) { + TEST_ERROR; + } + } + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY{ + if (buf) { + HDfree(buf); + } + (void)H5Sclose(dspace_id); + for (i = 0; i < CONCURRENT_COUNT; i++) { + (void)H5Dclose(bundle[i].dset_id); + (void)H5Fclose(bundle[i].file_id); + (void)H5Pclose(bundle[i].fapl_id); + } + } H5E_END_TRY; + return -1; +} /* end test_concurrent_access() */ + + +/* --------------------------------------------------------------------------- + * Function: main + * + * Purpose: Run tests. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Jacob Smith + * 2019 + * --------------------------------------------------------------------------- + */ +int +main(void) +{ + int nerrors = 0; + + h5_reset(); + + g_log_stream = stdout; /* default debug/logging output stream */ + + HDprintf("Testing Mirror VFD functionality.\n"); + + /* -------------------- */ + /* SETUP */ + + /* Create directories for test-generated .h5 files + */ + if (nerrors == 0) { + if ((HDmkdir(MIRROR_RW_DIR, (mode_t)0755) < 0) && (errno != EEXIST)) { + nerrors++; + } + } + if (nerrors == 0) { + if ((HDmkdir(MIRROR_WO_DIR, (mode_t)0755) < 0) && (errno != EEXIST)) { + nerrors++; + } + } + + /* -------------------- */ + /* TESTS */ + /* Tests return negative values; `-=' increments nerrors count */ + + if (nerrors == 0) { + nerrors -= test_fapl_configuration(); + nerrors -= test_xmit_encode_decode(); + nerrors -= test_create_and_close(); + nerrors -= test_basic_dataset_write(); + nerrors -= test_chunked_dataset_write(); + nerrors -= test_on_disk_zoo(); + nerrors -= test_vanishing_datasets(); + nerrors -= test_concurrent_access(); + } + + if (nerrors) { + HDprintf("***** %d Mirror VFD TEST%s FAILED! *****\n", + nerrors, nerrors > 1 ? "S" : ""); + return EXIT_FAILURE; + } + + HDprintf("All Mirror Virtual File Driver tests passed.\n"); + return EXIT_SUCCESS; +} /* end main() */ + +#else /* H5_HAVE_MIRROR_VFD */ + +int +main(void) +{ + h5_reset(); + HDprintf("Testing Mirror VFD functionality.\n"); + HDprintf("SKIPPED - Mirror VFD not built.\n"); + return EXIT_SUCCESS; +} + +#endif /* H5_HAVE_MIRROR_VFD */ + + diff --git a/test/mount.c b/test/mount.c index 181b8cb..2af1483 100644 --- a/test/mount.c +++ b/test/mount.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Wednesday, October 7, 1998 * * Purpose: Tests file mounting. diff --git a/test/mtime.c b/test/mtime.c index 06f576b..04e302e 100644 --- a/test/mtime.c +++ b/test/mtime.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Thursday, July 30, 1998 * * Purpose: Determines if the modification time message is working diff --git a/test/ntypes.c b/test/ntypes.c index d3d10eb..f1260ad 100644 --- a/test/ntypes.c +++ b/test/ntypes.c @@ -2675,7 +2675,7 @@ error: * Return: Success: 0 * Failure: -1 * - * Programmer: pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente * September 3, 2004 * * Modifications: diff --git a/test/ohdr.c b/test/ohdr.c index 8deec10..36a7742 100644 --- a/test/ohdr.c +++ b/test/ohdr.c @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Robb Matzke <matzke@llnl.gov> +/* Programmer: Robb Matzke * Tuesday, November 24, 1998 */ #include "h5test.h" diff --git a/test/pool.c b/test/pool.c index 1851d6e..59f9201 100644 --- a/test/pool.c +++ b/test/pool.c @@ -11,7 +11,7 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> +/* Programmer: Quincey Koziol * Tuesday, May 3, 2005 */ #include "h5test.h" @@ -636,10 +636,10 @@ test_allocate_random(void) /* Initialize random number seed */ curr_time = HDtime(NULL); -#ifdef QAK +#if 0 curr_time=1115412944; HDfprintf(stderr,"curr_time=%lu\n",(unsigned long)curr_time); -#endif /* QAK */ +#endif HDsrandom((unsigned)curr_time); /* Create a memory pool */ diff --git a/test/space_overflow.c b/test/space_overflow.c index 15be9ba..82ddb3b 100644 --- a/test/space_overflow.c +++ b/test/space_overflow.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Monday, October 26, 1998 * * Purpose: Create a dataset with a simple data space that has the diff --git a/test/stab.c b/test/stab.c index 9afe28e..c3f8dbf 100644 --- a/test/stab.c +++ b/test/stab.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Tuesday, November 24, 1998 */ @@ -171,7 +171,7 @@ test_misc(hid_t fcpl, hid_t fapl, hbool_t new_format) * * Failure: number of errors * - * Programmer: Robb Matzke <matzke@llnl.gov> 2002-03-28 + * Programmer: Robb Matzke 2002-03-28 * * Modifications: *------------------------------------------------------------------------- diff --git a/test/swmr.c b/test/swmr.c index 700a68c..dde6eac 100644 --- a/test/swmr.c +++ b/test/swmr.c @@ -90,7 +90,7 @@ static int test_file_lock_concur(hid_t fapl); static int test_file_lock_swmr_concur(hid_t fapl); /* Test file lock environment variable */ -static int test_file_lock_env_var(hid_t fapl); +static int test_file_locking(hid_t in_fapl, hbool_t turn_locking_on, hbool_t env_var_override); /* Tests for SWMR VFD flag */ static int test_swmr_vfd_flag(void); @@ -4255,8 +4255,11 @@ test_file_lock_same(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags--single process access"); + /* Set locking in the fapl */ if((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if(H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -4415,9 +4418,11 @@ test_file_lock_swmr_same(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags + SWMR flags--single process access"); - /* Get a copy of the parameter in_fapl */ + /* Set locking in the fapl */ if((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if(H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -4725,8 +4730,11 @@ test_file_lock_concur(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combinations of flags--concurrent access"); + /* Set locking in the fapl */ if((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if(H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -5101,8 +5109,11 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Output message about test being performed */ TESTING("File open with different combintations of flags + SWMR flags--concurrent access"); + /* Set locking in the fapl */ if((fapl = H5Pcopy(in_fapl)) < 0) FAIL_STACK_ERROR + if(H5Pset_file_locking(fapl, TRUE, TRUE) < 0) + FAIL_STACK_ERROR /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[2], fapl, filename, sizeof(filename)); @@ -5133,7 +5144,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5154,7 +5165,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5204,13 +5215,13 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Fork child process */ if((childpid = HDfork()) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; - /* Close unused write end for out_pdf */ + /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) HDexit(EXIT_FAILURE); @@ -5229,7 +5240,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5242,7 +5253,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Open the test file */ if((fid = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR /* Notify child process */ notify = 1; @@ -5255,7 +5266,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Wait for child process to complete */ if(HDwaitpid(childpid, &child_status, child_wait_option) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR /* Check if child terminated normally */ if(WIFEXITED(child_status)) { @@ -5283,7 +5294,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5304,7 +5315,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5353,11 +5364,11 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Fork child process */ if((childpid = HDfork()) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5378,7 +5389,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5391,7 +5402,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Open the test file */ if((fid = H5Fopen(filename, H5F_ACC_RDWR|H5F_ACC_SWMR_WRITE, fapl)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR /* Notify child process */ notify = 1; @@ -5404,7 +5415,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Wait for child process to complete */ if(HDwaitpid(childpid, &child_status, child_wait_option) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR /* Check if child terminated normally */ if(WIFEXITED(child_status)) { @@ -5427,11 +5438,11 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Fork child process */ if((childpid = HDfork()) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5455,7 +5466,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) HDexit(EXIT_SUCCESS); } - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5468,7 +5479,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Open the test file */ if((fid = H5Fopen(filename, H5F_ACC_RDWR|H5F_ACC_SWMR_WRITE, fapl)) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR /* Notify child process */ notify = 1; @@ -5481,7 +5492,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Wait for child process to complete */ if(HDwaitpid(childpid, &child_status, child_wait_option) < 0) - FAIL_STACK_ERROR + FAIL_STACK_ERROR /* Check if child terminated normally */ if(WIFEXITED(child_status)) { @@ -5508,7 +5519,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5529,7 +5540,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5583,7 +5594,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5597,14 +5608,14 @@ test_file_lock_swmr_concur(hid_t in_fapl) /* Open the test file */ H5E_BEGIN_TRY { - child_fid = H5Fopen(filename, H5F_ACC_RDWR, H5P_DEFAULT); + child_fid = H5Fopen(filename, H5F_ACC_RDWR, fapl); } H5E_END_TRY; /* Should fail */ if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5658,7 +5669,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5733,7 +5744,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5811,7 +5822,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5834,7 +5845,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) HDexit(EXIT_SUCCESS); } - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5888,7 +5899,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -5909,7 +5920,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(child_fid == FAIL) HDexit(EXIT_SUCCESS); - /* Close the pipe */ + /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); @@ -5921,7 +5932,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) FAIL_STACK_ERROR /* Open the test file */ - if((fid = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + if((fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) FAIL_STACK_ERROR /* Notify child process */ @@ -5963,7 +5974,7 @@ test_file_lock_swmr_concur(hid_t in_fapl) if(childpid == 0) { /* Child process */ hid_t child_fid; /* File ID */ - int child_notify = 0; + int child_notify = 0; /* Close unused write end for out_pdf */ if(HDclose(out_pdf[1]) < 0) @@ -6058,7 +6069,7 @@ error: ** *****************************************************************/ static int -test_file_lock_env_var(hid_t in_fapl) +test_file_locking(hid_t in_fapl, hbool_t turn_locking_on, hbool_t env_var_override) { #if !(defined(H5_HAVE_FORK) && defined(H5_HAVE_WAITPID)) SKIPPED(); @@ -6073,18 +6084,40 @@ test_file_lock_env_var(hid_t in_fapl) int child_wait_option=0; /* Options passed to waitpid */ int out_pdf[2]; int notify = 0; + int exit_status = 0; + herr_t ret; + + if (turn_locking_on && env_var_override) + TESTING("File locking: ON w/ env var override") + else if (turn_locking_on && !env_var_override) + TESTING("File locking: ON") + else if (!turn_locking_on && env_var_override) + TESTING("File locking: OFF w/ env var override") + else + TESTING("File locking: OFF") - - TESTING("File locking environment variable"); - - - /* Set the environment variable */ - if(HDsetenv("HDF5_USE_FILE_LOCKING", "FALSE", TRUE) < 0) + /* Copy the incoming fapl */ + if((fapl = H5Pcopy(in_fapl)) < 0) TEST_ERROR - if((fapl = H5Pcopy(in_fapl)) < 0) + /* Set locking in the fapl */ + if(H5Pset_file_locking(fapl, turn_locking_on ? TRUE : FALSE, TRUE) < 0) TEST_ERROR + /* If requested, set the environment variable */ + if (env_var_override) { + if(HDsetenv("HDF5_USE_FILE_LOCKING", turn_locking_on ? "FALSE" : "TRUE", TRUE) < 0) + TEST_ERROR + if(H5F__reparse_file_lock_variable_test() < 0) + TEST_ERROR + } + else { + if(HDsetenv("HDF5_USE_FILE_LOCKING", "", TRUE) < 0) + TEST_ERROR + if(H5F__reparse_file_lock_variable_test() < 0) + TEST_ERROR + } + /* Set the filename to use for this test (dependent on fapl) */ h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); @@ -6096,10 +6129,8 @@ test_file_lock_env_var(hid_t in_fapl) if(H5Fclose(fid) < 0) TEST_ERROR - /* Open a file for read-only and then read-write. This would - * normally fail due to the file locking scheme but should - * pass when the environment variable is set to disable file - * locking. + /* Open a file for read-only and then read-write. This will fail + * when the locking scheme is turned on. */ /* Create 1 pipe */ @@ -6114,7 +6145,7 @@ test_file_lock_env_var(hid_t in_fapl) /* Child process */ - hid_t child_fid; /* File ID */ + hid_t child_fid = H5I_INVALID_HID; /* File ID */ int child_notify = 0; /* Close unused write end for out_pdf */ @@ -6125,18 +6156,23 @@ test_file_lock_env_var(hid_t in_fapl) while(child_notify != 1) { if(HDread(out_pdf[0], &child_notify, sizeof(int)) < 0) HDexit(EXIT_FAILURE); - } /* end while */ + } - /* Open the test file */ - if((child_fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) - TEST_ERROR + /* Open and close the test file */ + H5E_BEGIN_TRY { + child_fid = H5Fopen(filename, H5F_ACC_RDWR, fapl); + ret = H5Fclose(child_fid); + } H5E_END_TRY; /* Close the pipe */ if(HDclose(out_pdf[0]) < 0) HDexit(EXIT_FAILURE); - HDexit(EXIT_SUCCESS); - } /* end if */ + if(H5I_INVALID_HID == child_fid || FAIL == ret) + HDexit(EXIT_FAILURE); + else + HDexit(EXIT_SUCCESS); + } /* end child process work */ /* close unused read end for out_pdf */ if(HDclose(out_pdf[0]) < 0) @@ -6159,15 +6195,28 @@ test_file_lock_env_var(hid_t in_fapl) if(HDwaitpid(childpid, &child_status, child_wait_option) < 0) TEST_ERROR - /* Check if child terminated normally */ - if(WIFEXITED(child_status)) { - /* Check exit status of the child */ - if(WEXITSTATUS(child_status) != 0) - TEST_ERROR - } /* end if */ + /* Check exit status of the child */ + if(WIFEXITED(child_status)) + exit_status = WEXITSTATUS(child_status); else TEST_ERROR + /* The child process should have passed or failed as follows: + * + * locks on: FAIL + * locks off: PASS + * locks on, env var override: PASS + * locks off, env var override: FAIL + */ + if(turn_locking_on && !env_var_override && (0 == exit_status)) + TEST_ERROR + else if(!turn_locking_on && !env_var_override && (0 != exit_status)) + TEST_ERROR + else if(turn_locking_on && env_var_override && (0 != exit_status)) + TEST_ERROR + else if(!turn_locking_on && env_var_override && (0 == exit_status)) + TEST_ERROR + /* Close the file */ if(H5Fclose(fid) < 0) TEST_ERROR @@ -6191,7 +6240,7 @@ error: #endif /* !(defined(H5_HAVE_FORK && defined(H5_HAVE_WAITPID)) */ -} /* end test_file_lock_env_var() */ +} /* end test_file_locking() */ static int @@ -7018,7 +7067,7 @@ error: H5Fclose(fid3); } H5E_END_TRY; - return -1; + return 1; } /* test_multiple_same() */ @@ -7035,6 +7084,7 @@ main(void) char *driver = NULL; /* VFD string (from env variable) */ char *lock_env_var = NULL; /* file locking env var pointer */ hbool_t use_file_locking; /* read from env var */ + hbool_t file_locking_enabled = FALSE; /* Checks if the file system supports locks */ /* Skip this test if SWMR I/O is not supported for the VFD specified * by the environment variable. @@ -7055,6 +7105,13 @@ main(void) else use_file_locking = TRUE; + /* Check if file locking is enabled on this file system */ + if(use_file_locking) + if(h5_check_if_file_locking_enabled(&file_locking_enabled) < 0) { + HDprintf("Error when determining if file locks are enabled\n"); + return EXIT_FAILURE; + } + /* Set up */ h5_reset(); @@ -7096,7 +7153,7 @@ main(void) nerrors += test_append_flush_dataset_fixed(fapl); nerrors += test_append_flush_dataset_multiple(fapl); - if(use_file_locking) { + if(use_file_locking && file_locking_enabled) { /* * Tests for: * file open flags--single process access @@ -7126,7 +7183,12 @@ main(void) /* This test changes the HDF5_USE_FILE_LOCKING environment variable * so it should be run last. */ - nerrors += test_file_lock_env_var(fapl); + if (use_file_locking && file_locking_enabled) { + nerrors += test_file_locking(fapl, TRUE, TRUE); + nerrors += test_file_locking(fapl, TRUE, FALSE); + nerrors += test_file_locking(fapl, FALSE, TRUE); + nerrors += test_file_locking(fapl, FALSE, FALSE); + } if(nerrors) goto error; diff --git a/test/swmr_addrem_writer.c b/test/swmr_addrem_writer.c index 71e4929..7ad74ed 100644 --- a/test/swmr_addrem_writer.c +++ b/test/swmr_addrem_writer.c @@ -88,26 +88,6 @@ open_skeleton(const char *filename, unsigned verbose) if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) return -1; -#ifdef QAK - /* Increase the initial size of the metadata cache */ - { - H5AC_cache_config_t mdc_config; - - mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; - H5Pget_mdc_config(fapl, &mdc_config); - HDfprintf(stderr, "mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size); - HDfprintf(stderr, "mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length); - mdc_config.set_initial_size = 1; - mdc_config.initial_size = 16 * 1024 * 1024; - /* mdc_config.epoch_length = 5000; */ - H5Pset_mdc_config(fapl, &mdc_config); - } -#endif /* QAK */ - -#ifdef QAK - H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024)); -#endif /* QAK */ - /* Open the file */ if((fid = H5Fopen(filename, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, fapl)) < 0) return -1; diff --git a/test/swmr_generator.c b/test/swmr_generator.c index f6a4fe2..7fb08df 100644 --- a/test/swmr_generator.c +++ b/test/swmr_generator.c @@ -121,38 +121,10 @@ gen_skeleton(const char *filename, hbool_t verbose, hbool_t swmr_write, if(!HDstrcmp(index_type, "b2")) max_dims[0] = H5S_UNLIMITED; -#ifdef QAK - /* Increase the initial size of the metadata cache */ - { - H5AC_cache_config_t mdc_config; - - mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; - H5Pget_mdc_config(fapl, &mdc_config); - HDfprintf(stderr, "mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size); - HDfprintf(stderr, "mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length); - mdc_config.set_initial_size = 1; - mdc_config.initial_size = 16 * 1024 * 1024; - /* mdc_config.epoch_length = 5000; */ - H5Pset_mdc_config(fapl, &mdc_config); - } -#endif /* QAK */ - -#ifdef QAK - H5Pset_small_data_block_size(fapl, (hsize_t)(50 * CHUNK_SIZE * DTYPE_SIZE)); -#endif /* QAK */ - -#ifdef QAK - H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024)); -#endif /* QAK */ - /* Create file creation property list */ if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) return -1; -#ifdef QAK - H5Pset_link_phase_change(fcpl, 0, 0); -#endif /* QAK */ - /* Emit informational message */ if(verbose) HDfprintf(stderr, "Creating file\n"); diff --git a/test/swmr_remove_writer.c b/test/swmr_remove_writer.c index 2bebab9..504ea8a 100644 --- a/test/swmr_remove_writer.c +++ b/test/swmr_remove_writer.c @@ -90,26 +90,6 @@ open_skeleton(const char *filename, unsigned verbose, unsigned old) return -1; } -#ifdef QAK -/* Increase the initial size of the metadata cache */ - { - H5AC_cache_config_t mdc_config; - - mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; - H5Pget_mdc_config(fapl, &mdc_config); - HDfprintf(stderr, "mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size); - HDfprintf(stderr, "mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length); - mdc_config.set_initial_size = 1; - mdc_config.initial_size = 16 * 1024 * 1024; - /* mdc_config.epoch_length = 5000; */ - H5Pset_mdc_config(fapl, &mdc_config); - } -#endif /* QAK */ - -#ifdef QAK - H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024)); -#endif /* QAK */ - /* Open the file */ if((fid = H5Fopen(filename, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, fapl)) < 0) return -1; diff --git a/test/swmr_sparse_writer.c b/test/swmr_sparse_writer.c index 5173c71..1892fe2 100644 --- a/test/swmr_sparse_writer.c +++ b/test/swmr_sparse_writer.c @@ -87,26 +87,6 @@ open_skeleton(const char *filename, unsigned verbose) if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) return -1; -#ifdef QAK - /* Increase the initial size of the metadata cache */ - { - H5AC_cache_config_t mdc_config; - - mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; - H5Pget_mdc_config(fapl, &mdc_config); - HDfprintf(stderr, "mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size); - HDfprintf(stderr,"mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length); - mdc_config.set_initial_size = 1; - mdc_config.initial_size = 16 * 1024 * 1024; - /* mdc_config.epoch_length = 5000; */ - H5Pset_mdc_config(fapl, &mdc_config); - } -#endif /* QAK */ - -#ifdef QAK - H5Pset_fapl_log(fapl, "append.log", H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024)); -#endif /* QAK */ - /* Open the file */ if((fid = H5Fopen(filename, H5F_ACC_RDWR | H5F_ACC_SWMR_WRITE, fapl)) < 0) return -1; diff --git a/test/swmr_start_write.c b/test/swmr_start_write.c index fc7e7a5..0527229 100644 --- a/test/swmr_start_write.c +++ b/test/swmr_start_write.c @@ -85,16 +85,6 @@ create_file(const char *filename, hbool_t verbose, FILE *verbose_file, if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) return -1; -#ifdef QAK - if(verbose) { - char verbose_name[1024]; - - HDsnprintf(verbose_name, sizeof(verbose_name), "swmr_start_write.log.%u", random_seed); - - H5Pset_fapl_log(fapl, verbose_name, H5FD_LOG_ALL, (size_t)(512 * 1024 * 1024)); - } /* end if */ -#endif /* QAK */ - /* Create file creation property list */ if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) return -1; diff --git a/test/swmr_writer.c b/test/swmr_writer.c index d4387aa..656a5b2 100644 --- a/test/swmr_writer.c +++ b/test/swmr_writer.c @@ -90,22 +90,6 @@ open_skeleton(const char *filename, hbool_t verbose, FILE *verbose_file, return -1; } -#ifdef QAK - /* Increase the initial size of the metadata cache */ - { - H5AC_cache_config_t mdc_config; - - mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; - H5Pget_mdc_config(fapl, &mdc_config); - HDfprintf(stderr, "mdc_config.initial_size = %lu\n", (unsigned long)mdc_config.initial_size); - HDfprintf(stderr, "mdc_config.epoch_length = %lu\n", (unsigned long)mdc_config.epoch_length); - mdc_config.set_initial_size = 1; - mdc_config.initial_size = 16 * 1024 * 1024; - /* mdc_config.epoch_length = 5000; */ - H5Pset_mdc_config(fapl, &mdc_config); - } -#endif /* QAK */ - if(use_log_vfd) { char verbose_name[1024]; diff --git a/test/tattr.c b/test/tattr.c index b11fd0d..2a5e613 100644 --- a/test/tattr.c +++ b/test/tattr.c @@ -531,7 +531,7 @@ test_attr_flush(hid_t fapl) ret=H5Aread(att, H5T_NATIVE_DOUBLE, &rdata); CHECK(ret, FAIL, "H5Awrite"); - if(!H5_DBL_ABS_EQUAL(rdata, H5_DOUBLE(0.0))) + if(!H5_DBL_ABS_EQUAL(rdata, (double)0.0f)) TestErrPrintf("attribute value wrong: rdata=%f, should be %f\n",rdata,(double)0.0F); ret=H5Fflush(fil, H5F_SCOPE_GLOBAL); @@ -540,7 +540,7 @@ test_attr_flush(hid_t fapl) ret=H5Aread(att, H5T_NATIVE_DOUBLE, &rdata); CHECK(ret, FAIL, "H5Awrite"); - if(!H5_DBL_ABS_EQUAL(rdata, H5_DOUBLE(0.0))) + if(!H5_DBL_ABS_EQUAL(rdata, (double)0.0f)) TestErrPrintf("attribute value wrong: rdata=%f, should be %f\n",rdata,(double)0.0F); ret=H5Awrite(att, H5T_NATIVE_DOUBLE, &wdata); @@ -6560,13 +6560,6 @@ attr_iterate2_cb(hid_t loc_id, const char *attr_name, const H5A_info_t *info, char attrname[NAME_BUF_SIZE]; /* Object name */ H5A_info_t my_info; /* Local attribute info */ -#ifdef QAK -HDfprintf(stderr, "attr_name = '%s'\n", attr_name); -if(info) - HDfprintf(stderr, "info->corder = %u\n", (unsigned)info->corder); -HDfprintf(stderr, "op_data->curr = %Hd\n", op_data->curr); -#endif /* QAK */ - /* Increment # of times the callback was called */ op_data->ncalled++; diff --git a/test/tchecksum.c b/test/tchecksum.c index 6e509fb..82dc135 100644 --- a/test/tchecksum.c +++ b/test/tchecksum.c @@ -15,7 +15,7 @@ * * Created: tchecksum.c * Aug 21 2006 - * Quincey Koziol <koziol@hdfgroup.org> + * Quincey Koziol * * Purpose: Test internal checksum routine(s) * diff --git a/test/test_mirror.sh.in b/test/test_mirror.sh.in new file mode 100644 index 0000000..9635c38 --- /dev/null +++ b/test/test_mirror.sh.in @@ -0,0 +1,100 @@ +#! /bin/bash +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Tests for the Mirror VFD feature. +# +# Created: +# Jacob Smith, 2019-12-30 + +############################################################################### +## test parameters +############################################################################### + +nerrors=0 + +SERVER_VERBOSITY="--verbosity=1" +SERVER_PORT="--port=3000" + + +############################################################################### +## Main +############################################################################### + +## TODO: arguments for main port, port range, verbosity? +# Parse options (none accepted at this time) +while [ $# -gt 0 ]; do + case "$1" in + *) # unknown option + echo "$0: Unknown option ($1)" + exit 1 + ;; + esac +done + + + +RUN_DIR=mirror_vfd_test +MIRROR_UTILS=../utils/mirror_vfd # TODO: presupposes from test/ + +mkdir $RUN_DIR + +# Copy program files into dedicated test directory +for FILE in $MIRROR_UTILS/mirror_* ; do + case "$FILE" in + *.o) continue ;; # Don't copy .o files + esac + cp $FILE $RUN_DIR +done +cp mirror_vfd $RUN_DIR + +# With the --disable-shared option, program files are built in their main +# directories; otherwise they are built in dir/.libs with a corresponding +# wrapper script. Copy these libs builds if appropriate. +if [ -f $MIRROR_UTILS/.libs/mirror_server ] ; then + RUN_LIBS=$RUN_DIR/.libs + mkdir $RUN_LIBS + for FILE in $MIRROR_UTILS/.libs/mirror_* ; do + case "$FILE" in + *.o) continue ;; # Don't copy .o files + esac + cp $FILE $RUN_LIBS + done + cp .libs/mirror_vfd $RUN_LIBS +fi + +cd $RUN_DIR + +echo "Launching Mirror Server" +SERVER_ARGS="$SERVER_PORT $SERVER_VERBOSITY" +./mirror_server $SERVER_ARGS & + +./mirror_vfd +nerrors=$? + +echo "Stopping Mirror Server" +./mirror_server_stop $SERVER_PORT + +############################################################################### +## Report and exit +############################################################################### +cd .. +if test $nerrors -eq 0 ; then + echo "Mirror VFD tests passed." + if test -z "$HDF5_NOCLEANUP" ; then + rm -rf $RUN_DIR + fi + exit 0 +else + echo "Mirror VFD tests FAILED." + exit 1 +fi + diff --git a/test/testframe.c b/test/testframe.c index 3c2a335..cfb62ec 100644 --- a/test/testframe.c +++ b/test/testframe.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu> + * Programmer: Quincey Koziol * Tuesday, January 6, 2004 * * Purpose: Provides support functions for the testing framework. diff --git a/test/timer.c b/test/timer.c new file mode 100644 index 0000000..1014e75 --- /dev/null +++ b/test/timer.c @@ -0,0 +1,414 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Dana Robinson + * May, 2011 + * + * Purpose: Tests the operation of the platform-independent timers. + */ + +#include "h5test.h" + + + + +/*------------------------------------------------------------------------- + * Function: test_time_formatting + * + * Purpose: Tests time string creation. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_time_formatting(void) +{ + char *s = NULL; + + TESTING("Time string formats"); + + /* < 0, N/A */ + s = H5_timer_get_time_string(-1.0F); + if(NULL == s || HDstrcmp(s, "N/A") != 0) + TEST_ERROR; + HDfree(s); + + /* 0 0 */ + s = H5_timer_get_time_string(0.0F); + if(NULL == s || HDstrcmp(s, "0.0 s") != 0) + TEST_ERROR; + HDfree(s); + + /* < 1 us nanoseconds */ + s = H5_timer_get_time_string(123.0E-9F); + if(NULL == s || HDstrcmp(s, "123 ns") != 0) + TEST_ERROR; + HDfree(s); + + /* < 1 ms microseconds */ + s = H5_timer_get_time_string(23.456E-6F); + if(NULL == s || HDstrcmp(s, "23.5 us") != 0) + TEST_ERROR; + HDfree(s); + + /* < 1 s milliseconds */ + s = H5_timer_get_time_string(4.56789E-3F); + if(NULL == s || HDstrcmp(s, "4.6 ms") != 0) + TEST_ERROR; + HDfree(s); + + /* < 1 min seconds */ + s = H5_timer_get_time_string(3.14F); + if(NULL == s || HDstrcmp(s, "3.14 s") != 0) + TEST_ERROR; + HDfree(s); + + /* < 1 hr mins, secs */ + s = H5_timer_get_time_string(2521.0F); + if(NULL == s || HDstrcmp(s, "42 m 1 s") != 0) + TEST_ERROR; + HDfree(s); + + /* < 1 d hrs, mins, secs */ + s = H5_timer_get_time_string(9756.0F); + if(NULL == s || HDstrcmp(s, "2 h 42 m 36 s") != 0) + TEST_ERROR; + HDfree(s); + + /* > 1 d days, hrs, mins, secs */ + s = H5_timer_get_time_string(280802.0F); + if(NULL == s || HDstrcmp(s, "3 d 6 h 0 m 2 s") != 0) + TEST_ERROR; + HDfree(s); + + PASSED(); + return 0; + +error: + if(s) + HDfree(s); + return -1; + +} + + +/*------------------------------------------------------------------------- + * Function: test_timer_system_user + * + * Purpose: Tests the ability to get system and user times from the + * timers. + * Some platforms may require special code to get system and + * user times. If we do not support that particular platform + * dependent functionality, this test is skipped. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_timer_system_user(void) +{ + int i; + char *buf = NULL; + H5_timer_t timer; + H5_timevals_t times; + herr_t err; + + TESTING("system/user times"); + + err = H5_timer_init(&timer); + if(err < 0) + TEST_ERROR; + + err = H5_timer_start(&timer); + if(err < 0) + TEST_ERROR; + + /* The system and user times may not be present on some systems. They + * will be -1.0 if they are not. + */ + if(timer.initial.system < (double)0.0f || timer.initial.user < (double)0.0f) { + SKIPPED(); + printf("NOTE: No suitable way to get system/user times on this platform.\n"); + return 0; + } + + /* Do some fake work */ + for(i=0; i < 1024; i++) { + buf = (char *)HDmalloc(1024 * (size_t)i); + HDfree(buf); + } + + err = H5_timer_stop(&timer); + if(err < 0) + TEST_ERROR; + + err = H5_timer_get_times(timer, ×); + if(err < 0) + TEST_ERROR; + + /* System and user times should be non-negative. */ + if(times.system < (double)0.0f || times.user < (double)0.0f) + TEST_ERROR; + + PASSED(); + return 0; + +error: + return -1; +} + + +/*------------------------------------------------------------------------- + * Function: test_timer_elapsed + * + * Purpose: Tests the ability to get elapsed times from the timers. + * We should always be able to get an elapsed time, + * regardless of the time libraries or platform. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Dana Robinson + * May 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_timer_elapsed(void) +{ + int i; + char *buf = NULL; + H5_timer_t timer; + H5_timevals_t times; + herr_t err; + + TESTING("elapsed times"); + + err = H5_timer_init(&timer); + if(err < 0) + TEST_ERROR; + + err = H5_timer_start(&timer); + if(err < 0) + TEST_ERROR; + + /* Do some fake work */ + for(i=0; i < 1024; i++) { + buf = (char *)HDmalloc(1024 * (size_t)i); + HDfree(buf); + } + + err = H5_timer_stop(&timer); + if(err < 0) + TEST_ERROR; + + err = H5_timer_get_times(timer, ×); + if(err < 0) + TEST_ERROR; + + /* Elapsed time should be non-negative. */ + if(times.elapsed < (double)0.0f) + TEST_ERROR; + + PASSED(); + return 0; + +error: + return -1; +} + + + +static herr_t +test_timer_functionality(void) +{ + int i; + char *buf = NULL; + H5_timer_t timer; + + H5_timevals_t times; + double prev_etime; + double prev_total_etime; + + herr_t err; + + TESTING("timer functionality"); + + /***************** + * CHECK STARTUP * + *****************/ + + /* Timer should be running after start */ + err = H5_timer_init(&timer); + if(err < 0 || timer.is_running) + TEST_ERROR; + + /* Times should be initialized to zero */ + err = H5_timer_get_times(timer, ×); + if(err < 0 || !H5_DBL_ABS_EQUAL(times.elapsed, (double)0.0f)) + TEST_ERROR; + + err = H5_timer_get_total_times(timer, ×); + if(err < 0 || !H5_DBL_ABS_EQUAL(times.elapsed, (double)0.0f)) + TEST_ERROR; + + + /******************** + * CHECK START/STOP * + ********************/ + + /* Running state should change after start */ + err = H5_timer_start(&timer); + if(err < 0 || !timer.is_running) + TEST_ERROR; + + /* Do some fake work */ + for(i=0; i < 1024; i++) { + buf = (char *)HDmalloc(1024 * (size_t)i); + HDfree(buf); + } + + /* Running state should change after stop */ + err = H5_timer_stop(&timer); + if(err < 0 || timer.is_running) + TEST_ERROR; + + /* Times should be positive and non-negative */ + err = H5_timer_get_times(timer, ×); + if(err < 0 || times.elapsed < (double)0.0f) + TEST_ERROR; + + err = H5_timer_get_total_times(timer, ×); + if(err < 0 || times.elapsed < (double)0.0f) + TEST_ERROR; + + + /********************** + * CHECK INTERRUPTING * + **********************/ + + /* Timer should change stat and refresh to 0s */ + err = H5_timer_init(&timer); + if(err < 0 || timer.is_running) + TEST_ERROR; + + err = H5_timer_get_times(timer, ×); + if(err < 0 || !H5_DBL_ABS_EQUAL(times.elapsed, (double)0.0f)) + TEST_ERROR; + + err = H5_timer_get_total_times(timer, ×); + if(err < 0 || !H5_DBL_ABS_EQUAL(times.elapsed, (double)0.0f)) + TEST_ERROR; + + /* Timer state should flip */ + err = H5_timer_start(&timer); + if(err < 0 || !timer.is_running) + TEST_ERROR; + + /* Do some fake work */ + for(i=0; i < 1024; i++) { + buf = (char *)HDmalloc(1024 * (size_t)i); + HDfree(buf); + } + + /* Times should be non-negative */ + err = H5_timer_get_times(timer, ×); + if(err < 0 || times.elapsed < (double)0.0f) + TEST_ERROR; + prev_etime = times.elapsed; + + err = H5_timer_get_total_times(timer, ×); + if(err < 0 || times.elapsed < (double)0.0f) + TEST_ERROR; + prev_total_etime = times.elapsed; + + /* Do some fake work */ + for(i=0; i < 1024; i++) { + buf = (char *)HDmalloc(1024 * (size_t)i); + HDfree(buf); + } + + /* State should flip on stop */ + err = H5_timer_stop(&timer); + if(err < 0 || timer.is_running) + TEST_ERROR; + + /* Times should be >= than the cached intermediate times */ + err = H5_timer_get_times(timer, ×); + if(err < 0 || times.elapsed < prev_etime) + TEST_ERROR; + + err = H5_timer_get_total_times(timer, ×); + if(err < 0 || times.elapsed < prev_total_etime) + TEST_ERROR; + + + PASSED(); + return 0; + +error: + return -1; +} + + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Tests the basic functionality of the platform-independent + * timers + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Dana Robinson + * May, 2011 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + int nerrors = 0; + + h5_reset(); + + printf("Testing platform-independent timer functionality.\n"); + + nerrors += test_time_formatting() < 0 ? 1 : 0; + nerrors += test_timer_system_user() < 0 ? 1 : 0; + nerrors += test_timer_elapsed() < 0 ? 1 : 0; + nerrors += test_timer_functionality() < 0 ? 1 : 0; + + if(nerrors) { + printf("***** %d platform-independent timer TEST%s FAILED! *****\n", + nerrors, nerrors > 1 ? "S" : ""); + return 1; + } else { + printf("All platform-independent timer tests passed.\n"); + return 0; + } +} + diff --git a/test/unlink.c b/test/unlink.c index 89c59c3..c599b49 100644 --- a/test/unlink.c +++ b/test/unlink.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Friday, September 25, 1998 * * Purpose: Test unlinking operations. diff --git a/test/use_append_chunk_mirror.c b/test/use_append_chunk_mirror.c new file mode 100644 index 0000000..6ee01c0 --- /dev/null +++ b/test/use_append_chunk_mirror.c @@ -0,0 +1,403 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* HACKED VERSION + * Demonstrate SWMR with a mirrored file. + * + * Must be built with SERVER_IP as the IP address of the target system + * with a running mirror server, and SERVER_PORT as the primary server port. + * + * In addition to the local file, 'shinano.h5' will be created on the remote + * system, mirroring the local file. The file location will be local to + * Server's/Writer's invocation directory. + * + * Template for demonstration purposes: + * + * # Launch mirror server on remote machine (in foreground to easily stop) + * REMOTE(1)$ ./mirror_server /path/to/mirror_worker + * + * # Launch chunk writer with plenty of chunks. + * LOCAL(1)$ ./use_append_chunk_mirror -l w -n 10000 + * + * # Wait one second for files to be created. + * + * # Launch chunk readers on both files. + * LOCAL(2)$ ./use_append_chunk_mirror -l r -n 10000 + * REMOTE(2)$ ./use_append_chunk_mirror -l r -n 10000 -f shinano.h5 + * + * # Hard-stop the server. + * REMOTE(1)$ ^C + * # alt, softer shutdown using echo and nc + * echo "SHUTDOWN" | nc localhost 3000 + */ + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Use Case 1.7 Appending a single chunk + * Description: + * Appending a single chunk of raw data to a dataset along an unlimited + * dimension within a pre-created file and reading the new data back. + * Goal: + * Read data appended by the Writer to a pre-existing dataset in a + * file. The dataset has one or more unlimited dimensions. The data is + * appended by a hyperslab that is contained in one chunk (for example, + * appending 2-dim planes along the slowest changing dimension in the + * 3-dim dataset). + * Level: + * User Level + * Guarantees: + * o Readers will see the modified dimension sizes after the Writer + * finishes HDF5 metadata updates and issues H5Fflush or H5Oflush calls. + * o Readers will see newly appended data after the Writer finishes + * the flush operation. + * + * Preconditions: + * o Readers are not allowed to modify the file. + * o All datasets that are modified by the Writer exist when the Writer + * opens the file. + * o All datasets that are modified by the Writer exist when a Reader + * opens the file. + * o Data is written by a hyperslab contained in one chunk. + * + * Main Success Scenario: + * 1. An application creates a file with required objects (groups, + * datasets, and attributes). + * 2. The Writer application opens the file and datasets in the file + * and starts adding data along the unlimited dimension using a hyperslab + * selection that corresponds to an HDF5 chunk. + * 3. A Reader opens the file and a dataset in a file, and queries + * the sizes of the dataset; if the extent of the dataset has changed, + * reads the appended data back. + * + * Discussion points: + * 1. Since the new data is written to the file, and metadata update + * operation of adding pointer to the newly written chunk is atomic and + * happens after the chunk is on the disk, only two things may happen + * to the Reader: + * o The Reader will not see new data. + * o The Reader will see all new data written by Writer. + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Created: Jacob Smith, 2019 */ + +#include "use.h" + +/* This test uses many POSIX things that are not available on + * Windows. We're using a check for fork(2) here as a proxy for + * all POSIX/Unix/Linux things until this test can be made + * more platform-independent. + */ +#ifdef H5_HAVE_FORK + +#ifdef H5_HAVE_MIRROR_VFD + +#define THIS_PROGNAME "use_append_chunk_mirror" + +#define CONNECT_WITH_JELLY 0 + +#if CONNECT_WITH_JELLY +#define SERVER_IP "10.10.10.248" /* hard-coded IP address */ +#else +#define SERVER_IP "127.0.0.1" /* localhost */ +#endif /* CONNECT_WITH_JELLY */ +#define SERVER_PORT 3000 /* hard-coded port number */ +#define MIRROR_FILE_NAME "shinano.h5" /* hard-coded duplicate/mirror filename */ + +static options_t UC_opts; /* Use Case Options */ + +/* Setup parameters for the use case. + * Return: 0 succeed; -1 fail. + */ +int +setup_parameters(int argc, char * const argv[], options_t * opts) +{ + /* use case defaults */ + HDmemset(opts, 0, sizeof(options_t)); + opts->chunksize = Chunksize_DFT; + opts->use_swmr = TRUE; + opts->iterations = 1; + opts->chunkplanes = 1; + opts->progname = THIS_PROGNAME; + + if (parse_option(argc, argv, opts) < 0) + return(-1); + + opts->chunkdims[0] = opts->chunkplanes; + opts->chunkdims[1] = opts->chunkdims[2] = opts->chunksize; + + opts->dims[0] = 0; + opts->max_dims[0] = H5S_UNLIMITED; + opts->dims[1] = opts->dims[2] = opts->max_dims[1] = opts->max_dims[2] = opts->chunksize; + + if (opts->nplanes == 0) + opts->nplanes = (hsize_t)opts->chunksize; + + show_parameters(opts); + return(0); +} /* setup_parameters() */ + + +/* Overall Algorithm: + * Parse options from user; + * Generate/pre-created test files needed and close it; + * fork: child process becomes the reader process; + * while parent process continues as the writer process; + * both run till ending conditions are met. + */ +int +main(int argc, char *argv[]) +{ + pid_t childpid=0; + pid_t mypid, tmppid; + int child_status; + int child_wait_option=0; + int ret_value = 0; + int child_ret_value; + hbool_t send_wait = FALSE; + hid_t fid = -1; /* File ID */ + H5FD_mirror_fapl_t mirr_fa; + H5FD_splitter_vfd_config_t split_fa; + hid_t mirr_fapl_id = H5I_INVALID_HID; + + if (setup_parameters(argc, argv, &UC_opts) < 0) { + Hgoto_error(1); + } + + mirr_fa.magic = H5FD_MIRROR_FAPL_MAGIC; + mirr_fa.version = H5FD_MIRROR_CURR_FAPL_T_VERSION; + mirr_fa.handshake_port = SERVER_PORT; + HDstrncpy(mirr_fa.remote_ip, SERVER_IP, H5FD_MIRROR_MAX_IP_LEN); + + + split_fa.wo_fapl_id = H5I_INVALID_HID; + split_fa.rw_fapl_id = H5I_INVALID_HID; + split_fa.magic = H5FD_SPLITTER_MAGIC; + split_fa.version = H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION; + split_fa.log_file_path[0] = '\0'; /* none */ + split_fa.ignore_wo_errs = FALSE; + HDstrncpy(split_fa.wo_path, MIRROR_FILE_NAME, H5FD_SPLITTER_PATH_MAX); + + /* Determine the need to send/wait message file*/ + if (UC_opts.launch == UC_READWRITE) { + HDunlink(WRITER_MESSAGE); + send_wait = TRUE; + } + + /* ==============================================================*/ + /* UC_READWRITE: create datafile, launch both reader and writer. */ + /* UC_WRITER: create datafile, skip reader, launch writer. */ + /* UC_READER: skip create, launch reader, exit. */ + /* ==============================================================*/ + /* =========== */ + /* Create file */ + /* =========== */ + if (UC_opts.launch != UC_READER) { + HDprintf("Creating skeleton data file for test...\n"); + + /* Prepare mirror child driver */ + mirr_fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (mirr_fapl_id == H5I_INVALID_HID) { + HDfprintf(stderr, "can't create creation mirror FAPL\n"); + Hgoto_error(1); + } + if (H5Pset_fapl_mirror(mirr_fapl_id, &mirr_fa) < 0) { + HDfprintf(stderr, "can't set creation mirror FAPL\n"); + H5Eprint2(H5E_DEFAULT, stdout); + Hgoto_error(1); + } + + /* Prepare parent "splitter" driver in UC_opts */ + split_fa.wo_fapl_id = mirr_fapl_id; + split_fa.rw_fapl_id = H5P_DEFAULT; + UC_opts.fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (UC_opts.fapl_id == H5I_INVALID_HID) { + HDfprintf(stderr, "can't create creation FAPL\n"); + Hgoto_error(1); + } + if (H5Pset_fapl_splitter(UC_opts.fapl_id, &split_fa) < 0) { + HDfprintf(stderr, "can't set creation FAPL\n"); + H5Eprint2(H5E_DEFAULT, stdout); + Hgoto_error(1); + } + + if (H5Pset_libver_bounds(UC_opts.fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) { + HDfprintf(stderr, "can't set creation FAPL libver bounds\n"); + Hgoto_error(1); + } + + /* Create file */ + if (create_uc_file(&UC_opts) < 0) { + HDfprintf(stderr, "***encounter error\n"); + Hgoto_error(1); + } else { + HDprintf("File created.\n"); + } + + /* Close FAPLs to prevent issues with forking later */ + if (H5Pclose(UC_opts.fapl_id) < 0) { + HDfprintf(stderr, "can't close creation FAPL\n"); + Hgoto_error(1); + } + UC_opts.fapl_id = H5I_INVALID_HID; + if (H5Pclose(mirr_fapl_id) < 0) { + HDfprintf(stderr, "can't close creation mirror FAPL\n"); + Hgoto_error(1); + } + mirr_fapl_id = H5I_INVALID_HID; + } + + /* ============ */ + /* Fork process */ + /* ============ */ + if (UC_opts.launch == UC_READWRITE) { + if ((childpid = HDfork()) < 0) { + HDperror("fork"); + Hgoto_error(1); + } + } + mypid = HDgetpid(); + + /* ============= */ + /* launch reader */ + /* ============= */ + if (UC_opts.launch != UC_WRITER) { + /* child process -- launch the reader */ + /* reader only opens the one file -- separate reader needed for mirrored file 'shinano.h5' */ + if (0 == childpid) { + HDprintf("%d: launch reader process\n", mypid); + + UC_opts.fapl_id = H5P_DEFAULT; + if (read_uc_file(send_wait, &UC_opts) < 0) { + HDfprintf(stderr, "read_uc_file encountered error (%d)\n", mypid); + HDexit(1); + } + + HDexit(0); + } + } + + /* ============= */ + /* launch writer */ + /* ============= */ + /* this process continues to launch the writer */ + HDprintf("%d: continue as the writer process\n", mypid); + + /* Prepare mirror child driver */ + mirr_fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (mirr_fapl_id == H5I_INVALID_HID) { + HDfprintf(stderr, "can't create creation mirror FAPL\n"); + Hgoto_error(1); + } + if (H5Pset_fapl_mirror(mirr_fapl_id, &mirr_fa) < 0) { + HDfprintf(stderr, "can't set creation mirror FAPL\n"); + H5Eprint2(H5E_DEFAULT, stdout); + Hgoto_error(1); + } + + /* Prepare parent "splitter" driver in UC_opts */ + split_fa.wo_fapl_id = mirr_fapl_id; + split_fa.rw_fapl_id = H5P_DEFAULT; + UC_opts.fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (UC_opts.fapl_id == H5I_INVALID_HID) { + HDfprintf(stderr, "can't create creation FAPL\n"); + Hgoto_error(1); + } + if (H5Pset_fapl_splitter(UC_opts.fapl_id, &split_fa) < 0) { + HDfprintf(stderr, "can't set creation FAPL\n"); + H5Eprint2(H5E_DEFAULT, stdout); + Hgoto_error(1); + } + + if (UC_opts.use_swmr) { + if (H5Pset_libver_bounds(UC_opts.fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) { + HDfprintf(stderr, "can't set write FAPL libver bounds\n"); + Hgoto_error(1); + } + } + + if ((fid = H5Fopen(UC_opts.filename, H5F_ACC_RDWR | (UC_opts.use_swmr ? H5F_ACC_SWMR_WRITE : 0), UC_opts.fapl_id)) < 0) { + HDfprintf(stderr, "H5Fopen failed\n"); + Hgoto_error(1); + } + + if (write_uc_file(send_wait, fid, &UC_opts) < 0) { + HDfprintf(stderr, "write_uc_file encountered error\n"); + Hgoto_error(1); + } + + if (H5Fclose(fid) < 0) { + HDfprintf(stderr, "Failed to close write\n"); + Hgoto_error(1); + } + + if (H5Pclose(UC_opts.fapl_id) < 0) { + HDfprintf(stderr, "can't close write FAPL\n"); + Hgoto_error(1); + } + + if (H5Pclose(mirr_fapl_id) < 0) { + HDfprintf(stderr, "can't close write mirror FAPL\n"); + Hgoto_error(1); + } + + /* ================================================ */ + /* If readwrite, collect exit code of child process */ + /* ================================================ */ + if (UC_opts.launch == UC_READWRITE) { + if ((tmppid = HDwaitpid(childpid, &child_status, child_wait_option)) < 0) { + HDperror("waitpid"); + Hgoto_error(1); + } + + if (WIFEXITED(child_status)) { + if ((child_ret_value = WEXITSTATUS(child_status)) != 0) { + HDprintf("%d: child process exited with non-zero code (%d)\n", + mypid, child_ret_value); + Hgoto_error(2); + } + } else { + HDprintf("%d: child process terminated abnormally\n", mypid); + Hgoto_error(2); + } + } + +done: + if (ret_value != 0) { + HDprintf("Error(s) encountered\n"); + } else { + HDprintf("All passed\n"); + } + + return(ret_value); +} + +#else /* H5_HAVE_MIRROR_VFD */ + +int +main(void) +{ + HDfprintf(stderr, "Mirror VFD is not built. Skipping.\n"); + return EXIT_SUCCESS; +} /* end main() */ + +#endif /* H5_HAVE_MIRROR_VFD */ + +#else /* H5_HAVE_FORK */ + +int +main(void) +{ + HDfprintf(stderr, "Non-POSIX platform. Skipping.\n"); + return EXIT_SUCCESS; +} /* end main() */ + +#endif /* H5_HAVE_FORK */ + @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Neil Fortner <nfortne2@hdfgroup.org> + * Programmer: Neil Fortner * Monday, February 16, 2015 * * Purpose: Tests datasets with virtual layout. diff --git a/test/vds_env.c b/test/vds_env.c index 3d5b5dd..0d0891f 100644 --- a/test/vds_env.c +++ b/test/vds_env.c @@ -11,7 +11,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Neil Fortner <nfortne2@hdfgroup.org> + * Programmer: Neil Fortner * Monday, February 16, 2015 * * Purpose: Tests datasets with virtual layout. @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu<slu@ncsa.uiuc.edu> + * Programmer: Raymond Lu * Tuesday, Sept 24, 2002 * * Purpose: Tests the basic features of Virtual File Drivers @@ -25,6 +25,7 @@ #define FAMILY_SIZE (1*KB) #define FAMILY_SIZE2 (5*KB) #define MULTI_SIZE 128 +#define SPLITTER_SIZE 8 /* dimensions of a dataset */ #define CORE_INCREMENT (4*KB) #define CORE_PAGE_SIZE (1024*KB) @@ -59,6 +60,9 @@ const char *FILENAME[] = { "windows_file", /*8*/ "new_multi_file_v16",/*9*/ "ro_s3_file", /*10*/ + "splitter_rw_file", /*11*/ + "splitter_wo_file", /*12*/ + "splitter.log", /*13*/ NULL }; @@ -66,7 +70,46 @@ const char *FILENAME[] = { #define COMPAT_BASENAME "family_v16_" #define MULTI_COMPAT_BASENAME "multi_file_v16" +#define SPLITTER_DATASET_NAME "dataset" + +/* Macro: HEXPRINT() + * Helper macro to pretty-print hexadecimal output of a buffer of known size. + * Each line has the address of the first printed byte, and four columns of + * four-byte data. + */ +static int __k; +#define HEXPRINT(size, buf) \ +for (__k = 0; __k < (size); __k++) { \ + if (__k % 16 == 0) { \ + HDprintf("\n%04x", __k); \ + } \ + HDprintf((__k%4 == 0) ? " %02X" : " %02X", (unsigned char)(buf)[__k]); \ +} /* end #define HEXPRINT() */ + +/* Helper structure to pass around dataset information. + */ +struct splitter_dataset_def { + void *buf; /* contents of dataset */ + const char *dset_name; /* dataset name, always added to root group */ + hid_t mem_type_id; /* datatype */ + const hsize_t *dims; /* dimensions */ + int n_dims; /* rank */ +}; + +static int splitter_prepare_file_paths(H5FD_splitter_vfd_config_t *vfd_config, + char *filename_rw_out); +static int splitter_create_single_file_at(const char *filename, hid_t fapl_id, + const struct splitter_dataset_def *data); +static int splitter_compare_expected_data(hid_t file_id, + const struct splitter_dataset_def *data); +static int run_splitter_test(const struct splitter_dataset_def *data, + hbool_t ignore_wo_errors, hbool_t provide_logfile_path, + hid_t sub_fapl_ids[2]); +static int splitter_RO_test(const struct splitter_dataset_def *data, + hid_t child_fapl_id); +static int splitter_tentative_open_test(hid_t child_fapl_id); +static int file_exists(const char *filename, hid_t fapl_id); /*------------------------------------------------------------------------- * Function: test_sec2 @@ -1148,6 +1191,123 @@ H5_GCC_DIAG_ON(format-nonliteral) /*------------------------------------------------------------------------- + * Function: test_family_member_fapl + * + * Purpose: Actually use the member fapl input to the member vfd. + * + * Return: SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 21 May 2019 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_family_member_fapl(void) +{ + hid_t file = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t memb_fapl_id = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t dset = H5I_INVALID_HID; + char filename[1024]; + char dname[] = "dataset"; + unsigned i = 0; + unsigned j = 0; + int **buf = NULL; + int *buf_data = NULL; + hsize_t dims[2] = {FAMILY_NUMBER, FAMILY_SIZE}; + + TESTING("Family member FAPL"); + + /* Set up data array */ + if(NULL == (buf_data = (int *)HDcalloc(FAMILY_NUMBER * FAMILY_SIZE, sizeof(int)))) + TEST_ERROR; + if(NULL == (buf = (int **)HDcalloc(FAMILY_NUMBER, sizeof(buf_data)))) + TEST_ERROR; + for (i = 0; i < FAMILY_NUMBER; i++) + buf[i] = buf_data + (i * FAMILY_SIZE); + + if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) == H5I_INVALID_HID) + TEST_ERROR; + + if((memb_fapl_id = H5Pcreate(H5P_FILE_ACCESS)) == H5I_INVALID_HID) + TEST_ERROR; + + if (H5Pset_fapl_sec2(memb_fapl_id) == FAIL) + TEST_ERROR; + if (H5Pset_fapl_family(fapl_id, (hsize_t)FAMILY_SIZE, memb_fapl_id) == FAIL) + TEST_ERROR; + + h5_fixname(FILENAME[2], fapl_id, filename, sizeof(filename)); + + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) == H5I_INVALID_HID) + TEST_ERROR; + + if((space = H5Screate_simple(2, dims, NULL)) == H5I_INVALID_HID) + TEST_ERROR; + + /* Create and write to dataset, then close file. + */ + if((dset = H5Dcreate2(file, dname, H5T_NATIVE_INT, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) == H5I_INVALID_HID) + TEST_ERROR; + + for (i = 0; i < FAMILY_NUMBER; i++) { + for (j = 0; j < FAMILY_SIZE; j++) { + buf[i][j] = (int)((i * 10000) + j); + } + } + + if (H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf_data) == FAIL) + TEST_ERROR; + + if (H5Dclose(dset) == FAIL) + TEST_ERROR; + if (H5Sclose(space) == FAIL) + TEST_ERROR; + if (H5Fclose(file) == FAIL) + TEST_ERROR; + + /* "Close" member FAPL at top level and re-open file. + * Should succeed, with library managing reference count properly + */ + if (H5Pclose(memb_fapl_id) == FAIL) + TEST_ERROR; + + if ((file = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) == H5I_INVALID_HID) + TEST_ERROR; + + if (H5Fclose(file) == FAIL) + TEST_ERROR; + + h5_delete_test_file(FILENAME[2], fapl_id); + + if (H5Pclose(fapl_id) == FAIL) + TEST_ERROR; + + HDfree(buf); + HDfree(buf_data); + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Sclose(space); + H5Dclose(dset); + H5Pclose(memb_fapl_id); + H5Pclose(fapl_id); + H5Fclose(file); + } H5E_END_TRY; + + HDfree(buf); + HDfree(buf_data); + + return FAIL; +} /* end test_family_member_fapl() */ + + +/*------------------------------------------------------------------------- * Function: test_multi_opens * * Purpose: Private function for test_multi() to tests wrong ways of @@ -2062,6 +2222,1104 @@ error: #endif /* H5_HAVE_ROS3_VFD */ } /* end test_ros3() */ + +/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + * Macro: SPLITTER_TEST_FAULT() + * + * utility macro, helps create stack-like backtrace on error. + * requires defined in the calling function: + * * variable `int ret_value` (return -1 on error)` + * * label `done` for exit on fault + * - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + */ +#define SPLITTER_TEST_FAULT(mesg) { \ + H5_FAILED(); \ + AT(); \ + HDfprintf(stderr, mesg); \ + H5Eprint2(H5E_DEFAULT, stderr); \ + fflush(stderr); \ + ret_value = -1; \ + goto done; \ +} + + +/*------------------------------------------------------------------------- + * Function: compare_splitter_config_info + * + * Purpose: Helper function to compare configuration info found in a + * FAPL against a canonical structure. + * + * Return: Success: 0, if config info in FAPL matches info structure. + * Failure: -1, if difference detected. + * + *------------------------------------------------------------------------- + */ +static int +compare_splitter_config_info(hid_t fapl_id, H5FD_splitter_vfd_config_t *info) +{ + int ret_value = 0; + H5FD_splitter_vfd_config_t *fetched_info = NULL; + + if (NULL == (fetched_info = HDcalloc(1, sizeof(H5FD_splitter_vfd_config_t)))) + SPLITTER_TEST_FAULT("memory allocation for fetched_info struct failed"); + + fetched_info->magic = H5FD_SPLITTER_MAGIC; + fetched_info->version = H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION; + fetched_info->rw_fapl_id = H5I_INVALID_HID; + fetched_info->wo_fapl_id = H5I_INVALID_HID; + + if (H5Pget_fapl_splitter(fapl_id, fetched_info) < 0) { + SPLITTER_TEST_FAULT("can't get splitter info"); + } + if (info->rw_fapl_id == H5P_DEFAULT) { + if (H5Pget_driver(fetched_info->rw_fapl_id) != H5Pget_driver(H5P_FILE_ACCESS_DEFAULT)) { + SPLITTER_TEST_FAULT("Read-Write driver mismatch (default)\n"); + } + } + else { + if (H5Pget_driver(fetched_info->rw_fapl_id) != H5Pget_driver(info->rw_fapl_id)) { + SPLITTER_TEST_FAULT("Read-Write driver mismatch\n"); + } + } + if (info->wo_fapl_id == H5P_DEFAULT) { + if (H5Pget_driver(fetched_info->wo_fapl_id) != H5Pget_driver(H5P_FILE_ACCESS_DEFAULT)) { + SPLITTER_TEST_FAULT("Write-Only driver mismatch (default)\n"); + } + } + else { + if (H5Pget_driver(fetched_info->wo_fapl_id) != H5Pget_driver(info->wo_fapl_id)) { + SPLITTER_TEST_FAULT("Write-Only driver mismatch\n"); + } + } + if ( (HDstrlen(info->wo_path) != HDstrlen(fetched_info->wo_path)) || + HDstrncmp(info->wo_path, fetched_info->wo_path, H5FD_SPLITTER_PATH_MAX)) + { + HDfprintf(stderr, "MISMATCH: '%s' :: '%s'\n", info->wo_path, fetched_info->wo_path); + HEXPRINT(H5FD_SPLITTER_PATH_MAX, info->wo_path); + HEXPRINT(H5FD_SPLITTER_PATH_MAX, fetched_info->wo_path); + SPLITTER_TEST_FAULT("Write-Only file path mismatch\n"); + } + +done: + HDfree(fetched_info); + + return ret_value; +} /* end compare_splitter_config_info() */ + + +/*------------------------------------------------------------------------- + * Function: run_splitter_test + * + * Purpose: Auxiliary function for test_splitter(). + * + * Return: Success: 0 + * Failure: -1 + * + * Description: + * Perform basic open-write-close with the Splitter VFD. + * Prior to operations, removes files from a previous run, + * if they exist. + * After writing, compares read-write and write-only files. + * Includes FAPL sanity testing. + * + *------------------------------------------------------------------------- + */ +static int +run_splitter_test(const struct splitter_dataset_def *data, + hbool_t ignore_wo_errors, + hbool_t provide_logfile_path, + hid_t sub_fapl_ids[2]) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t fapl_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; + hid_t space_id = H5I_INVALID_HID; + hid_t fapl_id_out = H5I_INVALID_HID; + hid_t fapl_id_cpy = H5I_INVALID_HID; + H5FD_splitter_vfd_config_t *vfd_config = NULL; + char *filename_rw = NULL; + FILE *logfile = NULL; + int ret_value = 0; + + if (NULL == (vfd_config = HDcalloc(1, sizeof(H5FD_splitter_vfd_config_t)))) + SPLITTER_TEST_FAULT("memory allocation for vfd_config struct failed"); + if (NULL == (filename_rw = HDcalloc(H5FD_SPLITTER_PATH_MAX + 1, sizeof(char)))) + SPLITTER_TEST_FAULT("memory allocation for filename_rw string failed"); + + vfd_config->magic = H5FD_SPLITTER_MAGIC; + vfd_config->version = H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION; + vfd_config->ignore_wo_errs = ignore_wo_errors; + vfd_config->rw_fapl_id = sub_fapl_ids[0]; + vfd_config->wo_fapl_id = sub_fapl_ids[1]; + + if (splitter_prepare_file_paths(vfd_config, filename_rw) < 0) { + SPLITTER_TEST_FAULT("can't prepare file paths\n"); + } + + if (provide_logfile_path == FALSE) { + vfd_config->log_file_path[0] = '\0'; /* reset as empty string */ + } + + /* Create a new fapl to use the SPLITTER file driver */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) == H5I_INVALID_HID) { + SPLITTER_TEST_FAULT("can't create FAPL ID\n"); + } + if (H5Pset_fapl_splitter(fapl_id, vfd_config) < 0) { + SPLITTER_TEST_FAULT("can't set splitter FAPL\n"); + } + if (H5Pget_driver(fapl_id) != H5FD_SPLITTER) { + SPLITTER_TEST_FAULT("set FAPL not SPLITTER\n"); + } + + if (compare_splitter_config_info(fapl_id, vfd_config) < 0) { + SPLITTER_TEST_FAULT("information mismatch\n"); + } + + /* + * Copy property list, light compare, and close the copy. + * Helps test driver-implemented FAPL-copying and library ID management. + */ + + fapl_id_cpy = H5Pcopy(fapl_id); + if (H5I_INVALID_HID == fapl_id_cpy) { + SPLITTER_TEST_FAULT("can't copy FAPL\n"); + } + if (compare_splitter_config_info(fapl_id_cpy, vfd_config) < 0) { + SPLITTER_TEST_FAULT("information mismatch\n"); + } + if (H5Pclose(fapl_id_cpy) < 0) { + SPLITTER_TEST_FAULT("can't close fapl copy\n"); + } + + /* + * Proceed with test. Create file. + */ + file_id = H5Fcreate(filename_rw, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + if (file_id < 0) { + SPLITTER_TEST_FAULT("can't create file\n"); + } + + /* + * Check driver from file + */ + + fapl_id_out = H5Fget_access_plist(file_id); + if (H5I_INVALID_HID == fapl_id_out) { + SPLITTER_TEST_FAULT("can't get file's FAPL\n"); + + } + if (H5Pget_driver(fapl_id_out) != H5FD_SPLITTER) { + SPLITTER_TEST_FAULT("wrong file FAPL driver\n"); + } + if (compare_splitter_config_info(fapl_id_out, vfd_config) < 0) { + SPLITTER_TEST_FAULT("information mismatch\n"); + } + if (H5Pclose(fapl_id_out) < 0) { + SPLITTER_TEST_FAULT("can't close file's FAPL\n"); + } + + /* + * Create and write the dataset + */ + + space_id = H5Screate_simple(data->n_dims, data->dims, NULL); + if (space_id < 0) { + SPLITTER_TEST_FAULT("can't create dataspace\n"); + } + dset_id = H5Dcreate2(file_id, data->dset_name, data->mem_type_id, space_id, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if (dset_id < 0) { + SPLITTER_TEST_FAULT("can't create dataset\n"); + } + if (H5Dwrite(dset_id, data->mem_type_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, data->buf) < 0) { + SPLITTER_TEST_FAULT("can't write data to dataset\n"); + } + + /* Close everything */ + if (H5Dclose(dset_id) < 0) { + SPLITTER_TEST_FAULT("can't close dset\n"); + } + if (H5Sclose(space_id) < 0) { + SPLITTER_TEST_FAULT("can't close space\n"); + } + if (H5Pclose(fapl_id) < 0) { + SPLITTER_TEST_FAULT("can't close fapl\n"); + } + if (H5Fclose(file_id) < 0) { + SPLITTER_TEST_FAULT("can't close file\n"); + } + + /* Verify that the R/W and W/O files are identical */ + if (h5_compare_file_bytes(filename_rw, vfd_config->wo_path) < 0) { + SPLITTER_TEST_FAULT("files are not byte-for-byte equivalent\n"); + } + + /* Verify existence of logfile if appropriate */ + logfile = fopen(vfd_config->log_file_path, "r"); + if ( (TRUE == provide_logfile_path && NULL == logfile) || + (FALSE == provide_logfile_path && NULL != logfile) ) + { + SPLITTER_TEST_FAULT("no logfile when one was expected\n"); + } + +done: + if (ret_value < 0) { + H5E_BEGIN_TRY { + H5Dclose(dset_id); + H5Sclose(space_id); + H5Pclose(fapl_id_out); + H5Pclose(fapl_id_cpy); + H5Pclose(fapl_id); + H5Fclose(file_id); + } H5E_END_TRY; + } + + if (logfile != NULL) + fclose(logfile); + + HDfree(vfd_config); + HDfree(filename_rw); + + return ret_value; +} /* end run_splitter_test() */ + + +/*------------------------------------------------------------------------- + * Function: driver_is_splitter_compatible + * + * Purpose: Determine whether the driver set in the FAPL ID is compatible + * with the Splitter VFD -- specificially, Write-Only channel. + * + * Return: Success: 0 + * Failure: -1 + * + * Description: Attempts to put the given FAPL ID as the W/O channel. + * Uses driver's own mechanisms to generate error, and catches + * error. + * + *------------------------------------------------------------------------- + */ +static int +driver_is_splitter_compatible(hid_t fapl_id) +{ + H5FD_splitter_vfd_config_t *vfd_config = NULL; + hid_t split_fapl_id = H5I_INVALID_HID; + herr_t ret = SUCCEED; + int ret_value = 0; + + if (NULL == (vfd_config = HDcalloc(1, sizeof(H5FD_splitter_vfd_config_t)))) { + FAIL_PUTS_ERROR("memory allocation for vfd_config struct failed"); + } + + if(H5I_INVALID_HID == (split_fapl_id = H5Pcreate(H5P_FILE_ACCESS))) { + FAIL_PUTS_ERROR("Can't create contained FAPL"); + } + vfd_config->magic = H5FD_SPLITTER_MAGIC; + vfd_config->version = H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION; + vfd_config->ignore_wo_errs = FALSE; + vfd_config->rw_fapl_id = H5P_DEFAULT; + vfd_config->wo_fapl_id = fapl_id; + HDstrncpy(vfd_config->wo_path, "nonesuch", H5FD_SPLITTER_PATH_MAX); + vfd_config->log_file_path[0] = '\0'; + + H5E_BEGIN_TRY { + ret = H5Pset_fapl_splitter(split_fapl_id, vfd_config); + } H5E_END_TRY; + if (SUCCEED == ret) { + ret_value = -1; + } + + if (H5Pclose(split_fapl_id) < 0) { + FAIL_PUTS_ERROR("Can't close contained FAPL") + } + split_fapl_id = H5I_INVALID_HID; + + HDfree(vfd_config); + + return ret_value; + +error: + H5E_BEGIN_TRY { + H5Pclose(split_fapl_id); + } H5E_END_TRY; + + HDfree(vfd_config); + + return -1; +} /* end driver_is_splitter_compatible() */ + + +/*------------------------------------------------------------------------- + * Function: splitter_RO_test + * + * Purpose: Verify Splitter VFD with the Read-Only access flag. + * + * Return: Success: 0 + * Failure: -1 + * + * Description: Attempt read-only opening of files with different + * permutations of files already existing on-disk. + * + *------------------------------------------------------------------------- + */ +static int +splitter_RO_test( + const struct splitter_dataset_def *data, + hid_t child_fapl_id) +{ + char *filename_rw = NULL; + H5FD_splitter_vfd_config_t *vfd_config = NULL; + hid_t fapl_id = H5I_INVALID_HID; + hid_t file_id = H5I_INVALID_HID; + int ret_value = 0; + + if (NULL == (vfd_config = HDcalloc(1, sizeof(H5FD_splitter_vfd_config_t)))) + SPLITTER_TEST_FAULT("memory allocation for vfd_config struct failed"); + if (NULL == (filename_rw = HDcalloc(H5FD_SPLITTER_PATH_MAX + 1, sizeof(char)))) + SPLITTER_TEST_FAULT("memory allocation for filename_rw string failed"); + + vfd_config->magic = H5FD_SPLITTER_MAGIC; + vfd_config->version = H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION; + vfd_config->ignore_wo_errs = FALSE; + vfd_config->rw_fapl_id = child_fapl_id; + vfd_config->wo_fapl_id = child_fapl_id; + + if (splitter_prepare_file_paths(vfd_config, filename_rw) < 0) { + SPLITTER_TEST_FAULT("can't prepare splitter file paths\n"); + } + + /* Create a new fapl to use the SPLITTER file driver */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (H5I_INVALID_HID == fapl_id) { + SPLITTER_TEST_FAULT("can't create FAPL ID\n"); + } + if (H5Pset_fapl_splitter(fapl_id, vfd_config) < 0) { + SPLITTER_TEST_FAULT("can't set splitter FAPL\n"); + } + if (H5Pget_driver(fapl_id) != H5FD_SPLITTER) { + SPLITTER_TEST_FAULT("set FAPL not SPLITTER\n"); + } + + /* Attempt R/O open when both files are nonexistent + * Should fail. + */ + + H5E_BEGIN_TRY { + file_id = H5Fopen(filename_rw, H5F_ACC_RDONLY, fapl_id); + } H5E_END_TRY; + if (file_id >= 0) { + SPLITTER_TEST_FAULT("R/O open on nonexistent files unexpectedly successful\n"); + } + + /* Attempt R/O open when only W/O file exists + * Should fail. + */ + + if (splitter_create_single_file_at(vfd_config->wo_path, vfd_config->wo_fapl_id, data) < 0) { + SPLITTER_TEST_FAULT("can't write W/O file\n"); + } + H5E_BEGIN_TRY { + file_id = H5Fopen(filename_rw, H5F_ACC_RDONLY, fapl_id); + } H5E_END_TRY; + if (file_id >= 0) { + SPLITTER_TEST_FAULT("R/O open with extant W/O file unexpectedly successful\n"); + } + HDremove(vfd_config->wo_path); + + /* Attempt R/O open when only R/W file exists + * Should fail. + */ + + if (splitter_create_single_file_at(filename_rw, vfd_config->rw_fapl_id, data) < 0) { + SPLITTER_TEST_FAULT("can't create R/W file\n"); + } + H5E_BEGIN_TRY { + file_id = H5Fopen(filename_rw, H5F_ACC_RDONLY, fapl_id); + } H5E_END_TRY; + if (file_id >= 0) { + SPLITTER_TEST_FAULT("R/O open with extant R/W file unexpectedly successful\n"); + } + + /* Attempt R/O open when both R/W and W/O files exist + */ + + if (splitter_create_single_file_at(vfd_config->wo_path, vfd_config->wo_fapl_id, data) < 0) { + SPLITTER_TEST_FAULT("can't create W/O file\n"); + } + file_id = H5Fopen(filename_rw, H5F_ACC_RDONLY, fapl_id); + if (file_id < 0) { + SPLITTER_TEST_FAULT("R/O open on two extant files failed\n"); + } + if (splitter_compare_expected_data(file_id, data) < 0) { + SPLITTER_TEST_FAULT("data mismatch in R/W file\n"); + } + if (H5Fclose(file_id) < 0) { + SPLITTER_TEST_FAULT("can't close file(s)\n"); + } + file_id = H5I_INVALID_HID; + + /* Cleanup + */ + + if (H5Pclose(fapl_id) < 0) { + SPLITTER_TEST_FAULT("can't close FAPL ID\n"); + } + fapl_id = H5I_INVALID_HID; + +done: + if (ret_value < 0) { + H5E_BEGIN_TRY { + H5Pclose(fapl_id); + H5Fclose(file_id); + } H5E_END_TRY; + } + + HDfree(vfd_config); + HDfree(filename_rw); + + return ret_value; +} /* end splitter_RO_test() */ + + +/*------------------------------------------------------------------------- + * Function: splitter_prepare_file_paths + * + * Purpose: Get file paths ready for use by the Splitter VFD tests. + * + * Return: Success: 0 + * Failure: -1 + * + * Description: + * Use h5_fixname to adjust the splitter-relevant file paths + * from those given in FILENAMES. + * + * REMOVES EXISTING FILES AT THE PATH LOCATIONS PRIOR TO RETURN. + * + * The relevant file paths will be set in filename_rw_out and + * inside the config structure (wo_path, log_file_path). + * + * `filename_rw_out` must be at least H5FD_SPLITTER_PATH_MAX+1 + * characters long. + * + * `vfd_config` must have its child FAPL IDs preset. + * + *------------------------------------------------------------------------- + */ +static int +splitter_prepare_file_paths(H5FD_splitter_vfd_config_t *vfd_config, char *filename_rw_out) +{ + int ret_value = 0; + + if (vfd_config == NULL || vfd_config->magic != H5FD_SPLITTER_MAGIC) { + SPLITTER_TEST_FAULT("invalid splitter config pointer\n"); + } + if (filename_rw_out == NULL) { + SPLITTER_TEST_FAULT("NULL filename_rw pointer\n"); + } + + /* TODO: sanity-check fapl IDs? */ + + /* Build the r/w file, w/o file, and the log file paths. + * Output is stored in the associated string pointers. + */ + h5_fixname(FILENAME[11], vfd_config->rw_fapl_id, filename_rw_out, H5FD_SPLITTER_PATH_MAX); + h5_fixname(FILENAME[12], vfd_config->wo_fapl_id, vfd_config->wo_path, H5FD_SPLITTER_PATH_MAX); + h5_fixname_no_suffix(FILENAME[13], vfd_config->wo_fapl_id, vfd_config->log_file_path, H5FD_SPLITTER_PATH_MAX); + + /* Delete any existing files on disk. + */ + HDremove(filename_rw_out); + HDremove(vfd_config->wo_path); + HDremove(vfd_config->log_file_path); + +done: + return ret_value; +} /* end splitter_prepare_file_paths() */ + + +/*------------------------------------------------------------------------- + * Function: splitter_crate_single_file_at + * + * Purpose: Create a file, optionally w/ dataset. + * + * Return: Success: 0 + * Failure: -1 + * + * Description: + * Create a file at the given location with the given FAPL, + * and write data as defined in `data` in a pre-determined location in the file. + * + * If the dataset definition pointer is NULL, no data is written + * to the file. + * + * Will always overwrite an existing file with the given name/path. + * + *------------------------------------------------------------------------- + */ +static int +splitter_create_single_file_at( + const char *filename, + hid_t fapl_id, + const struct splitter_dataset_def *data) +{ + hid_t file_id = H5I_INVALID_HID; + hid_t space_id = H5I_INVALID_HID; + hid_t dset_id = H5I_INVALID_HID; + int ret_value = 0; + + if (filename == NULL || *filename == '\0') { + SPLITTER_TEST_FAULT("filename is invalid\n"); + } + /* TODO: sanity-check fapl id? */ + + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + if (file_id < 0) { + SPLITTER_TEST_FAULT("can't create file\n"); + } + + if (data) { + /* TODO: sanity-check data, if it exists? */ + space_id = H5Screate_simple(data->n_dims, data->dims, NULL); + if (space_id < 0) { + SPLITTER_TEST_FAULT("can't create dataspace\n"); + } + + dset_id = H5Dcreate2( + file_id, + data->dset_name, + data->mem_type_id, + space_id, + H5P_DEFAULT, + H5P_DEFAULT, + H5P_DEFAULT); + if (dset_id < 0) { + SPLITTER_TEST_FAULT("can't create dataset\n"); + } + + if (H5Dwrite(dset_id, data->mem_type_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, data->buf) < 0) { + SPLITTER_TEST_FAULT("can't write data to dataset\n"); + } + + if (H5Dclose(dset_id) < 0) { + SPLITTER_TEST_FAULT("can't close dset\n"); + } + if (H5Sclose(space_id) < 0) { + SPLITTER_TEST_FAULT("can't close space\n"); + } + } /* end if data definition is provided */ + + if (H5Fclose(file_id) < 0) { + SPLITTER_TEST_FAULT("can't close file\n"); + } + +done: + if (ret_value < 0) { + H5E_BEGIN_TRY { + H5Dclose(dset_id); + H5Sclose(space_id); + H5Fclose(file_id); + } H5E_END_TRY; + } /* end if error */ + return ret_value; +} /* end splitter_create_single_file_at() */ + + +/*------------------------------------------------------------------------- + * Function: splitter_compare_expected_data + * + * Purpose: Compare data within a predermined dataset. + * + * Return: Success: 0 + * Failure: -1 + * + * Description: Read data from the file at a predetermined location, and + * compare its contents byte-for-byte with that expected in + * the `data` definition structure. + * + *------------------------------------------------------------------------- + */ +static int +splitter_compare_expected_data(hid_t file_id, + const struct splitter_dataset_def *data) +{ + hid_t dset_id = H5I_INVALID_HID; + int buf[SPLITTER_SIZE][SPLITTER_SIZE]; + int expected[SPLITTER_SIZE][SPLITTER_SIZE]; + size_t i = 0; + size_t j = 0; + int ret_value = 0; + + if (sizeof((void *)buf) != sizeof(data->buf)) { + SPLITTER_TEST_FAULT("invariant size of expected data does not match that received!\n"); + } + HDmemcpy(expected, data->buf, sizeof(expected)); + + dset_id = H5Dopen2(file_id, data->dset_name, H5P_DEFAULT); + if (dset_id < 0) { + SPLITTER_TEST_FAULT("can't open dataset\n"); + } + + if (H5Dread(dset_id, data->mem_type_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, (void *)buf) < 0) { + SPLITTER_TEST_FAULT("can't read dataset\n"); + } + + for (i=0; i < SPLITTER_SIZE; i++) { + for (j=0; j < SPLITTER_SIZE; j++) { + if (buf[i][j] != expected[i][j]) { + SPLITTER_TEST_FAULT("mismatch in expected data\n"); + } + } + } + + if (H5Dclose(dset_id) < 0) { + SPLITTER_TEST_FAULT("can't close dataset\n"); + } + +done: + if (ret_value < 0) { + H5E_BEGIN_TRY { + H5Dclose(dset_id); + } H5E_END_TRY; + } + return ret_value; +} /* end splitter_compare_expected_data() */ + + +/*------------------------------------------------------------------------- + * Function: splitter_tentative_open_test() + * + * Purpose: Verifies Splitter behavior with "tentative" H5F_open. + * + * Return: Success: 0 + * Failure: -1 + * + * Description: + * H5F_open() has a two-stage opening process when given a + * Read/Write access flag -- first it performs a "tentative + * open", where it checks to see whether files already exist + * on the system, done in such a way as to not "alter its state" + * (i.e., truncate). + * This can cause problems with the Splitter VFD, as the + * file on the R/W channel might exist already, but that on the + * W/O channel will not, and vice-versa. + * + * This test exists to verify that in any event, files will be + * created as required. + * + *------------------------------------------------------------------------- + */ +static int +splitter_tentative_open_test(hid_t child_fapl_id) +{ + const char *filename_tmp = "splitter_tmp.h5"; + char *filename_rw = NULL; + H5FD_splitter_vfd_config_t *vfd_config = NULL; + hid_t fapl_id = H5I_INVALID_HID; + hid_t file_id = H5I_INVALID_HID; + int buf[SPLITTER_SIZE][SPLITTER_SIZE]; /* for comparison */ + hsize_t dims[2] = { SPLITTER_SIZE, SPLITTER_SIZE }; /* for comparison */ + int i = 0; /* for comparison */ + int j = 0; /* for comparison */ + struct splitter_dataset_def data; /* for comparison */ + int ret_value = 0; + + if (NULL == (vfd_config = HDcalloc(1, sizeof(H5FD_splitter_vfd_config_t)))) + SPLITTER_TEST_FAULT("memory allocation for vfd_config struct failed"); + if (NULL == (filename_rw = HDcalloc(H5FD_SPLITTER_PATH_MAX + 1, sizeof(char)))) + SPLITTER_TEST_FAULT("memory allocation for filename_rw string failed"); + + /* pre-fill data buffer to write */ + for (i=0; i < SPLITTER_SIZE; i++) { + for (j=0; j < SPLITTER_SIZE; j++) { + buf[i][j] = i*100+j; + } + } + + /* Dataset info */ + data.buf = (void *)buf; + data.mem_type_id = H5T_NATIVE_INT; + data.dims = dims; + data.n_dims = 2; + data.dset_name = SPLITTER_DATASET_NAME; + + vfd_config->magic = H5FD_SPLITTER_MAGIC; + vfd_config->version = H5FD_CURR_SPLITTER_VFD_CONFIG_VERSION; + vfd_config->ignore_wo_errs = FALSE; + vfd_config->rw_fapl_id = child_fapl_id; + vfd_config->wo_fapl_id = child_fapl_id; + + if (splitter_prepare_file_paths(vfd_config, filename_rw) < 0) { + SPLITTER_TEST_FAULT("can't prepare splitter file paths\n"); + } + + /* Create a new fapl to use the SPLITTER file driver */ + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) == H5I_INVALID_HID) { + SPLITTER_TEST_FAULT("can't create FAPL ID\n"); + } + if (H5Pset_fapl_splitter(fapl_id, vfd_config) < 0) { + SPLITTER_TEST_FAULT("can't set splitter FAPL\n"); + } + if (H5Pget_driver(fapl_id) != H5FD_SPLITTER) { + SPLITTER_TEST_FAULT("set FAPL not SPLITTER\n"); + } + + /* Create instance of file on disk. + * Will be copied verbatim as needed, to avoid issues where differences in + * the creation time would befoul comparisons. + */ + if (splitter_create_single_file_at(filename_tmp, child_fapl_id, &data) < 0) { + SPLITTER_TEST_FAULT("can't write W/O file\n"); + } + + /* + * H5Fopen() with RDWR access. + * Neither file exist already + * Should fail. + */ + + H5E_BEGIN_TRY { + file_id = H5Fopen(filename_rw, H5F_ACC_RDWR, fapl_id); + } H5E_END_TRY; + if (file_id != H5I_INVALID_HID) { + SPLITTER_TEST_FAULT("open with both nonexistent files unexpectedly succeeded\n"); + } + if (file_exists(filename_rw, child_fapl_id)) { + SPLITTER_TEST_FAULT("R/W file unexpectedly created\n"); + } + if (file_exists(vfd_config->wo_path, child_fapl_id)) { + SPLITTER_TEST_FAULT("W/O file unexpectedly created\n"); + } + + /* + * H5Fopen() with RDWR access. + * Only W/O file present. + * Should fail. + */ + + if (h5_duplicate_file_by_bytes(filename_tmp, vfd_config->wo_path) < 0) { + SPLITTER_TEST_FAULT("Can't create W/O file copy.\n"); + } + H5E_BEGIN_TRY { + file_id = H5Fopen(filename_rw, H5F_ACC_RDWR, fapl_id); + } H5E_END_TRY; + if (file_id != H5I_INVALID_HID) { + SPLITTER_TEST_FAULT("open with nonexistent R/W file unexpectedly succeeded\n"); + } + if (file_exists(filename_rw, child_fapl_id)) { + SPLITTER_TEST_FAULT("R/W file unexpectedly created\n"); + } + if (!file_exists(vfd_config->wo_path, child_fapl_id)) { + SPLITTER_TEST_FAULT("W/O file mysteriously disappeared\n"); + } + HDremove(vfd_config->wo_path); + if (file_exists(vfd_config->wo_path, child_fapl_id)) { + SPLITTER_TEST_FAULT("failed to remove W/O file\n"); + } + + /* + * H5Fopen() with RDWR access. + * Only R/W file present. + * Should fail. + */ + + if (h5_duplicate_file_by_bytes(filename_tmp, filename_rw) < 0) { + SPLITTER_TEST_FAULT("Can't create R/W file copy.\n"); + } + H5E_BEGIN_TRY { + file_id = H5Fopen(filename_rw, H5F_ACC_RDWR, fapl_id); + } H5E_END_TRY; + if (file_id != H5I_INVALID_HID) { + SPLITTER_TEST_FAULT("open with nonexistent W/O unexpectedly succeeded\n"); + } + if (!file_exists(filename_rw, child_fapl_id)) { + SPLITTER_TEST_FAULT("R/W file mysteriously disappeared\n"); + } + if (file_exists(vfd_config->wo_path, child_fapl_id)) { + SPLITTER_TEST_FAULT("W/O file unexpectedly created\n"); + } + + /* + * H5Fopen() with RDWR access. + * Both files present. + */ + + if (h5_duplicate_file_by_bytes(filename_tmp, vfd_config->wo_path) < 0) { + SPLITTER_TEST_FAULT("Can't create W/O file copy.\n"); + } + file_id = H5Fopen(filename_rw, H5F_ACC_RDWR, fapl_id); + if (file_id == H5I_INVALID_HID) { + SPLITTER_TEST_FAULT("file-open failed with both present\n"); + } + /* Open successful; close file then inspect presence again */ + if (H5Fclose(file_id) < 0) { + SPLITTER_TEST_FAULT("can't close file ID\n"); + } + if (!file_exists(filename_rw, child_fapl_id)) { + SPLITTER_TEST_FAULT("R/W file mysteriously disappared\n"); + } + if (!file_exists(vfd_config->wo_path, child_fapl_id)) { + SPLITTER_TEST_FAULT("W/O file mysteriously disappeared\n"); + } + + /* + * H5Fcreate() with TRUNC access. + * Both files present. + */ + + file_id = H5Fcreate(filename_rw, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + if (file_id == H5I_INVALID_HID) { + SPLITTER_TEST_FAULT("file-open failed with both present\n"); + } + /* Open successful; close file then inspect presence again */ + if (H5Fclose(file_id) < 0) { + SPLITTER_TEST_FAULT("can't close file ID\n"); + } + if (!file_exists(filename_rw, child_fapl_id)) { + SPLITTER_TEST_FAULT("R/W file mysteriously disappared\n"); + } + if (!file_exists(vfd_config->wo_path, child_fapl_id)) { + SPLITTER_TEST_FAULT("W/O file mysteriously disappeared\n"); + } + if (h5_compare_file_bytes(filename_rw, vfd_config->wo_path) < 0) { + SPLITTER_TEST_FAULT("files are not byte-for-byte equivalent\n"); + } + HDremove(filename_rw); + HDremove(vfd_config->wo_path); + + /* + * H5Fcreate() with TRUNC access. + * R/W already exists. + */ + + if (h5_duplicate_file_by_bytes(filename_tmp, filename_rw) < 0) { + SPLITTER_TEST_FAULT("Can't create R/W file copy.\n"); + } + if (file_exists(vfd_config->wo_path, child_fapl_id)) { + SPLITTER_TEST_FAULT("failed to remove W/O file\n"); + } + file_id = H5Fcreate(filename_rw, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + if (file_id == H5I_INVALID_HID) { + SPLITTER_TEST_FAULT("file-open failed with both present\n"); + } + /* Open successful; close file then inspect presence again */ + if (H5Fclose(file_id) < 0) { + SPLITTER_TEST_FAULT("can't close file ID\n"); + } + if (!file_exists(filename_rw, child_fapl_id)) { + SPLITTER_TEST_FAULT("R/W file mysteriously disappared\n"); + } + if (!file_exists(vfd_config->wo_path, child_fapl_id)) { + SPLITTER_TEST_FAULT("W/O file mysteriously disappeared\n"); + } + if (h5_compare_file_bytes(filename_rw, vfd_config->wo_path) < 0) { + SPLITTER_TEST_FAULT("files are not byte-for-byte equivalent\n"); + } + HDremove(filename_rw); + HDremove(vfd_config->wo_path); + + /* + * H5Fcreate() with TRUNC access. + * Only W/O present. + */ + + if (h5_duplicate_file_by_bytes(filename_tmp, vfd_config->wo_path) < 0) { + SPLITTER_TEST_FAULT("Can't create W/O file copy.\n"); + } + if (file_exists(filename_rw, child_fapl_id)) { + SPLITTER_TEST_FAULT("failed to remove R/W file\n"); + } + file_id = H5Fcreate(filename_rw, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + if (file_id == H5I_INVALID_HID) { + SPLITTER_TEST_FAULT("file-open failed with both present\n"); + } + /* Open successful; close file then inspect presence again */ + if (H5Fclose(file_id) < 0) { + SPLITTER_TEST_FAULT("can't close file ID\n"); + } + if (!file_exists(filename_rw, child_fapl_id)) { + SPLITTER_TEST_FAULT("R/W file mysteriously disappared\n"); + } + if (!file_exists(vfd_config->wo_path, child_fapl_id)) { + SPLITTER_TEST_FAULT("W/O file mysteriously disappeared\n"); + } + if (h5_compare_file_bytes(filename_rw, vfd_config->wo_path) < 0) { + SPLITTER_TEST_FAULT("files are not byte-for-byte equivalent\n"); + } + HDremove(filename_rw); + HDremove(vfd_config->wo_path); + + /* H5Fcreate with both files absent is tested elsewhere */ + + /* + * Cleanup + */ + + if (H5Pclose(fapl_id) < 0) { + SPLITTER_TEST_FAULT("can't close splitter FAPL ID\n"); + } + +done: + if (ret_value < 0) { + H5E_BEGIN_TRY { + H5Pclose(fapl_id); + H5Fclose(file_id); + } H5E_END_TRY; + } + + HDfree(vfd_config); + HDfree(filename_rw); + + return ret_value; +} /* end splitter_tentative_open_test() */ + + +/*------------------------------------------------------------------------- + * Function: file_exists() + * + * Purpose: Determine whether a file exists on-system + * + * Return: Non-zero (1) if it exists (H5Fopen successful), + * zero (0) if absent (cannot be opened). + * + * Description: Attempt H5Fopen with the given FAPL ID and RDONLY access flag. + * + *------------------------------------------------------------------------- + */ +static int +file_exists(const char *filename, hid_t fapl_id) +{ + hid_t file_id = H5I_INVALID_HID; + int ret_value = 0; + + H5E_BEGIN_TRY { + file_id = H5Fopen(filename, H5F_ACC_RDONLY, fapl_id); + } H5E_END_TRY; + if (file_id != H5I_INVALID_HID) { + ret_value = 1; + if (H5Fclose(file_id) < 0) { + FAIL_PUTS_ERROR("can't close file ID\n"); + } + } + + return ret_value; + +error: + H5E_BEGIN_TRY { + H5Fclose(file_id); + } H5E_END_TRY; + return ret_value; +} /* end file_exists() */ + + +/*------------------------------------------------------------------------- + * Function: test_splitter + * + * Purpose: Tests the Splitter VFD + * + * Return: Success: 0 + * Failure: -1 + * + * Description: + * This test function uses the Splitter VFD to produce a r/w + * file and a w/o file. It will verify that the two files + * are identical. + * + *------------------------------------------------------------------------- + */ +static herr_t +test_splitter(void) +{ + int buf[SPLITTER_SIZE][SPLITTER_SIZE]; + hsize_t dims[2] = { SPLITTER_SIZE, SPLITTER_SIZE }; + hid_t child_fapl_id = H5I_INVALID_HID; + int i = 0; + int j = 0; + struct splitter_dataset_def data; + + TESTING("SPLITTER file driver"); + + /* pre-fill data buffer to write */ + for (i=0; i < SPLITTER_SIZE; i++) { + for (j=0; j < SPLITTER_SIZE; j++) { + buf[i][j] = i*100+j; + } + } + + /* Dataset info */ + data.buf = (void *)buf; + data.mem_type_id = H5T_NATIVE_INT; + data.dims = dims; + data.n_dims = 2; + data.dset_name = SPLITTER_DATASET_NAME; + + /* Stand-in for manual FAPL creation + * Enables verification with arbitrary VFDs via `make check-vfd` + */ + child_fapl_id = h5_fileaccess(); + if (child_fapl_id < 0) { + TEST_ERROR; + } + + if (!driver_is_splitter_compatible(child_fapl_id)) { + SKIPPED(); + HDprintf(" given driver is not Splitter W/O compatible.\n"); + return 0; + } + + /* Test Read-Only access, including when a file on the W/O channel + * does not exist. + */ + if (splitter_RO_test(&data, child_fapl_id) < 0) { + TEST_ERROR; + } + + /* Test opening of files when the W/O channel does not exist. + */ + if (splitter_tentative_open_test(child_fapl_id) < 0) { + TEST_ERROR; + } + + /* Test file creation, utilizing different child FAPLs (default vs. + * specified), logfile, and Write Channel error ignoring behavior. + */ + for (i=0; i < 4; i++) { + hbool_t ignore_wo_errors = (i & 1) ? TRUE : FALSE; + hbool_t provide_logfile_path = (i & 2) ? TRUE : FALSE; + hid_t child_fapl_ids[2] = { H5P_DEFAULT, H5P_DEFAULT }; + + /* Test child driver definition/default combination */ + for (j=0; j < 4; j++) { + + child_fapl_ids[0] = (j & 1) ? child_fapl_id : H5P_DEFAULT; + child_fapl_ids[1] = (j & 2) ? child_fapl_id : H5P_DEFAULT; + + if (run_splitter_test(&data, ignore_wo_errors, provide_logfile_path, child_fapl_ids) < 0) { + TEST_ERROR; + } + + } /* end for child fapl definition/pairing */ + + } /* end for behavior-flag loops */ + +/* TODO: SWMR open? */ +/* Concurrent opens with both drivers using the Splitter */ + + if (H5Pclose(child_fapl_id) == FAIL) { + TEST_ERROR; + } + + PASSED(); + return 0; + +error: + if (child_fapl_id != H5I_INVALID_HID) + H5Pclose(child_fapl_id); + + return -1; +} /* end test_splitter() */ + +#undef SPLITTER_TEST_FAULT + + /*------------------------------------------------------------------------- * Function: main * @@ -2085,18 +3343,20 @@ main(void) nerrors += test_direct() < 0 ? 1 : 0; nerrors += test_family() < 0 ? 1 : 0; nerrors += test_family_compat() < 0 ? 1 : 0; + nerrors += test_family_member_fapl() < 0 ? 1 : 0; nerrors += test_multi() < 0 ? 1 : 0; nerrors += test_multi_compat() < 0 ? 1 : 0; nerrors += test_log() < 0 ? 1 : 0; nerrors += test_stdio() < 0 ? 1 : 0; nerrors += test_windows() < 0 ? 1 : 0; nerrors += test_ros3() < 0 ? 1 : 0; + nerrors += test_splitter() < 0 ? 1 : 0; if(nerrors) { HDprintf("***** %d Virtual File Driver TEST%s FAILED! *****\n", nerrors, nerrors > 1 ? "S" : ""); return EXIT_FAILURE; - } /* end if */ + } HDprintf("All Virtual File Driver tests passed.\n"); diff --git a/tools/src/h5diff/h5diff_main.c b/tools/src/h5diff/h5diff_main.c index 33d6570..e14447d 100644 --- a/tools/src/h5diff/h5diff_main.c +++ b/tools/src/h5diff/h5diff_main.c @@ -26,7 +26,7 @@ * Return: An exit status of 0 means no differences were found, 1 means some * differences were found. * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente * * Date: May 9, 2003 * diff --git a/tools/src/h5diff/ph5diff_main.c b/tools/src/h5diff/ph5diff_main.c index c473c8b..2336378 100644 --- a/tools/src/h5diff/ph5diff_main.c +++ b/tools/src/h5diff/ph5diff_main.c @@ -31,7 +31,7 @@ static void ph5diff_worker(int ); * Return: An exit status of 0 means no differences were found, 1 means some * differences were found. * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente * * Date: May 9, 2003 * diff --git a/tools/src/h5format_convert/h5format_convert.c b/tools/src/h5format_convert/h5format_convert.c index fda5fe9..bd02e3b 100644 --- a/tools/src/h5format_convert/h5format_convert.c +++ b/tools/src/h5format_convert/h5format_convert.c @@ -64,7 +64,7 @@ static struct long_options l_opts[] = { { NULL, 0, '\0' } }; - + /*------------------------------------------------------------------------- * Function: usage * @@ -185,7 +185,7 @@ error: return(-1); ; } /* parse_command_line() */ - + /*------------------------------------------------------------------------- * Function: leave * @@ -387,7 +387,7 @@ error: return -1; } /* end convert_dsets_cb() */ - + /*------------------------------------------------------------------------- * Function: main * diff --git a/tools/src/h5import/h5import.c b/tools/src/h5import/h5import.c index b11fe7f..4a642e0 100644 --- a/tools/src/h5import/h5import.c +++ b/tools/src/h5import/h5import.c @@ -3735,8 +3735,8 @@ static int getExternalFilename(struct Input *in, FILE *strm) return (-1); } - in->externFilename = (char *) HDmalloc ((size_t) (HDstrlen(temp)) * sizeof(char)); - (void) HDstrcpy(in->externFilename, temp); + in->externFilename = (char *) HDmalloc ((size_t) (HDstrlen(temp) + 1) * sizeof(char)); + (void) HDstrncpy(in->externFilename, temp, HDstrlen(temp) + 1); return (0); } diff --git a/tools/src/h5ls/h5ls.c b/tools/src/h5ls/h5ls.c index 8be1a11..c4c3122 100644 --- a/tools/src/h5ls/h5ls.c +++ b/tools/src/h5ls/h5ls.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Monday, March 23, 1998 */ diff --git a/tools/src/h5repack/h5repack.c b/tools/src/h5repack/h5repack.c index 9347eff..77f2272 100644 --- a/tools/src/h5repack/h5repack.c +++ b/tools/src/h5repack/h5repack.c @@ -522,6 +522,8 @@ done: */ if (TRUE == h5tools_detect_vlen(wtype_id)) H5Dvlen_reclaim(wtype_id, space_id, H5P_DEFAULT, buf); + + /* Free buf */ HDfree(buf); } diff --git a/tools/src/h5repack/h5repack_parse.c b/tools/src/h5repack/h5repack_parse.c index 95cacc1..50ccd7a 100644 --- a/tools/src/h5repack/h5repack_parse.c +++ b/tools/src/h5repack/h5repack_parse.c @@ -478,7 +478,7 @@ obj_list_t* parse_filter(const char *str, unsigned *n_objs, filter_info_t *filt, * Example: * "AA,B,CDE:CHUNK=10X10" * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * Programmer: Pedro Vicente * * Date: December 30, 2003 * diff --git a/tools/src/misc/h5debug.c b/tools/src/misc/h5debug.c index 58a99a4..351c577 100644 --- a/tools/src/misc/h5debug.c +++ b/tools/src/misc/h5debug.c @@ -15,7 +15,7 @@ * * Created: debug.c * Jul 18 1997 - * Robb Matzke <matzke@llnl.gov> + * Robb Matzke * * Purpose: Debugs an existing HDF5 file at a low level. * diff --git a/tools/src/misc/h5mkgrp.c b/tools/src/misc/h5mkgrp.c index 2b4d57f..2046373 100644 --- a/tools/src/misc/h5mkgrp.c +++ b/tools/src/misc/h5mkgrp.c @@ -46,7 +46,7 @@ typedef struct mkgrp_opt_t { mkgrp_opt_t params_g; /* Command line parameter settings */ - + /*------------------------------------------------------------------------- * Function: leave * @@ -78,7 +78,7 @@ leave(int ret) HDexit(ret); } /* end leave() */ - + /*------------------------------------------------------------------------- * Function: usage * @@ -104,7 +104,7 @@ usage(const char *prog) PRINTVALSTREAM(rawoutstream, "\n"); } /* end usage() */ - + /*------------------------------------------------------------------------- * Function: parse_command_line * @@ -200,7 +200,7 @@ parse_command_line(int argc, const char *argv[], mkgrp_opt_t *options) return 0; } /* parse_command_line() */ - + /*------------------------------------------------------------------------- * Function: main * diff --git a/tools/src/misc/h5repart.c b/tools/src/misc/h5repart.c index d516fa0..371cf2c 100644 --- a/tools/src/misc/h5repart.c +++ b/tools/src/misc/h5repart.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Wednesday, May 13, 1998 * * Purpose: Repartitions a file family. This program can be used to diff --git a/tools/test/h5copy/testh5copy.sh.in b/tools/test/h5copy/testh5copy.sh.in index 2440ca4..ebbcb6c 100644 --- a/tools/test/h5copy/testh5copy.sh.in +++ b/tools/test/h5copy/testh5copy.sh.in @@ -13,7 +13,7 @@ # # Tests for the h5copy tool # -# Pedro Vicente Nunes (pvn@hdfgroup.org), Albert Cheng (acheng@hdfgroup.org) +# Pedro Vicente Nunes, Albert Cheng # Thursday, July 20, 2006 # @@ -200,6 +200,7 @@ TOOLTEST() fi runh5diff=no fi + if [ "$3" = -o ]; then outputfile=$4 else @@ -219,6 +220,7 @@ TOOLTEST() $RUNSERIAL $H5COPY_BIN $@ ) > $actualout 2> $actualerr RET=$? + if [ $RET != 0 ]; then echo "*FAILED*" echo "failed result is:" @@ -268,6 +270,7 @@ TOOLTEST_PREFILL() $RUNSERIAL $H5COPY_BIN -i $inputfile -o $outputfile -v -s $grp_name -d $grp_name2 ) > $actualout 2> $actualerr RET=$? + if [ $RET != 0 ]; then echo "*FAILED*" echo "failed result is:" @@ -282,6 +285,7 @@ TOOLTEST_PREFILL() $RUNSERIAL $H5COPY_BIN -i $inputfile -o $outputfile -v -s $obj_name -d $obj_name2 ) > $actualout 2> $actualerr RET=$? + if [ $RET != 0 ]; then echo "*FAILED*" echo "failed result is:" @@ -313,6 +317,7 @@ TOOLTEST_SAME() else runh5diff=no fi + if [ "$3" = -o ]; then outputfile=$4 else @@ -330,6 +335,7 @@ TOOLTEST_SAME() $RUNSERIAL $H5COPY_BIN -i $inputfile -o $outputfile -v -s $grp_name -d $grp_name ) > $actualout 2> $actualerr RET=$? + if [ $RET != 0 ]; then echo "*FAILED*" echo "failed result is:" @@ -344,6 +350,7 @@ TOOLTEST_SAME() $RUNSERIAL $H5COPY_BIN -i $outputfile -o $outputfile -v -s $grp_name -d $grp_name2 ) > $actualout 2> $actualerr RET=$? + if [ $RET != 0 ]; then echo "*FAILED*" echo "failed result is:" @@ -406,6 +413,7 @@ TOOLTEST_FAIL() if [ "$1" = -i ]; then inputfile=$2 fi + if [ "$3" = -o ]; then outputfile=$4 fi @@ -417,8 +425,8 @@ TOOLTEST_FAIL() #echo "#############################" $RUNSERIAL $H5COPY_BIN $@ ) > $actualout 2> $actualerr - RET=$? + # save actualout and actualerr in case they are needed later. cp $actualout $actualout_sav STDOUT_FILTER $actualout diff --git a/tools/test/h5diff/h5diff_plugin.sh.in b/tools/test/h5diff/h5diff_plugin.sh.in index 341cba5..ffc43da 100644 --- a/tools/test/h5diff/h5diff_plugin.sh.in +++ b/tools/test/h5diff/h5diff_plugin.sh.in @@ -135,24 +135,24 @@ CLEAN_TESTFILES_AND_TESTDIR() # -h print help page while [ $# -gt 0 ]; do case "$1" in - -p) # reset the tool name and bin to run ph5diff tests - TESTNAME=ph5diff - H5DIFF=../../src/h5diff/ph5diff # The tool name - H5DIFF_BIN=`pwd`/$H5DIFF # The path of the tool binary - pmode=yes - shift - ;; + -p) # reset the tool name and bin to run ph5diff tests + TESTNAME=ph5diff + H5DIFF=../../src/h5diff/ph5diff # The tool name + H5DIFF_BIN=`pwd`/$H5DIFF # The path of the tool binary + pmode=yes + shift + ;; -h) # print help page - echo "$0 [-p] [-h]" - echo " -p run ph5diff tests" - echo " -h print help page" - shift - exit 0 - ;; + echo "$0 [-p] [-h]" + echo " -p run ph5diff tests" + echo " -h print help page" + shift + exit 0 + ;; *) # unknown option echo "$0: Unknown option ($1)" - exit 1 - ;; + exit 1 + ;; esac done @@ -218,19 +218,21 @@ TOOLTEST() { # Run test. TESTING $H5DIFF $@ ( - #echo "#############################" - #echo "Expected output for '$H5DIFF $@'" - #echo "#############################" - cd $TESTDIR - eval $ENVCMD $RUNCMD $H5DIFF_BIN "$@" + #echo "#############################" + #echo "Expected output for '$H5DIFF $@'" + #echo "#############################" + cd $TESTDIR + eval $ENVCMD $RUNCMD $H5DIFF_BIN "$@" ) >$actual 2>$actual_err EXIT_CODE=$? + # save actual and actual_err in case they are needed later. cp $actual $actual_sav STDOUT_FILTER $actual cp $actual_err $actual_err_sav STDERR_FILTER $actual_err cat $actual_err >> $actual + # don't add exit code check in pmode, as it causes failure. (exit code # is from mpirun not tool) # if any problem occurs relate to an exit code, it will be caught in @@ -260,27 +262,29 @@ TOOLTEST() { actual_sorted=actual_sorted sort $expect -o $expect_sorted sort $actual -o $actual_sorted + # remove "EXIT CODE:" line from expect file. test for exit code # is done by serial mode. grep -v "EXIT CODE:" $expect_sorted > $expect_sorted.noexit mv $expect_sorted.noexit $expect_sorted - if $CMP $expect_sorted $actual_sorted; then - echo " PASSED" - else - echo "*FAILED*" - nerrors="`expr $nerrors + 1`" - if test yes = "$verbose"; then - echo "====Expected result ($expect_sorted) differs from actual result ($actual_sorted)" - $DIFF $expect_sorted $actual_sorted |sed 's/^/ /' - echo "====The actual output ($actual_sav)" - sed 's/^/ /' < $actual_sav - echo "====The actual stderr ($actual_err_sav)" - sed 's/^/ /' < $actual_err_sav - echo "====End of actual stderr ($actual_err_sav)" - echo "" + + if $CMP $expect_sorted $actual_sorted; then + echo " PASSED" + else + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + if test yes = "$verbose"; then + echo "====Expected result ($expect_sorted) differs from actual result ($actual_sorted)" + $DIFF $expect_sorted $actual_sorted |sed 's/^/ /' + echo "====The actual output ($actual_sav)" + sed 's/^/ /' < $actual_sav + echo "====The actual stderr ($actual_err_sav)" + sed 's/^/ /' < $actual_err_sav + echo "====End of actual stderr ($actual_err_sav)" + echo "" + fi fi fi - fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then @@ -306,18 +310,20 @@ TOOLTEST_ERR() { # Run test. TESTING $H5DIFF $@ ( - #echo "#############################" - #echo "Expected output for '$H5DIFF $@'" - #echo "#############################" - cd $TESTDIR - eval $ENVCMD $RUNCMD $H5DIFF_BIN "$@" + #echo "#############################" + #echo "Expected output for '$H5DIFF $@'" + #echo "#############################" + cd $TESTDIR + eval $ENVCMD $RUNCMD $H5DIFF_BIN "$@" ) >$actual 2>$actual_err EXIT_CODE=$? + # save actual and actual_err in case they are needed later. cp $actual $actual_sav STDOUT_FILTER $actual cp $actual_err $actual_err_sav STDERR_FILTER $actual_err + # don't add exit code check in pmode, as it causes failure. (exit code # is from mpirun not tool) # if any problem occurs relate to an exit code, it will be caught in @@ -348,23 +354,24 @@ TOOLTEST_ERR() { sort $expect_err -o $expect_sorted sort $actual_err -o $actual_sorted mv $expect_sorted.noexit $expect_sorted - if $CMP $expect_sorted $actual_sorted; then - echo " PASSED" - else - echo "*FAILED*" - nerrors="`expr $nerrors + 1`" - if test yes = "$verbose"; then - echo "====Expected result ($expect_sorted) differs from actual result ($actual_sorted)" - $DIFF $expect_sorted $actual_sorted |sed 's/^/ /' - echo "====The actual output ($actual_sav)" - sed 's/^/ /' < $actual_sav - echo "====The actual stderr ($actual_err_sav)" - sed 's/^/ /' < $actual_err_sav - echo "====End of actual stderr ($actual_err_sav)" - echo "" + + if $CMP $expect_sorted $actual_sorted; then + echo " PASSED" + else + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + if test yes = "$verbose"; then + echo "====Expected result ($expect_sorted) differs from actual result ($actual_sorted)" + $DIFF $expect_sorted $actual_sorted |sed 's/^/ /' + echo "====The actual output ($actual_sav)" + sed 's/^/ /' < $actual_sav + echo "====The actual stderr ($actual_err_sav)" + sed 's/^/ /' < $actual_err_sav + echo "====End of actual stderr ($actual_err_sav)" + echo "" + fi fi fi - fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then diff --git a/tools/test/h5diff/testh5diff.sh.in b/tools/test/h5diff/testh5diff.sh.in index 962f09c..efc2722 100644 --- a/tools/test/h5diff/testh5diff.sh.in +++ b/tools/test/h5diff/testh5diff.sh.in @@ -486,18 +486,20 @@ TOOLTEST() { # Run test. TESTING $H5DIFF $@ ( - #echo "#############################" - #echo "Expected output for '$H5DIFF $@'" - #echo "#############################" - cd $TESTDIR - eval $RUNCMD $H5DIFF_BIN "$@" + #echo "#############################" + #echo "Expected output for '$H5DIFF $@'" + #echo "#############################" + cd $TESTDIR + eval $RUNCMD $H5DIFF_BIN "$@" ) >$actual 2>$actual_err EXIT_CODE=$? + # save actual and actual_err in case they are needed later. cp $actual $actual_sav STDOUT_FILTER $actual cp $actual_err $actual_err_sav STDERR_FILTER $actual_err + # don't add exit code check in pmode, as it causes failure. (exit code # is from mpirun not tool) # if any problem occurs relate to an exit code, it will be caught in @@ -531,23 +533,24 @@ TOOLTEST() { # is done by serial mode. grep -v "EXIT CODE:" $expect_sorted > $expect_sorted.noexit mv $expect_sorted.noexit $expect_sorted - if $CMP $expect_sorted $actual_sorted; then - echo " PASSED" - else - echo "*FAILED*" - nerrors="`expr $nerrors + 1`" - if test yes = "$verbose"; then - echo "====Expected result ($expect_sorted) differs from actual result ($actual_sorted)" - $DIFF $expect_sorted $actual_sorted |sed 's/^/ /' - echo "====The actual output ($actual_sav)" - sed 's/^/ /' < $actual_sav - echo "====The actual stderr ($actual_err_sav)" - sed 's/^/ /' < $actual_err_sav - echo "====End of actual stderr ($actual_err_sav)" - echo "" + + if $CMP $expect_sorted $actual_sorted; then + echo " PASSED" + else + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + if test yes = "$verbose"; then + echo "====Expected result ($expect_sorted) differs from actual result ($actual_sorted)" + $DIFF $expect_sorted $actual_sorted |sed 's/^/ /' + echo "====The actual output ($actual_sav)" + sed 's/^/ /' < $actual_sav + echo "====The actual stderr ($actual_err_sav)" + sed 's/^/ /' < $actual_err_sav + echo "====End of actual stderr ($actual_err_sav)" + echo "" + fi fi fi - fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then diff --git a/tools/test/h5dump/h5dump_plugin.sh.in b/tools/test/h5dump/h5dump_plugin.sh.in index 6a00a16..d940ab3 100644 --- a/tools/test/h5dump/h5dump_plugin.sh.in +++ b/tools/test/h5dump/h5dump_plugin.sh.in @@ -181,8 +181,8 @@ TOOLTEST() { # Run test. TESTING $H5DUMP $@ ( - cd $TESTDIR - $ENVCMD $RUNSERIAL $H5DUMP_BIN "$@" + cd $TESTDIR + $ENVCMD $RUNSERIAL $H5DUMP_BIN "$@" ) >$actual 2>$actual_err # save actual and actual_err in case they are needed later. @@ -192,24 +192,24 @@ TOOLTEST() { STDERR_FILTER $actual_err cat $actual_err >> $actual - if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.ddl) missing" + nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual > /dev/null 2>&1 ; then - echo " PASSED" + echo " PASSED" else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $caseless $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $caseless $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_sav $actual_err_sav $actual_ext + rm -f $actual $actual_err $actual_sav $actual_err_sav $actual_ext fi } diff --git a/tools/test/h5dump/testh5dump.sh.in b/tools/test/h5dump/testh5dump.sh.in index 02d5da0..1704e1e 100644 --- a/tools/test/h5dump/testh5dump.sh.in +++ b/tools/test/h5dump/testh5dump.sh.in @@ -458,14 +458,14 @@ TESTING() { TOOLTEST() { # check if caseless compare and diff requested if [ "$1" = ignorecase ]; then - caseless="-i" - # replace cmp with diff which runs much longer. - xCMP="$DIFF -i" - shift + caseless="-i" + # replace cmp with diff which runs much longer. + xCMP="$DIFF -i" + shift else - caseless="" - # stick with faster cmp if ignorecase is not requested. - xCMP="$CMP" + caseless="" + # stick with faster cmp if ignorecase is not requested. + xCMP="$CMP" fi expect="$TESTDIR/$1" @@ -478,8 +478,8 @@ TOOLTEST() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err # save actual and actual_err in case they are needed later. @@ -488,24 +488,24 @@ TOOLTEST() { cp $actual_err $actual_err_sav STDERR_FILTER $actual_err - if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.ddl) missing" + nerrors="`expr $nerrors + 1`" elif $xCMP $expect $actual > /dev/null 2>&1 ; then - echo " PASSED" + echo " PASSED" else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $caseless $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $caseless $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_sav $actual_err_sav $actual_ext + rm -f $actual $actual_err $actual_sav $actual_err_sav $actual_ext fi } @@ -527,41 +527,41 @@ TOOLTEST2() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" - elif $CMP $expect $actual; then - if [ ! -f $expectdata ]; then - # Create the expect data file if it doesn't yet exist. + # Create the expect file if it doesn't yet exist. echo " CREATED" - cp $actualdata $expectdata - echo " Expected data (*.exp) missing" + cp $actual $expect + echo " Expected result (*.ddl) missing" nerrors="`expr $nerrors + 1`" - elif $CMP $expectdata $actualdata; then - echo " PASSED" - else + elif $CMP $expect $actual; then + if [ ! -f $expectdata ]; then + # Create the expect data file if it doesn't yet exist. + echo " CREATED" + cp $actualdata $expectdata + echo " Expected data (*.exp) missing" + nerrors="`expr $nerrors + 1`" + elif $CMP $expectdata $actualdata; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected datafile (*.exp) differs from actual datafile (*.txt)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expectdata $actualdata |sed 's/^/ /' + fi + else echo "*FAILED*" - echo " Expected datafile (*.exp) differs from actual datafile (*.txt)" + echo " Expected result (*.ddl) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expectdata $actualdata |sed 's/^/ /' - fi - else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actualdata $actual_err + rm -f $actual $actualdata $actual_err fi } @@ -585,56 +585,55 @@ TOOLTEST2A() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" - elif $CMP $expect $actual; then - if [ ! -f $expectdata ]; then - # Create the expect data file if it doesn't yet exist. + # Create the expect file if it doesn't yet exist. echo " CREATED" - cp $actualdata $expectdata - echo " Expected data (*.exp) missing" + cp $actual $expect + echo " Expected result (*.ddl) missing" nerrors="`expr $nerrors + 1`" - elif $DIFF $expectdata $actualdata; then - if [ ! -f $expectmeta ]; then - # Create the expect meta file if it doesn't yet exist. - echo " CREATED" - cp $actualmeta $expectmeta - echo " Expected metafile (*.ddl) missing" - nerrors="`expr $nerrors + 1`" - elif $CMP $expectmeta $actualmeta; then - echo " PASSED" + elif $CMP $expect $actual; then + if [ ! -f $expectdata ]; then + # Create the expect data file if it doesn't yet exist. + echo " CREATED" + cp $actualdata $expectdata + echo " Expected data (*.exp) missing" + nerrors="`expr $nerrors + 1`" + elif $DIFF $expectdata $actualdata; then + if [ ! -f $expectmeta ]; then + # Create the expect meta file if it doesn't yet exist. + echo " CREATED" + cp $actualmeta $expectmeta + echo " Expected metafile (*.ddl) missing" + nerrors="`expr $nerrors + 1`" + elif $CMP $expectmeta $actualmeta; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected metafile (*.ddl) differs from actual metafile (*.txt)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expectmeta $actualmeta |sed 's/^/ /' + fi else - echo "*FAILED*" - echo " Expected metafile (*.ddl) differs from actual metafile (*.txt)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expectmeta $actualmeta |sed 's/^/ /' + echo "*FAILED*" + echo " Expected datafile (*.exp) differs from actual datafile (*.txt)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expectdata $actualdata |sed 's/^/ /' fi - else + else echo "*FAILED*" - echo " Expected datafile (*.exp) differs from actual datafile (*.txt)" + echo " Expected result (*.ddl) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expectdata $actualdata |sed 's/^/ /' - fi - else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actualdata $actual_err $actualmeta + rm -f $actual $actualdata $actual_err $actualmeta fi - } # same as TOOLTEST2 but only compares the generated data file to the expected data file @@ -651,28 +650,28 @@ TOOLTEST2B() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err if [ ! -f $expectdata ]; then - # Create the expect data file if it doesn't yet exist. - echo " CREATED" - cp $actualdata $expectdata - echo " Expected data (*.exp) missing" - nerrors="`expr $nerrors + 1`" + # Create the expect data file if it doesn't yet exist. + echo " CREATED" + cp $actualdata $expectdata + echo " Expected data (*.exp) missing" + nerrors="`expr $nerrors + 1`" elif $CMP $expectdata $actualdata; then - echo " PASSED" + echo " PASSED" else - echo "*FAILED*" - echo " Expected datafile (*.exp) differs from actual datafile (*.txt)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expectdata $actualdata |sed 's/^/ /' + echo "*FAILED*" + echo " Expected datafile (*.exp) differs from actual datafile (*.txt)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expectdata $actualdata |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actualdata $actual_err + rm -f $actual $actualdata $actual_err fi } @@ -692,8 +691,8 @@ TOOLTEST3() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err # save actual and actual_err in case they are needed later. @@ -712,23 +711,23 @@ TOOLTEST3() { $actual_err > $actual_ext if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.ddl) missing" + nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual; then - echo " PASSED" + echo " PASSED" else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_sav $actual_err_sav + rm -f $actual $actual_err $actual_sav $actual_err_sav fi } @@ -750,8 +749,8 @@ TOOLTEST4() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $ENVCMD $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $ENVCMD $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err # save actual and actual_err in case they are needed later. @@ -770,30 +769,30 @@ TOOLTEST4() { $actual_err > $actual_ext if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.ddl) missing" + nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual; then - if $CMP $expect_err $actual_ext; then - echo " PASSED" - else - echo "*FAILED*" - echo " Expected result (*.err) differs from actual result (*.oerr)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect_err $actual_ext |sed 's/^/ /' - fi + if $CMP $expect_err $actual_ext; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.err) differs from actual result (*.oerr)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect_err $actual_ext |sed 's/^/ /' + fi else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_sav $actual_err_sav + rm -f $actual $actual_err $actual_sav $actual_err_sav fi } @@ -815,8 +814,8 @@ TOOLTEST5() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $ENVCMD $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $ENVCMD $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err # save actual and actual_err in case they are needed later. @@ -835,33 +834,33 @@ TOOLTEST5() { $actual_err > $actual_ext if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.ddl) missing" + nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual; then - if $CMP $expect_err $actual_ext; then - echo " PASSED" - else - echo "*FAILED*" - echo " Expected result (*.err) differs from actual result (*.oerr)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect_err $actual_ext |sed 's/^/ /' - fi + if $CMP $expect_err $actual_ext; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.err) differs from actual result (*.oerr)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect_err $actual_ext |sed 's/^/ /' + fi else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_sav $actual_err_sav + rm -f $actual $actual_err $actual_sav $actual_err_sav fi - } + # ADD_HELP_TEST TOOLTEST_HELP() { @@ -873,27 +872,27 @@ TOOLTEST_HELP() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err if [ ! -f $expectdata ]; then - # Create the expect data file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect-CREATED - echo " Expected output (*.txt) missing" - nerrors="`expr $nerrors + 1`" + # Create the expect data file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect-CREATED + echo " Expected output (*.txt) missing" + nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual; then - echo " PASSED" + echo " PASSED" else - echo "*FAILED*" - echo " Expected output (*.txt) differs from actual output (*.out)" - nerrors="`expr $nerrors + 1`" + echo "*FAILED*" + echo " Expected output (*.txt) differs from actual output (*.out)" + nerrors="`expr $nerrors + 1`" fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err + rm -f $actual $actual_err fi } @@ -913,14 +912,16 @@ GREPTEST() # Run test. TESTING $DUMPER -p $@ ( - cd $TESTDIR - $ENVCMD $RUNSERIAL $DUMPER_BIN -p "$@" + cd $TESTDIR + $ENVCMD $RUNSERIAL $DUMPER_BIN -p "$@" ) >$actual 2>$actual_err + if [ "$txttype" = "ERRTXT" ]; then $GREP "$expectdata" $actual_err > /dev/null else $GREP "$expectdata" $actual > /dev/null fi + if [ $? -eq 0 ]; then echo " PASSED" else @@ -930,7 +931,7 @@ GREPTEST() # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err + rm -f $actual $actual_err fi } @@ -949,14 +950,16 @@ GREPTEST2() # Run test. TESTING $DUMPER -p $@ ( - cd $TESTDIR - $ENVCMD $RUNSERIAL $DUMPER_BIN -p "$@" + cd $TESTDIR + $ENVCMD $RUNSERIAL $DUMPER_BIN -p "$@" ) >$actual 2>$actual_err + if [ "$txttype" = "ERRTXT" ]; then $GREP "$expectdata" $actual_err > /dev/null else $GREP "$expectdata" $actual > /dev/null fi + if [ $? -eq 0 ]; then echo " PASSED" else @@ -966,21 +969,21 @@ GREPTEST2() # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err + rm -f $actual $actual_err fi } # Print a "SKIP" message SKIP() { - TESTING $DUMPER $@ + TESTING $DUMPER $@ echo " -SKIP-" } # Print a line-line message left justified in a field of 70 characters # PRINT_H5DIFF() { - SPACES=" " - echo " Running h5diff $* $SPACES" | cut -c1-70 | tr -d '\012' + SPACES=" " + echo " Running h5diff $* $SPACES" | cut -c1-70 | tr -d '\012' } @@ -990,10 +993,11 @@ DIFFTEST() { PRINT_H5DIFF $@ ( - cd $TESTDIR - $RUNSERIAL $H5DIFF_BIN "$@" -q + cd $TESTDIR + $RUNSERIAL $H5DIFF_BIN "$@" -q ) RET=$? + if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" @@ -1007,8 +1011,8 @@ DIFFTEST() # beginning with the word "Verifying". # PRINT_H5IMPORT() { - SPACES=" " - echo " Running h5import $* $SPACES" | cut -c1-70 | tr -d '\012' + SPACES=" " + echo " Running h5import $* $SPACES" | cut -c1-70 | tr -d '\012' } # Call the h5import tool @@ -1018,15 +1022,16 @@ IMPORTTEST() # remove the output hdf5 file if it exists hdf5_file="$TESTDIR/$5" if [ -f $hdf5_file ]; then - rm -f $hdf5_file + rm -f $hdf5_file fi PRINT_H5IMPORT $@ ( - cd $TESTDIR - $RUNSERIAL $H5IMPORT_BIN "$@" + cd $TESTDIR + $RUNSERIAL $H5IMPORT_BIN "$@" ) RET=$? + if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" @@ -1131,9 +1136,9 @@ TOOLTEST tcomp-4.ddl --enable-error-stack tcompound_complex.h5 TOOLTEST tcompound_complex2.ddl --enable-error-stack tcompound_complex2.h5 # tests for bitfields and opaque data types if test $WORDS_BIGENDIAN != "yes"; then -TOOLTEST tbitnopaque_le.ddl --enable-error-stack tbitnopaque.h5 + TOOLTEST tbitnopaque_le.ddl --enable-error-stack tbitnopaque.h5 else -TOOLTEST tbitnopaque_be.ddl --enable-error-stack tbitnopaque.h5 + TOOLTEST tbitnopaque_be.ddl --enable-error-stack tbitnopaque.h5 fi #test for the nested compound type @@ -1311,12 +1316,12 @@ TOOLTEST tallfilters.ddl --enable-error-stack -H -p -d all tfilters.h5 TOOLTEST tuserfilter.ddl --enable-error-stack -H -p -d myfilter tfilters.h5 if test $USE_FILTER_DEFLATE = "yes" ; then - # data read internal filters - TOOLTEST treadintfilter.ddl --enable-error-stack -d deflate -d shuffle -d fletcher32 -d nbit -d scaleoffset tfilters.h5 - if test $USE_FILTER_SZIP = "yes"; then - # data read - TOOLTEST treadfilter.ddl --enable-error-stack -d all -d szip tfilters.h5 - fi + # data read internal filters + TOOLTEST treadintfilter.ddl --enable-error-stack -d deflate -d shuffle -d fletcher32 -d nbit -d scaleoffset tfilters.h5 + if test $USE_FILTER_SZIP = "yes"; then + # data read + TOOLTEST treadfilter.ddl --enable-error-stack -d all -d szip tfilters.h5 + fi fi # test for displaying objects with very long names @@ -1373,9 +1378,9 @@ TOOLTEST tbin4.ddl --enable-error-stack -d double -b FILE -o out4.bin tbin # Clean up binary output files if test -z "$HDF5_NOCLEANUP"; then - rm -f out[1-4].bin - rm -f out1.h5 - rm -f out3.h5 + rm -f out[1-4].bin + rm -f out1.h5 + rm -f out3.h5 fi # test for dataset region references @@ -1387,7 +1392,7 @@ TOOLTEST2 tbinregR.exp --enable-error-stack -d /Dataset1 -s 0 -R -y -o tbinregR. # Clean up text output files if test -z "$HDF5_NOCLEANUP"; then - rm -f tbinregR.txt + rm -f tbinregR.txt fi # tests for group creation order diff --git a/tools/test/h5dump/testh5dumppbits.sh.in b/tools/test/h5dump/testh5dumppbits.sh.in index febce2c..ff0659a 100644 --- a/tools/test/h5dump/testh5dumppbits.sh.in +++ b/tools/test/h5dump/testh5dumppbits.sh.in @@ -236,26 +236,25 @@ TOOLTEST() { cp $actual_err $actual_err_sav STDERR_FILTER $actual_err - if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.ddl) missing" + nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual; then - echo " PASSED" + echo " PASSED" else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_sav $actual_err_sav $actual_ext + rm -f $actual $actual_err $actual_sav $actual_err_sav $actual_ext fi - } @@ -274,36 +273,36 @@ TOOLTEST2() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" - elif $CMP $expect $actual; then - if [ ! -f $expectdata ]; then - # Create the expect data file if it doesn't yet exist. + # Create the expect file if it doesn't yet exist. echo " CREATED" - cp $actualdata $expectdata - echo " Expected data (*.exp) missing" + cp $actual $expect + echo " Expected result (*.ddl) missing" nerrors="`expr $nerrors + 1`" - elif $CMP $expectdata $actualdata; then - echo " PASSED" - else + elif $CMP $expect $actual; then + if [ ! -f $expectdata ]; then + # Create the expect data file if it doesn't yet exist. + echo " CREATED" + cp $actualdata $expectdata + echo " Expected data (*.exp) missing" + nerrors="`expr $nerrors + 1`" + elif $CMP $expectdata $actualdata; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected datafile (*.exp) differs from actual datafile (*.txt)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expectdata $actualdata |sed 's/^/ /' + fi + else echo "*FAILED*" - echo " Expected datafile (*.exp) differs from actual datafile (*.txt)" + echo " Expected result (*.ddl) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expectdata $actualdata |sed 's/^/ /' - fi - else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file @@ -328,8 +327,8 @@ TOOLTEST3() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err # save actual and actual_err in case they are needed later. @@ -348,23 +347,23 @@ TOOLTEST3() { $actual_err > $actual_ext if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.ddl) missing" + nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual; then - echo " PASSED" + echo " PASSED" else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_sav $actual_err_sav + rm -f $actual $actual_err $actual_sav $actual_err_sav fi } @@ -385,8 +384,8 @@ TOOLTEST4() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err # save actual and actual_err in case they are needed later. @@ -405,30 +404,30 @@ TOOLTEST4() { $actual_err > $actual_ext if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.ddl) missing" + nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual; then - if $CMP $expect_err $actual_ext; then - echo " PASSED" - else - echo "*FAILED*" - echo " Expected result (*.err) differs from actual result (*.oerr)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect_err $actual_ext |sed 's/^/ /' - fi + if $CMP $expect_err $actual_ext; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.err) differs from actual result (*.oerr)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect_err $actual_ext |sed 's/^/ /' + fi else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_sav $actual_err_sav + rm -f $actual $actual_err $actual_sav $actual_err_sav fi } @@ -442,8 +441,8 @@ SKIP() { # Print a line-line message left justified in a field of 70 characters # PRINT_H5DIFF() { - SPACES=" " - echo " Running h5diff $* $SPACES" | cut -c1-70 | tr -d '\012' + SPACES=" " + echo " Running h5diff $* $SPACES" | cut -c1-70 | tr -d '\012' } @@ -453,10 +452,11 @@ DIFFTEST() { PRINT_H5DIFF $@ ( - cd $TESTDIR - $RUNSERIAL $H5DIFF_BIN "$@" -q + cd $TESTDIR + $RUNSERIAL $H5DIFF_BIN "$@" -q ) RET=$? + if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" @@ -470,8 +470,8 @@ DIFFTEST() # beginning with the word "Verifying". # PRINT_H5IMPORT() { - SPACES=" " - echo " Running h5import $* $SPACES" | cut -c1-70 | tr -d '\012' + SPACES=" " + echo " Running h5import $* $SPACES" | cut -c1-70 | tr -d '\012' } # Call the h5import tool @@ -481,22 +481,22 @@ IMPORTTEST() # remove the output hdf5 file if it exists hdf5_file="$TESTDIR/$5" if [ -f $hdf5_file ]; then - rm -f $hdf5_file + rm -f $hdf5_file fi PRINT_H5IMPORT $@ ( - cd $TESTDIR - $RUNSERIAL $H5IMPORT_BIN "$@" + cd $TESTDIR + $RUNSERIAL $H5IMPORT_BIN "$@" ) RET=$? + if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" fi - } diff --git a/tools/test/h5dump/testh5dumpvds.sh.in b/tools/test/h5dump/testh5dumpvds.sh.in index f89234e..29ff238 100644 --- a/tools/test/h5dump/testh5dumpvds.sh.in +++ b/tools/test/h5dump/testh5dumpvds.sh.in @@ -205,8 +205,8 @@ TOOLTEST() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err # save actual and actual_err in case they are needed later. @@ -216,26 +216,25 @@ TOOLTEST() { STDERR_FILTER $actual_err cat $actual_err >> $actual - if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.ddl) missing" + nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual; then - echo " PASSED" + echo " PASSED" else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_sav $actual_err_sav $actual_ext + rm -f $actual $actual_err $actual_sav $actual_err_sav $actual_ext fi - } @@ -254,42 +253,42 @@ TOOLTEST2() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err cat $actual_err >> $actual if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" - elif $CMP $expect $actual; then - if [ ! -f $expectdata ]; then - # Create the expect data file if it doesn't yet exist. + # Create the expect file if it doesn't yet exist. echo " CREATED" - cp $actualdata $expectdata - echo " Expected data (*.exp) missing" + cp $actual $expect + echo " Expected result (*.ddl) missing" nerrors="`expr $nerrors + 1`" - elif $CMP $expectdata $actualdata; then - echo " PASSED" - else + elif $CMP $expect $actual; then + if [ ! -f $expectdata ]; then + # Create the expect data file if it doesn't yet exist. + echo " CREATED" + cp $actualdata $expectdata + echo " Expected data (*.exp) missing" + nerrors="`expr $nerrors + 1`" + elif $CMP $expectdata $actualdata; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected datafile (*.exp) differs from actual datafile (*.txt)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expectdata $actualdata |sed 's/^/ /' + fi + else echo "*FAILED*" - echo " Expected datafile (*.exp) differs from actual datafile (*.txt)" + echo " Expected result (*.ddl) differs from actual result (*.out)" nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expectdata $actualdata |sed 's/^/ /' - fi - else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actualdata $actual_err + rm -f $actual $actualdata $actual_err fi } @@ -309,8 +308,8 @@ TOOLTEST3() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err # save actual and actual_err in case they are needed later. @@ -326,27 +325,27 @@ TOOLTEST3() { -e 's/[1-9]*\.[0-9]*\.[0-9]*[^)]*/version (number)/' \ -e 's/H5Eget_auto[1-2]*/H5Eget_auto(1 or 2)/' \ -e 's/H5Eset_auto[1-2]*/H5Eset_auto(1 or 2)/' \ - $actual_err > $actual_ext + $actual_err > $actual_ext cat $actual_ext >> $actual if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ddl) missing" - nerrors="`expr $nerrors + 1`" + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.ddl) missing" + nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual; then - echo " PASSED" + echo " PASSED" else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_sav $actual_err_sav + rm -f $actual $actual_err $actual_sav $actual_err_sav fi } @@ -367,8 +366,8 @@ TOOLTEST4() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err # save actual and actual_err in case they are needed later. @@ -384,47 +383,46 @@ TOOLTEST4() { -e 's/[1-9]*\.[0-9]*\.[0-9]*[^)]*/version (number)/' \ -e 's/H5Eget_auto[1-2]*/H5Eget_auto(1 or 2)/' \ -e 's/H5Eset_auto[1-2]*/H5Eset_auto(1 or 2)/' \ - $actual_err > $actual_ext + $actual_err > $actual_ext #cat $actual_ext >> $actual if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect elif $CMP $expect $actual; then - if $CMP $expect_err $actual_ext; then - echo " PASSED" - else - echo "*FAILED*" - echo " Expected result (*.err) differs from actual result (*.oerr)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect_err $actual_ext |sed 's/^/ /' - fi + if $CMP $expect_err $actual_ext; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.err) differs from actual result (*.oerr)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect_err $actual_ext |sed 's/^/ /' + fi else - echo "*FAILED*" - echo " Expected result (*.ddl) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err $actual_sav $actual_err_sav + rm -f $actual $actual_err $actual_sav $actual_err_sav fi - } # Print a "SKIP" message SKIP() { - TESTING $DUMPER $@ + TESTING $DUMPER $@ echo " -SKIP-" } # Print a line-line message left justified in a field of 70 characters # PRINT_H5DIFF() { - SPACES=" " - echo " Running h5diff $* $SPACES" | cut -c1-70 | tr -d '\012' + SPACES=" " + echo " Running h5diff $* $SPACES" | cut -c1-70 | tr -d '\012' } @@ -434,25 +432,25 @@ DIFFTEST() { PRINT_H5DIFF $@ ( - cd $TESTDIR - $RUNSERIAL $H5DIFF_BIN "$@" -q + cd $TESTDIR + $RUNSERIAL $H5DIFF_BIN "$@" -q ) RET=$? + if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" fi - } # Print a line-line message left justified in a field of 70 characters # beginning with the word "Verifying". # PRINT_H5IMPORT() { - SPACES=" " - echo " Running h5import $* $SPACES" | cut -c1-70 | tr -d '\012' + SPACES=" " + echo " Running h5import $* $SPACES" | cut -c1-70 | tr -d '\012' } # Call the h5import tool @@ -462,22 +460,22 @@ IMPORTTEST() # remove the output hdf5 file if it exists hdf5_file="$TESTDIR/$5" if [ -f $hdf5_file ]; then - rm -f $hdf5_file + rm -f $hdf5_file fi PRINT_H5IMPORT $@ ( - cd $TESTDIR - $RUNSERIAL $H5IMPORT_BIN "$@" + cd $TESTDIR + $RUNSERIAL $H5IMPORT_BIN "$@" ) RET=$? + if [ $RET != 0 ] ; then echo "*FAILED*" nerrors="`expr $nerrors + 1`" else echo " PASSED" fi - } @@ -491,7 +489,7 @@ COPY_TESTFILES_TO_TESTDIR ####### test for dataset vds ###### - # Data read +# Data read if test $USE_FILTER_DEFLATE = "yes" ; then TOOLTEST tvds-1.ddl --enable-error-stack 1_vds.h5 TOOLTEST tvds-2.ddl --enable-error-stack 2_vds.h5 @@ -504,7 +502,7 @@ if test $USE_FILTER_DEFLATE = "yes" ; then TOOLTEST vds-gap2.ddl --vds-gap-size=2 --enable-error-stack vds-eiger.h5 fi - # Layout read +# Layout read if test $USE_FILTER_DEFLATE = "yes" ; then TOOLTEST tvds_layout-1.ddl -p --enable-error-stack 1_vds.h5 TOOLTEST tvds_layout-2.ddl -p --enable-error-stack 2_vds.h5 diff --git a/tools/test/h5dump/testh5dumpxml.sh.in b/tools/test/h5dump/testh5dumpxml.sh.in index f7af300..52a13c2 100644 --- a/tools/test/h5dump/testh5dumpxml.sh.in +++ b/tools/test/h5dump/testh5dumpxml.sh.in @@ -262,36 +262,35 @@ TOOLTEST() { # Run test. TESTING $DUMPER $@ ( - cd $TESTDIR - $RUNSERIAL $DUMPER_BIN "$@" + cd $TESTDIR + $RUNSERIAL $DUMPER_BIN "$@" ) >$actual 2>$actual_err - if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.xml) missing" - nerrors="`expr $nerrors + 1`" + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.xml) missing" + nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual; then - echo " PASSED" + echo " PASSED" else - echo "*FAILED*" - echo " Expected result (*.xml) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result (*.xml) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err + rm -f $actual $actual_err fi } # Print a "SKIP" message SKIP() { - TESTING $DUMPER $@ - echo " -SKIP-" + TESTING $DUMPER $@ + echo " -SKIP-" } diff --git a/tools/test/h5ls/h5ls_plugin.sh.in b/tools/test/h5ls/h5ls_plugin.sh.in index c89269d..28370e6 100644 --- a/tools/test/h5ls/h5ls_plugin.sh.in +++ b/tools/test/h5ls/h5ls_plugin.sh.in @@ -139,8 +139,8 @@ CLEAN_TESTFILES_AND_TESTDIR() # Print a $* message left justified in a field of 70 characters # MESSAGE() { - SPACES=" " - echo "$* $SPACES" | cut -c1-70 | tr -d '\012' + SPACES=" " + echo "$* $SPACES" | cut -c1-70 | tr -d '\012' } # Print a line-line message left justified in a field of 70 characters @@ -206,7 +206,7 @@ TOOLTEST() { echo "" fi elif [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. + # Create the expect file if it doesn't yet exist. echo " CREATED" cp $actual $expect echo " Expected result (*.ls) missing" diff --git a/tools/test/h5repack/h5repacktst.c b/tools/test/h5repack/h5repacktst.c index 911572e..5cd76ef 100644 --- a/tools/test/h5repack/h5repacktst.c +++ b/tools/test/h5repack/h5repacktst.c @@ -3872,7 +3872,7 @@ out: * * Purpose: write datasets in LOC_ID * -* Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu +* Programmer: Pedro Vicente * * Date: November 12, 2003 * @@ -4736,7 +4736,7 @@ out: * * Purpose: write attributes in LOC_ID (dataset, group, named datatype) * -* Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu +* Programmer: Pedro Vicente * * Date: November 12, 2003 * @@ -5927,7 +5927,7 @@ out: * * Purpose: utility function to create and write a dataset in LOC_ID * -* Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu +* Programmer: Pedro Vicente * * Date: November 12, 2003 * @@ -5963,7 +5963,7 @@ out: * * Purpose: utility function to create and write a dataset in LOC_ID * -* Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu +* Programmer: Pedro Vicente * * Date: November 12, 2003 * @@ -6011,7 +6011,7 @@ out: * * Purpose: utility function to write an attribute in LOC_ID * -* Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu +* Programmer: Pedro Vicente * * Date: November 12, 2003 * diff --git a/tools/test/misc/h5perf_gentest.c b/tools/test/misc/h5perf_gentest.c index f50e5fb..511a9e2 100644 --- a/tools/test/misc/h5perf_gentest.c +++ b/tools/test/misc/h5perf_gentest.c @@ -14,7 +14,7 @@ creates a large number of attributes, groups, and datasets by specifying -a, -g, -d options respectively. Using "-h" option to see details. - Programmer: Peter Cao <xcao@hdfgroup.org>, Jan. 2013 + Programmer: Peter Cao, Jan. 2013 ****************************************************************************/ #include "hdf5.h" @@ -128,7 +128,7 @@ int main (int argc, char *argv[]) Return: Non-negative on success/Negative on failure - Programmer: Peter Cao <xcao@hdfgroup.org>, Jan. 2013 + Programmer: Peter Cao, Jan. 2013 ****************************************************************************/ herr_t create_perf_test_file(const char *fname, int ngrps, int ndsets, int nattrs, hsize_t nrows, hsize_t dim0, hsize_t chunk, int vlen, diff --git a/tools/test/misc/h5repart_gentest.c b/tools/test/misc/h5repart_gentest.c index bd94104..520069f 100644 --- a/tools/test/misc/h5repart_gentest.c +++ b/tools/test/misc/h5repart_gentest.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu<slu@ncsa.uiuc.edu> + * Programmer: Raymond Lu * June 1, 2005 * * Purpose: Generate a family file of 1024 bytes for each member diff --git a/tools/test/misc/testh5mkgrp.sh.in b/tools/test/misc/testh5mkgrp.sh.in index 3ad1f71..7dc8155 100644 --- a/tools/test/misc/testh5mkgrp.sh.in +++ b/tools/test/misc/testh5mkgrp.sh.in @@ -13,7 +13,7 @@ # # Tests for the h5mkgrp tool # -# Quincey Koziol (koziol@hdfgroup.org) +# Quincey Koziol # Tuesday, February 13, 2007 # @@ -150,10 +150,11 @@ TOOLTEST() { TESTING $H5MKGRP $@ ( - cd $TESTDIR - $RUNSERIAL $H5MKGRP_BIN $@ + cd $TESTDIR + $RUNSERIAL $H5MKGRP_BIN $@ ) > output.out RET=$? + if [ $RET != 0 ]; then echo "*FAILED*" echo "failed result is:" @@ -164,7 +165,7 @@ TOOLTEST() # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f output.out + rm -f output.out fi fi } @@ -181,8 +182,8 @@ H5LSTEST() # any unexpected output from that stream too. VERIFY_H5LS $@ ( - cd $TESTDIR - $RUNSERIAL $H5LS_BIN $H5LS_ARGS $@ + cd $TESTDIR + $RUNSERIAL $H5LS_BIN $H5LS_ARGS $@ ) 2>&1 |sed 's/Modified:.*/Modified: XXXX-XX-XX XX:XX:XX XXX/' >$actual # save actual in case it is needed later. @@ -190,25 +191,25 @@ H5LSTEST() STDOUT_FILTER $actual STDERR_FILTER $actual - if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.ls) missing" - nerrors="`expr $nerrors + 1`" - elif $CMP $expect $actual; then - echo " PASSED" - else - echo "*FAILED*" - echo " Expected result (*.ls) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' - fi - - # Clean up output file - if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_sav - fi + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.ls) missing" + nerrors="`expr $nerrors + 1`" + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ls) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_sav + fi } # Single run of tool @@ -258,30 +259,30 @@ CMPTEST() # any unexpected output from that stream too. TESTING $H5MKGRP $@ ( - cd $TESTDIR - $RUNSERIAL $H5MKGRP_BIN $@ + cd $TESTDIR + $RUNSERIAL $H5MKGRP_BIN $@ ) >$actual 2>$actual_err cat $actual_err >> $actual - if [ ! -f $expect ]; then - # Create the expect file if it doesn't yet exist. - echo " CREATED" - cp $actual $expect - echo " Expected result (*.txt) missing" - nerrors="`expr $nerrors + 1`" - elif $CMP $expect $actual; then - echo " PASSED" - else - echo "*FAILED*" - echo " Expected result (*.txt) differs from actual result (*.out)" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' - fi - - # Clean up output file - if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err - fi + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + echo " Expected result (*.txt) missing" + nerrors="`expr $nerrors + 1`" + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.txt) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actual $actual_err + fi } ############################################################################## diff --git a/tools/test/perform/CMakeLists.txt b/tools/test/perform/CMakeLists.txt index c05698e..3c45e85 100644 --- a/tools/test/perform/CMakeLists.txt +++ b/tools/test/perform/CMakeLists.txt @@ -73,6 +73,21 @@ else () endif () set_target_properties (iopipe PROPERTIES FOLDER perform) +#-- Adding test for chunk_cache +set (chunk_cache_SOURCES + ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/chunk_cache.c +) +add_executable (chunk_cache ${chunk_cache_SOURCES}) +target_include_directories (chunk_cache PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") +if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (chunk_cache STATIC) + target_link_libraries (chunk_cache PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) +else () + TARGET_C_PROPERTIES (chunk_cache SHARED) + target_link_libraries (chunk_cache PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) +endif () +set_target_properties (chunk_cache PROPERTIES FOLDER perform) + #-- Adding test for overhead set (overhead_SOURCES ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/overhead.c diff --git a/tools/test/perform/Makefile.am b/tools/test/perform/Makefile.am index 5a89a66..39800d7 100644 --- a/tools/test/perform/Makefile.am +++ b/tools/test/perform/Makefile.am @@ -50,12 +50,12 @@ if BUILD_PARALLEL_CONDITIONAL TEST_PROG_PARA=h5perf perf endif # Serial test programs. -TEST_PROG = iopipe chunk overhead zip_perf perf_meta h5perf_serial $(BUILD_ALL_PROGS) +TEST_PROG = iopipe chunk chunk_cache overhead zip_perf perf_meta h5perf_serial $(BUILD_ALL_PROGS) # check_PROGRAMS will be built but not installed. Do not any executable # that is in bin_PROGRAMS already. Otherwise, it will be removed twice in # "make clean" and some systems, e.g., AIX, do not like it. -check_PROGRAMS= iopipe chunk overhead zip_perf perf_meta $(BUILD_ALL_PROGS) perf +check_PROGRAMS= iopipe chunk chunk_cache overhead zip_perf perf_meta $(BUILD_ALL_PROGS) perf h5perf_SOURCES=pio_perf.c pio_engine.c h5perf_serial_SOURCES=sio_perf.c sio_engine.c diff --git a/tools/test/perform/chunk_cache.c b/tools/test/perform/chunk_cache.c new file mode 100644 index 0000000..d7c56af --- /dev/null +++ b/tools/test/perform/chunk_cache.c @@ -0,0 +1,394 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: check the performance of chunk cache in these two cases (HDFFV-10601): + * 1. partial chunks exist along any dimension. + * 2. number of slots in chunk cache is smaller than the number of chunks + * in the fastest-growing dimension. + */ +#include "hdf5.h" +#include "H5private.h" +#include "h5test.h" + +#define FILENAME "chunk_cache_perf.h5" + +#define RANK 2 + +#define DSET1_NAME "partial_chunks" +#define DSET1_DIM1 9 * 1000 +#define DSET1_DIM2 9 +#define CHUNK1_DIM1 2 * 1000 +#define CHUNK1_DIM2 2 + +#define DSET2_NAME "hash_value" +#define DSET2_DIM1 300 +#define DSET2_DIM2 600 +#define CHUNK2_DIM1 100 +#define CHUNK2_DIM2 100 + +#define RDCC_NSLOTS 5 +#define RDCC_NBYTES 1024 * 1024 * 10 +#define RDCC_W0 0.75F + +#define FILTER_COUNTER 306 +static size_t nbytes_global; + +typedef struct test_time_t { + long tv_sec; + long tv_usec; +} test_time_t; + +/* Local function prototypes for the dummy filter */ +static size_t +counter (unsigned flags, size_t cd_nelmts, + const unsigned *cd_values, size_t nbytes, + size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class2_t H5Z_COUNTER[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + FILTER_COUNTER, /* Filter id number */ + 1, 1, /* Encoding and decoding enabled */ + "counter", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + counter, /* The actual filter function */ +}}; + +/*------------------------------------------------------------------------- + * Count number of bytes but don't do anything else. Keep + * track of the data of chunks being read from file into memory. + */ +static size_t +counter (unsigned H5_ATTR_UNUSED flags, size_t H5_ATTR_UNUSED cd_nelmts, + const unsigned H5_ATTR_UNUSED *cd_values, size_t nbytes, + size_t H5_ATTR_UNUSED *buf_size, void H5_ATTR_UNUSED **buf) +{ + nbytes_global += nbytes; + return nbytes; +} + +/*---------------------------------------------------------------------------*/ +static void +cleanup (void) +{ + if (!getenv ("HDF5_NOCLEANUP")) { + remove (FILENAME); + } +} + +/*------------------------------------------------------------------------------- + * Create a chunked dataset with partial chunks along either dimensions: + * dataset dimension: 9000 x 9 + * chunk dimension: 2000 x 2 + */ +static int create_dset1(hid_t file) +{ + hid_t dataspace = H5I_INVALID_HID, dataset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hsize_t dims[RANK] = {DSET1_DIM1, DSET1_DIM2}; + hsize_t chunk_dims[RANK] = {CHUNK1_DIM1, CHUNK1_DIM2}; + int **data; /* data for writing */ + + /* Create the data space. */ + if((dataspace = H5Screate_simple (RANK, dims, NULL)) < 0) + goto error; + + /* Modify dataset creation properties, i.e. enable chunking */ + if((dcpl = H5Pcreate (H5P_DATASET_CREATE)) < 0) + goto error; + if(H5Pset_chunk (dcpl, RANK, chunk_dims) < 0) + goto error; + + /* Set the dummy filter simply for counting the number of bytes being read into the memory */ + if(H5Zregister(H5Z_COUNTER) < 0) + goto error; + + if(H5Pset_filter(dcpl, FILTER_COUNTER, 0, 0, NULL) < 0) + goto error; + + /* Create a new dataset within the file using chunk creation properties. */ + if((dataset = H5Dcreate2 (file, DSET1_NAME, H5T_NATIVE_INT, dataspace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Create & fill array */ + H5TEST_ALLOCATE_2D_ARRAY(data, int, DSET1_DIM1, DSET1_DIM2); + H5TEST_FILL_2D_ARRAY(data, int, DSET1_DIM1, DSET1_DIM2); + + + /* Write data to dataset */ + if(H5Dwrite (dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, + H5P_DEFAULT, data) < 0) + goto error; + + /* Close resources */ + H5Dclose (dataset); + H5Pclose (dcpl); + H5Sclose (dataspace); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose (dataset); + H5Pclose (dcpl); + H5Sclose (dataspace); + } H5E_END_TRY; + + return 1; +} + +/*--------------------------------------------------------------------------- + * Create a chunked dataset for testing hash values: + * dataset dimensions: 300 x 600 + * chunk dimensions: 100 x 100 + */ +static int create_dset2(hid_t file) +{ + hid_t dataspace = H5I_INVALID_HID, dataset = H5I_INVALID_HID; + hid_t dcpl = H5I_INVALID_HID; + hsize_t dims[RANK] = {DSET2_DIM1, DSET2_DIM2}; + hsize_t chunk_dims[RANK] = {CHUNK2_DIM1, CHUNK2_DIM2}; + int **data; /* data for writing */ + + /* Create the data space. */ + if((dataspace = H5Screate_simple(RANK, dims, NULL)) < 0) + goto error; + + /* Modify dataset creation properties, i.e. enable chunking */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + if(H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) + goto error; + + /* Set the dummy filter simply for counting the number of bytes being read into the memory */ + if(H5Zregister(H5Z_COUNTER) < 0) + goto error; + if(H5Pset_filter(dcpl, FILTER_COUNTER, 0, 0, NULL) < 0) + goto error; + + /* Create a new dataset within the file using chunk creation properties. */ + if((dataset = H5Dcreate2(file, DSET2_NAME, H5T_NATIVE_INT, dataspace, + H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Create & fill array */ + H5TEST_ALLOCATE_2D_ARRAY(data, int, DSET2_DIM1, DSET2_DIM2); + H5TEST_FILL_2D_ARRAY(data, int, DSET2_DIM1, DSET2_DIM2); + + /* Write data to dataset */ + if(H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data) < 0) + goto error; + + /* Close resources */ + H5Dclose(dataset); + H5Pclose(dcpl); + H5Sclose(dataspace); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dataset); + H5Pclose(dcpl); + H5Sclose(dataspace); + } H5E_END_TRY; + + return 1; +} + +/*--------------------------------------------------------------------------- + * Check the performance of the chunk cache when partial chunks exist + * along the dataset dimensions. + */ +static int check_partial_chunks_perf(hid_t file) +{ + hid_t dataset = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; + hid_t memspace = H5I_INVALID_HID; + hid_t dapl = H5I_INVALID_HID; + + int rdata[DSET1_DIM2]; /* data for reading */ + int i; + + hsize_t row_rank = 1; + hsize_t row_dim[1] = {DSET1_DIM2}; + hsize_t start[RANK] = {0, 0}; + hsize_t count[RANK] = {1, DSET1_DIM2}; + double start_t, end_t; + + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + goto error; + if(H5Pset_chunk_cache (dapl, RDCC_NSLOTS, RDCC_NBYTES, RDCC_W0) < 0) + goto error; + + dataset = H5Dopen2 (file, DSET1_NAME, dapl); + + H5_CHECK_OVERFLOW(row_rank, hsize_t, int); + memspace = H5Screate_simple((int)row_rank, row_dim, NULL); + filespace = H5Dget_space(dataset); + + nbytes_global = 0; + + start_t = H5_get_time(); + + /* Read the data row by row */ + for(i = 0; i < DSET1_DIM1; i++) { + start[0] = (hsize_t)i; + if(H5Sselect_hyperslab(filespace, H5S_SELECT_SET, + start, NULL, count, NULL) < 0) + goto error; + + if(H5Dread (dataset, H5T_NATIVE_INT, memspace, filespace, + H5P_DEFAULT, rdata) < 0) + goto error; + } + + end_t = H5_get_time(); + + if((end_t - start_t) > (double)0.0f) + printf("1. Partial chunks: total read time is %lf; number of bytes being read from file is %lu\n", (end_t -start_t), nbytes_global); + else + printf("1. Partial chunks: no total read time because timer is not available; number of bytes being read from file is %lu\n", nbytes_global); + + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Pclose (dapl); + + return 0; +error: + H5E_BEGIN_TRY { + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Pclose (dapl); + } H5E_END_TRY; + return 1; +} + +/*--------------------------------------------------------------------------- + * Check the performance of chunk cache when the number of cache slots + * is smaller than the number of chunks along the fastest-growing + * dimension of the dataset. + */ +static int check_hash_value_perf(hid_t file) +{ + hid_t dataset = H5I_INVALID_HID; + hid_t filespace = H5I_INVALID_HID; + hid_t memspace = H5I_INVALID_HID; + hid_t dapl = H5I_INVALID_HID; + + int rdata[DSET2_DIM1]; /* data for reading */ + int i; + + hsize_t column_rank = 1; + hsize_t column_dim[1] = {DSET2_DIM1}; + hsize_t start[RANK] = {0, 0}; + hsize_t count[RANK] = {DSET2_DIM1, 1}; + double start_t, end_t; + + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + goto error; + if(H5Pset_chunk_cache (dapl, RDCC_NSLOTS, RDCC_NBYTES, RDCC_W0) < 0) + goto error; + + if((dataset = H5Dopen2 (file, DSET2_NAME, dapl)) < 0) + goto error; + + H5_CHECK_OVERFLOW(column_rank, hsize_t, int); + if((memspace = H5Screate_simple((int)column_rank, column_dim, NULL)) < 0) + goto error; + if((filespace = H5Dget_space(dataset)) < 0) + goto error; + + nbytes_global = 0; + + start_t = H5_get_time(); + + /* Read the data column by column */ + for(i = 0; i < DSET2_DIM2; i++) { + start[1] = (hsize_t)i; + if(H5Sselect_hyperslab(filespace, H5S_SELECT_SET, + start, NULL, count, NULL) < 0) + goto error; + + if(H5Dread (dataset, H5T_NATIVE_INT, memspace, filespace, + H5P_DEFAULT, rdata) < 0) + goto error; + } + + end_t = H5_get_time(); + + if((end_t - start_t) > (double)0.0f) + printf("2. Hash value: total read time is %lf; number of bytes being read from file is %lu\n", (end_t -start_t), nbytes_global); + else + printf("2. Hash value: no total read time because timer is not available; number of bytes being read from file is %lu\n", nbytes_global); + + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Pclose (dapl); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose (dataset); + H5Sclose (filespace); + H5Sclose (memspace); + H5Pclose (dapl); + } H5E_END_TRY; + return 1; +} + +/*------------------------------------------------------------------------------------- + * Purpose: check the performance of chunk cache in these two cases (HDFFV-10601): + * 1. partial chunks exist along any dimension. + * 2. number of slots in chunk cache is smaller than the number of chunks + * in the fastest-growing dimension. + *-------------------------------------------------------------------------------------*/ +int +main (void) +{ + hid_t file; /* handles */ + int nerrors = 0; + + /* Create a new file. If file exists its contents will be overwritten. */ + if((file = H5Fcreate (FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + nerrors += create_dset1(file); + nerrors += create_dset2(file); + + if(H5Fclose (file) < 0) + goto error; + + /* Re-open the file for testing performance. */ + if((file = H5Fopen (FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + goto error; + + nerrors += check_partial_chunks_perf(file); + nerrors += check_hash_value_perf(file); + + if(H5Fclose (file) < 0) + goto error; + + if (nerrors>0) goto error; + cleanup(); + return 0; + +error: + fprintf(stderr, "*** ERRORS DETECTED ***\n"); + return 1; +} diff --git a/tools/test/perform/iopipe.c b/tools/test/perform/iopipe.c index bf4728d..bb74a1b 100644 --- a/tools/test/perform/iopipe.c +++ b/tools/test/perform/iopipe.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Thursday, March 12, 1998 */ @@ -21,10 +21,6 @@ #include "H5private.h" -#ifdef H5_HAVE_SYS_TIMEB -#include <sys/timeb.h> -#endif - #define RAW_FILE_NAME "iopipe.raw" #define HDF5_FILE_NAME "iopipe.h5" @@ -56,25 +52,18 @@ * Programmer: Robb Matzke * Thursday, March 12, 1998 * - * Modifications: - * *------------------------------------------------------------------------- */ -#ifdef H5_HAVE_GETRUSAGE static void -print_stats (const char *prefix, +print_stats(const char *prefix, +#ifdef H5_HAVE_GETRUSAGE struct rusage *r_start, struct rusage *r_stop, - struct timeval *t_start, struct timeval *t_stop, - size_t nbytes) -#else /* H5_HAVE_GETRUSAGE */ -static void -print_stats (const char *prefix, - struct timeval *r_start, struct timeval *r_stop, - struct timeval *t_start, struct timeval *t_stop, - size_t nbytes) #endif /* H5_HAVE_GETRUSAGE */ + double t_start, double t_stop, + size_t nbytes) { - double e_time, bw; + double e_time; + char bw[16]; #ifdef H5_HAVE_GETRUSAGE double u_time, s_time; @@ -88,27 +77,16 @@ print_stats (const char *prefix, ((double)(r_start->ru_stime.tv_sec)+ (double)(r_start->ru_stime.tv_usec)/(double)1000000.0F); #endif -#ifndef H5_HAVE_SYS_TIMEB - e_time = ((double)(t_stop->tv_sec)+ - (double)(t_stop->tv_usec)/(double)1000000.0F) - - ((double)(t_start->tv_sec)+ - (double)(t_start->tv_usec)/(double)1000000.0F); -#else - e_time = ((double)(t_stop->tv_sec)+ - (double)(t_stop->tv_usec)/(double)1000.0F) - - ((double)(t_start->tv_sec)+ - (double)(t_start->tv_usec)/(double)1000.0F); -#endif - bw = (double)nbytes / e_time; + e_time = t_stop - t_start; + H5_bandwidth(bw, (double)nbytes, e_time); #ifdef H5_HAVE_GETRUSAGE - printf (HEADING "%1.2fuser %1.2fsystem %1.2felapsed %1.2fMB/s\n", - prefix, u_time, s_time, e_time, bw/(1024*1024)); + HDprintf(HEADING "%1.2fuser %1.2fsystem %1.2felapsed %s\n", + prefix, u_time, s_time, e_time, bw); #else - printf (HEADING "%1.2felapsed %1.2fMB/s\n", - prefix, e_time, bw/(1024*1024)); + HDprintf(HEADING "%1.2felapsed %s\n", + prefix, e_time, bw); #endif - } @@ -122,12 +100,10 @@ print_stats (const char *prefix, * Programmer: Robb Matzke * Thursday, March 12, 1998 * - * Modifications: - * *------------------------------------------------------------------------- */ static void -synchronize (void) +synchronize(void) { #ifdef H5_HAVE_SYSTEM #if defined(H5_HAVE_WIN32_API) && ! defined(__CYGWIN__) @@ -157,24 +133,20 @@ synchronize (void) * Programmer: Robb Matzke * Thursday, March 12, 1998 * - * Modifications: - * *------------------------------------------------------------------------- */ int -main (void) +main(void) { - static hsize_t size[2] = {REQUEST_SIZE_X, REQUEST_SIZE_Y}; - static unsigned nread = NREAD_REQUESTS, nwrite = NWRITE_REQUESTS; + hsize_t size[2] = {REQUEST_SIZE_X, REQUEST_SIZE_Y}; + unsigned nread = NREAD_REQUESTS, nwrite = NWRITE_REQUESTS; unsigned char *the_data = NULL; hid_t file, dset, file_space = H5I_INVALID_HID; #ifdef H5_HAVE_GETRUSAGE struct rusage r_start, r_stop; -#else - struct timeval r_start, r_stop; #endif - struct timeval t_start, t_stop; + double t_start, t_stop; int fd; unsigned u; herr_t H5_ATTR_NDEBUG_UNUSED status; @@ -184,322 +156,221 @@ main (void) hsize_t count[2]; -#ifdef H5_HAVE_SYS_TIMEB - struct _timeb *tbstart = malloc(sizeof(struct _timeb)); - struct _timeb *tbstop = malloc(sizeof(struct _timeb)); -#endif /* * The extra cast in the following statement is a bug workaround for the * Win32 version 5.0 compiler. * 1998-11-06 ptl */ - printf ("I/O request size is %1.1fMB\n", - (double)(hssize_t)(size[0]*size[1])/(double)1024.0F*(double)1024); + HDprintf("I/O request size is %1.1fMB\n", + (double)(hssize_t)(size[0] * size[1]) / (double)1024.0F * (double)1024); /* Open the files */ - file = H5Fcreate (HDF5_FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - HDassert (file>=0); - fd = HDopen (RAW_FILE_NAME, O_RDWR|O_CREAT|O_TRUNC, 0666); - HDassert (fd>=0); + file = H5Fcreate(HDF5_FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + HDassert(file >= 0); + fd = HDopen(RAW_FILE_NAME, O_RDWR|O_CREAT|O_TRUNC, 0666); + HDassert(fd >= 0); /* Create the dataset */ - file_space = H5Screate_simple (2, size, size); + file_space = H5Screate_simple(2, size, size); HDassert(file_space >= 0); dset = H5Dcreate2(file, "dset", H5T_NATIVE_UCHAR, file_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); HDassert(dset >= 0); - the_data = (unsigned char *)malloc((size_t)(size[0] * size[1])); + the_data = (unsigned char *)HDmalloc((size_t)(size[0] * size[1])); /* initial fill for lazy malloc */ HDmemset(the_data, 0xAA, (size_t)(size[0] * size[1])); + /* Fill raw */ - synchronize (); + synchronize(); #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_start); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_start, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstart); -#endif -#endif - HDfprintf (stderr, HEADING, "fill raw"); + t_start = H5_get_time(); + HDfprintf(stderr, HEADING, "fill raw"); for(u = 0; u < nwrite; u++) { - putc (PROGRESS, stderr); - HDfflush(stderr); - HDmemset(the_data, 0xAA, (size_t)(size[0]*size[1])); + HDputc(PROGRESS, stderr); + HDfflush(stderr); + HDmemset(the_data, 0xAA, (size_t)(size[0] * size[1])); } #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_stop); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_stop, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstop); - t_start.tv_sec = tbstart->time; - t_start.tv_usec = tbstart->millitm; - t_stop.tv_sec = tbstop->time; - t_stop.tv_usec = tbstop->millitm; -#endif -#endif - putc ('\n', stderr); - print_stats ("fill raw", - &r_start, &r_stop, &t_start, &t_stop, - (size_t)(nread*size[0]*size[1])); + t_stop = H5_get_time(); + HDputc('\n', stderr); + print_stats("fill raw", +#ifdef H5_HAVE_GETRUSAGE + &r_start, &r_stop, +#endif /* H5_HAVE_GETRUSAGE */ + t_start, t_stop, (size_t)(nread * size[0] * size[1])); /* Fill hdf5 */ - synchronize (); + synchronize(); #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_start); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_start, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstart); -#endif -#endif - HDfprintf (stderr, HEADING, "fill hdf5"); + t_start = H5_get_time(); + HDfprintf(stderr, HEADING, "fill hdf5"); for(u = 0; u < nread; u++) { - putc (PROGRESS, stderr); - HDfflush(stderr); - status = H5Dread (dset, H5T_NATIVE_UCHAR, file_space, file_space, - H5P_DEFAULT, the_data); - HDassert (status>=0); + HDputc(PROGRESS, stderr); + HDfflush(stderr); + status = H5Dread(dset, H5T_NATIVE_UCHAR, file_space, file_space, + H5P_DEFAULT, the_data); + HDassert(status >= 0); } #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_stop); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_stop, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstop); - t_start.tv_sec = tbstart->time; - t_start.tv_usec = tbstart->millitm; - t_stop.tv_sec = tbstop->time; - t_stop.tv_usec = tbstop->millitm; -#endif -#endif - putc ('\n', stderr); - print_stats ("fill hdf5", - &r_start, &r_stop, &t_start, &t_stop, - (size_t)(nread*size[0]*size[1])); + t_stop = H5_get_time(); + HDputc('\n', stderr); + print_stats("fill hdf5", +#ifdef H5_HAVE_GETRUSAGE + &r_start, &r_stop, +#endif /* H5_HAVE_GETRUSAGE */ + t_start, t_stop, (size_t)(nread * size[0] * size[1])); /* Write the raw dataset */ - synchronize (); + synchronize(); #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_start); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_start, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstart); -#endif -#endif - HDfprintf (stderr, HEADING, "out raw"); + t_start = H5_get_time(); + HDfprintf(stderr, HEADING, "out raw"); for(u = 0; u < nwrite; u++) { - putc (PROGRESS, stderr); - HDfflush(stderr); - offset = HDlseek (fd, (off_t)0, SEEK_SET); - HDassert (0==offset); - n = HDwrite (fd, the_data, (size_t)(size[0]*size[1])); - HDassert (n>=0 && (size_t)n==size[0]*size[1]); + HDputc(PROGRESS, stderr); + HDfflush(stderr); + offset = HDlseek(fd, (off_t)0, SEEK_SET); + HDassert(0 == offset); + n = HDwrite(fd, the_data, (size_t)(size[0] * size[1])); + HDassert(n >= 0 && (size_t)n == (size[0] * size[1])); } #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_stop); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_stop, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstop); - t_start.tv_sec = tbstart->time; - t_start.tv_usec = tbstart->millitm; - t_stop.tv_sec = tbstop->time; - t_stop.tv_usec = tbstop->millitm; -#endif -#endif - putc ('\n', stderr); - print_stats ("out raw", - &r_start, &r_stop, &t_start, &t_stop, - (size_t)(nread*size[0]*size[1])); + t_stop = H5_get_time(); + HDputc('\n', stderr); + print_stats("out raw", +#ifdef H5_HAVE_GETRUSAGE + &r_start, &r_stop, +#endif /* H5_HAVE_GETRUSAGE */ + t_start, t_stop, (size_t)(nread * size[0] * size[1])); /* Write the hdf5 dataset */ - synchronize (); + synchronize(); #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_start); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_start, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstart); -#endif -#endif - HDfprintf (stderr, HEADING, "out hdf5"); + t_start = H5_get_time(); + HDfprintf(stderr, HEADING, "out hdf5"); for(u = 0; u < nwrite; u++) { - putc (PROGRESS, stderr); - HDfflush(stderr); - status = H5Dwrite (dset, H5T_NATIVE_UCHAR, H5S_ALL, H5S_ALL, - H5P_DEFAULT, the_data); - HDassert (status>=0); + HDputc(PROGRESS, stderr); + HDfflush(stderr); + status = H5Dwrite(dset, H5T_NATIVE_UCHAR, H5S_ALL, H5S_ALL, + H5P_DEFAULT, the_data); + HDassert(status >= 0); } #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_stop); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_stop, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstop); - t_start.tv_sec = tbstart->time; - t_start.tv_usec = tbstart->millitm; - t_stop.tv_sec = tbstop->time; - t_stop.tv_usec = tbstop->millitm; -#endif -#endif - putc ('\n', stderr); - print_stats ("out hdf5", - &r_start, &r_stop, &t_start, &t_stop, - (size_t)(nread*size[0]*size[1])); + t_stop = H5_get_time(); + HDputc('\n', stderr); + print_stats("out hdf5", +#ifdef H5_HAVE_GETRUSAGE + &r_start, &r_stop, +#endif /* H5_HAVE_GETRUSAGE */ + t_start, t_stop, (size_t)(nread * size[0] * size[1])); /* Read the raw dataset */ - synchronize (); + synchronize(); #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_start); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_start, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstart); -#endif -#endif - HDfprintf (stderr, HEADING, "in raw"); + t_start = H5_get_time(); + HDfprintf(stderr, HEADING, "in raw"); for(u = 0; u < nread; u++) { - putc (PROGRESS, stderr); - HDfflush(stderr); - offset = HDlseek (fd, (off_t)0, SEEK_SET); - HDassert (0==offset); - n = HDread (fd, the_data, (size_t)(size[0]*size[1])); - HDassert (n>=0 && (size_t)n==size[0]*size[1]); + HDputc(PROGRESS, stderr); + HDfflush(stderr); + offset = HDlseek(fd, (off_t)0, SEEK_SET); + HDassert(0 == offset); + n = HDread(fd, the_data, (size_t)(size[0] * size[1])); + HDassert(n >= 0 && (size_t)n == (size[0] * size[1])); } #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_stop); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_stop, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstop); - t_start.tv_sec = tbstart->time; - t_start.tv_usec = tbstart->millitm; - t_stop.tv_sec = tbstop->time; - t_stop.tv_usec = tbstop->millitm; -#endif -#endif - putc ('\n', stderr); - print_stats ("in raw", - &r_start, &r_stop, &t_start, &t_stop, - (size_t)(nread*size[0]*size[1])); + t_stop = H5_get_time(); + HDputc('\n', stderr); + print_stats("in raw", +#ifdef H5_HAVE_GETRUSAGE + &r_start, &r_stop, +#endif /* H5_HAVE_GETRUSAGE */ + t_start, t_stop, (size_t)(nread * size[0] * size[1])); /* Read the hdf5 dataset */ - synchronize (); + synchronize(); #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_start); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_start, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstart); -#endif -#endif - HDfprintf (stderr, HEADING, "in hdf5"); + t_start = H5_get_time(); + HDfprintf(stderr, HEADING, "in hdf5"); for(u = 0; u < nread; u++) { - putc (PROGRESS, stderr); - HDfflush(stderr); - status = H5Dread (dset, H5T_NATIVE_UCHAR, file_space, file_space, - H5P_DEFAULT, the_data); - HDassert (status>=0); + HDputc(PROGRESS, stderr); + HDfflush(stderr); + status = H5Dread(dset, H5T_NATIVE_UCHAR, file_space, file_space, + H5P_DEFAULT, the_data); + HDassert(status >= 0); } #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_stop); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_stop, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstop); - t_start.tv_sec = tbstart->time; - t_start.tv_usec = tbstart->millitm; - t_stop.tv_sec = tbstop->time; - t_stop.tv_usec = tbstop->millitm; -#endif -#endif - putc ('\n', stderr); - print_stats ("in hdf5", - &r_start, &r_stop, &t_start, &t_stop, - (size_t)(nread*size[0]*size[1])); + t_stop = H5_get_time(); + HDputc('\n', stderr); + print_stats("in hdf5", +#ifdef H5_HAVE_GETRUSAGE + &r_start, &r_stop, +#endif /* H5_HAVE_GETRUSAGE */ + t_start, t_stop, (size_t)(nread * size[0] * size[1])); /* Read hyperslab */ - HDassert (size[0]>20 && size[1]>20); + HDassert(size[0] > 20 && size[1] > 20); start[0] = start[1] = 10; - count[0] = count[1] = size[0]-20; - status = H5Sselect_hyperslab (file_space, H5S_SELECT_SET, start, NULL, count, NULL); - HDassert (status>=0); - synchronize (); + count[0] = count[1] = size[0] - 20; + status = H5Sselect_hyperslab(file_space, H5S_SELECT_SET, start, NULL, count, NULL); + HDassert(status >= 0); + synchronize(); #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_start); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_start, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstart); -#endif -#endif - HDfprintf (stderr, HEADING, "in hdf5 partial"); + t_start = H5_get_time(); + HDfprintf(stderr, HEADING, "in hdf5 partial"); for(u = 0; u < nread; u++) { - putc (PROGRESS, stderr); - HDfflush(stderr); - status = H5Dread (dset, H5T_NATIVE_UCHAR, file_space, file_space, - H5P_DEFAULT, the_data); - HDassert (status>=0); + HDputc(PROGRESS, stderr); + HDfflush(stderr); + status = H5Dread(dset, H5T_NATIVE_UCHAR, file_space, file_space, + H5P_DEFAULT, the_data); + HDassert(status >= 0); } #ifdef H5_HAVE_GETRUSAGE HDgetrusage(RUSAGE_SELF, &r_stop); #endif -#ifdef H5_HAVE_GETTIMEOFDAY - HDgettimeofday(&t_stop, NULL); -#else -#ifdef H5_HAVE_SYS_TIMEB - _ftime(tbstop); - t_start.tv_sec = tbstart->time; - t_start.tv_usec = tbstart->millitm; - t_stop.tv_sec = tbstop->time; - t_stop.tv_usec = tbstop->millitm; -#endif -#endif - putc('\n', stderr); + t_stop = H5_get_time(); + HDputc('\n', stderr); print_stats("in hdf5 partial", - &r_start, &r_stop, &t_start, &t_stop, - (size_t)(nread*count[0]*count[1])); - - +#ifdef H5_HAVE_GETRUSAGE + &r_start, &r_stop, +#endif /* H5_HAVE_GETRUSAGE */ + t_start, t_stop, (size_t)(nread * size[0] * size[1])); /* Close everything */ HDclose(fd); + H5Dclose(dset); H5Sclose(file_space); H5Fclose(file); - free(the_data); + + HDfree(the_data); return 0; } diff --git a/tools/test/perform/overhead.c b/tools/test/perform/overhead.c index 58558a5..bb3aff5 100644 --- a/tools/test/perform/overhead.c +++ b/tools/test/perform/overhead.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> + * Programmer: Robb Matzke * Monday, September 28, 1998 * * Purpose: Creates a chunked dataset and measures the storage overhead. diff --git a/tools/test/perform/perf_meta.c b/tools/test/perform/perf_meta.c index b56f074..77248cc 100644 --- a/tools/test/perform/perf_meta.c +++ b/tools/test/perform/perf_meta.c @@ -12,7 +12,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu <slu@ncsa.uiuc.edu> + * Programmer: Raymond Lu * Friday, Oct 3, 2004 * * Purpose: Tests performance of metadata diff --git a/tools/test/perform/sio_engine.c b/tools/test/perform/sio_engine.c index 2562ab8..aa3a316 100644 --- a/tools/test/perform/sio_engine.c +++ b/tools/test/perform/sio_engine.c @@ -1268,8 +1268,7 @@ done: * 'temp' in the code below, but early (4.4.7, at least) gcc only * allows diagnostic pragmas to be toggled outside of functions. */ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wformat-nonliteral" +H5_GCC_DIAG_OFF(format-nonliteral) static void do_cleanupfile(iotype iot, char *filename) { @@ -1331,5 +1330,5 @@ do_cleanupfile(iotype iot, char *filename) } } } -#pragma GCC diagnostic pop +H5_GCC_DIAG_ON(format-nonliteral) diff --git a/tools/test/perform/sio_perf.c b/tools/test/perform/sio_perf.c index a56558e..d2eb3fc 100644 --- a/tools/test/perform/sio_perf.c +++ b/tools/test/perform/sio_perf.c @@ -303,7 +303,7 @@ typedef struct _minmax { /* local functions */ static hsize_t parse_size_directive(const char *size); -static struct options *parse_command_line(int argc, char *argv[]); +static struct options *parse_command_line(int argc, const char *argv[]); static void run_test_loop(struct options *options); static int run_test(iotype iot, parameters parms, struct options *opts); static void output_all_info(minmax *mm, int count, int indent_level); @@ -324,7 +324,7 @@ static void report_parameters(struct options *opts); * Modifications: */ int -main(int argc, char **argv) +main(int argc, const char *argv[]) { int exit_value = EXIT_SUCCESS; struct options *opts = NULL; @@ -944,7 +944,7 @@ report_parameters(struct options *opts) * Added multidimensional testing (Christian Chilan, April, 2008) */ static struct options * -parse_command_line(int argc, char *argv[]) +parse_command_line(int argc, const char *argv[]) { int opt; struct options *cl_opts; @@ -984,7 +984,7 @@ parse_command_line(int argc, char *argv[]) cl_opts->h5_extendable = FALSE; /* Use extendable dataset */ cl_opts->verify = FALSE; /* No Verify data correctness by default */ - while ((opt = get_option(argc, (const char **)argv, s_opts, l_opts)) != EOF) { + while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { case 'a': cl_opts->h5_alignment = parse_size_directive(opt_arg); diff --git a/tools/test/perform/sio_standalone.h b/tools/test/perform/sio_standalone.h index 45f6d25..99e13bc 100644 --- a/tools/test/perform/sio_standalone.h +++ b/tools/test/perform/sio_standalone.h @@ -228,7 +228,7 @@ H5_DLL int HDfprintf (FILE *stream, const char *fmt, ...); #define HDgetpwnam(S) getpwnam(S) #define HDgetpwuid(U) getpwuid(U) #define HDgetrusage(X,S) getrusage(X,S) -#define HDgets(S) gets(S) +/* Don't define a macro for gets() - it was removed in C11 */ #ifdef H5_HAVE_WIN32_API H5_DLL int Wgettimeofday(struct timeval *tv, struct timezone *tz); #define HDgettimeofday(V,Z) Wgettimeofday(V,Z) diff --git a/tools/test/perform/zip_perf.c b/tools/test/perform/zip_perf.c index e301bb3..8f1f584 100644 --- a/tools/test/perform/zip_perf.c +++ b/tools/test/perform/zip_perf.c @@ -552,7 +552,7 @@ do_write_test(unsigned long file_size, unsigned long min_buf_size, * Modifications: */ int -main(int argc, char **argv) +main(int argc, const char *argv[]) { unsigned long min_buf_size = 128 * ONE_KB, max_buf_size = ONE_MB; unsigned long file_size = 64 * ONE_MB; @@ -563,7 +563,7 @@ main(int argc, char **argv) /* Initialize h5tools lib */ h5tools_init(); - while ((opt = get_option(argc, (const char **)argv, s_opts, l_opts)) > 0) { + while ((opt = get_option(argc, argv, s_opts, l_opts)) > 0) { switch ((char)opt) { case '0': case '1': case '2': case '3': case '4': case '5': diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt new file mode 100644 index 0000000..2d5626e --- /dev/null +++ b/utils/CMakeLists.txt @@ -0,0 +1,4 @@ +cmake_minimum_required (VERSION 3.10) +project (HDF5_UTILS C) + +add_subdirectory (mirror_vfd) diff --git a/utils/COPYING b/utils/COPYING new file mode 100644 index 0000000..4a23112 --- /dev/null +++ b/utils/COPYING @@ -0,0 +1,12 @@ + + Copyright by The HDF Group. + All rights reserved. + + The files and subdirectories in this directory are part of HDF5. + The full HDF5 copyright notice, including terms governing use, + modification, and redistribution, is contained in the COPYING file + which can be found at the root of the source code distribution tree + or in https://support.hdfgroup.org/ftp/HDF5/releases. If you do + not have access to either file, you may request a copy from + help@hdfgroup.org. + diff --git a/utils/Makefile.am b/utils/Makefile.am new file mode 100644 index 0000000..b4409ac --- /dev/null +++ b/utils/Makefile.am @@ -0,0 +1,26 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +## +# +# Utils HDF5 Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +CONFIG=ordered + +# All subdirectories +SUBDIRS=mirror_vfd + +include $(top_srcdir)/config/conclude.am diff --git a/utils/mirror_vfd/CMakeLists.txt b/utils/mirror_vfd/CMakeLists.txt new file mode 100644 index 0000000..1926352 --- /dev/null +++ b/utils/mirror_vfd/CMakeLists.txt @@ -0,0 +1,64 @@ +cmake_minimum_required (VERSION 3.10) +project (HDF5_UTILS_MIRRORVFD C) + +#----------------------------------------------------------------------------- +# Add the mirror_server executable +#----------------------------------------------------------------------------- + +set (mirror_server_SOURCES + ${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_remote.c + ${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_server.c + ${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_writer.c) +add_executable (mirror_server ${mirror_server_SOURCES}) +target_include_directories (mirror_server PRIVATE "${HDF5_UITLS_DIR};${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") +if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (mirror_server STATIC) + target_link_libraries (mirror_server PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) +else () + TARGET_C_PROPERTIES (mirror_server SHARED) + target_link_libraries (mirror_server PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) +endif () +set_target_properties (mirror_server PROPERTIES FOLDER utils) +set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};mirror_server") +set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} mirror_server) + +#----------------------------------------------------------------------------- +# Add the mirror_server_stop executable +#----------------------------------------------------------------------------- + +set (mirror_server_stop_SOURCES ${HDF5_UTILS_MIRRORVFD_SOURCE_DIR}/mirror_server_stop.c) +add_executable (mirror_server_stop ${mirror_server_stop_SOURCES}) +target_include_directories (mirror_server_stop PRIVATE "${HDF5_UITLS_DIR};${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>") +if (NOT BUILD_SHARED_LIBS) + TARGET_C_PROPERTIES (mirror_server_stop STATIC) + target_link_libraries (mirror_server_stop PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) +else () + TARGET_C_PROPERTIES (mirror_server_stop SHARED) + target_link_libraries (mirror_server_stop PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) +endif () +set_target_properties (mirror_server_stop PROPERTIES FOLDER utils) +set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};mirror_server_stop") +set (H5_DEP_EXECUTABLES ${H5_DEP_EXECUTABLES} mirror_server_stop) + +############################################################################## +############################################################################## +### I N S T A L L A T I O N ### +############################################################################## +############################################################################## + +#----------------------------------------------------------------------------- +# Rules for Installation of tools using make Install target +#----------------------------------------------------------------------------- +if (HDF5_EXPORTED_TARGETS) + foreach (exec ${H5_DEP_EXECUTABLES}) + INSTALL_PROGRAM_PDB (${exec} ${HDF5_INSTALL_BIN_DIR} utilsapplications) + endforeach () + + install ( + TARGETS + ${H5_DEP_EXECUTABLES} + EXPORT + ${HDF5_EXPORTED_TARGETS} + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT utilsapplications + ) +endif () diff --git a/utils/mirror_vfd/Makefile.am b/utils/mirror_vfd/Makefile.am new file mode 100644 index 0000000..96d3104 --- /dev/null +++ b/utils/mirror_vfd/Makefile.am @@ -0,0 +1,30 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +AM_CPPFLAGS+=-I$(top_srcdir)/src + +bin_PROGRAMS = mirror_server mirror_server_stop + +mirror_server_SOURCES = mirror_server.c mirror_writer.c mirror_remote.c +#mirror_writer_SOURCES = mirror_writer.c mirror_remote.c + +# All programs depend on the hdf5 library +LDADD=$(LIBHDF5) + +include $(top_srcdir)/config/conclude.am diff --git a/utils/mirror_vfd/mirror_remote.c b/utils/mirror_vfd/mirror_remote.c new file mode 100644 index 0000000..81a3625 --- /dev/null +++ b/utils/mirror_vfd/mirror_remote.c @@ -0,0 +1,225 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Common operations for "remote" processes for the Mirror VFD. + * + * Jacob Smith, 2020-03-06 + */ + +#include "mirror_remote.h" + +#ifdef H5_HAVE_MIRROR_VFD + + +/* --------------------------------------------------------------------------- + * Function: mirror_log + * + * Purpose: Write message to the logging stream/file. + * If logging info pointer is NULL, uses logging defaults. + * ---------------------------------------------------------------------------- + */ +void +mirror_log(struct mirror_log_info *info, + unsigned int level, + const char *format, + ...) +{ + FILE *stream = MIRROR_LOG_DEFAULT_STREAM; + unsigned int verbosity = MIRROR_LOG_DEFAULT_VERBOSITY; + hbool_t custom = FALSE; + + if (info != NULL && info->magic == MIRROR_LOG_INFO_MAGIC) { + stream = info->stream; + verbosity = info->verbosity; + custom = TRUE; + } + + if (level == V_NONE) { + return; + } + else + if (level <= verbosity) { + if (custom == TRUE && info->prefix[0] != '\0') { + HDfprintf(stream, "%s", info->prefix); + } + + switch (level) { + case (V_ERR) : + HDfprintf(stream, "ERROR "); + break; + case (V_WARN) : + HDfprintf(stream, "WARNING "); + break; + default: + break; + } + + if (format != NULL) { + va_list args; + HDva_start(args, format); + HDvfprintf(stream, format, args); + HDva_end(args); + } + + HDfprintf(stream, "\n"); + HDfflush(stream); + } /* end if sufficiently verbose to print */ +} /* end mirror_log() */ + + +/* --------------------------------------------------------------------------- + * Function: session_log_bytes + * + * Purpose: "Pretty-print" raw binary data to logging stream/file. + * If info pointer is NULL, uses logging defaults. + * ---------------------------------------------------------------------------- + */ +void +mirror_log_bytes(struct mirror_log_info *info, + unsigned int level, + size_t n_bytes, + const unsigned char *buf) +{ + FILE *stream = MIRROR_LOG_DEFAULT_STREAM; + unsigned int verbosity = MIRROR_LOG_DEFAULT_VERBOSITY; + + if (buf == NULL) { + return; + } + + if (info != NULL && info->magic == MIRROR_LOG_INFO_MAGIC) { + stream = info->stream; + verbosity = info->verbosity; + } + + if (level <= verbosity) { + size_t bytes_written = 0; + const unsigned char *b = NULL; + + /* print whole lines */ + while ((n_bytes - bytes_written) >= 32) { + b = buf + bytes_written; /* point to region in buffer */ + HDfprintf(stream, + "%04zX %02X%02X%02X%02X %02X%02X%02X%02X" \ + " %02X%02X%02X%02X %02X%02X%02X%02X" \ + " %02X%02X%02X%02X %02X%02X%02X%02X" \ + " %02X%02X%02X%02X %02X%02X%02X%02X\n", + bytes_written, + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], + b[8], b[9], b[10],b[11], b[12],b[13],b[14],b[15], + b[16],b[17],b[18],b[19], b[20],b[21],b[22],b[23], + b[24],b[25],b[26],b[27], b[28],b[29],b[30],b[31]); + bytes_written += 32; + } + + /* start partial line */ + if (n_bytes > bytes_written) { + HDfprintf(stream, "%04zX ", bytes_written); + } + + /* partial line blocks */ + while ((n_bytes - bytes_written) >= 4) { + HDfprintf(stream, " %02X%02X%02X%02X", + buf[bytes_written], buf[bytes_written+1], + buf[bytes_written+2], buf[bytes_written+3]); + bytes_written += 4; + } + + /* block separator before partial block */ + if (n_bytes > bytes_written) { + HDfprintf(stream, " "); + } + + /* partial block individual bytes */ + while (n_bytes > bytes_written) { + HDfprintf(stream, "%02X", buf[bytes_written++]); + } + + /* end partial line */ + HDfprintf(stream, "\n"); + } /* end if suitably verbose to log */ +} /* end mirror_log_bytes() */ + + +/* --------------------------------------------------------------------------- + * Function: mirror_log_init + * + * Purpose: Prepare a loginfo_t structure for use. + * + * Return: Success: Pointer to newly-ceated info. + * Failure: NULL. Either unable to allocate or cannot open file. + * ---------------------------------------------------------------------------- + */ +loginfo_t * +mirror_log_init(char *path, char *prefix, unsigned int verbosity) +{ + loginfo_t *info = NULL; + + info = (loginfo_t *)HDmalloc(sizeof(loginfo_t)); + if (info != NULL) { + info->magic = MIRROR_LOG_INFO_MAGIC; + info->verbosity = verbosity; + info->stream = MIRROR_LOG_DEFAULT_STREAM; + info->prefix[0] = '\0'; + + if (prefix && *prefix) { + HDstrncpy(info->prefix, prefix, MIRROR_LOG_PREFIX_MAX); + } + + if (path && *path) { + FILE *f = NULL; + f = HDfopen(path, "w"); + if (NULL == f) { + HDfprintf(MIRROR_LOG_DEFAULT_STREAM, + "WARN custom logging path could not be opened: %s\n", + path); + info->magic += 1; + HDfree(info); + } + else { + info->stream = f; + } + } + + } /* end if able to allocate */ + + return info; +} /* end mirror_log_init() */ + + +/* --------------------------------------------------------------------------- + * Function: mirror_log_term + * + * Purpose: Shut down and clean up a loginfo_t structure. + * + * Return: Success: SUCCEED. Resources released. + * Failure: FAIL. Indeterminite state. + * ---------------------------------------------------------------------------- + */ +herr_t +mirror_log_term(loginfo_t *info) +{ + if (info == NULL || info->magic != MIRROR_LOG_INFO_MAGIC) { + return FAIL; + } + if (info->stream != stderr || info->stream != stdout) { + if (HDfclose(info->stream) < 0) { + return FAIL; + } + } + info->magic += 1; + HDfree(info); + return SUCCEED; +} /* end mirror_log_term() */ + +#endif /* H5_HAVE_MIRROR_VFD */ + diff --git a/utils/mirror_vfd/mirror_remote.h b/utils/mirror_vfd/mirror_remote.h new file mode 100644 index 0000000..67e95a5 --- /dev/null +++ b/utils/mirror_vfd/mirror_remote.h @@ -0,0 +1,53 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Common definitions for "remote" processes for the Mirror VFD. + * + * Jacob Smith, 2020-03-06 + */ + +#include "hdf5.h" +#include "H5private.h" + +#ifdef H5_HAVE_MIRROR_VFD + +#include "H5FDmirror_priv.h" /* Private header for the mirror VFD */ + +#define V_NONE 0 +#define V_ERR 1 +#define V_WARN 2 +#define V_INFO 3 +#define V_ALL 4 + +#define MIRROR_LOG_DEFAULT_STREAM stdout +#define MIRROR_LOG_DEFAULT_VERBOSITY V_WARN +#define MIRROR_LOG_PREFIX_MAX 79 +#define MIRROR_LOG_INFO_MAGIC 0x569D589A + +typedef struct mirror_log_info { + uint32_t magic; + FILE *stream; + unsigned int verbosity; + char prefix[MIRROR_LOG_PREFIX_MAX+1]; +} loginfo_t; + +void mirror_log(loginfo_t *info, unsigned int level, + const char *format, ...); +void mirror_log_bytes(loginfo_t *info, unsigned int level, + size_t n_bytes, const unsigned char *buf); +loginfo_t *mirror_log_init(char *path, char *prefix, unsigned int verbosity); +int mirror_log_term(loginfo_t *loginfo); + +herr_t run_writer(int socketfd, H5FD_mirror_xmit_open_t *xmit_open); + +#endif /* H5_HAVE_MIRROR_VFD */ + diff --git a/utils/mirror_vfd/mirror_server.c b/utils/mirror_vfd/mirror_server.c new file mode 100644 index 0000000..db7cf04 --- /dev/null +++ b/utils/mirror_vfd/mirror_server.c @@ -0,0 +1,666 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * "Server" application to associate a Mirror VFD Driver with a Writer. + * + * Server waits on a dedicated port for a Driver to attempt to connect. + * When connection is made, reads a message from the Driver. + * If message is "SHUTDOWN", Server closes connection and terminates. + * Else, if it receives an encoded OPEN xmit (from Driver), the Server forks + * itself; the child becomes a dedicated Writer and maintains connection with + * the Driver instance, and the parent process remains a Server and returns + * to listening for incoming requests. + * Else, the message is not recognized and is ignored. + * + * + * + * mirror_server [args] + * + * Primary server for coordinating mirror VFD connections with the remote + * process. + * + * args: + * --help, -h Print help message and exit. + * --port=N Positive integer for primary listening port. + * --verbosity=N Debugging verbosity + * 0: none + * 1: errors + * 2: details + * 3: all + * --logpath=S File path to direct debugging output, if any. + * Default of none prints output to stdout. + * + */ + +#include "mirror_remote.h" + +#ifdef H5_HAVE_MIRROR_VFD + +#define MAXBUF 2048 /* max buffer length. */ +#define LISTENQ 80 /* max pending mirrorS requests */ +#define DEFAULT_PORT 3000 /* default listening port */ +#define MAX_PORT_LOOPS 20 /* max iteratations through port range */ +#define PORT_LOOP_RETRY_DELAY 1 /* seconds to wait between port scans */ + +/* semi-unique "magic" numbers to sanity-check structure pointers */ +#define OP_ARGS_MAGIC 0xCF074379u +#define SERVER_RUN_MAGIC 0x741B459Au + + +/* --------------------------------------------------------------------------- + * Structure: struct op_args + * + * Purpose: Convenience structure for holding arguments from command-line. + * + * `magic` (uint32_t) + * Semi-unique number to help validate a pointer to this struct type. + * Must be OP_ARGS_MAGIC to be considered valid. + * + * `help` (int) + * Flag that the help argument was present in the command line. + * + * `main_port` (int) + * Flag that the help argument was present in the command line. + * + * `verbosity` (int) + * Number between 0 (none) and 4 (all) that controls how much detail + * the program prints as a course of logging. + * + * `log_prepend_serv` (int) + * Flag that the logging messages should have 'S- ' at the start of each + * line. + * + * `log_prepend_type` (int) + * Flag that the logging messages should have the assocaited verbosity + * level present in the line (e.g., "WARN", "ERROR", or "INFO"). + * + * `log_path` (char *) + * Path string from the command line, giving the absolute path + * for the file for logging output. Can be empty. + * + * `writer_log_path` (char *) + * Path string from the command line, giving the absolute path + * for the file for writer's logging output. Can be empty. + * + * --------------------------------------------------------------------------- + */ +struct op_args { + uint32_t magic; + int help; + int main_port; + int verbosity; + int log_prepend_serv; + int log_prepend_type; + char log_path[PATH_MAX+1]; + char writer_log_path[PATH_MAX+1]; +}; + + +/* --------------------------------------------------------------------------- + * Structure: struct server_run + * + * Purpose: Convenience structure for holding information about a server + * in operation. + * + * `magic` (uint32_t) + * Semi-unique number to help validate a pointer to this struct type. + * Must be SERVER_RUN_MAGIC to be considered valid. + * + * `log_stream` (FILE *) + * File handle where logging output is directed. + * By default, is stdout. + * + * `opts` (struct opt_args) + * Contained structure, holds the server's configuration. + * + * `listenfd` (int) + * File descriptor of the listening socket. + * + * --------------------------------------------------------------------------- + */ +struct server_run { + uint32_t magic; + struct op_args opts; + struct mirror_log_info *loginfo; + int listenfd; +}; + + +/* --------------------------------------------------------------------------- + * Function: mybzero + * + * Purpose: Introduce bzero without neededing it on the system. + * + * Programmer: Jacob Smith + * 2020-03-30 + * --------------------------------------------------------------------------- + */ +static void mybzero(void *dest, size_t size) +{ + size_t i = 0; + char *s = NULL; + HDassert(dest); + s = (char *)dest; + for (i = 0; i < size; i++) { + *(s+i) = 0; + } +} /* end mybzero() */ + + +/* --------------------------------------------------------------------------- + * Function: usage + * + * Purpose: Print the usage message to stdout. + * --------------------------------------------------------------------------- + */ +static void +usage(void) +{ + HDfprintf(stdout, + "mirror_server [options]\n" \ + "\n" \ + "Application for providing Mirror Writer process to " \ + " Mirror VFD on file-open.\n" \ + "Listens on a dedicated socket; forks as a Writer upon receipt" \ + " of a valid OPEN xmit.\n" \ + "\n" \ + "Options:\n" \ + "--help [-h] : Print this help message and quit.\n" \ + "--logpath=PATH : File path for logging output " \ + "(default none, to stdout).\n" \ + "--port=PORT : Primary port (default %d).\n" \ + "--verbosity=NUM : Debug printing level " \ + "0..4, (default %d).\n", + DEFAULT_PORT, + MIRROR_LOG_DEFAULT_VERBOSITY); +} /* end usage() */ + + +/* --------------------------------------------------------------------------- + * Function: parse_args + * + * Purpose: Read command line options and store results in args_out + * structure. Fails in event of unrecognized option. + * + * Return: 0 on success, -1 on failure. + * --------------------------------------------------------------------------- + */ +static int +parse_args(int argc, char **argv, struct op_args *args_out) +{ + int i; + + /* preset default values + */ + args_out->main_port = DEFAULT_PORT; + args_out->help = 0; + args_out->log_prepend_serv = 1; + args_out->log_prepend_type = 1; + args_out->verbosity = MIRROR_LOG_DEFAULT_VERBOSITY; + /* preset empty strings */ + mybzero(args_out->log_path, PATH_MAX+1); + mybzero(args_out->writer_log_path, PATH_MAX+1); + + if (argv == NULL || *argv == NULL) { + mirror_log(NULL, V_ERR, "invalid argv pointer"); + return -1; + } + + /* Loop over arguments after program name and writer_path */ + for (i=2; i < argc; i++) { + if (!HDstrncmp(argv[i], "-h", 3) || !HDstrncmp(argv[i], "--help", 7)) { + mirror_log(NULL, V_INFO, "found help argument"); + args_out->help = 1; + return 0; + } /* end if help */ + else + if (!HDstrncmp(argv[i], "--port=", 7)) { + mirror_log(NULL, V_INFO, "parsing 'main_port' (%s)", argv[i]+7); + args_out->main_port = HDatoi(argv[i]+7); + } /* end if port */ + else + if (!HDstrncmp(argv[i], "--verbosity=", 12)) { + mirror_log(NULL, V_INFO, "parsing 'verbosity' (%s)", argv[i]+12); + args_out->verbosity = HDatoi(argv[i]+12); + } /* end if verbosity */ + else + if (!HDstrncmp(argv[i], "--logpath=", 10)) { + mirror_log(NULL, V_INFO, "parsing 'logpath' (%s)", argv[i]+10); + HDstrncpy(args_out->log_path, argv[i]+10, PATH_MAX); + } /* end if logpath */ + else { + mirror_log(NULL, V_ERR, "unrecognized argument: %s", argv[i]); + return -1; + } /* end if unrecognized argument */ + } /* end for each arg after the path to writer "receiver process" */ + + mirror_log(NULL, V_INFO, "all args parsed"); + + return 0; +} /* end parse_args() */ + + +/* --------------------------------------------------------------------------- + * Function: prepare_listening_socket + * + * Purpose: Configure and open a socket. + * In event of error, attempts to undo its processes. + * + * Return: Success: non-negative (the file descriptor of the socket) + * Failure: -1 + * --------------------------------------------------------------------------- + */ +static int +prepare_listening_socket(struct server_run *run) +{ + struct sockaddr_in server_addr; + int _true = 1; /* needed for setsockopt() */ + int ret_value = -1; + int ret = 0; /* for checking return value of function calls */ + + if (run == NULL || run->magic != SERVER_RUN_MAGIC) { + mirror_log(NULL, V_ERR, "invalid server_run pointer"); + return -1; + } + + mirror_log(run->loginfo, V_INFO, "preparing socket"); + + server_addr.sin_family = AF_INET; + server_addr.sin_addr.s_addr = HDhtonl(INADDR_ANY); + server_addr.sin_port = HDhtons((uint16_t)run->opts.main_port); + + mirror_log(run->loginfo, V_INFO, "socket()"); + ret_value = HDsocket(AF_INET, SOCK_STREAM, 0); + if (ret_value < 0) { + mirror_log(run->loginfo, V_ERR, "listening socket:%d", ret_value); + goto error; + } + + mirror_log(run->loginfo, V_ALL, "setsockopt()"); + HDsetsockopt(ret_value, SOL_SOCKET, SO_REUSEADDR, &_true, sizeof(int)); + + mirror_log(run->loginfo, V_INFO, "bind()"); + ret = HDbind(ret_value, (struct sockaddr *)&server_addr, + sizeof(server_addr)); + if (ret < 0) { + mirror_log(run->loginfo, V_ERR, "bind() %s", HDstrerror(errno)); + goto error; + } + + mirror_log(run->loginfo, V_INFO, "listen()"); + ret = HDlisten(ret_value, LISTENQ); + if (ret < 0) { + mirror_log(run->loginfo, V_ERR, "H5FD server listen:%d", ret); + goto error; + } + + return ret_value; + +error: + if (ret_value >= 0) { + HDshutdown(ret_value, SHUT_RDWR); + HDclose(ret_value); + } + return -1; +} /* end prepare_listening_socket() */ + + +/* --------------------------------------------------------------------------- + * Function: init_server_run + * + * Purpose: Set up server_run struct with default and specified values. + * + * Return: Zero (0) if successful, -1 if an error occurred. + * --------------------------------------------------------------------------- + */ +static struct server_run * +init_server_run(int argc, char **argv) +{ + struct server_run *run; + + run = (struct server_run *)HDmalloc(sizeof(struct server_run)); + if (run == NULL) { + mirror_log(NULL, V_ERR, "can't allocate server_run struct"); + return NULL; + } + + run->magic = (uint32_t)SERVER_RUN_MAGIC; + run->opts.magic = (uint32_t)OP_ARGS_MAGIC; + run->listenfd = -1; + + if (parse_args(argc, argv, &(run->opts)) < 0) { + mirror_log(NULL, V_ERR, "can't parse arguments"); + usage(); + goto error; + } + + if (run->opts.help) { + usage(); + return run; /* early exit */ + } + + run->loginfo = mirror_log_init(run->opts.log_path, "s- ", + run->opts.verbosity); + + run->listenfd = prepare_listening_socket(run); + if (run->listenfd < 0) { + mirror_log(NULL, V_ERR, "can't prepare listening socket"); + goto error; + } + + return run; + +error: + if (run != NULL) { + HDfree(run); + } + return NULL; + +} /* end init_server_run() */ + + +/* --------------------------------------------------------------------------- + * Function: term_server_run + * + * Purpose: Close opened items in a sever_run and release the pointer. + * + * Return: Zero (0) if successful, -1 if an error occurred. + * --------------------------------------------------------------------------- + */ +static int +term_server_run(struct server_run *run) +{ + if (run == NULL || run->magic != SERVER_RUN_MAGIC) { + mirror_log(NULL, V_ERR, "invalid server_run pointer"); + return -1; + } + + mirror_log(run->loginfo, V_INFO, "shutting down"); + + if (run->listenfd >= 0) { + HDshutdown(run->listenfd, SHUT_RDWR); /* TODO: error-checking? */ + HDclose(run->listenfd); /* TODO: error-checking? */ + run->listenfd = -1; + } + + if (mirror_log_term(run->loginfo) < 0) { + mirror_log(NULL, V_ERR, "can't close logging stream"); + return -1; /* doesn't solve the problem, but informs of error */ + } + run->loginfo = NULL; + + (run->magic)++; + (run->opts.magic)++; + HDfree(run); + return 0; +} /* end term_server_run() */ + + +/* --------------------------------------------------------------------------- + * Function: accept_connection + * + * Purpose: Main working loop; process requests as they are received. + * Does nothing if the run option help is set. + * + * Return: -1 on error, else a non-negative file descriptor of the socket. + * --------------------------------------------------------------------------- + */ +static int +accept_connection(struct server_run *run) +{ + struct sockaddr_in client_addr; /**/ + socklen_t clilen; /**/ + struct hostent *host_port = NULL; /**/ + char *hostaddrp; /**/ + int connfd = -1; /* connection file descriptor */ + + if (run == NULL || run->magic != SERVER_RUN_MAGIC) { + mirror_log(NULL, V_ERR, "invalid server_run pointer"); + return -1; + } + + /*------------------------------*/ + /* accept a connection on a socket */ + clilen = sizeof(client_addr); + connfd = HDaccept(run->listenfd, (struct sockaddr *)&client_addr, &clilen); + if (connfd < 0) { + mirror_log(run->loginfo, V_ERR, "accept:%d", connfd); + goto error; + } + mirror_log(run->loginfo, V_INFO, "connection achieved"); + + /*------------------------------*/ + /* get client address information */ + host_port = HDgethostbyaddr( + (const char *)&client_addr.sin_addr.s_addr, + sizeof(client_addr.sin_addr.s_addr), + AF_INET); + if (host_port == NULL) { + mirror_log(run->loginfo, V_ERR, "gethostbyaddr()"); + goto error; + } + + /* function has the string space statically scoped -- OK until next call */ + hostaddrp = HDinet_ntoa(client_addr.sin_addr); + /* TODO? proper error-checking */ + + mirror_log(run->loginfo, V_INFO, + "server connected with %s (%s)", + host_port->h_name, + hostaddrp); + + return connfd; + +error: + if (connfd >= 0) { + close(connfd); + } + return -1; +} /* end accept_connection() */ + + +/* --------------------------------------------------------------------------- + * Function: wait_for_child + * + * Purpose: Signal handler to reap zombie processes. + * --------------------------------------------------------------------------- + */ +static void +wait_for_child(int sig) +{ + while (HDwaitpid(-1, NULL, WNOHANG) > 0); +} /* end wait_for_child() */ + + +/* --------------------------------------------------------------------------- + * Function: handle_requests + * + * Purpose: Main working loop; process requests as they are received. + * Does nothing if the run option `help` is set. + * + * Return: -1 on error, else 0 for successful operation. + * --------------------------------------------------------------------------- + */ +static int +handle_requests(struct server_run *run) +{ + int connfd = -1; /**/ + char mybuf[H5FD_MIRROR_XMIT_OPEN_SIZE]; /**/ + int ret; /* general-purpose error-checking */ + int pid; /* process ID of fork */ + struct sigaction sa; + int ret_value = 0; + + if (run == NULL || run->magic != SERVER_RUN_MAGIC) { + mirror_log(NULL, V_ERR, "invalid server_run pointer"); + return -1; + } + + if (run->opts.help) { + return 0; + } + + if (run->listenfd < 0) { + mirror_log(NULL, V_ERR, "invalid listening socket"); + return -1; + } + + /* Set up the signal handler */ + sa.sa_handler = wait_for_child; + HDsigemptyset(&sa.sa_mask); + sa.sa_flags = SA_RESTART; + if (HDsigaction(SIGCHLD, &sa, NULL) == -1) { + perror("sigaction"); + return 1; + } + + /* Keep listening for attempts to connect. + */ + + while (1) { /* infinite loop, exited via break or goto */ + mirror_log(run->loginfo, V_INFO, "server waiting for connections..."); + + connfd = -1; + + connfd = accept_connection(run); + if (connfd < 0) { + mirror_log(run->loginfo, V_ERR, "unable to receive connection"); + goto error; + } + + /* Read handshake from port connection. + */ + + ret = (int)HDread(connfd, &mybuf, H5FD_MIRROR_XMIT_OPEN_SIZE); + if (-1 == ret) { + mirror_log(run->loginfo, V_ERR, "read:%d", ret); + goto error; + } + mirror_log(run->loginfo, V_INFO, "received %d bytes", ret); + mirror_log(run->loginfo, V_ALL, "```"); + mirror_log_bytes(run->loginfo, V_ALL, ret, + (const unsigned char *)mybuf); + mirror_log(run->loginfo, V_ALL, "```"); + + /* Respond to handshake message. + */ + + if (!HDstrncmp("SHUTDOWN", mybuf, 8)) { + /* Stop operation if told to stop */ + mirror_log(run->loginfo, V_INFO, "received SHUTDOWN!", ret); + HDclose(connfd); + connfd = -1; + goto done; + } /* end if explicit "SHUTDOWN" directive */ + else + if (H5FD_MIRROR_XMIT_OPEN_SIZE == ret) { + H5FD_mirror_xmit_open_t xopen; + + mirror_log(run->loginfo, V_INFO, + "probable OPEN xmit received"); + + H5FD_mirror_xmit_decode_open(&xopen, (const unsigned char *)mybuf); + if (FALSE == H5FD_mirror_xmit_is_open(&xopen)) { + mirror_log(run->loginfo, V_WARN, + "expected OPEN xmit was malformed"); + HDclose(connfd); + continue; + } + + mirror_log(run->loginfo, V_INFO, + "probable OPEN xmit confirmed"); + + pid = HDfork(); + if (pid < 0) { /* fork error */ + mirror_log(run->loginfo, V_ERR, "cannot fork"); + goto error; + } /* end if fork error */ + else + if (pid == 0) { /* child process (writer side of fork) */ + mirror_log(run->loginfo, V_INFO, + "executing writer"); + if (run_writer(connfd, &xopen) < 0) { + HDprintf("can't run writer\n"); + } + else { + HDprintf("writer OK\n"); + } + HDclose(connfd); + + HDexit(EXIT_SUCCESS); + } /* end if writer side of fork */ + else { /* parent process (server side of fork) */ + mirror_log(run->loginfo, V_INFO, "tidying up from handshake"); + HDclose(connfd); + } /* end if server side of fork */ + + } /* end else-if valid request for service */ + else { + /* Ignore unrecognized messages */ + HDclose(connfd); + continue; + } /* end else (not a valid message, to be ignored) */ + + } /* end while listening for new connections */ + +done: + if (connfd >= 0) { + mirror_log(run->loginfo, V_WARN, "connfd still open upon cleanup"); + HDclose(connfd); + } + + return ret_value; + +error: + if (connfd >= 0) { + HDclose(connfd); + } + return -1; +} /* end handle_requests() */ + + +/* ------------------------------------------------------------------------- */ +int +main(int argc, char **argv) +{ + struct server_run *run; + + run = init_server_run(argc, argv); + if (NULL == run) { + mirror_log(NULL, V_ERR, "can't initialize run"); + HDexit(EXIT_FAILURE); + } + + if (handle_requests(run) < 0) { + mirror_log(run->loginfo, V_ERR, "problem handling requests"); + } + + if (term_server_run(run) < 0) { + mirror_log(NULL, V_ERR, "problem closing server run"); + HDexit(EXIT_FAILURE); + } + + HDexit(EXIT_SUCCESS); +} /* end main() */ + +#else /* H5_HAVE_MIRROR_VFD */ + +int +main(void) +{ + HDprintf("Mirror VFD was not built -- cannot launch server.\n"); + HDexit(EXIT_FAILURE); +} + +#endif /* H5_HAVE_MIRROR_VFD */ + diff --git a/utils/mirror_vfd/mirror_server_stop.c b/utils/mirror_vfd/mirror_server_stop.c new file mode 100644 index 0000000..f2414e0 --- /dev/null +++ b/utils/mirror_vfd/mirror_server_stop.c @@ -0,0 +1,214 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Stop the mirror server + * Exists for cross-platform, optionally remote shutdown. + */ + +#include "H5private.h" /* System compatability call-wrapper macros */ + +#ifdef H5_HAVE_MIRROR_VFD + +#define MSHS_OPTS_MAGIC 0x613B1C15u /* sanity-checking constant */ +#define MSHS_IP_STR_SIZE 20 +#define MSHS_DEFAULT_IP "127.0.0.1" +#define MSHS_DEFAULT_PORTNO 3000 + + +/* ---------------------------------------------------------------------------- + * Structure: struct mshs_opts + * + * Purpose: Convenience structure to hold options as parsed from the + * command line. + * + * `magic` (uint32_t) + * Semi-unique constant to help verify pointer integrity. + * + * `help` (int) + * Flag that the help argument was present. + * + * `portno` (int) + * Port number, as received from arguments. + * + * `ip` (char *) + * IP address string as received from arguments. + * + * ---------------------------------------------------------------------------- + */ +struct mshs_opts { + uint32_t magic; + int help; + int portno; + char ip[MSHS_IP_STR_SIZE + 1]; +}; + + +/* ---------------------------------------------------------------------------- + * Function: usage + * + * Purpose: Print usage message to stdout. + * ---------------------------------------------------------------------------- + */ +static void +usage(void) +{ + HDprintf("mirror_server_halten_sie [options]\n" \ + "System-independent Mirror Server shutdown program.\n" \ + "Sends shutdown message to Mirror Server at given IP:port\n" \ + "\n" \ + "Options:\n" \ + " -h | --help Print this usage message and exit.\n" \ + " --ip=ADDR IP Address of remote server (defaut %s)\n" \ + " --port=PORT Handshake port of remote server (default %d)\n", + MSHS_DEFAULT_IP, + MSHS_DEFAULT_PORTNO); +} /* end usage() */ + + +/* ---------------------------------------------------------------------------- + * Function: parse_args + * + * Purpose: Parse command-line arguments, populating the options struct + * pointer as appropriate. + * Default values will be set for unspecified options. + * + * Return: 0 on success, negative (-1) if error. + * ---------------------------------------------------------------------------- + */ +static int +parse_args(int argc, char **argv, struct mshs_opts *opts) +{ + int i = 0; + + opts->magic = MSHS_OPTS_MAGIC; + opts->help = 0; + opts->portno = MSHS_DEFAULT_PORTNO; + HDstrncpy(opts->ip, MSHS_DEFAULT_IP, MSHS_IP_STR_SIZE); + + for (i=1; i < argc; i++) { /* start with first possible option argument */ + if (!HDstrncmp(argv[i], "-h", 3) || !HDstrncmp(argv[i], "--help", 7)) { + opts->help = 1; + } + else + if (!HDstrncmp(argv[i], "--ip=", 5)) { + HDstrncpy(opts->ip, argv[i]+5, MSHS_IP_STR_SIZE); + } + else + if (!HDstrncmp(argv[i], "--port=", 7)) { + opts->portno = HDatoi(argv[i]+7); + } + else { + HDprintf("Unrecognized option: '%s'\n", argv[i]); + usage(); + opts->magic++; /* invalidate for sanity */ + return -1; + } + } /* end for each argument from command line */ + + /* auto-replace 'localhost' with numeric IP */ + if (!HDstrncmp(opts->ip, "localhost", 10)) { /* include null terminator */ + HDstrncpy(opts->ip, "127.0.0.1", MSHS_IP_STR_SIZE); + } + + return 0; +} /* end parse_args() */ + + +/* ---------------------------------------------------------------------------- + * Function: send_shutdown + * + * Purpose: Create socket and send shutdown signal to remote server. + * + * Return: 0 on success, negative (-1) if error. + * ---------------------------------------------------------------------------- + */ +static int +send_shutdown(struct mshs_opts *opts) +{ + int live_socket; + struct sockaddr_in target_addr; + + if (opts->magic != MSHS_OPTS_MAGIC) { + HDprintf("invalid options structure\n"); + return -1; + } + + live_socket = HDsocket(AF_INET, SOCK_STREAM, 0); + if (live_socket < 0) { + HDprintf("ERROR socket()\n"); + return -1; + } + + target_addr.sin_family = AF_INET; + target_addr.sin_port = HDhtons((uint16_t)opts->portno); + target_addr.sin_addr.s_addr = HDinet_addr(opts->ip); + HDmemset(target_addr.sin_zero, '\0', sizeof(target_addr.sin_zero)); + + if (HDconnect(live_socket, (struct sockaddr *)&target_addr, + (socklen_t)sizeof(target_addr)) + < 0) + { + HDprintf("ERROR connect() (%d)\n%s\n", errno, HDstrerror(errno)); + return -1; + } + + if (HDwrite(live_socket, "SHUTDOWN", 9) == -1) { + HDprintf("ERROR write() (%d)\n%s\n", errno, HDstrerror(errno)); + return -1; + } + + if (HDclose(live_socket) < 0) { + HDprintf("ERROR close() can't close socket\n"); + return -1; + } + + return 0; +} /* end send_shutdown() */ + + +/* ------------------------------------------------------------------------- */ +int +main(int argc, char **argv) +{ + struct mshs_opts opts; + + if (parse_args(argc, argv, &opts) < 0) { + HDprintf("Unable to parse arguments\n"); + HDexit(EXIT_FAILURE); + } + + if (opts.help) { + usage(); + HDexit(EXIT_FAILURE); + } + + if (send_shutdown(&opts) < 0) { + HDprintf("Unable to send shutdown command\n"); + HDexit(EXIT_FAILURE); + } + + HDexit(EXIT_SUCCESS); +} /* end main() */ + +#else /* H5_HAVE_MIRROR_VFD */ + + +/* ------------------------------------------------------------------------- */ +int +main(void) +{ + HDprintf("Mirror VFD not built -- unable to perform shutdown.\n"); + HDexit(EXIT_FAILURE); +} + +#endif /* H5_HAVE_MIRROR_VFD */ diff --git a/utils/mirror_vfd/mirror_writer.c b/utils/mirror_vfd/mirror_writer.c new file mode 100644 index 0000000..e1ab1b2 --- /dev/null +++ b/utils/mirror_vfd/mirror_writer.c @@ -0,0 +1,1103 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Remote writer process for the mirror (socket) VFD. + * + * Writer is started with arguments for slaved port. + * Awaits a connection on the socket. + * Handles instructions from the master 'Driver' process. + * + * Current implementation uses Sec2 as the underlying driver when opening a + * file. This is reflected in the source (H5FDmirror.c) of the Mirror driver. + */ + +#include "mirror_remote.h" + +#ifdef H5_HAVE_MIRROR_VFD + +#define HEXDUMP_XMITS 1 /* Toggle whether to print xmit bytes-blob */ + /* in detailed logging */ +#define HEXDUMP_WRITEDATA 0 /* Toggle whether to print bytes to write */ + /* in detailed logging */ +#define LISTENQ 80 /* max pending Driver requests */ + +#define MW_SESSION_MAGIC 0x88F36B32u +#define MW_SOCK_COMM_MAGIC 0xDF10A157u +#define MW_OPTS_MAGIC 0x3BA8B462u + + +/* --------------------------------------------------------------------------- + * Structure: struct mirror_session + * + * Bundle of information used to manage the operation of this remote Writer + * in a "session" with the Driver process. + * + * magic (uint32_t) + * Semi-unique "magic" number used to sanity-check a structure for + * validity. MUST equal MW_SESSION_MAGIC to be valid. + * + * sockfd (int) + * File descriptor to the socket. + * Used for receiving bytes from and writing bytes to the Driver + * across the network. + * If not NULL, should be a valid descriptor. + * + * token (uint32t) + * Number used to help sanity-check received transmission from the Writer. + * Each Driver/Writer pairing should have a semi-unique "token" to help + * guard against commands from the wrong entity. + * + * xmit_count (uint32_t) + * Record of trasmissions received from the Driver. While the transmission + * protocol should be trustworthy, this serves as an additional guard. + * Starts a 0 and should be incremented for each one-way transmission. + * + * file (H5FD_t *) + * Virtual File handle for the hdf5 file. + * Set on file open if H5Fopen() is successful. If NULL, it is invalid. + * + * log_verbosity (unsigned int) + * The verbosity level for logging. Should be set to one of the values + * defined at the top of this file. + * + * log_stream (FILE *) + * File pointer to which logging output is written. Starts (and ends) + * with a default stream, such as stdout, but can be overridden at + * runtime. + * + * reply (H5FD_mirror_xmit_reply_t) + * Structure space for persistent reply data. + * Should be initialized with basic header info (magic, version, op), + * then with session info (token, xmit count), and finally with specific + * reply info (update xmit_count, status code, and message) before + * transmission. + * + * ---------------------------------------------------------------------------- + */ +struct mirror_session { + uint32_t magic; + int sockfd; + uint32_t token; + uint32_t xmit_count; + H5FD_t *file; + loginfo_t *loginfo; + H5FD_mirror_xmit_reply_t reply; +}; + + +/* --------------------------------------------------------------------------- + * Structure: struct sock_comm + * + * Structure for placing the data read and pre-processed from Driver in an + * organized fashion. Useful for pre-processing a received xmit. + * + * magic (uint32_t) + * Semi-unique number to sanity-check structure pointer and validity. + * Must equal MW_SOCK_COMM_MAGIC to be valid. + * + * recd_die (int) + * "Boolean" flag indicating that an explicit shutdown/kill/die command + * was received. Potentially useful for debugging and or "manual" + * operation of the program. + * 0 indicates normal operation, non-0 (1) indicates to die. + * + * xmit_recd (H5FD_mirror_xmit_t *) + * Structure pointer for the "xmit header" as decoded from the raw + * binary stream read from the socket. + * + * raw (char *) + * Pointer to a raw byte array, for storing data as read from the + * socket. Bytes buffer is decoded into xmit_t header and derivative + * structures. + * + * raw_size (size_t) + * Give the size of the `raw` buffer. + * + * --------------------------------------------------------------------------- + */ +struct sock_comm { + uint32_t magic; + int recd_die; + H5FD_mirror_xmit_t *xmit_recd; + char *raw; + size_t raw_size; +}; + + +/* --------------------------------------------------------------------------- + * Structure: struct mirror_writer_opts + * + * Container for default values and options as parsed from the command line. + * Currently rather vestigal, but may be expanded and/or moved to be set by + * Server and passed around as an argument. + * + * magic (uint32_t) + * Semi-unique number to sanity-check structure pointer and validity. + * Must equal MW_OPTS_MAGIC to be valid. + * + * logpath (char *) + * String pointer. Allocated at runtime. + * Specifies file location for logging output. + * May be NULL -- uses default output (e.g., stdout). + * + * ---------------------------------------------------------------------------- + */ +struct mirror_writer_opts { + uint32_t magic; + char *logpath; +}; + +static void mybzero(void *dest, size_t size); + +static int do_open(struct mirror_session *session, + const H5FD_mirror_xmit_open_t *xmit_open); + + +/* --------------------------------------------------------------------------- + * Function: mybzero + * + * Purpose: Introduce bzero without neededing it on the system. + * + * Programmer: Jacob Smith + * 2020-03-30 + * --------------------------------------------------------------------------- + */ +static void mybzero(void *dest, size_t size) +{ + size_t i = 0; + char *s = NULL; + HDassert(dest); + s = (char *)dest; + for (i = 0; i < size; i++) { + *(s+i) = 0; + } +} /* end mybzero() */ + + +/* --------------------------------------------------------------------------- + * Function: session_init + * + * Purpose: Populate mirror_session structure with default and + * options-drived values. + * + * Return: An allocated mirror_session structure pointer on success, + * else NULL. + * ---------------------------------------------------------------------------- + */ +static struct mirror_session * +session_init(struct mirror_writer_opts *opts) +{ + struct mirror_session *session = NULL; + + mirror_log(NULL, V_INFO, "session_init()"); + + if (NULL == opts || opts->magic != MW_OPTS_MAGIC) { + mirror_log(NULL, V_ERR, "invalid opts pointer"); + goto error; + } + + session = (struct mirror_session *)HDmalloc(sizeof(struct mirror_session)); + if (session == NULL) { + mirror_log(NULL, V_ERR, "can't allocate session structure"); + goto error; + } + + session->magic = MW_SESSION_MAGIC; + session->sockfd = -1; + session->xmit_count = 0; + session->token = 0; + session->file = NULL; + + session->reply.pub.magic = H5FD_MIRROR_XMIT_MAGIC; + session->reply.pub.version = H5FD_MIRROR_XMIT_CURR_VERSION; + session->reply.pub.op = H5FD_MIRROR_OP_REPLY; + session->reply.pub.session_token = 0; + mybzero(session->reply.message, H5FD_MIRROR_STATUS_MESSAGE_MAX); + + /* Options-derived population + */ + + session->loginfo = mirror_log_init(opts->logpath, "W- ", + MIRROR_LOG_DEFAULT_VERBOSITY); + + return session; + +error: + if (session) { + HDfree(session); + } + return NULL; +} /* end session_init() */ + + +/* --------------------------------------------------------------------------- + * Function: session_stop + * + * Purpose: Stop and clean up a session. + * Only do this as part of program termination or aborting startup. + * + * Return: 0 on success, or negative sum of errors encountered. + * ---------------------------------------------------------------------------- + */ +static int +session_stop(struct mirror_session *session) +{ + int ret_value = 0; + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_INFO, "session_stop()"); + + /* Close HDF5 file if it is still open (probably in error) */ + if (session->file) { + mirror_log(session->loginfo, V_WARN, "HDF5 file still open at cleanup"); + if (H5FDclose(session->file) < 0) { + mirror_log(session->loginfo, V_ERR, "H5FDclose() during cleanup!"); + ret_value--; + } + } + + /* Socket will be closed by parent side of server fork after exit */ + + /* Close custom logging stream */ + if (mirror_log_term(session->loginfo) < 0) { + mirror_log(NULL, V_ERR, "Problem closing logging stream"); + ret_value--; + } + session->loginfo = NULL; + + /* Invalidate and release structure */ + session->magic++; + HDfree(session); + + return ret_value; +} /* end session_stop() */ + + +/* --------------------------------------------------------------------------- + * Function: session_start + * + * Purpose: Initiate session, open files. + * + * Return: Success: A valid mirror_session pointer which must later be + * cleaned up with session_stop(). + * Failure: NULL, after cleaning up after itself. + * --------------------------------------------------------------------------- + */ +static struct mirror_session * +session_start(int socketfd, const H5FD_mirror_xmit_open_t *xmit_open) +{ + struct mirror_session *session = NULL; + struct mirror_writer_opts opts; +#if 0 /* TODO: behaviro option */ + char logpath[H5FD_MIRROR_XMIT_FILEPATH_MAX] = ""; +#endif + + mirror_log(NULL, V_INFO, "session_start()"); + + if (FALSE == H5FD_mirror_xmit_is_open(xmit_open)) { + mirror_log(NULL, V_ERR, "invalid OPEN xmit"); + return NULL; + } + + opts.magic = MW_OPTS_MAGIC; +#if 0 /* TODO: behavior option */ + HDsnprintf(logpath, H5FD_MIRROR_XMIT_FILEPATH_MAX, "%s.log", + xmit_open->filename); + opts.logpath = logpath; +#else + opts.logpath = NULL; +#endif + + session = session_init(&opts); + if (NULL == session) { + mirror_log(NULL, V_ERR, "can't instantiate session"); + goto error; + } + + session->sockfd = socketfd; + + if (do_open(session, xmit_open) < 0) { + mirror_log(NULL, V_ERR, "unable to open file"); + goto error; + } + + return session; + +error: + if (session != NULL) { + if (session_stop(session) < 0) { + mirror_log(NULL, V_WARN, "Can't abort session init"); + } + session = NULL; + } + return NULL; +} + + +/* --------------------------------------------------------------------------- + * Function: _xmit_reply + * + * Purpose: Common operations to send a reply xmit through the session. + * + * Return: 0 on success, -1 if error. + * ---------------------------------------------------------------------------- + */ +static int +_xmit_reply(struct mirror_session *session) +{ + unsigned char xmit_buf[H5FD_MIRROR_XMIT_REPLY_SIZE]; + H5FD_mirror_xmit_reply_t *reply = &(session->reply); + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_ALL, "_xmit_reply()"); + + reply->pub.xmit_count = session->xmit_count++; + if (H5FD_mirror_xmit_encode_reply(xmit_buf, + (const H5FD_mirror_xmit_reply_t *)reply) + != H5FD_MIRROR_XMIT_REPLY_SIZE) + { + mirror_log(session->loginfo, V_ERR, "can't encode reply"); + return -1; + } + + mirror_log(session->loginfo, V_ALL, "reply xmit data\n```"); + mirror_log_bytes(session->loginfo, V_ALL, H5FD_MIRROR_XMIT_REPLY_SIZE, + (const unsigned char *)xmit_buf); + mirror_log(session->loginfo, V_ALL, "```"); + + if (HDwrite(session->sockfd, xmit_buf, H5FD_MIRROR_XMIT_REPLY_SIZE) < 0) { + mirror_log(session->loginfo, V_ERR, "can't write reply to Driver"); + return -1; + } + + return 0; +} /* end _xmit_reply() */ + + +/* --------------------------------------------------------------------------- + * Function: reply_ok + * + * Purpose: Send an OK reply through the session. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +reply_ok(struct mirror_session *session) +{ + H5FD_mirror_xmit_reply_t *reply = &(session->reply); + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_ALL, "reply_ok()"); + + reply->status = H5FD_MIRROR_STATUS_OK; + mybzero(reply->message, H5FD_MIRROR_STATUS_MESSAGE_MAX); + return _xmit_reply(session); +} /* end reply_ok() */ + + +/* --------------------------------------------------------------------------- + * Function: reply_error + * + * Purpose: Send an ERROR reply with message through the session. + * Message may be cut short if it would overflow the available + * buffer in the xmit. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +reply_error(struct mirror_session *session, + const char *msg) +{ + H5FD_mirror_xmit_reply_t *reply = &(session->reply); + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_ALL, "reply_error(%s)", msg); + + reply->status = H5FD_MIRROR_STATUS_ERROR; + HDsnprintf(reply->message, H5FD_MIRROR_STATUS_MESSAGE_MAX-1, "%s", msg); + return _xmit_reply(session); +} /* end reply_error() */ + + +/* --------------------------------------------------------------------------- + * Function: do_close + * + * Purpose: Handle an CLOSE operation. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_close(struct mirror_session *session) +{ + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_INFO, "do_close()"); + + if (NULL == session->file) { + mirror_log(session->loginfo, V_ERR, "no file to close!"); + reply_error(session, "no file to close"); + return -1; + } + + if (H5FDclose(session->file) < 0) { + mirror_log(session->loginfo, V_ERR, "H5FDclose()"); + reply_error(session, "H5FDclose()"); + return -1; + } + session->file = NULL; + + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; +} /* end do_close() */ + + +/* --------------------------------------------------------------------------- + * Function: do_lock + * + * Purpose: Handle a LOCK operation. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_lock(struct mirror_session *session, + const unsigned char *xmit_buf) +{ + size_t decode_ret = 0; + H5FD_mirror_xmit_lock_t xmit_lock; + + HDassert(session && (session->magic == MW_SESSION_MAGIC) && xmit_buf); + + mirror_log(session->loginfo, V_INFO, "do_lock()"); + + decode_ret = H5FD_mirror_xmit_decode_lock(&xmit_lock, xmit_buf); + if (H5FD_MIRROR_XMIT_LOCK_SIZE != decode_ret) { + mirror_log(session->loginfo, V_ERR, "can't decode set-eoa xmit"); + reply_error(session, "remote xmit_eoa_t decoding size failure"); + return -1; + } + + if (!H5FD_mirror_xmit_is_lock(&xmit_lock)) { + mirror_log(session->loginfo, V_ERR, "not a set-eoa xmit"); + reply_error(session, "remote xmit_eoa_t decode failure"); + return -1; + } + mirror_log(session->loginfo, V_INFO, "lock rw: (%d)", xmit_lock.rw); + + if (H5FDlock(session->file, (hbool_t)xmit_lock.rw) < 0) { + mirror_log(session->loginfo, V_ERR, "H5FDlock()"); + reply_error(session, "remote H5FDlock() failure"); + return -1; + } + + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; +} /* end do_lock() */ + + +/* --------------------------------------------------------------------------- + * Function: do_open + * + * Purpose: Handle an OPEN operation. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_open(struct mirror_session *session, + const H5FD_mirror_xmit_open_t *xmit_open) +{ + hid_t fapl_id = H5I_INVALID_HID; + unsigned _flags = 0; + haddr_t _maxaddr = HADDR_UNDEF; + + HDassert(session && (session->magic == MW_SESSION_MAGIC) && xmit_open && + TRUE == H5FD_mirror_xmit_is_open(xmit_open)); + + mirror_log(session->loginfo, V_INFO, "do_open()"); + + if (0 != xmit_open->pub.xmit_count) { + mirror_log(session->loginfo, V_ERR, "open with xmit count not zero!"); + reply_error(session, "initial transmission count not zero"); + goto error; + } + if (0 != session->token) { + mirror_log(session->loginfo, V_ERR, "open with token already set!"); + reply_error(session, "initial session token not zero"); + goto error; + } + + session->xmit_count = 1; + session->token = xmit_open->pub.session_token; + session->reply.pub.session_token = session->token; + + _flags = (unsigned)xmit_open->flags; + _maxaddr = (haddr_t)xmit_open->maxaddr; + + /* Check whether the native size_t on the remote machine (Driver) is larger + * than that on the local machine; if so, issue a warning. + * The blob is always an 8-byte bitfield -- check its contents. + */ + if (xmit_open->size_t_blob > (uint64_t)((size_t)(-1))) { + mirror_log(session->loginfo, V_WARN, + "Driver size_t is larger than our own"); + } + + mirror_log(session->loginfo, V_INFO, + "to open file %s (flags %d) (maxaddr %d)", + xmit_open->filename, _flags, _maxaddr); + + /* Explicitly use Sec2 as the underlying driver for now. + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (fapl_id < 0) { + mirror_log(session->loginfo, V_ERR, "can't create FAPL"); + reply_error(session, "H5Pcreate() failure"); + goto error; + } + if (H5Pset_fapl_sec2(fapl_id) < 0) { + mirror_log(session->loginfo, V_ERR, "can't set FAPL as Sec2"); + reply_error(session, "H5Pset_fapl_sec2() failure"); + goto error; + } + + session->file = H5FDopen(xmit_open->filename, _flags, fapl_id, _maxaddr); + if (NULL == session->file) { + mirror_log(session->loginfo, V_ERR, "H5FDopen()"); + reply_error(session, "remote H5FDopen() failure"); + goto error; + } + + /* FAPL is set and in use; clean up */ + if (H5Pclose(fapl_id) < 0) { + mirror_log(session->loginfo, V_ERR, "can't set FAPL as Sec2"); + reply_error(session, "H5Pset_fapl_sec2() failure"); + goto error; + } + + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; + +error: + if (fapl_id > 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + return -1; +} /* end do_open() */ + + +/* --------------------------------------------------------------------------- + * Function: do_set_eoa + * + * Purpose: Handle a SET_EOA operation. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_set_eoa(struct mirror_session *session, + const unsigned char *xmit_buf) +{ + size_t decode_ret = 0; + H5FD_mirror_xmit_eoa_t xmit_seoa; + + HDassert(session && (session->magic == MW_SESSION_MAGIC) && xmit_buf); + + mirror_log(session->loginfo, V_INFO, "do_set_eoa()"); + + decode_ret = H5FD_mirror_xmit_decode_set_eoa(&xmit_seoa, xmit_buf); + if (H5FD_MIRROR_XMIT_EOA_SIZE != decode_ret) { + mirror_log(session->loginfo, V_ERR, "can't decode set-eoa xmit"); + reply_error(session, "remote xmit_eoa_t decoding size failure"); + return -1; + } + + if (!H5FD_mirror_xmit_is_set_eoa(&xmit_seoa)) { + mirror_log(session->loginfo, V_ERR, "not a set-eoa xmit"); + reply_error(session, "remote xmit_eoa_t decode failure"); + return -1; + } + + mirror_log(session->loginfo, V_INFO, "set EOA addr %d", + xmit_seoa.eoa_addr); + + if (H5FDset_eoa(session->file, (H5FD_mem_t)xmit_seoa.type, + (haddr_t)xmit_seoa.eoa_addr) + < 0) + { + mirror_log(session->loginfo, V_ERR, "H5FDset_eoa()"); + reply_error(session, "remote H5FDset_eoa() failure"); + return -1; + } + + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; +} /* end do_set_eoa() */ + + +/* --------------------------------------------------------------------------- + * Function: do_truncate + * + * Purpose: Handle a TRUNCATE operation. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_truncate(struct mirror_session *session) +{ + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_INFO, "do_truncate()"); + + /* default DXPL ID (0), 0 for "FALSE" closing -- both probably unused */ + if (H5FDtruncate(session->file, 0, 0) < 0) { + mirror_log(session->loginfo, V_ERR, "H5FDtruncate()"); + reply_error(session, "remote H5FDtruncate() failure"); + return -1; + } + + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; +} /* end do_truncate() */ + + +/* --------------------------------------------------------------------------- + * Function: do_unlock + * + * Purpose: Handle an UNLOCK operation. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_unlock(struct mirror_session *session) +{ + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_INFO, "do_unlock()"); + + if (H5FDunlock(session->file) < 0) { + mirror_log(session->loginfo, V_ERR, "H5FDunlock()"); + reply_error(session, "remote H5FDunlock() failure"); + return -1; + } + + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; +} /* end do_unlock() */ + + +/* --------------------------------------------------------------------------- + * Function: do_write + * + * Purpose: Handle a WRITE operation. + * Receives command, replies; receives & writes data, replies. + * + * It is known that this results in suboptimal performance, + * but handling both small and very, very large write buffers + * with a single "over the wire" exchange + * poses design challenges not worth tackling as of March 2020. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +do_write(struct mirror_session *session, + const unsigned char *xmit_buf) +{ + size_t decode_ret = 0; + haddr_t addr = 0; + haddr_t sum_bytes_written = 0; + H5FD_mem_t type = 0; + char *buf = NULL; + ssize_t nbytes_in_packet = 0; + H5FD_mirror_xmit_write_t xmit_write; + + HDassert(session && (session->magic == MW_SESSION_MAGIC) && xmit_buf); + + mirror_log(session->loginfo, V_INFO, "do_write()"); + + if (NULL == session->file) { + mirror_log(session->loginfo, V_ERR, "no open file!"); + reply_error(session, "no file open on remote"); + return -1; + } + + decode_ret = H5FD_mirror_xmit_decode_write(&xmit_write, xmit_buf); + if (H5FD_MIRROR_XMIT_WRITE_SIZE != decode_ret) { + mirror_log(session->loginfo, V_ERR, "can't decode write xmit"); + reply_error(session, "remote xmit_write_t decoding size failure"); + return -1; + } + + if (!H5FD_mirror_xmit_is_write(&xmit_write)) { + mirror_log(session->loginfo, V_ERR, "not a write xmit"); + reply_error(session, "remote xmit_write_t decode failure"); + return -1; + } + + addr = (haddr_t)xmit_write.offset; + type = (H5FD_mem_t)xmit_write.type; + + /* Allocate the buffer once -- re-use between loops. + */ + buf = (char *)HDmalloc(sizeof(char) * H5FD_MIRROR_DATA_BUFFER_MAX); + if (NULL == buf) { + mirror_log(session->loginfo, V_ERR, "can't allocate databuffer"); + reply_error(session, "can't allocate buffer for receiving data"); + return -1; + } + + /* got write signal; ready for data */ + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + mirror_log(session->loginfo, V_INFO, "to write %zu bytes at %zu", + xmit_write.size, + addr); + + /* The given write may be: + * 1. larger than the allowed single buffer size + * 2. larger than the native size_t of this system + * + * Handle all cases by looping, ingesting as much of the stream as possible + * and writing that part to the file. + */ + sum_bytes_written = 0; + do { + nbytes_in_packet = HDread(session->sockfd, buf, + H5FD_MIRROR_DATA_BUFFER_MAX); + if (-1 == nbytes_in_packet) { + mirror_log(session->loginfo, V_ERR, "can't read into databuffer"); + reply_error(session, "can't read data buffer"); + return -1; + } + + mirror_log(session->loginfo, V_INFO, "received %zd bytes", + nbytes_in_packet); + if (HEXDUMP_WRITEDATA) { + mirror_log(session->loginfo, V_ALL, "DATA:\n```"); + mirror_log_bytes(session->loginfo, V_ALL, nbytes_in_packet, + (const unsigned char *)buf); + mirror_log(session->loginfo, V_ALL, "```"); + } + + mirror_log(session->loginfo, V_INFO, "writing %zd bytes at %zu", + nbytes_in_packet, + (addr + sum_bytes_written)); + + if (H5FDwrite(session->file, type, H5P_DEFAULT, + (addr + sum_bytes_written), (size_t)nbytes_in_packet, buf) + < 0) + { + mirror_log(session->loginfo, V_ERR, "H5FDwrite()"); + reply_error(session, "remote H5FDwrite() failure"); + return -1; + } + + sum_bytes_written += (haddr_t)nbytes_in_packet; + + } while (sum_bytes_written < xmit_write.size); /* end while ingesting */ + + HDfree(buf); + + /* signal that we're done here and a-ok */ + if (reply_ok(session) < 0) { + mirror_log(session->loginfo, V_ERR, "can't reply"); + reply_error(session, "ok reply failed; session contaminated"); + return -1; + } + + return 0; +} /* end do_write() */ + + +/* --------------------------------------------------------------------------- + * Function: receive_communique + * + * Purpose: Accept bytes from the socket, check for emergency shutdown, and + * sanity-check received bytes. + * The raw bytes read are stored in the sock_comm structure at + * comm->raw. + * The raw bytes are decoded and a xmit_t (header) struct pointer + * in comm is populated at comm->xmit_recd. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +receive_communique( + struct mirror_session *session, + struct sock_comm *comm) +{ + ssize_t read_ret = 0; + size_t decode_ret; + H5FD_mirror_xmit_t *X = comm->xmit_recd; + + HDassert((session != NULL) && \ + (session->magic == MW_SESSION_MAGIC) && \ + (comm != NULL) && \ + (comm->magic == MW_SOCK_COMM_MAGIC) && \ + (comm->xmit_recd != NULL) && \ + (comm->raw != NULL) && \ + (comm->raw_size >= H5FD_MIRROR_XMIT_BUFFER_MAX) ); + + mirror_log(session->loginfo, V_INFO, "receive_communique()"); + + mybzero(comm->raw, comm->raw_size); + comm->recd_die = 0; + + mirror_log(session->loginfo, V_INFO, "ready to receive"); /* TODO */ + + read_ret = HDread(session->sockfd, comm->raw, H5FD_MIRROR_XMIT_BUFFER_MAX); + if (-1 == read_ret) { + mirror_log(session->loginfo, V_ERR, "read:%zd", read_ret); + goto error; + } + + mirror_log(session->loginfo, V_INFO, "received %zd bytes", read_ret); + if (HEXDUMP_XMITS) { + mirror_log(session->loginfo, V_ALL, "```", read_ret); + mirror_log_bytes(session->loginfo, V_ALL, (size_t)read_ret, + (const unsigned char *)comm->raw); + mirror_log(session->loginfo, V_ALL, "```"); + } /* end if hexdump transmissions received */ + + /* old-fashioned manual kill (for debugging) */ + if (!HDstrncmp("GOODBYE", comm->raw, 7)) { + mirror_log(session->loginfo, V_INFO, "received GOODBYE"); + comm->recd_die = 1; + goto done; + } + + decode_ret = H5FD_mirror_xmit_decode_header(X, + (const unsigned char *)comm->raw); + if (H5FD_MIRROR_XMIT_HEADER_SIZE != decode_ret) { + mirror_log(session->loginfo, V_ERR, + "header decode size mismatch: expected (%z), got (%z)", + H5FD_MIRROR_XMIT_HEADER_SIZE, decode_ret); + /* Try to tell Driver that it should stop */ + reply_error(session, "xmit size mismatch"); + goto error; + } + + if (!H5FD_mirror_xmit_is_xmit(X)) { + mirror_log(session->loginfo, V_ERR, "bad magic: 0x%X", X->magic); + /* Try to tell Driver that it should stop */ + reply_error(session, "bad magic"); + goto error; + } + + if (session->xmit_count != X->xmit_count) { + mirror_log(session->loginfo, V_ERR, + "xmit_count mismatch exp:%d recd:%d", + session->xmit_count, X->xmit_count); + /* Try to tell Driver that it should stop */ + reply_error(session, "xmit_count mismatch"); + goto error; + } + + if ( (session->token > 0) && (session->token != X->session_token) ) { + mirror_log(session->loginfo, V_ERR, "wrong session"); + /* Try to tell Driver that it should stop */ + reply_error(session, "wrong session"); + goto error; + } + + session->xmit_count++; + +done: + return 0; + +error: + return -1; +} /* end receive_communique() */ + + +/* --------------------------------------------------------------------------- + * Function: process_instructions + * + * Purpose: Receive and handle all instructions from Driver. + * + * Return: 0 on success, -1 if error. + * --------------------------------------------------------------------------- + */ +static int +process_instructions(struct mirror_session *session) +{ + struct sock_comm comm; + char xmit_buf[H5FD_MIRROR_XMIT_BUFFER_MAX]; /* raw bytes */ + H5FD_mirror_xmit_t xmit_recd; /* for decoded xmit header */ + + HDassert(session && (session->magic == MW_SESSION_MAGIC)); + + mirror_log(session->loginfo, V_INFO, "process_instructions()"); + + comm.magic = MW_SOCK_COMM_MAGIC; + comm.recd_die = 0; /* Flag for program to terminate */ + comm.xmit_recd = &xmit_recd; + comm.raw = xmit_buf; + comm.raw_size = sizeof(xmit_buf); + + while (1) { /* sill-listening infinite loop */ + + /* Use convenience structure for raw/decoded info in/out */ + if (receive_communique(session, &comm) < 0) { + mirror_log(session->loginfo, V_ERR, "problem reading socket"); + return -1; + } + + if (comm.recd_die) { + goto done; + } + + switch(xmit_recd.op) { + case H5FD_MIRROR_OP_CLOSE: + if (do_close(session) < 0) { + return -1; + } + goto done; + case H5FD_MIRROR_OP_LOCK: + if (do_lock(session, (const unsigned char *)xmit_buf) < 0) { + return -1; + } + break; + case H5FD_MIRROR_OP_OPEN: + mirror_log(session->loginfo, V_ERR, "OPEN xmit during session"); + reply_error(session, "illegal OPEN xmit during session"); + return -1; + case H5FD_MIRROR_OP_SET_EOA: + if (do_set_eoa(session, (const unsigned char *)xmit_buf) < 0) { + return -1; + } + break; + case H5FD_MIRROR_OP_TRUNCATE: + if (do_truncate(session) < 0) { + return -1; + } + break; + case H5FD_MIRROR_OP_UNLOCK: + if (do_unlock(session) < 0) { + return -1; + } + break; + case H5FD_MIRROR_OP_WRITE: + if (do_write(session, (const unsigned char *)xmit_buf) < 0) { + return -1; + } + break; + default: + mirror_log(session->loginfo, V_ERR, "unrecognized transmission"); + reply_error(session, "unrecognized transmission"); + return -1; + } /* end switch (xmit_recd.op) */ + + } /* end while still listening */ + +done: + comm.magic = 0; /* invalidate structure, on principle */ + xmit_recd.magic = 0; /* invalidate structure, on principle */ + return 0; +} /* end process_instructions() */ + + +/* --------------------------------------------------------------------------- + * Function: run_writer + * + * Purpose: Initiate Writer operations. + * + * Receives as parameters a socket which has accepted the + * connection to the Driver and the OPEN xmit (which must be + * decoded into the structure and verified prior to being passed + * to this function). + * + * Is not responsible for closing or cleaning up any of the + * received parameters. + * + * Return: Success: SUCCEED + * Failure: FAIL + * --------------------------------------------------------------------------- + */ +herr_t +run_writer(int socketfd, H5FD_mirror_xmit_open_t *xmit_open) +{ + struct mirror_session *session = NULL; + int ret_value = SUCCEED; + + session = session_start(socketfd, xmit_open); + if (NULL == session) { + mirror_log(NULL, V_ERR, "Can't start session -- aborting"); + ret_value = FAIL; + } + else { + if (process_instructions(session) < 0) { + mirror_log(session->loginfo, V_ERR, + "problem processing instructions"); + ret_value = FAIL; + } + if (session_stop(session) < 0) { + mirror_log(NULL, V_ERR, "Can't stop session -- going down hard"); + ret_value = FAIL; + } + } + + return ret_value; +} /* end run_writer */ + +#endif /* H5_HAVE_MIRROR_VFD */ + |