From a4d6a270783d69c663357c31fcd6196b98706dfb Mon Sep 17 00:00:00 2001 From: Allen Byrne Date: Mon, 23 Sep 2019 15:30:07 -0500 Subject: HDFFV-10903 merge S3 from dev --- CMakeLists.txt | 25 + MANIFEST | 34 + config/cmake/ConfigureChecks.cmake | 38 +- config/cmake/FindHDFS.cmake | 70 + config/cmake/H5pubconf.h.in | 31 + config/cmake/HDFCompilerFlags.cmake | 84 +- config/cmake/libhdf5.settings.cmake.in | 36 +- config/cmake/scripts/CTestScript.cmake | 20 +- config/cmake/scripts/HDF5config.cmake | 101 +- config/cmake/scripts/HPC/bsub-HDF5options.cmake | 31 + config/cmake/scripts/HPC/qsub-HDF5options.cmake | 42 + config/cmake/scripts/HPC/raybsub-HDF5options.cmake | 32 + config/cmake/scripts/HPC/sbatch-HDF5options.cmake | 43 + config/cmake/wait_H5Tinit.cmake | 11 + config/cmake_ext_mod/runTest.cmake | 9 + config/toolchain/build32.cmake | 36 + config/toolchain/crayle.cmake | 10 + release_docs/README_HDF5_CMake | 23 + release_docs/README_HPC | 206 ++ release_docs/RELEASE.txt | 25 + release_docs/USING_CMake_Examples.txt | 5 +- src/CMakeLists.txt | 261 +- src/H5FD.c | 1054 +++--- src/H5FDcore.h | 2 +- src/H5FDhdfs.c | 1876 ++++++++++ src/H5FDhdfs.h | 123 + src/H5FDprivate.h | 16 +- src/H5FDpublic.h | 73 +- src/H5FDros3.c | 1787 ++++++++++ src/H5FDros3.h | 106 + src/H5FDs3comms.c | 3593 ++++++++++++++++++++ src/H5FDs3comms.h | 610 ++++ src/Makefile.am | 14 +- src/Makefile.in | 11 +- src/libhdf5.settings.in | 88 +- test/CMakeLists.txt | 4 + test/Makefile.am | 6 +- test/dtypes.c | 1307 +++---- test/extend.c | 22 +- test/filter_fail.c | 12 +- test/flush1.c | 207 +- test/flush2.c | 65 +- test/h5test.c | 39 +- test/hdfs.c | 1767 ++++++++++ test/hyperslab.c | 12 +- test/istore.c | 17 +- test/links.c | 4 +- test/links_env.c | 12 +- test/mtime.c | 66 +- test/objcopy.c | 6 +- test/reserved.c | 80 +- test/ros3.c | 1937 +++++++++++ test/s3comms.c | 2730 +++++++++++++++ test/testframe.c | 5 +- test/vfd.c | 950 ++++-- tools/CMakeLists.txt | 9 + tools/h5copy/h5copy.c | 8 +- tools/h5copy/h5copygentest.c | 20 +- tools/h5diff/h5diffgentest.c | 162 +- tools/h5diff/ph5diff_main.c | 26 +- tools/h5dump/h5dump.c | 328 +- tools/h5dump/h5dump_ddl.c | 31 +- tools/h5dump/h5dump_defines.h | 24 +- tools/h5dump/h5dump_xml.c | 5 +- tools/h5dump/h5dumpgentest.c | 99 +- tools/h5import/h5importtest.c | 2 +- tools/h5jam/h5jam.c | 42 +- tools/h5jam/h5jamgentest.c | 12 +- tools/h5jam/h5unjam.c | 12 +- tools/h5ls/h5ls.c | 1216 ++++--- tools/h5repack/h5repack.c | 64 +- tools/h5repack/h5repack.h | 28 +- tools/h5repack/h5repack_copy.c | 34 +- tools/h5repack/h5repack_filters.c | 16 +- tools/h5repack/h5repack_main.c | 4 +- tools/h5repack/h5repack_refs.c | 34 +- tools/h5repack/testfiles/h5repack-help.txt | 4 +- tools/h5stat/h5stat.c | 386 ++- tools/h5stat/testfiles/h5stat_help1.ddl | 10 + tools/h5stat/testfiles/h5stat_help2.ddl | 10 + tools/h5stat/testfiles/h5stat_nofile.ddl | 10 + tools/lib/h5diff_array.c | 11 +- tools/lib/h5tools_dump.c | 26 +- tools/lib/h5tools_utils.c | 485 ++- tools/lib/h5tools_utils.h | 13 + tools/libtest/CMakeLists.txt | 18 + tools/libtest/CMakeTests.cmake | 49 + tools/libtest/Makefile.am | 34 + tools/libtest/h5tools_test_utils.c | 1265 +++++++ tools/misc/h5debug.c | 173 +- tools/misc/h5mkgrp.c | 6 +- tools/misc/h5repart.c | 585 ++-- tools/testfiles/h5dump-help.txt | 10 + tools/testfiles/help-1.ls | 9 + tools/testfiles/help-2.ls | 9 + tools/testfiles/help-3.ls | 9 + .../pbits/tnofilename-with-packed-bits.ddl | 10 + tools/testfiles/pbits/tpbitsIncomplete.ddl | 10 + tools/testfiles/pbits/tpbitsLengthExceeded.ddl | 10 + tools/testfiles/pbits/tpbitsLengthPositive.ddl | 10 + tools/testfiles/pbits/tpbitsMaxExceeded.ddl | 10 + tools/testfiles/pbits/tpbitsOffsetExceeded.ddl | 10 + tools/testfiles/pbits/tpbitsOffsetNegative.ddl | 10 + tools/testfiles/textlinksrc-nodangle-1.ls | 9 + tools/testfiles/tgroup-1.ls | 9 + tools/testfiles/tudlink-2.ddl | 1 - 106 files changed, 21731 insertions(+), 3430 deletions(-) create mode 100644 config/cmake/FindHDFS.cmake create mode 100644 config/cmake/scripts/HPC/bsub-HDF5options.cmake create mode 100644 config/cmake/scripts/HPC/qsub-HDF5options.cmake create mode 100644 config/cmake/scripts/HPC/raybsub-HDF5options.cmake create mode 100644 config/cmake/scripts/HPC/sbatch-HDF5options.cmake create mode 100644 config/cmake/wait_H5Tinit.cmake create mode 100644 config/toolchain/crayle.cmake create mode 100644 release_docs/README_HDF5_CMake create mode 100644 release_docs/README_HPC create mode 100644 src/H5FDhdfs.c create mode 100644 src/H5FDhdfs.h create mode 100644 src/H5FDros3.c create mode 100644 src/H5FDros3.h create mode 100644 src/H5FDs3comms.c create mode 100644 src/H5FDs3comms.h create mode 100644 test/hdfs.c create mode 100644 test/ros3.c create mode 100644 test/s3comms.c create mode 100644 tools/libtest/CMakeLists.txt create mode 100644 tools/libtest/CMakeTests.cmake create mode 100644 tools/libtest/Makefile.am create mode 100644 tools/libtest/h5tools_test_utils.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 4fe71ff..63a0dfd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,10 @@ cmake_minimum_required (VERSION 3.10) project (HDF5 C) +if(POLICY CMP0074) + cmake_policy(SET CMP0074 NEW) +endif() + #----------------------------------------------------------------------------- # Instructions for use : Normal Build # @@ -469,6 +473,27 @@ include (${HDF_RESOURCES_DIR}/HDFCompilerFlags.cmake) set (CMAKE_MODULE_PATH ${HDF_RESOURCES_DIR} ${HDF_RESOURCES_EXT_DIR} ${CMAKE_MODULE_PATH}) #----------------------------------------------------------------------------- +# Option to Enable HDFS +#----------------------------------------------------------------------------- +option (HDF5_ENABLE_HDFS "Enable HDFS" OFF) +if (HDF5_ENABLE_HDFS) + find_package(JNI REQUIRED) + if (JNI_FOUND) + set (H5_HAVE_LIBJVM 1) + endif () + find_package(HDFS REQUIRED) + if (HDFS_FOUND) + set (H5_HAVE_LIBHDFS 1) + set (H5_HAVE_HDFS_H 1) + if (NOT MSVC) + list (APPEND LINK_LIBS -pthread) + endif () + else () + message (FATAL_ERROR "Set to use libhdfs library, but could not find or use libhdfs. Please verify that the path to HADOOP_HOME is valid, and/or reconfigure without HDF5_ENABLE_HDFS") + endif () +endif () + +#----------------------------------------------------------------------------- # Option to Enable MPI Parallel #----------------------------------------------------------------------------- option (HDF5_ENABLE_PARALLEL "Enable parallel build (requires MPI)" OFF) diff --git a/MANIFEST b/MANIFEST index eeae62a..463042d 100644 --- a/MANIFEST +++ b/MANIFEST @@ -93,6 +93,17 @@ ./bin/timekeeper _DO_NOT_DISTRIBUTE_ ./bin/trace ./bin/yodconfigure +./bin/batch/ctestP.lsf.in.cmake +./bin/batch/ctestP.sl.in.cmake +./bin/batch/ctestS.lsf.in.cmake +./bin/batch/ctestS.sl.in.cmake +./bin/batch/knl_ctestP.sl.in.cmake +./bin/batch/knl_ctestS.sl.in.cmake +./bin/batch/knl_H5detect.sl.in.cmake +./bin/batch/ctest.qsub.in.cmake +./bin/batch/ray_ctestP.lsf.in.cmake +./bin/batch/ray_ctestS.lsf.in.cmake +./bin/batch/raybsub ./config/COPYING ./config/BlankForm @@ -466,6 +477,8 @@ ./release_docs/INSTALL_parallel ./release_docs/INSTALL_Warnings.txt ./release_docs/INSTALL_Windows.txt +./release_docs/README_HDF5_CMake +./release_docs/README_HPC ./release_docs/RELEASE.txt ./release_docs/USING_HDF5_CMake.txt ./release_docs/USING_HDF5_VS.txt @@ -591,9 +604,13 @@ ./src/H5FDmpio.h ./src/H5FDmulti.c ./src/H5FDmulti.h +./src/H5FDros3.c +./src/H5FDros3.h ./src/H5FDpkg.h ./src/H5FDprivate.h ./src/H5FDpublic.h +./src/H5FDs3comms.h +./src/H5FDs3comms.c ./src/H5FDsec2.c ./src/H5FDsec2.h ./src/H5FDspace.c @@ -921,6 +938,7 @@ ./test/group_old.h5 ./test/h5test.c ./test/h5test.h +./test/hdfs.c ./test/hyperslab.c ./test/istore.c ./test/le_data.h5 @@ -940,7 +958,9 @@ ./test/objcopy.c ./test/plugin.c ./test/reserved.c +./test/ros3.c ./test/pool.c +./test/s3comms.c ./test/set_extent.c # ====distribute this for now. See HDFFV-8236==== ./test/space_overflow.c @@ -1191,6 +1211,10 @@ ./tools/lib/io_timer.c ./tools/lib/io_timer.h +./tools/libtest/Makefile.am +./tools/libtest/Makefile.in +./tools/libtest/h5tools_test_utils.c + ./tools/misc/Makefile.am ./tools/misc/Makefile.in ./tools/misc/h5debug.c @@ -2341,6 +2365,7 @@ ./config/cmake/ConfigureChecks.cmake ./config/cmake/CPack.Info.plist.in ./config/cmake/CTestCustom.cmake +./config/cmake/FindHDFS.cmake ./config/cmake/H5cxx_config.h.in ./config/cmake/H5pubconf.h.in ./config/cmake/hdf5-config.cmake.in @@ -2357,6 +2382,7 @@ ./config/cmake/README.txt.cmake.in ./config/cmake/userblockTest.cmake ./config/cmake/vfdTest.cmake +./config/cmake/wait_H5Tinit.cmake ./config/cmake_ext_mod/ConfigureChecks.cmake ./config/cmake_ext_mod/CTestCustom.cmake @@ -2458,6 +2484,8 @@ ./tools/h5stat/CMakeLists.txt ./tools/h5stat/CMakeTests.cmake ./tools/lib/CMakeLists.txt +./tools/libtest/CMakeLists.txt +./tools/libtest/CMakeTests.cmake ./tools/misc/CMakeLists.txt ./tools/misc/CMakeTestsMkgrp.cmake ./tools/misc/CMakeTestsRepart.cmake @@ -2470,3 +2498,9 @@ ./config/cmake/scripts/CTestScript.cmake ./config/cmake/scripts/HDF5config.cmake ./config/cmake/scripts/HDF5options.cmake + +# CMake-specific HPC Scripts +./config/cmake/scripts/HPC/sbatch-HDF5options.cmake +./config/cmake/scripts/HPC/bsub-HDF5options.cmake +./config/cmake/scripts/HPC/qsub-HDF5options.cmake +./config/cmake/scripts/HPC/raybsub-HDF5options.cmake diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 3c23375..938042a 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -47,7 +47,7 @@ MARK_AS_ADVANCED (HDF5_METADATA_TRACE_FILE) # conversions. If not, some hard conversions will still be prefered even # though the data may be wrong (for example, some compilers don't # support denormalized floating values) to maximize speed. -# +#----------------------------------------------------------------------------- option (HDF5_WANT_DATA_ACCURACY "IF data accuracy is guaranteed during data conversions" ON) if (HDF5_WANT_DATA_ACCURACY) set (${HDF_PREFIX}_WANT_DATA_ACCURACY 1) @@ -59,7 +59,7 @@ MARK_AS_ADVANCED (HDF5_WANT_DATA_ACCURACY) # checked and data conversion exceptions are returned. This is mainly # for the speed optimization of hard conversions. Soft conversions can # actually benefit little. -# +#----------------------------------------------------------------------------- option (HDF5_WANT_DCONV_EXCEPTION "exception handling functions is checked during data conversions" ON) if (HDF5_WANT_DCONV_EXCEPTION) set (${HDF_PREFIX}_WANT_DCONV_EXCEPTION 1) @@ -68,7 +68,7 @@ MARK_AS_ADVANCED (HDF5_WANT_DCONV_EXCEPTION) # ---------------------------------------------------------------------- # Check if they would like the function stack support compiled in -# +#----------------------------------------------------------------------------- option (HDF5_ENABLE_CODESTACK "Enable the function stack tracing (for developer debugging)." OFF) if (HDF5_ENABLE_CODESTACK) set (${HDF_PREFIX}_HAVE_CODESTACK 1) @@ -89,7 +89,7 @@ set (${HDF_PREFIX}_HAVE_TMPFILE 1) # TODO -------------------------------------------------------------------------- # Should the Default Virtual File Driver be compiled? # This is hard-coded now but option should added to match configure -# +#----------------------------------------------------------------------------- set (${HDF_PREFIX}_DEFAULT_VFD H5FD_SEC2) if (NOT DEFINED "${HDF_PREFIX}_DEFAULT_PLUGINDIR") @@ -106,6 +106,7 @@ if (WINDOWS) # Set the flag to indicate that the machine has window style pathname, # that is, "drive-letter:\" (e.g. "C:") or "drive-letter:/" (e.g. "C:/"). # (This flag should be _unset_ for all machines, except for Windows) + #----------------------------------------------------------------------- set (${HDF_PREFIX}_HAVE_WINDOW_PATH 1) endif () @@ -168,6 +169,25 @@ if (NOT WINDOWS) endif () endif () +#----------------------------------------------------------------------------- +# Check if ROS3 driver can be built +#----------------------------------------------------------------------------- +option (HDF5_ENABLE_ROS3_VFD "Build the ROS3 Virtual File Driver" OFF) + if (HDF5_ENABLE_ROS3_VFD) + # CMake version 3.13 fixed FindCURL module + if(CMAKE_VERSION VERSION_LESS "3.13.0" AND WIN32) + MESSAGE(FATAL_ERROR "Windows builds for this option requires a minimum of CMake 3.13") + endif () + find_package(CURL REQUIRED) + find_package(OpenSSL REQUIRED) + if (${CURL_FOUND} AND ${OPENSSL_FOUND}) + set (${HDF_PREFIX}_HAVE_ROS3_VFD 1) + list (APPEND LINK_LIBS ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES}) + INCLUDE_DIRECTORIES (${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR}) + else () + message (STATUS "The Read-Only S3 VFD was requested but cannot be built.\nPlease check that openssl and cURL are available on your\nsystem, and/or re-configure without option HDF5_ENABLE_ROS3_VFD.") + endif () +endif () #----------------------------------------------------------------------------- # Macro to determine the various conversion capabilities @@ -215,7 +235,7 @@ endmacro () # is 0x004733ce17af227f, not the same as the library's conversion to 0x004733ce17af2282. # The machine's conversion gets the correct value. We define the macro and disable # this kind of test until we figure out what algorithm they use. -# +#----------------------------------------------------------------------------- H5ConversionTests (${HDF_PREFIX}_LDOUBLE_TO_LONG_SPECIAL "Checking IF your system converts long double to (unsigned) long values with special algorithm") # ---------------------------------------------------------------------- # Set the flag to indicate that the machine is using a special algorithm @@ -224,7 +244,7 @@ H5ConversionTests (${HDF_PREFIX}_LDOUBLE_TO_LONG_SPECIAL "Checking IF your syst # when the bit sequences are 003fff..., 007fff..., 00ffff..., 01ffff..., # ..., 7fffff..., the compiler uses a unknown algorithm. We define a # macro and skip the test for now until we know about the algorithm. -# +#----------------------------------------------------------------------------- H5ConversionTests (${HDF_PREFIX}_LONG_TO_LDOUBLE_SPECIAL "Checking IF your system can convert (unsigned) long to long double values with special algorithm") # ---------------------------------------------------------------------- # Set the flag to indicate that the machine can accurately convert @@ -234,7 +254,7 @@ H5ConversionTests (${HDF_PREFIX}_LONG_TO_LDOUBLE_SPECIAL "Checking IF your syste # start to go wrong on these two machines. Adjusting it higher to # 0x4351ccf385ebc8a0dfcc... or 0x4351ccf385ebc8a0ffcc... will make the converted # values wildly wrong. This test detects this wrong behavior and disable the test. -# +#----------------------------------------------------------------------------- H5ConversionTests (${HDF_PREFIX}_LDOUBLE_TO_LLONG_ACCURATE "Checking IF correctly converting long double to (unsigned) long long values") # ---------------------------------------------------------------------- # Set the flag to indicate that the machine can accurately convert @@ -242,9 +262,9 @@ H5ConversionTests (${HDF_PREFIX}_LDOUBLE_TO_LLONG_ACCURATE "Checking IF correctl # all machines, except for Mac OS 10.4, when the bit sequences are 003fff..., # 007fff..., 00ffff..., 01ffff..., ..., 7fffff..., the converted values are twice # as big as they should be. -# +#----------------------------------------------------------------------------- H5ConversionTests (${HDF_PREFIX}_LLONG_TO_LDOUBLE_CORRECT "Checking IF correctly converting (unsigned) long long to long double values") # ---------------------------------------------------------------------- # Check if pointer alignments are enforced -# +#----------------------------------------------------------------------------- H5ConversionTests (${HDF_PREFIX}_NO_ALIGNMENT_RESTRICTIONS "Checking IF alignment restrictions are strictly enforced") diff --git a/config/cmake/FindHDFS.cmake b/config/cmake/FindHDFS.cmake new file mode 100644 index 0000000..e401a94 --- /dev/null +++ b/config/cmake/FindHDFS.cmake @@ -0,0 +1,70 @@ + +# DerivedFrom: https://github.com/cloudera/Impala/blob/cdh5-trunk/cmake_modules/FindHDFS.cmake +# - Find HDFS (hdfs.h and libhdfs.so) +# This module defines +# Hadoop_VERSION, version string of ant if found +# HDFS_INCLUDE_DIR, directory containing hdfs.h +# HDFS_LIBRARIES, location of libhdfs.so +# HDFS_FOUND, whether HDFS is found. + +exec_program($ENV{HADOOP_HOME}/bin/hadoop ARGS version OUTPUT_VARIABLE Hadoop_VERSION + RETURN_VALUE Hadoop_RETURN) + +# currently only looking in HADOOP_HOME +find_path(HDFS_INCLUDE_DIR hdfs.h PATHS + $ENV{HADOOP_HOME}/include/ + # make sure we don't accidentally pick up a different version + NO_DEFAULT_PATH +) + +if ("${CMAKE_SIZEOF_VOID_P}" STREQUAL "8") + set(arch_hint "x64") +elseif ("$ENV{LIB}" MATCHES "(amd64|ia64)") + set(arch_hint "x64") +else () + set(arch_hint "x86") +endif() + +message(STATUS "Architecture: ${arch_hint}") + +if ("${arch_hint}" STREQUAL "x64") + set(HDFS_LIB_PATHS $ENV{HADOOP_HOME}/lib/native) +else () + set(HDFS_LIB_PATHS $ENV{HADOOP_HOME}/lib/native) +endif () + +message(STATUS "HDFS_LIB_PATHS: ${HDFS_LIB_PATHS}") + +find_library(HDFS_LIB NAMES hdfs PATHS + ${HDFS_LIB_PATHS} + # make sure we don't accidentally pick up a different version + NO_DEFAULT_PATH +) + +if (HDFS_LIB) + set(HDFS_FOUND TRUE) + set(HDFS_LIBRARIES ${HDFS_LIB}) + set(HDFS_STATIC_LIB ${HDFS_LIB_PATHS}/${CMAKE_STATIC_LIBRARY_PREFIX}hdfs${CMAKE_STATIC_LIBRARY_SUFFIX}) + + add_library(hdfs_static STATIC IMPORTED) + set_target_properties(hdfs_static PROPERTIES IMPORTED_LOCATION ${HDFS_STATIC_LIB}) +else () + set(HDFS_FOUND FALSE) +endif () + +if (HDFS_FOUND) + if (NOT HDFS_FIND_QUIETLY) + message(STATUS "${Hadoop_VERSION}") + message(STATUS "HDFS_INCLUDE_DIR: ${HDFS_INCLUDE_DIR}") + message(STATUS "HDFS_LIBRARIES: ${HDFS_LIBRARIES}") + message(STATUS "hdfs_static: ${HDFS_STATIC_LIB}") + endif () +else () + message(FATAL_ERROR "HDFS includes and libraries NOT found." + "(${HDFS_INCLUDE_DIR}, ${HDFS_LIB})") +endif () + +mark_as_advanced( + HDFS_LIBRARIES + HDFS_INCLUDE_DIR +) diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index 5906f34..15a16b4 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -78,6 +78,9 @@ /* Define if the function stack tracing code is to be compiled in */ #cmakedefine H5_HAVE_CODESTACK @H5_HAVE_CODESTACK@ +/* Define to 1 if you have the header file. */ +#cmakedefine H5_HAVE_CURL_H @H5_HAVE_CURL_H@ + /* Define if Darwin or Mac OS X */ #cmakedefine H5_HAVE_DARWIN @H5_HAVE_DARWIN@ @@ -156,6 +159,9 @@ /* Define to 1 if you have the `gettimeofday' function. */ #cmakedefine H5_HAVE_GETTIMEOFDAY @H5_HAVE_GETTIMEOFDAY@ +/* Define to 1 if you have the header file. */ +#cmakedefine H5_HAVE_HDFS_H @H5_HAVE_HDFS_H@ + /* Define if the compiler understands inline */ #cmakedefine H5_HAVE_INLINE @H5_HAVE_INLINE@ @@ -172,12 +178,24 @@ /* Define to 1 if you have the header file. */ #cmakedefine H5_HAVE_IO_H @H5_HAVE_IO_H@ +/* Define to 1 if you have the `crypto' library (-lcrypto). */ +#cmakedefine H5_HAVE_LIBCRYPTO @H5_HAVE_LIBCRYPTO@ + +/* Define to 1 if you have the `curl' library (-lcurl). */ +#cmakedefine H5_HAVE_LIBCURL @H5_HAVE_LIBCURL@ + /* Define to 1 if you have the `dl' library (-ldl). */ #cmakedefine H5_HAVE_LIBDL @H5_HAVE_LIBDL@ /* Define to 1 if you have the `dmalloc' library (-ldmalloc). */ #cmakedefine H5_HAVE_LIBDMALLOC @H5_HAVE_LIBDMALLOC@ +/* Proceed to build with libhdfs */ +#cmakedefine H5_HAVE_LIBHDFS @H5_HAVE_LIBHDFS@ + +/* Define to 1 if you have the `jvm' library (-ljvm). */ +#cmakedefine H5_HAVE_LIBJVM @H5_HAVE_LIBJVM@ + /* Define to 1 if you have the `m' library (-lm). */ #cmakedefine H5_HAVE_LIBM @H5_HAVE_LIBM@ @@ -235,6 +253,15 @@ /* Define if MPI_Info_c2f and MPI_Info_f2c exists */ #cmakedefine H5_HAVE_MPI_MULTI_LANG_Info @H5_HAVE_MPI_MULTI_LANG_Info@ +/* Define to 1 if you have the header file. */ +#cmakedefine H5_HAVE_OPENSSL_EVP_H @H5_HAVE_OPENSSL_EVP_H@ + +/* Define to 1 if you have the header file. */ +#cmakedefine H5_HAVE_OPENSSL_HMAC_H @H5_HAVE_OPENSSL_HMAC_H@ + +/* Define to 1 if you have the header file. */ +#cmakedefine H5_HAVE_OPENSSL_SHA_H @H5_HAVE_OPENSSL_SHA_H@ + /* Define if we have parallel support */ #cmakedefine H5_HAVE_PARALLEL @H5_HAVE_PARALLEL@ @@ -247,6 +274,10 @@ /* Define to 1 if you have the `rand_r' function. */ #cmakedefine H5_HAVE_RAND_R @H5_HAVE_RAND_R@ +/* Define whether the Read-Only S3 virtual file driver (VFD) should be + compiled */ +#cmakedefine H5_HAVE_ROS3_VFD @H5_HAVE_ROS3_VFD@ + /* Define to 1 if you have the `round' function. */ #cmakedefine H5_HAVE_ROUND @H5_HAVE_ROUND@ diff --git a/config/cmake/HDFCompilerFlags.cmake b/config/cmake/HDFCompilerFlags.cmake index 82d63af..adace89 100644 --- a/config/cmake/HDFCompilerFlags.cmake +++ b/config/cmake/HDFCompilerFlags.cmake @@ -143,7 +143,7 @@ if (NOT MSVC AND CMAKE_COMPILER_IS_GNUCC) # # Technically, variable-length arrays are part of the C99 standard, but # we should approach them a bit cautiously... -QAK - set (H5_CFLAGS1 "${H5_CFLAGS1} -Wlogical-op -Wlarger-than=2048 -Wvla") + set (H5_CFLAGS1 "${H5_CFLAGS1} -Wlogical-op -Wlarger-than=2560 -Wvla") # Append more extra warning flags that only gcc 4.4+ know about set (H5_CFLAGS1 "${H5_CFLAGS1} -Wsync-nand -Wframe-larger-than=16384 -Wpacked-bitfield-compat") @@ -210,9 +210,10 @@ if (NOT MSVC AND CMAKE_COMPILER_IS_GNUCC) # Append more extra warning flags that only gcc 9.x+ know about if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0) - set (H5_CFLAGS4 "${H5_CFLAGS4} Wattribute-alias=2 -Wmissing-profile") + set (H5_CFLAGS4 "${H5_CFLAGS4} -Wattribute-alias=2 -Wmissing-profile") endif () - +elseif (CMAKE_C_COMPILER_ID STREQUAL "PGI") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Minform=inform") endif () #----------------------------------------------------------------------------- @@ -339,3 +340,80 @@ endif () if (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_LOADED) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmessage-length=0") endif () + +#----------------------------------------------------------------------------- +# Option for --enable-asserts +# By default, CMake adds NDEBUG to CMAKE_${lang}_FLAGS for Release build types +# This option will force/override the default setting for all configurations +#----------------------------------------------------------------------------- +#option (HDF5_ENABLE_ASSERTS "Determines whether NDEBUG is defined to control assertions." OFF) +set (HDF5_ENABLE_ASSERTS "OFF" CACHE STRING "Determines whether NDEBUG is defined to control assertions (OFF NO YES)") +set_property (CACHE HDF5_ENABLE_ASSERTS PROPERTY STRINGS OFF NO YES) +if (HDF5_ENABLE_ASSERTS MATCHES "YES") + add_compile_options ("-UNDEBUG") +elseif (HDF5_ENABLE_ASSERTS MATCHES "NO") + add_compile_options ("-DNDEBUG") +endif () +MARK_AS_ADVANCED (HDF5_ENABLE_ASSERTS) + +#----------------------------------------------------------------------------- +# Option for --enable-symbols +# This option will force/override the default setting for all configurations +#----------------------------------------------------------------------------- +#option (HDF5_ENABLE_SYMBOLS "Add debug symbols to the library independent of the build mode and optimization level." OFF) +set (HDF5_ENABLE_SYMBOLS "OFF" CACHE STRING "Add debug symbols to the library independent of the build mode and optimization level (OFF NO YES)") +set_property (CACHE HDF5_ENABLE_SYMBOLS PROPERTY STRINGS OFF NO YES) +if (HDF5_ENABLE_SYMBOLS MATCHES "YES") + if (CMAKE_C_COMPILER_ID STREQUAL "Intel") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g") + elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fno-omit-frame-pointer") + endif () + if(CMAKE_CXX_COMPILER_LOADED) + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") + endif () + endif () +elseif (HDF5_ENABLE_SYMBOLS MATCHES "NO") + if (CMAKE_C_COMPILER_ID STREQUAL "Intel") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-s") + elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -s") + endif () + if(CMAKE_CXX_COMPILER_LOADED) + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + set (CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -Wl,-s") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -s") + endif () + endif () +endif () +MARK_AS_ADVANCED (HDF5_ENABLE_SYMBOLS) + +#----------------------------------------------------------------------------- +# Option for --enable-profiling +# This option will force/override the default setting for all configurations +#----------------------------------------------------------------------------- +option (HDF5_ENABLE_PROFILING "Enable profiling flags independently from the build mode." OFF) +if (HDF5_ENABLE_PROFILING) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PROFILE_CFLAGS}") + if(CMAKE_CXX_COMPILER_LOADED) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PROFILE_CXXFLAGS}") + endif () +endif () +MARK_AS_ADVANCED (HDF5_ENABLE_PROFILING) + +#----------------------------------------------------------------------------- +# Option for --enable-optimization +# This option will force/override the default setting for all configurations +#----------------------------------------------------------------------------- +option (HDF5_ENABLE_OPTIMIZATION "Enable optimization flags/settings independently from the build mode" OFF) +if (HDF5_ENABLE_OPTIMIZATION) + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OPTIMIZE_CFLAGS}") + if(CMAKE_CXX_COMPILER_LOADED) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPTIMIZE_CXXFLAGS}") + endif () +endif () +MARK_AS_ADVANCED (HDF5_ENABLE_OPTIMIZATION) diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in index c3efbdd..7bfadb1 100644 --- a/config/cmake/libhdf5.settings.cmake.in +++ b/config/cmake/libhdf5.settings.cmake.in @@ -10,32 +10,40 @@ General Information: Host system: @CMAKE_HOST_SYSTEM@ Uname information: @CMAKE_SYSTEM_NAME@ Byte sex: @BYTESEX@ - Libraries: @BUILD_NAME_EXT@ Installation point: @CMAKE_INSTALL_PREFIX@ Compiling Options: ------------------ - Compilation Mode: @CMAKE_BUILD_TYPE@ @CMAKE_C_COMPILER_VERSION@ - C Compiler: @CMAKE_C_COMPILER@ - CFLAGS: @CMAKE_C_FLAGS@ - H5_CFLAGS: @H5_CFLAGS@ - AM_CFLAGS: @AM_CFLAGS@ - CPPFLAGS: @CPPFLAGS@ - H5_CPPFLAGS: @H5_CPPFLAGS@ - AM_CPPFLAGS: @AM_CPPFLAGS@ - Shared C Library: @H5_ENABLE_SHARED_LIB@ - Static C Library: @H5_ENABLE_STATIC_LIB@ + Build Mode: @CMAKE_BUILD_TYPE@ + Debugging Symbols: @HDF5_ENABLE_SYMBOLS@ + Asserts: @HDF5_ENABLE_ASSERTS@ + Profiling: @HDF5_ENABLE_PROFILING@ + Optimization Level: @HDF5_ENABLE_OPTIMIZATION@ + +Linking Options: +---------------- + Libraries: @BUILD_NAME_EXT@ Statically Linked Executables: @BUILD_STATIC_EXECS@ LDFLAGS: @CMAKE_SHARED_LINKER_FLAGS@ + H5_LDFLAGS: @H5_LDFLAGS@ AM_LDFLAGS: @AM_LDFLAGS@ Extra libraries: @LINK_LIBS@ Archiver: @CMAKE_AR@ Ranlib: @CMAKE_RANLIB@ - Debugged Packages: @DEBUG_PKG@ - API Tracing: @HDF5_ENABLE_TRACE@ Languages: ---------- + C: YES + C Compiler: @CMAKE_C_COMPILER@ @CMAKE_C_COMPILER_VERSION@ + CPPFLAGS: @CPPFLAGS@ + H5_CPPFLAGS: @H5_CPPFLAGS@ + AM_CPPFLAGS: @AM_CPPFLAGS@ + CFLAGS: @CMAKE_C_FLAGS@ + H5_CFLAGS: @H5_CFLAGS@ + AM_CFLAGS: @AM_CFLAGS@ + Shared C Library: @H5_ENABLE_SHARED_LIB@ + Static C Library: @H5_ENABLE_STATIC_LIB@ + Fortran: @HDF5_BUILD_FORTRAN@ @BUILD_FORTRAN_CONDITIONAL_TRUE@ Fortran Compiler: @CMAKE_Fortran_COMPILER@ @CMAKE_Fortran_COMPILER_VERSION@ @BUILD_FORTRAN_CONDITIONAL_TRUE@ Fortran 2003 Compiler: @HDF5_ENABLE_F2003@ @@ -65,6 +73,8 @@ Features: I/O filters (external): @EXTERNAL_FILTERS@ MPE: @H5_HAVE_LIBLMPE@ Direct VFD: @H5_HAVE_DIRECT@ + (Read-Only) S3 VFD: @H5_HAVE_ROS3_VFD@ + (Read-Only) HDFS VFD: @H5_HAVE_LIBHDFS@ dmalloc: @H5_HAVE_LIBDMALLOC@ Clear file buffers before write: @HDF5_Enable_Clear_File_Buffers@ Using memory checker: @HDF5_ENABLE_USING_MEMCHECKER@ diff --git a/config/cmake/scripts/CTestScript.cmake b/config/cmake/scripts/CTestScript.cmake index 8ff52ec..dc3939e 100755 --- a/config/cmake/scripts/CTestScript.cmake +++ b/config/cmake/scripts/CTestScript.cmake @@ -277,7 +277,15 @@ message (STATUS "Dashboard script configuration:\n${vars}\n") ctest_test (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND ${ctest_test_args} RETURN_VALUE res) else () file(STRINGS ${CTEST_BINARY_DIRECTORY}/Testing/TAG TAG_CONTENTS REGEX "^2([0-9]+)[-]([0-9]+)$") - execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME}) + if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub") + execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME}) + else () + if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "qsub") + execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME} ctestS.out) + else () + execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME}) + endif() + endif () message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml") execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET) while(result) @@ -292,7 +300,15 @@ message (STATUS "Dashboard script configuration:\n${vars}\n") file (RENAME ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/SerialTest.xml) file (RENAME ${CTEST_BINARY_DIRECTORY}/Testing/Temporary/LastTest_${TAG_CONTENTS}.log ${CTEST_BINARY_DIRECTORY}/Testing/Temporary/LastTest_${TAG_CONTENTS}_Serial.log) unset(result CACHE) - execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}) + if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub") + execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}) + else () + if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "qsub") + execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME} ctestP.out) + else () + execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}) + endif () + endif () message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml") execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET) while(result) diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake index 9d84ba6..185a86e 100755 --- a/config/cmake/scripts/HDF5config.cmake +++ b/config/cmake/scripts/HDF5config.cmake @@ -9,56 +9,62 @@ # If you do not have access to either file, you may request a copy from # help@hdfgroup.org. # -############################################################################################# -### ${CTEST_SCRIPT_ARG} is of the form OPTION=VALUE ### +############################################################################################# +### ${CTEST_SCRIPT_ARG} is of the form OPTION=VALUE ### ### BUILD_GENERATOR required [Unix, VS2017, VS201764, VS2015, VS201564, VS2013, VS201364] ### -### ctest -S HDF518config.cmake,BUILD_GENERATOR=VS201764 -C Release -VV -O hdf518.log ### -############################################################################################# - -cmake_minimum_required (VERSION 3.10) -############################################################################ -# Usage: -# ctest -S HDF518config.cmake,OPTION=VALUE -C Release -VV -O test.log -# where valid options for OPTION are: -# BUILD_GENERATOR - The cmake build generator: -# Unix * Unix Makefiles +### ctest -S HDF518config.cmake,BUILD_GENERATOR=VS201764 -C Release -VV -O hdf518.log ### +############################################################################################# + +cmake_minimum_required (VERSION 3.10) +############################################################################ +# Usage: +# ctest -S HDF518config.cmake,OPTION=VALUE -C Release -VV -O test.log +# where valid options for OPTION are: +# BUILD_GENERATOR - The cmake build generator: +# Unix * Unix Makefiles # VS2017 * Visual Studio 15 2017 # VS201764 * Visual Studio 15 2017 Win64 -# VS2015 * Visual Studio 14 2015 -# VS201564 * Visual Studio 14 2015 Win64 -# VS2013 * Visual Studio 12 2013 -# VS201364 * Visual Studio 12 2013 Win64 -# -# INSTALLDIR - root folder where hdf5 is installed -# CTEST_CONFIGURATION_TYPE - Release, Debug, etc -# CTEST_SOURCE_NAME - source folder -############################################################################## - -set (CTEST_SOURCE_VERSION "1.8.22") -set (CTEST_SOURCE_VERSEXT "-snap3") - -############################################################################## -# handle input parameters to script. -#BUILD_GENERATOR - which CMake generator to use, required -#INSTALLDIR - HDF5-1.8 root folder -#CTEST_CONFIGURATION_TYPE - Release, Debug, RelWithDebInfo -#CTEST_SOURCE_NAME - name of source folder; HDF5-1.8.x +# VS2015 * Visual Studio 14 2015 +# VS201564 * Visual Studio 14 2015 Win64 +# VS2013 * Visual Studio 12 2013 +# VS201364 * Visual Studio 12 2013 Win64 +# +# INSTALLDIR - root folder where hdf5 is installed +# CTEST_CONFIGURATION_TYPE - Release, Debug, etc +# CTEST_SOURCE_NAME - source folder +############################################################################## + +set (CTEST_SOURCE_VERSION "1.8.22") +set (CTEST_SOURCE_VERSEXT "-snap3") + +############################################################################## +# handle input parameters to script. +#BUILD_GENERATOR - which CMake generator to use, required +#INSTALLDIR - HDF5-1.8 root folder +#CTEST_CONFIGURATION_TYPE - Release, Debug, RelWithDebInfo +#CTEST_SOURCE_NAME - name of source folder; HDF5-1.8.x #MODEL - CDash group name +#HPC - run alternate configurations for HPC machines; sbatch, bsub, raybsub, qsub #MPI - enable MPI -if (DEFINED CTEST_SCRIPT_ARG) - # transform ctest script arguments of the form - # script.ctest,var1=value1,var2=value2 - # to variables with the respective names set to the respective values - string (REPLACE "," ";" script_args "${CTEST_SCRIPT_ARG}") - foreach (current_var ${script_args}) - if ("${current_var}" MATCHES "^([^=]+)=(.+)$") - set ("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}") - endif () - endforeach () -endif () - -# build generator must be defined -if (NOT DEFINED BUILD_GENERATOR) +if (DEFINED CTEST_SCRIPT_ARG) + # transform ctest script arguments of the form + # script.ctest,var1=value1,var2=value2 + # to variables with the respective names set to the respective values + string (REPLACE "," ";" script_args "${CTEST_SCRIPT_ARG}") + foreach (current_var ${script_args}) + if ("${current_var}" MATCHES "^([^=]+)=(.+)$") + set ("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}") + endif () + endforeach () +endif () + +#HPC - run alternate configurations for HPC machines +if (DEFINED HPC) + set (BUILD_GENERATOR "Unix") +endif () + +# build generator must be defined +if (NOT DEFINED BUILD_GENERATOR) message (FATAL_ERROR "BUILD_GENERATOR must be defined - Unix, VS2017, or VS201764, VS2015, VS201564, VS2013, VS201364") endif () @@ -94,6 +100,7 @@ endif () ################################################################### ######### Following describes compiler ############ +if (NOT DEFINED HPC) if (NOT DEFINED BUILD_GENERATOR) message (FATAL_ERROR "BUILD_GENERATOR must be defined - Unix, VS2017, or VS201764, VS2015, VS201564, VS2013, VS201364") endif () @@ -169,6 +176,10 @@ endif () set (ENV{CXXFLAGS} "${RR_WARNINGS_CXX} ${RR_FLAGS_CXX}") endif () endif () +else () + set (CTEST_SITE "${SITE_OS_NAME}") + set (CTEST_CMAKE_GENERATOR "Unix Makefiles") +endif () ################################################################### ################################################################### diff --git a/config/cmake/scripts/HPC/bsub-HDF5options.cmake b/config/cmake/scripts/HPC/bsub-HDF5options.cmake new file mode 100644 index 0000000..7473e8a --- /dev/null +++ b/config/cmake/scripts/HPC/bsub-HDF5options.cmake @@ -0,0 +1,31 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +############################################################################################# +#### Change default configuration of options in config/cmake/cacheinit.cmake file ### +#### format: set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ") ### +############################################################################################# +if (DEFINED MPI) + # maximum parallel processor count for build and test #### + set (MAX_PROC_COUNT 8) +endif() +############################################################################################# +### options to run test scripts in batch commands +set (LOCAL_BATCH_SCRIPT_COMMAND "bsub") +set (LOCAL_BATCH_TEST "TRUE") +set (LOCAL_BATCH_SCRIPT_NAME "ctestS.lsf") +set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "ctestP.lsf") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_TEST:BOOL=ON") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_NAME:STRING=${LOCAL_BATCH_SCRIPT_NAME}") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_PARALLEL_NAME:STRING=${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_EXECUTABLE:STRING=srun") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_NUMPROC_FLAG:STRING=-n") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_MAX_NUMPROCS:STRING=6") diff --git a/config/cmake/scripts/HPC/qsub-HDF5options.cmake b/config/cmake/scripts/HPC/qsub-HDF5options.cmake new file mode 100644 index 0000000..34eba1b --- /dev/null +++ b/config/cmake/scripts/HPC/qsub-HDF5options.cmake @@ -0,0 +1,42 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +############################################################################################# +#### Change default configuration of options in config/cmake/cacheinit.cmake file ### +#### format: set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ") ### +############################################################################################# +if (DEFINED MPI) + # maximum parallel processor count for build and test #### + set (MAX_PROC_COUNT 8) +endif() +############################################################################################# +### options to run test scripts in batch commands +set (LOCAL_BATCH_SCRIPT_NAME "ctest.qsub") +set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "ctest.qsub") +if (DEFINED KNL) + ### some additions and alternatives to cross compile on haswell for knl + set (COMPUTENODE_HWCOMPILE_MODULE "craype-mic-knl") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake") +endif () +set (LOCAL_BATCH_SCRIPT_COMMAND "qsub") +set (LOCAL_BATCH_TEST "TRUE") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_TEST:BOOL=ON") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_NAME:STRING=${LOCAL_BATCH_SCRIPT_NAME}") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_PARALLEL_NAME:STRING=${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_EXECUTABLE:STRING=aprun") +# Option to suppress writing job statistics; to avoid issues with h5diff comparisons. +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_PREFLAGS:STRING=-q") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_NUMPROC_FLAG:STRING=-n") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_MAX_NUMPROCS:STRING=6") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DACCOUNT_ID:STRING=${LOCAL_BATCH_SCRIPT_ARGS}") + +############################################################################################# +############################################################################################# diff --git a/config/cmake/scripts/HPC/raybsub-HDF5options.cmake b/config/cmake/scripts/HPC/raybsub-HDF5options.cmake new file mode 100644 index 0000000..fa1ec4a --- /dev/null +++ b/config/cmake/scripts/HPC/raybsub-HDF5options.cmake @@ -0,0 +1,32 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +############################################################################################# +#### Change default configuration of options in config/cmake/cacheinit.cmake file ### +#### format: set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ") ### +############################################################################################# +if (DEFINED MPI) + # maximum parallel processor count for build and test #### + set (MAX_PROC_COUNT 8) +endif() +############################################################################################# +### options to run test scripts in batch commands +set (LOCAL_BATCH_SCRIPT_COMMAND "raybsub") +set (LOCAL_BATCH_TEST "TRUE") +set (LOCAL_BATCH_SCRIPT_NAME "ray_ctestS.lsf") +set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "ray_ctestP.lsf") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_TEST:BOOL=ON") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_COMMAND:STRING=${LOCAL_BATCH_SCRIPT_COMMAND}") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_NAME:STRING=${LOCAL_BATCH_SCRIPT_NAME}") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_PARALLEL_NAME:STRING=${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_EXECUTABLE:STRING=mpirun") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_NUMPROC_FLAG:STRING=-np") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_MAX_NUMPROCS:STRING=6") diff --git a/config/cmake/scripts/HPC/sbatch-HDF5options.cmake b/config/cmake/scripts/HPC/sbatch-HDF5options.cmake new file mode 100644 index 0000000..3205a1c --- /dev/null +++ b/config/cmake/scripts/HPC/sbatch-HDF5options.cmake @@ -0,0 +1,43 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +############################################################################################# +#### Change default configuration of options in config/cmake/cacheinit.cmake file ### +#### format: set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ") ### +############################################################################################# +if (DEFINED MPI) + # maximum parallel processor count for build and test #### + set (MAX_PROC_COUNT 8) +endif() +############################################################################################# +### options to run test scripts in batch commands +if (DEFINED KNL) + ### some additions and alternatives to cross compile on haswell for knl + set (COMPILENODE_HWCOMPILE_MODULE "craype-haswell") + set (COMPUTENODE_HWCOMPILE_MODULE "craype-mic-knl") + set (LOCAL_BATCH_SCRIPT_NAME "knl_ctestS.sl") + set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "knl_ctestP.sl") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake") +else () + set (LOCAL_BATCH_SCRIPT_NAME "ctestS.sl") + set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "ctestP.sl") +endif () +set (LOCAL_BATCH_SCRIPT_COMMAND "sbatch") +set (LOCAL_BATCH_TEST "TRUE") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_TEST:BOOL=ON") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_NAME:STRING=${LOCAL_BATCH_SCRIPT_NAME}") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_PARALLEL_NAME:STRING=${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_EXECUTABLE:STRING=srun") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_NUMPROC_FLAG:STRING=-n") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_MAX_NUMPROCS:STRING=6") + +############################################################################################# +############################################################################################# diff --git a/config/cmake/wait_H5Tinit.cmake b/config/cmake/wait_H5Tinit.cmake new file mode 100644 index 0000000..c0b4d06 --- /dev/null +++ b/config/cmake/wait_H5Tinit.cmake @@ -0,0 +1,11 @@ +cmake_minimum_required (VERSION 3.10) + +message(STATUS "Check for existence of ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c") +execute_process(COMMAND ls ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c RESULT_VARIABLE H5TI_result OUTPUT_QUIET ERROR_QUIET) +while(H5TI_result) + ctest_sleep(30) + message(STATUS "Checking again for existence of ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c") + execute_process(COMMAND ls ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c RESULT_VARIABLE H5TI_result OUTPUT_QUIET ERROR_QUIET) +endwhile(H5TI_result) +file (TOUCH "${HDF5_BINARY_DIR}/H5Tinit_created") +message(STATUS "Found ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c") diff --git a/config/cmake_ext_mod/runTest.cmake b/config/cmake_ext_mod/runTest.cmake index e601653..6f633f3 100644 --- a/config/cmake_ext_mod/runTest.cmake +++ b/config/cmake_ext_mod/runTest.cmake @@ -343,6 +343,15 @@ if (TEST_GREP_COMPARE) endif () endif () +# dump the output unless nodisplay option is set +if (TEST_SKIP_COMPARE AND NOT TEST_NO_DISPLAY) + file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM) + execute_process ( + COMMAND ${CMAKE_COMMAND} -E echo ${TEST_STREAM} + RESULT_VARIABLE TEST_RESULT + ) +endif () + # everything went fine... message (STATUS "${TEST_PROGRAM} Passed") diff --git a/config/toolchain/build32.cmake b/config/toolchain/build32.cmake index d078956..deb5899 100644 --- a/config/toolchain/build32.cmake +++ b/config/toolchain/build32.cmake @@ -3,6 +3,42 @@ if (WIN32) set (CMAKE_GENERATOR_PLATFORM "x86") elseif(APPLE) set (CMAKE_OSX_ARCHITECTURES "i386") +elseif(MINGW) + set (CMAKE_SYSTEM_NAME Windows) + set (CMAKE_C_COMPILER i686-w64-mingw32-gcc) + set (CMAKE_CXX_COMPILER i686-w64-mingw32-g++) + set (CMAKE_RC_COMPILER i686-w64-mingw32-windres) + set (CMAKE_Fortran_COMPILER i686-w64-mingw32-gfortran) + + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "c++ flags") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32" CACHE STRING "c flags") + + set (LIB32 /usr/lib) # Fedora + + if (EXISTS "/usr/lib32") + set (LIB32 /usr/lib32) # Arch, Solus + endif () + + set (CMAKE_SYSTEM_LIBRARY_PATH ${LIB32} CACHE STRING "system library search path" FORCE) + set (CMAKE_LIBRARY_PATH ${LIB32} CACHE STRING "library search path" FORCE) + + # this is probably unlikely to be needed, but just in case + set (CMAKE_EXE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "executable linker flags" FORCE) + set (CMAKE_SHARED_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "shared library linker flags" FORCE) + set (CMAKE_MODULE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "module linker flags" FORCE) + + # on Fedora and Arch and similar, point pkgconfig at 32 bit .pc files. We have + # to include the regular system .pc files as well (at the end), because some + # are not always present in the 32 bit directory + if (EXISTS "${LIB32}/pkgconfig") + set (ENV{PKG_CONFIG_LIBDIR} ${LIB32}/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/usr/lib64/pkgconfig) + endif () + + set (CMAKE_FIND_ROOT_PATH /usr/i686-w64-mingw32) + set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) + set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) + set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) + set (CMAKE_CROSSCOMPILING_EMULATOR wine32) else () set (CMAKE_SYSTEM_NAME Linux) diff --git a/config/toolchain/crayle.cmake b/config/toolchain/crayle.cmake new file mode 100644 index 0000000..bf7cf69 --- /dev/null +++ b/config/toolchain/crayle.cmake @@ -0,0 +1,10 @@ +# The following line will use cross-compiling +set(CMAKE_SYSTEM_NAME Linux) + +set(CMAKE_COMPILER_VENDOR "CrayLinuxEnvironment") + +set(CMAKE_C_COMPILER cc) +set(CMAKE_Fortran_COMPILER ftn) + +# the following is used if cross-compiling +set(CMAKE_CROSSCOMPILING_EMULATOR "") diff --git a/release_docs/README_HDF5_CMake b/release_docs/README_HDF5_CMake new file mode 100644 index 0000000..9580fc7 --- /dev/null +++ b/release_docs/README_HDF5_CMake @@ -0,0 +1,23 @@ +This tar file contains + + build-unix.sh script to build HDF5 with CMake on unix machines + build-unix-hpc.sh script to build HDF5 with CMake on unix machines and run + tests with batch scripts (sbatch). + CTestScript.cmake + HDF5config.cmake CMake scripts for building HDF5 + HDF5options.cmake + hdf5-1.8.22 HDF5 1.8.22 source + SZip.tar.gz source for building SZIP + ZLib.tar.gz source for building ZLIB + +For more information about building HDF5 with CMake, see USING_HDF5_CMake.txt in +hdf5-1.8.22/release_docs, or +https://portal.hdfgroup.org/display/support/Building+HDF5+with+CMake. + +For more information about building HDF5 with CMake on HPC machines, including +cross compiling on Cray XC40, see README_HPC in hdf5-1.8.22/release_docs. + + + + + diff --git a/release_docs/README_HPC b/release_docs/README_HPC new file mode 100644 index 0000000..97f088c --- /dev/null +++ b/release_docs/README_HPC @@ -0,0 +1,206 @@ +************************************************************************ +* Using CMake to build and test HDF5 source on HPC machines * +************************************************************************ + + Contents + +Section I: Prerequisites +Section II: Obtain HDF5 source +Section III: Using ctest command to build and test +Section IV: Cross compiling +Section V: Manual alternatives +Section VI: Other cross compiling options + +************************************************************************ + +======================================================================== +I. Prerequisites +======================================================================== + 1. Create a working directory that is accessible from the compute nodes for + running tests; the working directory should be in a scratch space or a + parallel file system space since testing will use this space. Building + from HDF5 source in a 'home' directory typically results in test + failures and should be avoided. + + 2. Load modules for desired compilers, module for cmake version 3.10 or greater, + and set any needed environment variables for compilers (i.e., CC, FC, CXX). + Unload any problematic modules (i.e., craype-hugepages2M). + +======================================================================== +II. Obtain HDF5 source +======================================================================== +Obtain HDF5 source code from the HDF5 repository using a git command or +from a release tar file in a working directory: + + git clone https://git@bitbucket.hdfgroup.org/scm/hdffv/hdf5.git + [-b branch] [source directory] + +If no branch is specified, then the 'develop' version will be checked out. +If no source directory is specified, then the source will be located in the +'hdf5' directory. The Cmake scripts expect the source to be in a directory +named hdf5-, where 'version string' uses the format '1.xx.xx'. +For example, for the current 'develop' version, the "hdf5" directory should +be renamed "hdf5-1.8.22", or for the first hdf5_1_8_22 pre-release version, +it should be renamed "hdf5-1.8.22-pre1". + +If the version number is not known a priori, the version string +can be obtained by running bin/h5vers in the top level directory of the source clone, and +the source directory renamed 'hdf5-'. + +Release or snapshot tar files may also be extracted and used. + +======================================================================== +III. Using ctest command to build and test +======================================================================== + +The ctest command [1]: + + ctest -S HDF5config.cmake,BUILD_GENERATOR=Unix -C Release -V -O hdf5.log + +will configure, build, test and package HDF5 from the downloaded source +after the setup steps outlined below are followed. + +CMake option variables are available to allow running test programs in batch +scripts on compute nodes and to cross-compile for compute node hardware using +a cross-compiling emulator. The setup steps will make default settings for +parallel or serial only builds available to the CMake command. + + 1. For the current 'develop' version the "hdf5" directory should be renamed + "hdf5-1.8.22". + + 2. Three cmake script files need to be copied to the working directory, or + have symbolic links to them, created in the working directory: + + hdf5-1.8.22/config/cmake/scripts/HDF5config.cmake + hdf5-1.8.22/config/cmake/scripts/CTestScript.cmake + hdf5-1.8.22/config/cmake/scripts/HDF5options.cmake + + should be copied to the working directory. + + 3. The resulting contents of the working directory are then: + + CTestScript.cmake + HDF5config.cmake + HDF5options.cmake + hdf5-1.8.22 + + Additionally, when the ctest command runs [1], it will add a build directory + in the working directory. + + 4. The following options (among others) can be added to the ctest + command [1], following '-S HDF5config.cmake,' and separated by ',': + + HPC=sbatch (or 'bsub' or 'raybsub') indicates which type of batch + files to use for running tests. If omitted, test + will run on the local machine or login node. + + KNL=true to cross-compile for KNL compute nodes on CrayXC40 + (see section IV) + + MPI=true enables parallel, disables c++, java, and threadsafe + + LOCAL_BATCH_SCRIPT_ARGS="--account=" to supply user account + information for batch jobs + + The HPC options will add BUILD_GENERATOR=Unix for the three HPC options. + An example ctest command for a parallel build on a system using sbatch is + + ctest -S HDF5config.cmake,HPC=sbatch,MPI=true -C Release -V -O hdf5.log + + Adding the option 'KNL=true' to the above list will compile for KNL nodes, + for example, on 'mutrino' and other CrayXC40 machines. + + Changing -V to -VV will produce more logging information in HDF5.log. + + More detailed CMake information can be found in the HDF5 source in + release_docs/INSTALL_CMake.txt. + +======================================================================== +IV. Cross-compiling +======================================================================== +For cross-compiling on Cray, set environment variables CC=cc, FC=ftn +and CXX=CC (for c++) after all compiler modules are loaded since switching +compiler modules may unset or reset these variables. + +CMake provides options for cross-compiling. To cross-compile for KNL hardware +on mutrino and other CrayXC40 machines, add HPC=sbatch,KNL=true to the +ctest command line. This will set the following options from the +config/cmake/scripts/HPC/sbatch-HDF5options.cmake file: + + set (COMPILENODE_HWCOMPILE_MODULE "craype-haswell") + set (COMPUTENODE_HWCOMPILE_MODULE "craype-mic-knl") + set (LOCAL_BATCH_SCRIPT_NAME "knl_ctestS.sl") + set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "knl_ctestP.sl") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake") + +On the Cray XC40 the craype-haswell module is needed for configuring, and the +craype-mic-knl module is needed for building to run on the KNL nodes. CMake +with the above options will swap modules after configuring is complete, +but before compiling programs for KNL. + +The sbatch script arguments for running jobs on KNL nodes may differ on CrayXC40 +machines other than mutrino. The batch scripts knl_ctestS.sl and knl_ctestP.sl +have the correct arguments for mutrino: "#SBATCH -p knl -C quad,cache". For +cori, another CrayXC40, that line is replaced by "#SBATCH -C knl,quad,cache". +For cori (and other machines), the values in LOCAL_BATCH_SCRIPT_NAME and +LOCAL_BATCH_SCRIPT_PARALLEL_NAME in the config/cmake/scripts/HPC/sbatch-HDF5options.cmake +file can be replaced by cori_knl_ctestS.sl and cori_knl_ctestS.sl, or the lines +can be edited in the batch files in hdf5-1.8.22/bin/batch. + +======================================================================== +V. Manual alternatives +======================================================================== +If using ctest is undesirable, one can create a build directory and run the cmake +configure command, for example + +"/projects/Mutrino/hpcsoft/cle6.0/common/cmake/3.10.2/bin/cmake" +-C "/hdf5-1.8.22/config/cmake/cacheinit.cmake" +-DCMAKE_BUILD_TYPE:STRING=Release -DHDF5_BUILD_FORTRAN:BOOL=ON +-DHDF5_BUILD_JAVA:BOOL=OFF +-DCMAKE_INSTALL_PREFIX:PATH=/HDF_Group/HDF5/1.8.22 +-DHDF5_ENABLE_Z_LIB_SUPPORT:BOOL=OFF -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF +-DHDF5_ENABLE_PARALLEL:BOOL=ON -DHDF5_BUILD_CPP_LIB:BOOL=OFF +-DHDF5_ENABLE_THREADSAFE:BOOL=OFF +-DHDF5_PACKAGE_EXTLIBS:BOOL=ON -DLOCAL_BATCH_TEST:BOOL=ON +-DMPIEXEC_EXECUTABLE:STRING=srun -DMPIEXEC_NUMPROC_FLAG:STRING=-n +-DMPIEXEC_MAX_NUMPROCS:STRING=6 +-DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake +-DLOCAL_BATCH_SCRIPT_NAME:STRING=knl_ctestS.sl +-DLOCAL_BATCH_SCRIPT_PARALLEL_NAME:STRING=knl_ctestP.sl -DSITE:STRING=mutrino +-DBUILDNAME:STRING=par-knl_GCC493-SHARED-Linux-4.4.156-94.61.1.16335.0.PTF.1107299-default-x86_64 +"-GUnix Makefiles" "" "/hdf5-1.8.22" + +followed by make and batch jobs to run tests. + +To cross-compile on CrayXC40, run the configure command with the craype-haswell +module loaded, then switch to the craype-mic-knl module for the build process. + +Tests on machines using slurm can be run with + +"sbatch -p knl -C quad,cache ctestS.sl" + +or + +"sbatch -p knl -C quad,cache ctestP.sl" + +for parallel builds. + +Tests on machines using LSF will typically use "bsub ctestS.lsf", etc. + +======================================================================== +VI. Other cross compiling options +======================================================================== +Settings for two other cross-compiling options are also in the config/toolchain +files which do not seem to be necessary with the Cray PrgEnv-* modules + +1. HDF5_USE_PREGEN. This option, along with the HDF5_USE_PREGEN_DIR CMake + variable would allow the use of an appropriate H5Tinit.c file with type + information generated on a compute node to be used when cross compiling + for those compute nodes. The use of the variables in lines 110 and 111 + of HDF5options.cmake file seem to preclude needing this option with the + available Cray modules and CMake option. + +2. HDF5_BATCH_H5DETECT and associated CMake variables. This option when + properly configured will run H5detect in a batch job on a compute node + at the beginning of the CMake build process. It was also found to be + unnecessary with the available Cray modules and CMake options. diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 05fe857..23f3652 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -148,6 +148,31 @@ New Features Library ------- + - Add S3 and HDFS VFDs to HDF5 maintenance + + Fix windows requirements and java tests. Windows requires CMake 3.13. + Install openssl library (with dev files); + from "Shining Light Productions". msi package preferred. + + PATH should have been updated with the installation dir. + set ENV variable OPENSSL_ROOT_DIR to the installation dir. + set ENV variable OPENSSL_CONF to the cfg file, likely %OPENSSL_ROOT_DIR%\bin\openssl.cfg + Install libcurl library (with dev files); + download the latest released version using git: https://github.com/curl/curl.git + + Open a Visual Studio Command prompt + change to the libcurl root folder + run the "buildconf.bat" batch file + change to the winbuild directory + nmake /f Makefile.vc mode=dll MACHINE=x64 + copy libcurl-vc-x64-release-dll-ipv6-sspi-winssl dir to C:\curl (installation dir) + set ENV variable CURL_ROOT to C:\curl (installation dir) + update PATH ENV variable to %CURL_ROOT%\bin (installation bin dir). + the aws credentials file should be in %USERPROFILE%\.aws folder + set the ENV variable "HDF5_ROS3_TEST_BUCKET_URL=https://s3.us-east-2.amazonaws.com/hdf5ros3" + + (ADB - 2019/09/12, HDFFV-10854) + - Allow pre-generated H5Tinit.c and H5make_libsettings.c to be used. Rather than always running H5detect and generating H5Tinit.c and diff --git a/release_docs/USING_CMake_Examples.txt b/release_docs/USING_CMake_Examples.txt index f98b39e..a5ababa 100644 --- a/release_docs/USING_CMake_Examples.txt +++ b/release_docs/USING_CMake_Examples.txt @@ -58,8 +58,9 @@ Default installation process: with the CTEST_CONFIGURATION_TYPE script option. Note that this must be the same as the value used with the -C command line option. The default build configuration is defined to build and use static libraries. - Shared libraries can be used with the STATIC_ONLY script option set to "NO". - Other options can be changed by editing the HDF518_Examples.cmake file. + + Shared libraries and other options can be changed by editing the + HDF5_Examples_options.cmake file. If the defaults are okay, execute from this directory: ctest -S HDF518_Examples.cmake -C Release -V -O test.log diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 86deb10..544859f 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -4,7 +4,7 @@ project (HDF5_SRC C) #----------------------------------------------------------------------------- # List Source Files #----------------------------------------------------------------------------- -set (H5_SRCS +set (H5_SOURCES ${HDF5_SRC_DIR}/H5.c ${HDF5_SRC_DIR}/H5checksum.c ${HDF5_SRC_DIR}/H5dbg.c @@ -20,9 +20,9 @@ set (H5_HDRS #${HDF5_SRC_DIR}/H5version.h #${HDF5_SRC_DIR}/H5overflow.h ) -IDE_GENERATED_PROPERTIES ("H5" "${H5_HDRS}" "${H5_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5" "${H5_HDRS}" "${H5_SOURCES}" ) -set (H5A_SRCS +set (H5A_SOURCES ${HDF5_SRC_DIR}/H5A.c ${HDF5_SRC_DIR}/H5Abtree2.c ${HDF5_SRC_DIR}/H5Adense.c @@ -34,28 +34,28 @@ set (H5A_SRCS set (H5A_HDRS ${HDF5_SRC_DIR}/H5Apublic.h ) -IDE_GENERATED_PROPERTIES ("H5A" "${H5A_HDRS}" "${H5A_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5A" "${H5A_HDRS}" "${H5A_SOURCES}" ) -set (H5AC_SRCS +set (H5AC_SOURCES ${HDF5_SRC_DIR}/H5AC.c ) set (H5AC_HDRS ${HDF5_SRC_DIR}/H5ACpublic.h ) -IDE_GENERATED_PROPERTIES ("H5AC" "${H5AC_HDRS}" "${H5AC_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5AC" "${H5AC_HDRS}" "${H5AC_SOURCES}" ) -set (H5B_SRCS +set (H5B_SOURCES ${HDF5_SRC_DIR}/H5B.c ${HDF5_SRC_DIR}/H5Bcache.c ${HDF5_SRC_DIR}/H5Bdbg.c ) set (H5B_HDRS ) -IDE_GENERATED_PROPERTIES ("H5B" "${H5B_HDRS}" "${H5B_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5B" "${H5B_HDRS}" "${H5B_SOURCES}" ) -set (H5B2_SRCS +set (H5B2_SOURCES ${HDF5_SRC_DIR}/H5B2.c ${HDF5_SRC_DIR}/H5B2cache.c ${HDF5_SRC_DIR}/H5B2dbg.c @@ -66,10 +66,10 @@ set (H5B2_SRCS ) set (H5B2_HDRS ) -IDE_GENERATED_PROPERTIES ("H5B2" "${H5B2_HDRS}" "${H5B2_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5B2" "${H5B2_HDRS}" "${H5B2_SOURCES}" ) -set (H5C_SRCS +set (H5C_SOURCES ${HDF5_SRC_DIR}/H5C.c ) set (H5C_HDRS @@ -78,7 +78,7 @@ set (H5C_HDRS IDE_GENERATED_PROPERTIES ("H5C" "${H5C_HDRS}" "${H5C_SOURCES}" ) -set (H5CS_SRCS +set (H5CS_SOURCES ${HDF5_SRC_DIR}/H5CS.c ) set (H5CS_HDRS @@ -86,7 +86,7 @@ set (H5CS_HDRS IDE_GENERATED_PROPERTIES ("H5CS" "${H5CS_HDRS}" "${H5CS_SOURCES}" ) -set (H5D_SRCS +set (H5D_SOURCES ${HDF5_SRC_DIR}/H5D.c ${HDF5_SRC_DIR}/H5Dbtree.c ${HDF5_SRC_DIR}/H5Dchunk.c @@ -109,9 +109,9 @@ set (H5D_SRCS set (H5D_HDRS ${HDF5_SRC_DIR}/H5Dpublic.h ) -IDE_GENERATED_PROPERTIES ("H5D" "${H5D_HDRS}" "${H5D_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5D" "${H5D_HDRS}" "${H5D_SOURCES}" ) -set (H5E_SRCS +set (H5E_SOURCES ${HDF5_SRC_DIR}/H5E.c ${HDF5_SRC_DIR}/H5Edeprec.c ${HDF5_SRC_DIR}/H5Eint.c @@ -121,9 +121,9 @@ set (H5E_HDRS ${HDF5_SRC_DIR}/H5Epubgen.h ${HDF5_SRC_DIR}/H5Epublic.h ) -IDE_GENERATED_PROPERTIES ("H5E" "${H5E_HDRS}" "${H5E_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5E" "${H5E_HDRS}" "${H5E_SOURCES}" ) -set (H5F_SRCS +set (H5F_SOURCES ${HDF5_SRC_DIR}/H5F.c ${HDF5_SRC_DIR}/H5Faccum.c ${HDF5_SRC_DIR}/H5Fcwfs.c @@ -144,18 +144,21 @@ set (H5F_SRCS set (H5F_HDRS ${HDF5_SRC_DIR}/H5Fpublic.h ) -IDE_GENERATED_PROPERTIES ("H5F" "${H5F_HDRS}" "${H5F_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5F" "${H5F_HDRS}" "${H5F_SOURCES}" ) -set (H5FD_SRCS +set (H5FD_SOURCES ${HDF5_SRC_DIR}/H5FD.c ${HDF5_SRC_DIR}/H5FDcore.c ${HDF5_SRC_DIR}/H5FDdirect.c ${HDF5_SRC_DIR}/H5FDfamily.c + ${HDF5_SRC_DIR}/H5FDhdfs.c ${HDF5_SRC_DIR}/H5FDint.c ${HDF5_SRC_DIR}/H5FDlog.c ${HDF5_SRC_DIR}/H5FDmpi.c ${HDF5_SRC_DIR}/H5FDmpio.c ${HDF5_SRC_DIR}/H5FDmulti.c + ${HDF5_SRC_DIR}/H5FDros3.c + ${HDF5_SRC_DIR}/H5FDs3comms.c ${HDF5_SRC_DIR}/H5FDsec2.c ${HDF5_SRC_DIR}/H5FDspace.c ${HDF5_SRC_DIR}/H5FDstdio.c @@ -166,35 +169,38 @@ set (H5FD_HDRS ${HDF5_SRC_DIR}/H5FDcore.h ${HDF5_SRC_DIR}/H5FDdirect.h ${HDF5_SRC_DIR}/H5FDfamily.h + ${HDF5_SRC_DIR}/H5FDhdfs.h ${HDF5_SRC_DIR}/H5FDlog.h ${HDF5_SRC_DIR}/H5FDmpi.h ${HDF5_SRC_DIR}/H5FDmpio.h ${HDF5_SRC_DIR}/H5FDmulti.h ${HDF5_SRC_DIR}/H5FDpublic.h + ${HDF5_SRC_DIR}/H5FDros3.h + ${HDF5_SRC_DIR}/H5FDs3comms.c ${HDF5_SRC_DIR}/H5FDsec2.h ${HDF5_SRC_DIR}/H5FDstdio.h ${HDF5_SRC_DIR}/H5FDwindows.h ) -IDE_GENERATED_PROPERTIES ("H5FD" "${H5FD_HDRS}" "${H5FD_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5FD" "${H5FD_HDRS}" "${H5FD_SOURCES}" ) -set (H5FL_SRCS +set (H5FL_SOURCES ${HDF5_SRC_DIR}/H5FL.c ) set (H5FL_HDRS ) -IDE_GENERATED_PROPERTIES ("H5FL" "${H5FL_HDRS}" "${H5FL_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5FL" "${H5FL_HDRS}" "${H5FL_SOURCES}" ) -set (H5FO_SRCS +set (H5FO_SOURCES ${HDF5_SRC_DIR}/H5FO.c ) set (H5FO_HDRS ) -IDE_GENERATED_PROPERTIES ("H5FO" "${H5FO_HDRS}" "${H5FO_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5FO" "${H5FO_HDRS}" "${H5FO_SOURCES}" ) -set (H5FS_SRCS +set (H5FS_SOURCES ${HDF5_SRC_DIR}/H5FS.c ${HDF5_SRC_DIR}/H5FScache.c ${HDF5_SRC_DIR}/H5FSdbg.c @@ -205,9 +211,9 @@ set (H5FS_SRCS set (H5FS_HDRS ) -IDE_GENERATED_PROPERTIES ("H5FS" "${H5FS_HDRS}" "${H5FS_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5FS" "${H5FS_HDRS}" "${H5FS_SOURCES}" ) -set (H5G_SRCS +set (H5G_SOURCES ${HDF5_SRC_DIR}/H5G.c ${HDF5_SRC_DIR}/H5Gbtree2.c ${HDF5_SRC_DIR}/H5Gcache.c @@ -231,9 +237,9 @@ set (H5G_SRCS set (H5G_HDRS ${HDF5_SRC_DIR}/H5Gpublic.h ) -IDE_GENERATED_PROPERTIES ("H5G" "${H5G_HDRS}" "${H5G_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5G" "${H5G_HDRS}" "${H5G_SOURCES}" ) -set (H5HF_SRCS +set (H5HF_SOURCES ${HDF5_SRC_DIR}/H5HF.c ${HDF5_SRC_DIR}/H5HFbtree2.c ${HDF5_SRC_DIR}/H5HFcache.c @@ -254,9 +260,9 @@ set (H5HF_SRCS set (H5HF_HDRS ) -IDE_GENERATED_PROPERTIES ("H5HF" "${H5HF_HDRS}" "${H5HF_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5HF" "${H5HF_HDRS}" "${H5HF_SOURCES}" ) -set (H5HG_SRCS +set (H5HG_SOURCES ${HDF5_SRC_DIR}/H5HG.c ${HDF5_SRC_DIR}/H5HGcache.c ${HDF5_SRC_DIR}/H5HGdbg.c @@ -265,9 +271,9 @@ set (H5HG_SRCS set (H5HG_HDRS ) -IDE_GENERATED_PROPERTIES ("H5HG" "${H5HG_HDRS}" "${H5HG_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5HG" "${H5HG_HDRS}" "${H5HG_SOURCES}" ) -set (H5HL_SRCS +set (H5HL_SOURCES ${HDF5_SRC_DIR}/H5HL.c ${HDF5_SRC_DIR}/H5HLcache.c ${HDF5_SRC_DIR}/H5HLdbg.c @@ -276,38 +282,38 @@ set (H5HL_SRCS set (H5HL_HDRS ) -IDE_GENERATED_PROPERTIES ("H5HL" "${H5HL_HDRS}" "${H5HL_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5HL" "${H5HL_HDRS}" "${H5HL_SOURCES}" ) -set (H5HP_SRCS +set (H5HP_SOURCES ${HDF5_SRC_DIR}/H5HP.c ) set (H5HP_HDRS ) -IDE_GENERATED_PROPERTIES ("H5HP" "${H5HP_HDRS}" "${H5HP_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5HP" "${H5HP_HDRS}" "${H5HP_SOURCES}" ) -set (H5I_SRCS +set (H5I_SOURCES ${HDF5_SRC_DIR}/H5I.c ${HDF5_SRC_DIR}/H5Itest.c ) set (H5I_HDRS ${HDF5_SRC_DIR}/H5Ipublic.h ) -IDE_GENERATED_PROPERTIES ("H5I" "${H5I_HDRS}" "${H5I_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5I" "${H5I_HDRS}" "${H5I_SOURCES}" ) -set (H5L_SRCS +set (H5L_SOURCES ${HDF5_SRC_DIR}/H5L.c ${HDF5_SRC_DIR}/H5Lexternal.c ) set (H5L_HDRS ${HDF5_SRC_DIR}/H5Lpublic.h ) -IDE_GENERATED_PROPERTIES ("H5L" "${H5L_HDRS}" "${H5L_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5L" "${H5L_HDRS}" "${H5L_SOURCES}" ) -set (H5MF_SRCS +set (H5MF_SOURCES ${HDF5_SRC_DIR}/H5MF.c ${HDF5_SRC_DIR}/H5MFaggr.c ${HDF5_SRC_DIR}/H5MFdbg.c @@ -316,28 +322,28 @@ set (H5MF_SRCS set (H5MF_HDRS ) -IDE_GENERATED_PROPERTIES ("H5MF" "${H5MF_HDRS}" "${H5MF_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5MF" "${H5MF_HDRS}" "${H5MF_SOURCES}" ) -set (H5MM_SRCS +set (H5MM_SOURCES ${HDF5_SRC_DIR}/H5MM.c ) set (H5MM_HDRS ${HDF5_SRC_DIR}/H5MMpublic.h ) -IDE_GENERATED_PROPERTIES ("H5MM" "${H5MM_HDRS}" "${H5MM_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5MM" "${H5MM_HDRS}" "${H5MM_SOURCES}" ) -set (H5MP_SRCS +set (H5MP_SOURCES ${HDF5_SRC_DIR}/H5MP.c ${HDF5_SRC_DIR}/H5MPtest.c ) set (H5MP_HDRS ) -IDE_GENERATED_PROPERTIES ("H5MP" "${H5MP_HDRS}" "${H5MP_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5MP" "${H5MP_HDRS}" "${H5MP_SOURCES}" ) -set (H5O_SRCS +set (H5O_SOURCES ${HDF5_SRC_DIR}/H5O.c ${HDF5_SRC_DIR}/H5Oainfo.c ${HDF5_SRC_DIR}/H5Oalloc.c @@ -375,9 +381,9 @@ set (H5O_SRCS set (H5O_HDRS ${HDF5_SRC_DIR}/H5Opublic.h ) -IDE_GENERATED_PROPERTIES ("H5O" "${H5O_HDRS}" "${H5O_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5O" "${H5O_HDRS}" "${H5O_SOURCES}" ) -set (H5P_SRCS +set (H5P_SOURCES ${HDF5_SRC_DIR}/H5P.c ${HDF5_SRC_DIR}/H5Pacpl.c ${HDF5_SRC_DIR}/H5Pdapl.c @@ -400,9 +406,9 @@ set (H5P_SRCS set (H5P_HDRS ${HDF5_SRC_DIR}/H5Ppublic.h ) -IDE_GENERATED_PROPERTIES ("H5P" "${H5P_HDRS}" "${H5P_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5P" "${H5P_HDRS}" "${H5P_SOURCES}" ) -set (H5PL_SRCS +set (H5PL_SOURCES ${HDF5_SRC_DIR}/H5PL.c ) @@ -410,35 +416,36 @@ set (H5PL_HDRS ${HDF5_SRC_DIR}/H5PLextern.h ${HDF5_SRC_DIR}/H5PLpublic.h ) -IDE_GENERATED_PROPERTIES ("H5PL" "${H5PL_HDRS}" "${H5PL_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5PL" "${H5PL_HDRS}" "${H5PL_SOURCES}" ) -set (H5R_SRCS +set (H5R_SOURCES ${HDF5_SRC_DIR}/H5R.c ${HDF5_SRC_DIR}/H5Rdeprec.c ) set (H5R_HDRS ${HDF5_SRC_DIR}/H5Rpublic.h ) -IDE_GENERATED_PROPERTIES ("H5R" "${H5R_HDRS}" "${H5R_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5R" "${H5R_HDRS}" "${H5R_SOURCES}" ) -set (H5RC_SRCS +set (H5RC_SOURCES ${HDF5_SRC_DIR}/H5RC.c ) set (H5RC_HDRS ) -IDE_GENERATED_PROPERTIES ("H5RC" "${H5RC_HDRS}" "${H5RC_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5RC" "${H5RC_HDRS}" "${H5RC_SOURCES}" ) -set (H5RS_SRCS +set (H5RS_SOURCES ${HDF5_SRC_DIR}/H5RS.c ) set (H5RS_HDRS ) -IDE_GENERATED_PROPERTIES ("H5RS" "${H5RS_HDRS}" "${H5RS_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5RS" "${H5RS_HDRS}" "${H5RS_SOURCES}" ) -set (H5S_SRCS + +set (H5S_SOURCES ${HDF5_SRC_DIR}/H5S.c ${HDF5_SRC_DIR}/H5Sall.c ${HDF5_SRC_DIR}/H5Sdbg.c @@ -453,18 +460,18 @@ set (H5S_SRCS set (H5S_HDRS ${HDF5_SRC_DIR}/H5Spublic.h ) -IDE_GENERATED_PROPERTIES ("H5S" "${H5S_HDRS}" "${H5S_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5S" "${H5S_HDRS}" "${H5S_SOURCES}" ) -set (H5SL_SRCS +set (H5SL_SOURCES ${HDF5_SRC_DIR}/H5SL.c ) set (H5SL_HDRS ) -IDE_GENERATED_PROPERTIES ("H5SL" "${H5SL_HDRS}" "${H5SL_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5SL" "${H5SL_HDRS}" "${H5SL_SOURCES}" ) -set (H5SM_SRCS +set (H5SM_SOURCES ${HDF5_SRC_DIR}/H5SM.c ${HDF5_SRC_DIR}/H5SMbtree2.c ${HDF5_SRC_DIR}/H5SMcache.c @@ -474,18 +481,18 @@ set (H5SM_SRCS set (H5SM_HDRS ) -IDE_GENERATED_PROPERTIES ("H5SM" "${H5SM_HDRS}" "${H5SM_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5SM" "${H5SM_HDRS}" "${H5SM_SOURCES}" ) -set (H5ST_SRCS +set (H5ST_SOURCES ${HDF5_SRC_DIR}/H5ST.c ) set (H5ST_HDRS ) -IDE_GENERATED_PROPERTIES ("H5ST" "${H5ST_HDRS}" "${H5ST_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5ST" "${H5ST_HDRS}" "${H5ST_SOURCES}" ) -set (H5T_SRCS +set (H5T_SOURCES ${HDF5_SRC_DIR}/H5T.c ${HDF5_SRC_DIR}/H5Tarray.c ${HDF5_SRC_DIR}/H5Tbit.c @@ -514,34 +521,34 @@ set (H5T_SRCS set (H5T_HDRS ${HDF5_SRC_DIR}/H5Tpublic.h ) -IDE_GENERATED_PROPERTIES ("H5T" "${H5T_HDRS}" "${H5T_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5T" "${H5T_HDRS}" "${H5T_SOURCES}" ) -set (H5TS_SRCS +set (H5TS_SOURCES ${HDF5_SRC_DIR}/H5TS.c ) set (H5TS_HDRS ) -IDE_GENERATED_PROPERTIES ("H5TS" "${H5TS_HDRS}" "${H5TS_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5TS" "${H5TS_HDRS}" "${H5TS_SOURCES}" ) -set (H5VM_SRCS +set (H5VM_SOURCES ${HDF5_SRC_DIR}/H5VM.c ) set (H5VM_HDRS ) -IDE_GENERATED_PROPERTIES ("H5VM" "${H5VM_HDRS}" "${H5VM_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5VM" "${H5VM_HDRS}" "${H5VM_SOURCES}" ) -set (H5WB_SRCS +set (H5WB_SOURCES ${HDF5_SRC_DIR}/H5WB.c ) set (H5WB_HDRS ) -IDE_GENERATED_PROPERTIES ("H5WB" "${H5WB_HDRS}" "${H5WB_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5WB" "${H5WB_HDRS}" "${H5WB_SOURCES}" ) -set (H5Z_SRCS +set (H5Z_SOURCES ${HDF5_SRC_DIR}/H5Z.c ${HDF5_SRC_DIR}/H5Zdeflate.c ${HDF5_SRC_DIR}/H5Zfletcher32.c @@ -560,48 +567,48 @@ endif () set (H5Z_HDRS ${HDF5_SRC_DIR}/H5Zpublic.h ) -IDE_GENERATED_PROPERTIES ("H5Z" "${H5Z_HDRS}" "${H5Z_SRCS}" ) +IDE_GENERATED_PROPERTIES ("H5Z" "${H5Z_HDRS}" "${H5Z_SOURCES}" ) set (common_SRCS - ${H5_SRCS} - ${H5A_SRCS} - ${H5AC_SRCS} - ${H5B_SRCS} - ${H5B2_SRCS} - ${H5C_SRCS} - ${H5CS_SRCS} - ${H5D_SRCS} - ${H5E_SRCS} - ${H5F_SRCS} - ${H5FD_SRCS} - ${H5FL_SRCS} - ${H5FO_SRCS} - ${H5FS_SRCS} - ${H5G_SRCS} - ${H5HF_SRCS} - ${H5HG_SRCS} - ${H5HL_SRCS} - ${H5HP_SRCS} - ${H5I_SRCS} - ${H5L_SRCS} - ${H5MF_SRCS} - ${H5MM_SRCS} - ${H5MP_SRCS} - ${H5O_SRCS} - ${H5P_SRCS} - ${H5PL_SRCS} - ${H5R_SRCS} - ${H5RC_SRCS} - ${H5RS_SRCS} - ${H5S_SRCS} - ${H5SL_SRCS} - ${H5SM_SRCS} - ${H5ST_SRCS} - ${H5T_SRCS} - ${H5TS_SRCS} - ${H5VM_SRCS} - ${H5WB_SRCS} - ${H5Z_SRCS} + ${H5_SOURCES} + ${H5A_SOURCES} + ${H5AC_SOURCES} + ${H5B_SOURCES} + ${H5B2_SOURCES} + ${H5C_SOURCES} + ${H5CS_SOURCES} + ${H5D_SOURCES} + ${H5E_SOURCES} + ${H5F_SOURCES} + ${H5FD_SOURCES} + ${H5FL_SOURCES} + ${H5FO_SOURCES} + ${H5FS_SOURCES} + ${H5G_SOURCES} + ${H5HF_SOURCES} + ${H5HG_SOURCES} + ${H5HL_SOURCES} + ${H5HP_SOURCES} + ${H5I_SOURCES} + ${H5L_SOURCES} + ${H5MF_SOURCES} + ${H5MM_SOURCES} + ${H5MP_SOURCES} + ${H5O_SOURCES} + ${H5P_SOURCES} + ${H5PL_SOURCES} + ${H5R_SOURCES} + ${H5RC_SOURCES} + ${H5RS_SOURCES} + ${H5S_SOURCES} + ${H5SL_SOURCES} + ${H5SM_SOURCES} + ${H5ST_SOURCES} + ${H5T_SOURCES} + ${H5TS_SOURCES} + ${H5VM_SOURCES} + ${H5WB_SOURCES} + ${H5Z_SOURCES} ) set (H5_PUBLIC_HEADERS @@ -797,6 +804,27 @@ if (BUILD_SHARED_LIBS) file (MAKE_DIRECTORY "${HDF5_BINARY_DIR}/shared") endif () +if (LOCAL_BATCH_TEST) + if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub") + configure_file ( + ${HDF5_SOURCE_DIR}/bin/batch/${LOCAL_BATCH_SCRIPT_COMMAND} + ${HDF5_BINARY_DIR}/${LOCAL_BATCH_SCRIPT_COMMAND} ESCAPE_QUOTES @ONLY + ) + endif () + if (LOCAL_BATCH_SCRIPT_NAME) + configure_file ( + ${HDF5_SOURCE_DIR}/bin/batch/${LOCAL_BATCH_SCRIPT_NAME}.in.cmake + ${HDF5_BINARY_DIR}/${LOCAL_BATCH_SCRIPT_NAME} ESCAPE_QUOTES @ONLY + ) + endif () + if (LOCAL_BATCH_SCRIPT_PARALLEL_NAME) + configure_file ( + ${HDF5_SOURCE_DIR}/bin/batch/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}.in.cmake + ${HDF5_BINARY_DIR}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME} ESCAPE_QUOTES @ONLY + ) + endif () +endif () + if (NOT EXISTS "${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c") add_executable (H5detect ${HDF5_SRC_DIR}/H5detect.c) target_include_directories (H5detect PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") @@ -941,7 +969,6 @@ if (NOT ONLY_SHARED_LIBS) PUBLIC ${HDF_EXTRA_C_FLAGS} ${HDF_EXTRA_FLAGS} - $,DEBUG,NDEBUG> PRIVATE $<$:H5_DEBUG_API> # Enable tracing of the API $<$:H5Z_DEBUG;H5T_DEBUG;H5ST_DEBUG;H5S_DEBUG;H5O_DEBUG;H5I_DEBUG;H5HL_DEBUG;H5F_DEBUG;H5D_DEBUG;H5B2_DEBUG;H5AC_DEBUG> @@ -971,6 +998,7 @@ if (BUILD_SHARED_LIBS) add_library (${HDF5_LIBSH_TARGET} SHARED ${common_SRCS} ${shared_gen_SRCS} ${H5_PUBLIC_HEADERS} ${H5_PRIVATE_HEADERS} ${H5_GENERATED_HEADERS}) target_include_directories (${HDF5_LIBSH_TARGET} PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>" + PUBLIC "$<$:${HDFS_INCLUDE_DIR}>" INTERFACE "$/include>" ) target_compile_definitions(${HDF5_LIBSH_TARGET} @@ -978,7 +1006,6 @@ if (BUILD_SHARED_LIBS) "H5_BUILT_AS_DYNAMIC_LIB" ${HDF_EXTRA_C_FLAGS} ${HDF_EXTRA_FLAGS} - $,DEBUG,NDEBUG> PRIVATE $<$:H5_HAVE_THREADSAFE> $<$:H5_DEBUG_API> # Enable tracing of the API diff --git a/src/H5FD.c b/src/H5FD.c index 63b7dfc..11ef24a 100644 --- a/src/H5FD.c +++ b/src/H5FD.c @@ -15,7 +15,7 @@ * Programmer: Robb Matzke * Monday, July 26, 1999 * - * Purpose: The Virtual File Layer as described in documentation. + * Purpose: The Virtual File Layer as described in documentation. * This is the greatest common denominator for all types of * storage access whether a file, memory, network, etc. This * layer usually just dispatches the request to an actual @@ -26,35 +26,35 @@ /* Module Setup */ /****************/ -#define H5F_PACKAGE /*suppress error about including H5Fpkg */ -#define H5FD_PACKAGE /*suppress error about including H5FDpkg */ +#define H5F_PACKAGE /*suppress error about including H5Fpkg */ +#define H5FD_PACKAGE /*suppress error about including H5FDpkg */ /* Interface initialization */ -#define H5_INTERFACE_INIT_FUNC H5FD_init_interface +#define H5_INTERFACE_INIT_FUNC H5FD_init_interface /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Dprivate.h" /* Datasets */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fpkg.h" /* File access */ -#include "H5FDpkg.h" /* File Drivers */ -#include "H5FDcore.h" /* Files stored entirely in memory */ -#include "H5FDfamily.h" /* File families */ -#include "H5FDlog.h" /* sec2 driver with I/O logging (for debugging) */ -#include "H5FDmpi.h" /* MPI-based file drivers */ -#include "H5FDmulti.h" /* Usage-partitioned file family */ -#include "H5FDsec2.h" /* POSIX unbuffered file I/O */ -#include "H5FDstdio.h" /* Standard C buffered I/O */ +#include "H5private.h" /* Generic Functions */ +#include "H5Dprivate.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* File access */ +#include "H5FDpkg.h" /* File Drivers */ +#include "H5FDcore.h" /* Files stored entirely in memory */ +#include "H5FDfamily.h" /* File families */ +#include "H5FDlog.h" /* sec2 driver with I/O logging (for debugging) */ +#include "H5FDmpi.h" /* MPI-based file drivers */ +#include "H5FDmulti.h" /* Usage-partitioned file family */ +#include "H5FDsec2.h" /* POSIX unbuffered file I/O */ +#include "H5FDstdio.h" /* Standard C buffered I/O */ #ifdef H5_HAVE_WINDOWS #include "H5FDwindows.h" /* Windows buffered I/O */ #endif -#include "H5FDdirect.h" /* Direct file I/O */ -#include "H5Iprivate.h" /* IDs */ -#include "H5MMprivate.h" /* Memory management */ -#include "H5Pprivate.h" /* Property lists */ +#include "H5FDdirect.h" /* Direct file I/O */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Pprivate.h" /* Property lists */ /****************/ /* Local Macros */ @@ -113,23 +113,23 @@ static unsigned long H5FD_file_serial_no_g; /* File driver ID class */ static const H5I_class_t H5I_VFL_CLS[1] = {{ - H5I_VFL, /* ID class value */ - H5I_CLASS_REUSE_IDS, /* Class flags */ - 0, /* # of reserved IDs for class */ - (H5I_free_t)H5FD_free_cls /* Callback routine for closing objects of this class */ + H5I_VFL, /* ID class value */ + H5I_CLASS_REUSE_IDS, /* Class flags */ + 0, /* # of reserved IDs for class */ + (H5I_free_t)H5FD_free_cls /* Callback routine for closing objects of this class */ }}; - + /*------------------------------------------------------------------------- - * Function: H5FD_init + * Function: H5FD_init * - * Purpose: Initialize the interface from some other package. + * Purpose: Initialize the interface from some other package. * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, January 3, 2007 * *------------------------------------------------------------------------- @@ -146,17 +146,17 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_init() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_init_interface + * Function: H5FD_init_interface * - * Purpose: Initialize the virtual file layer. + * Purpose: Initialize the virtual file layer. * - * Return: Success: Non-negative + * Return: Success: Non-negative * - * Failure: Negative + * Failure: Negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, July 26, 1999 * * Modifications: @@ -171,7 +171,7 @@ H5FD_init_interface(void) FUNC_ENTER_NOAPI_NOINIT if(H5I_register_type(H5I_VFL_CLS) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize interface") + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize interface") /* Reset the file serial numbers */ H5FD_file_serial_no_g = 0; @@ -180,21 +180,21 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_init_interface() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_term_interface + * Function: H5FD_term_interface * - * Purpose: Terminate this interface: free all memory and reset global - * variables to their initial values. Release all ID groups - * associated with this interface. + * Purpose: Terminate this interface: free all memory and reset global + * variables to their initial values. Release all ID groups + * associated with this interface. * - * Return: Success: Positive if anything was done that might - * have affected other interfaces; zero - * otherwise. + * Return: Success: Positive if anything was done that might + * have affected other interfaces; zero + * otherwise. * - * Failure: Never fails. + * Failure: Never fails. * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Friday, February 19, 1999 * * Modifications: @@ -204,13 +204,13 @@ done: int H5FD_term_interface(void) { - int n = 0; + int n = 0; FUNC_ENTER_NOAPI_NOINIT_NOERR if(H5_interface_initialize_g) { - if(H5I_nmembers(H5I_VFL) > 0) { - (void)H5I_clear_type(H5I_VFL, FALSE, FALSE); + if(H5I_nmembers(H5I_VFL) > 0) { + (void)H5I_clear_type(H5I_VFL, FALSE, FALSE); /* Reset the VFL drivers, if they've been closed */ if(H5I_nmembers(H5I_VFL)==0) { @@ -232,33 +232,33 @@ H5FD_term_interface(void) } /* end if */ n++; /*H5I*/ - } /* end if */ + } /* end if */ else { /* Destroy the VFL driver id group */ - (void)H5I_dec_type_ref(H5I_VFL); + (void)H5I_dec_type_ref(H5I_VFL); n++; /*H5I*/ - /* Mark closed */ - H5_interface_initialize_g = 0; - } /* end else */ + /* Mark closed */ + H5_interface_initialize_g = 0; + } /* end else */ } /* end if */ FUNC_LEAVE_NOAPI(n) } /* end H5FD_term_interface() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_free_cls + * Function: H5FD_free_cls * - * Purpose: Frees a file driver class struct and returns an indication of - * success. This function is used as the free callback for the - * virtual file layer object identifiers (cf H5FD_init_interface). + * Purpose: Frees a file driver class struct and returns an indication of + * success. This function is used as the free callback for the + * virtual file layer object identifiers (cf H5FD_init_interface). * - * Return: Success: Non-negative + * Return: Success: Non-negative * - * Failure: Negative + * Failure: Negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, July 26, 1999 * * Modifications: @@ -275,22 +275,22 @@ H5FD_free_cls(H5FD_class_t *cls) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5FD_free_cls() */ - + /*------------------------------------------------------------------------- - * Function: H5FDregister + * Function: H5FDregister * - * Purpose: Registers a new file driver as a member of the virtual file - * driver class. Certain fields of the class struct are - * required and that is checked here so it doesn't have to be - * checked every time the field is accessed. + * Purpose: Registers a new file driver as a member of the virtual file + * driver class. Certain fields of the class struct are + * required and that is checked here so it doesn't have to be + * checked every time the field is accessed. * - * Return: Success: A file driver ID which is good until the - * library is closed or the driver is - * unregistered. + * Return: Success: A file driver ID which is good until the + * library is closed or the driver is + * unregistered. * - * Failure: A negative value. + * Failure: A negative value. * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, July 26, 1999 * * Modifications: @@ -303,26 +303,26 @@ H5FD_free_cls(H5FD_class_t *cls) hid_t H5FDregister(const H5FD_class_t *cls) { - hid_t ret_value; - H5FD_mem_t type; + hid_t ret_value; + H5FD_mem_t type; FUNC_ENTER_API(FAIL) H5TRACE1("i", "*x", cls); /* Check arguments */ if(!cls) - HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "null class pointer is disallowed") + HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "null class pointer is disallowed") if(!cls->open || !cls->close) - HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`open' and/or `close' methods are not defined") + HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`open' and/or `close' methods are not defined") if(!cls->get_eoa || !cls->set_eoa) - HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`get_eoa' and/or `set_eoa' methods are not defined") + HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`get_eoa' and/or `set_eoa' methods are not defined") if(!cls->get_eof) - HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`get_eof' method is not defined") + HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`get_eof' method is not defined") if(!cls->read || !cls->write) - HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`read' and/or `write' method is not defined") + HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`read' and/or `write' method is not defined") for (type=H5FD_MEM_DEFAULT; typefl_map[type]fl_map[type]>=H5FD_MEM_NTYPES) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid free-list mapping") + if(cls->fl_map[type]fl_map[type]>=H5FD_MEM_NTYPES) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid free-list mapping") /* Create the new class ID */ if((ret_value=H5FD_register(cls, sizeof(H5FD_class_t), TRUE)) < 0) @@ -332,22 +332,22 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5FDregister() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_register + * Function: H5FD_register * - * Purpose: Registers a new file driver as a member of the virtual file - * driver class. Certain fields of the class struct are - * required and that is checked here so it doesn't have to be - * checked every time the field is accessed. + * Purpose: Registers a new file driver as a member of the virtual file + * driver class. Certain fields of the class struct are + * required and that is checked here so it doesn't have to be + * checked every time the field is accessed. * - * Return: Success: A file driver ID which is good until the - * library is closed or the driver is - * unregistered. + * Return: Success: A file driver ID which is good until the + * library is closed or the driver is + * unregistered. * - * Failure: A negative value. + * Failure: A negative value. * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, July 26, 1999 * * Modifications: @@ -363,10 +363,10 @@ done: hid_t H5FD_register(const void *_cls, size_t size, hbool_t app_ref) { - const H5FD_class_t *cls = (const H5FD_class_t *)_cls; - H5FD_class_t *saved = NULL; - H5FD_mem_t type; - hid_t ret_value; + const H5FD_class_t *cls = (const H5FD_class_t *)_cls; + H5FD_class_t *saved = NULL; + H5FD_mem_t type; + hid_t ret_value; FUNC_ENTER_NOAPI(FAIL) @@ -381,7 +381,7 @@ H5FD_register(const void *_cls, size_t size, hbool_t app_ref) /* Copy the class structure so the caller can reuse or free it */ if(NULL == (saved = (H5FD_class_t *)H5MM_malloc(size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for file driver class struct") + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for file driver class struct") HDmemcpy(saved, cls, size); /* Create the new class ID */ @@ -396,20 +396,20 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_register() */ - + /*------------------------------------------------------------------------- - * Function: H5FDunregister + * Function: H5FDunregister * - * Purpose: Removes a driver ID from the library. This in no way affects - * file access property lists which have been defined to use - * this driver or files which are already opened under this - * driver. + * Purpose: Removes a driver ID from the library. This in no way affects + * file access property lists which have been defined to use + * this driver or files which are already opened under this + * driver. * - * Return: Success: Non-negative + * Return: Success: Non-negative * - * Failure: Negative + * Failure: Negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, July 26, 1999 * *------------------------------------------------------------------------- @@ -424,33 +424,33 @@ H5FDunregister(hid_t driver_id) /* Check arguments */ if(NULL == H5I_object_verify(driver_id, H5I_VFL)) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file driver") + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file driver") /* The H5FD_class_t struct will be freed by this function */ if(H5I_dec_app_ref(driver_id) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "unable to unregister file driver") + HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "unable to unregister file driver") done: FUNC_LEAVE_API(ret_value) } /* end H5FDunregister() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_get_class + * Function: H5FD_get_class * - * Purpose: Obtains a pointer to the driver struct containing all the - * callback pointers, etc. The PLIST_ID argument can be a file - * access property list, a data transfer property list, or a - * file driver identifier. + * Purpose: Obtains a pointer to the driver struct containing all the + * callback pointers, etc. The PLIST_ID argument can be a file + * access property list, a data transfer property list, or a + * file driver identifier. * - * Return: Success: Ptr to the driver information. The pointer is - * only valid as long as the driver remains - * registered or some file or property list - * exists which references the driver. + * Return: Success: Ptr to the driver information. The pointer is + * only valid as long as the driver remains + * registered or some file or property list + * exists which references the driver. * - * Failure: NULL + * Failure: NULL * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Friday, August 20, 1999 * *------------------------------------------------------------------------- @@ -458,12 +458,12 @@ done: H5FD_class_t * H5FD_get_class(hid_t id) { - H5FD_class_t *ret_value = NULL; + H5FD_class_t *ret_value = NULL; FUNC_ENTER_NOAPI(NULL) if(H5I_VFL == H5I_get_type(id)) - ret_value = (H5FD_class_t *)H5I_object(id); + ret_value = (H5FD_class_t *)H5I_object(id); else { H5P_genplist_t *plist; /* Property list pointer */ hid_t driver_id = -1; @@ -485,19 +485,19 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_get_class() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_sb_size + * Function: H5FD_sb_size * - * Purpose: Obtains the number of bytes required to store the driver file - * access data in the HDF5 superblock. + * Purpose: Obtains the number of bytes required to store the driver file + * access data in the HDF5 superblock. * - * Return: Success: Number of bytes required. + * Return: Success: Number of bytes required. * - * Failure: 0 if an error occurs or if the driver has no - * data to store in the superblock. + * Failure: 0 if an error occurs or if the driver has no + * data to store in the superblock. * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, August 16, 1999 * * Modifications: @@ -507,35 +507,35 @@ done: hsize_t H5FD_sb_size(H5FD_t *file) { - hsize_t ret_value=0; + hsize_t ret_value=0; FUNC_ENTER_NOAPI(0) HDassert(file && file->cls); if(file->cls->sb_size) - ret_value = (file->cls->sb_size)(file); + ret_value = (file->cls->sb_size)(file); done: FUNC_LEAVE_NOAPI(ret_value) } - + /*------------------------------------------------------------------------- - * Function: H5FD_sb_encode + * Function: H5FD_sb_encode * - * Purpose: Encode driver-specific data into the output arguments. The - * NAME is a nine-byte buffer which should get an - * eight-character driver name and/or version followed by a null - * terminator. The BUF argument is a buffer to receive the - * encoded driver-specific data. The size of the BUF array is - * the size returned by the H5FD_sb_size() call. + * Purpose: Encode driver-specific data into the output arguments. The + * NAME is a nine-byte buffer which should get an + * eight-character driver name and/or version followed by a null + * terminator. The BUF argument is a buffer to receive the + * encoded driver-specific data. The size of the BUF array is + * the size returned by the H5FD_sb_size() call. * - * Return: Success: Non-negative + * Return: Success: Non-negative * - * Failure: Negative + * Failure: Negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, August 16, 1999 * * Modifications: @@ -552,22 +552,22 @@ H5FD_sb_encode(H5FD_t *file, char *name/*out*/, uint8_t *buf) HDassert(file && file->cls); if(file->cls->sb_encode && (file->cls->sb_encode)(file, name/*out*/, buf/*out*/) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_encode request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_encode request failed") done: FUNC_LEAVE_NOAPI(ret_value) } - + /*------------------------------------------------------------------------- - * Function: H5FD_sb_decode + * Function: H5FD_sb_decode * - * Purpose: Decodes the driver information block. + * Purpose: Decodes the driver information block. * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, August 16, 1999 * *------------------------------------------------------------------------- @@ -581,25 +581,25 @@ H5FD_sb_decode(H5FD_t *file, const char *name, const uint8_t *buf) HDassert(file && file->cls); if(file->cls->sb_decode && (file->cls->sb_decode)(file, name, buf) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_decode request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_decode request failed") done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_sb_decode() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_pl_copy + * Function: H5FD_pl_copy * - * Purpose: Copies the driver-specific part of the a property list. + * Purpose: Copies the driver-specific part of the a property list. * This is common code, used by both the dataset transfer and * file access property list routines. * - * Return: Success: non-negative + * Return: Success: non-negative * - * Failure: negative + * Failure: negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, October 23, 2003 * * Modifications: @@ -638,18 +638,18 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_pl_copy() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_pl_close + * Function: H5FD_pl_close * - * Purpose: Closes a driver for a property list + * Purpose: Closes a driver for a property list * This is common code, used by both the dataset transfer and * file access property list routines. * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, October 23, 2003 * *------------------------------------------------------------------------- @@ -663,11 +663,11 @@ H5FD_pl_close(hid_t driver_id, herr_t (*free_func)(void *), void *pl) /* Allow driver to free or do it ourselves */ if(pl && free_func) { - if((free_func)(pl) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver free request failed") + if((free_func)(pl) < 0) + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver free request failed") } /* end if */ else - H5MM_xfree(pl); + H5MM_xfree(pl); /* Decrement reference count for driver */ if(H5I_dec_ref(driver_id) < 0) @@ -677,27 +677,27 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_pl_close() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_fapl_get + * Function: H5FD_fapl_get * - * Purpose: Gets the file access property list associated with a file. - * Usually the file will copy what it needs from the original - * file access property list when the file is created. The - * purpose of this function is to create a new file access - * property list based on the settings in the file, which may - * have been modified from the original file access property - * list. + * Purpose: Gets the file access property list associated with a file. + * Usually the file will copy what it needs from the original + * file access property list when the file is created. The + * purpose of this function is to create a new file access + * property list based on the settings in the file, which may + * have been modified from the original file access property + * list. * - * Return: Success: Pointer to a new file access property list - * with all members copied. If the file is - * closed then this property list lives on, and - * vice versa. + * Return: Success: Pointer to a new file access property list + * with all members copied. If the file is + * closed then this property list lives on, and + * vice versa. * - * Failure: NULL, including when the file has no - * properties. + * Failure: NULL, including when the file has no + * properties. * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Friday, August 13, 1999 * * Modifications: @@ -707,30 +707,30 @@ done: void * H5FD_fapl_get(H5FD_t *file) { - void *ret_value=NULL; + void *ret_value=NULL; FUNC_ENTER_NOAPI(NULL) HDassert(file); if(file->cls->fapl_get) - ret_value = (file->cls->fapl_get)(file); + ret_value = (file->cls->fapl_get)(file); done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_fapl_get() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_fapl_open + * Function: H5FD_fapl_open * - * Purpose: Mark a driver as used by a file access property list + * Purpose: Mark a driver as used by a file access property list * - * Return: Success: non-negative + * Return: Success: non-negative * - * Failure: negative + * Failure: negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, October 23, 2003 * * Modifications: @@ -766,18 +766,18 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_fapl_open() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_fapl_copy + * Function: H5FD_fapl_copy * - * Purpose: Copies the driver-specific part of the file access property - * list. + * Purpose: Copies the driver-specific part of the file access property + * list. * - * Return: Success: non-negative + * Return: Success: non-negative * - * Failure: negative + * Failure: negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Tuesday, August 3, 1999 * *------------------------------------------------------------------------- @@ -792,7 +792,7 @@ H5FD_fapl_copy(hid_t driver_id, const void *old_fapl, void **copied_fapl) /* Check args */ if(NULL == (driver = (H5FD_class_t *)H5I_object(driver_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a driver ID") + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a driver ID") /* Copy the file access property list */ if(H5FD_pl_copy(driver->fapl_copy, driver->fapl_size, old_fapl, copied_fapl) < 0) @@ -802,16 +802,16 @@ done: FUNC_LEAVE_NOAPI(ret_value) } - + /*------------------------------------------------------------------------- - * Function: H5FD_fapl_close + * Function: H5FD_fapl_close * - * Purpose: Closes a driver for a dataset transfer property list + * Purpose: Closes a driver for a dataset transfer property list * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Tuesday, August 3, 1999 * * Modifications: @@ -821,7 +821,7 @@ done: herr_t H5FD_fapl_close(hid_t driver_id, void *fapl) { - H5FD_class_t *driver = NULL; + H5FD_class_t *driver = NULL; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -840,56 +840,56 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_fapl_close() */ - + /*------------------------------------------------------------------------- - * Function: H5FDopen - * - * Purpose: Opens a file named NAME for the type(s) of access described - * by the bit vector FLAGS according to a file access property - * list FAPL_ID (which may be the constant H5P_DEFAULT). The - * file should expect to handle format addresses in the range [0, - * MAXADDR] (if MAXADDR is the undefined address then the caller - * doesn't care about the address range). - * - * Possible values for the FLAGS bits are: - * - * H5F_ACC_RDWR: Open the file for read and write access. If - * this bit is not set then open the file for - * read only access. It is permissible to open a - * file for read and write access when only read - * access is requested by the library (the - * library will never attempt to write to a file - * which it opened with only read access). - * - * H5F_ACC_CREATE: Create the file if it doesn't already exist. - * However, see H5F_ACC_EXCL below. - * - * H5F_ACC_TRUNC: Truncate the file if it already exists. This - * is equivalent to deleting the file and then - * creating a new empty file. - * - * H5F_ACC_EXCL: When used with H5F_ACC_CREATE, if the file - * already exists then the open should fail. - * Note that this is unsupported/broken with - * some file drivers (e.g., sec2 across nfs) and - * will contain a race condition when used to - * perform file locking. - * - * The MAXADDR is the maximum address which will be requested by - * the library during an allocation operation. Usually this is - * the same value as the MAXADDR field of the class structure, - * but it can be smaller if the driver is being used under some - * other driver. - * - * Note that when the driver `open' callback gets control that - * the public part of the file struct (the H5FD_t part) will be - * incomplete and will be filled in after that callback returns. - * - * Return: Success: Pointer to a new file driver struct. - * - * Failure: NULL - * - * Programmer: Robb Matzke + * Function: H5FDopen + * + * Purpose: Opens a file named NAME for the type(s) of access described + * by the bit vector FLAGS according to a file access property + * list FAPL_ID (which may be the constant H5P_DEFAULT). The + * file should expect to handle format addresses in the range [0, + * MAXADDR] (if MAXADDR is the undefined address then the caller + * doesn't care about the address range). + * + * Possible values for the FLAGS bits are: + * + * H5F_ACC_RDWR: Open the file for read and write access. If + * this bit is not set then open the file for + * read only access. It is permissible to open a + * file for read and write access when only read + * access is requested by the library (the + * library will never attempt to write to a file + * which it opened with only read access). + * + * H5F_ACC_CREATE: Create the file if it doesn't already exist. + * However, see H5F_ACC_EXCL below. + * + * H5F_ACC_TRUNC: Truncate the file if it already exists. This + * is equivalent to deleting the file and then + * creating a new empty file. + * + * H5F_ACC_EXCL: When used with H5F_ACC_CREATE, if the file + * already exists then the open should fail. + * Note that this is unsupported/broken with + * some file drivers (e.g., sec2 across nfs) and + * will contain a race condition when used to + * perform file locking. + * + * The MAXADDR is the maximum address which will be requested by + * the library during an allocation operation. Usually this is + * the same value as the MAXADDR field of the class structure, + * but it can be smaller if the driver is being used under some + * other driver. + * + * Note that when the driver `open' callback gets control that + * the public part of the file struct (the H5FD_t part) will be + * incomplete and will be filled in after that callback returns. + * + * Return: Success: Pointer to a new file driver struct. + * + * Failure: NULL + * + * Programmer: Robb Matzke * Tuesday, July 27, 1999 * * Modifications: @@ -899,7 +899,7 @@ done: H5FD_t * H5FDopen(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) { - H5FD_t *ret_value=NULL; + H5FD_t *ret_value=NULL; FUNC_ENTER_API(NULL) H5TRACE4("*x", "*sIuia", name, flags, fapl_id, maxaddr); @@ -912,50 +912,50 @@ H5FDopen(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list") if(NULL==(ret_value=H5FD_open(name, flags, fapl_id, maxaddr))) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "unable to open file") + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "unable to open file") done: FUNC_LEAVE_API(ret_value) } - + /*------------------------------------------------------------------------- - * Function: H5FD_open + * Function: H5FD_open * - * Purpose: Private version of H5FDopen() + * Purpose: Private version of H5FDopen() * - * Return: Success: Pointer to a new file driver struct + * Return: Success: Pointer to a new file driver struct * - * Failure: NULL + * Failure: NULL * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Wednesday, August 4, 1999 * * Modifications: * - * Raymond Lu - * Tuesday, Oct 23, 2001 - * Changed the file access list to the new generic property - * list. + * Raymond Lu + * Tuesday, Oct 23, 2001 + * Changed the file access list to the new generic property + * list. * *------------------------------------------------------------------------- */ H5FD_t * H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) { - H5FD_class_t *driver; /* VFD for file */ - H5FD_t *file = NULL; /* VFD file struct */ + H5FD_class_t *driver; /* VFD for file */ + H5FD_t *file = NULL; /* VFD file struct */ hid_t driver_id = -1; /* VFD ID */ H5P_genplist_t *plist; /* Property list pointer */ unsigned long driver_flags = 0; /* File-inspecific driver feature flags */ H5FD_file_image_info_t file_image_info; /* Initial file image */ - H5FD_t *ret_value; /* Return value */ + H5FD_t *ret_value; /* Return value */ FUNC_ENTER_NOAPI(NULL) /* Sanity check */ if(0 == maxaddr) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "zero format address range") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "zero format address range") /* Get file access property list */ if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id))) @@ -967,9 +967,9 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) /* Get driver info */ if(NULL == (driver = (H5FD_class_t *)H5I_object(driver_id))) - HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "invalid driver ID in file access property list") + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "invalid driver ID in file access property list") if(NULL == driver->open) - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "file driver has no `open' method") + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "file driver has no `open' method") /* Query driver flag */ H5FD_driver_query(driver, &driver_flags); @@ -988,7 +988,7 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) if(HADDR_UNDEF == maxaddr) maxaddr = driver->maxaddr; if(NULL == (file = (driver->open)(name, flags, fapl_id, maxaddr))) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "open failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "open failed") /* * Fill in public fields. We must increment the reference count on the @@ -1027,20 +1027,20 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_open() */ - + /*------------------------------------------------------------------------- - * Function: H5FDclose + * Function: H5FDclose * * Purpose: Closes the file by calling the driver `close' callback, which - * should free all driver-private data and free the file struct. - * Note that the public part of the file struct (the H5FD_t part) - * will be all zero during the driver close callback like during - * the `open' callback. + * should free all driver-private data and free the file struct. + * Note that the public part of the file struct (the H5FD_t part) + * will be all zero during the driver close callback like during + * the `open' callback. * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Tuesday, July 27, 1999 * *------------------------------------------------------------------------- @@ -1054,25 +1054,25 @@ H5FDclose(H5FD_t *file) H5TRACE1("e", "*x", file); if(!file || !file->cls) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer") if(H5FD_close(file) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close file") + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close file") done: FUNC_LEAVE_API(ret_value) } /* end H5FDclose() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_close + * Function: H5FD_close * - * Purpose: Private version of H5FDclose() + * Purpose: Private version of H5FDclose() * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Wednesday, August 4, 1999 * *------------------------------------------------------------------------- @@ -1105,25 +1105,25 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_close() */ - + /*------------------------------------------------------------------------- - * Function: H5FDcmp + * Function: H5FDcmp * - * Purpose: Compare the keys of two files using the file driver callback - * if the files belong to the same driver, otherwise sort the - * files by driver class pointer value. + * Purpose: Compare the keys of two files using the file driver callback + * if the files belong to the same driver, otherwise sort the + * files by driver class pointer value. * - * Return: Success: A value like strcmp() + * Return: Success: A value like strcmp() * - * Failure: Must never fail. If both file handles are - * invalid then they compare equal. If one file - * handle is invalid then it compares less than - * the other. If both files belong to the same - * driver and the driver doesn't provide a - * comparison callback then the file pointers - * themselves are compared. + * Failure: Must never fail. If both file handles are + * invalid then they compare equal. If one file + * handle is invalid then it compares less than + * the other. If both files belong to the same + * driver and the driver doesn't provide a + * comparison callback then the file pointers + * themselves are compared. * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Tuesday, July 27, 1999 * * Modifications: @@ -1133,7 +1133,7 @@ done: int H5FDcmp(const H5FD_t *f1, const H5FD_t *f2) { - int ret_value; + int ret_value; FUNC_ENTER_API(-1) /*return value is arbitrary*/ H5TRACE2("Is", "*x*x", f1, f2); @@ -1144,17 +1144,17 @@ done: FUNC_LEAVE_API(ret_value) } - + /*------------------------------------------------------------------------- - * Function: H5FD_cmp + * Function: H5FD_cmp * - * Purpose: Private version of H5FDcmp() + * Purpose: Private version of H5FDcmp() * - * Return: Success: A value like strcmp() + * Return: Success: A value like strcmp() * - * Failure: Must never fail. + * Failure: Must never fail. * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Wednesday, August 4, 1999 * * Modifications: @@ -1164,7 +1164,7 @@ done: int H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2) { - int ret_value; + int ret_value; FUNC_ENTER_NOAPI(-1) /*return value is arbitrary*/ @@ -1181,11 +1181,11 @@ H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2) /* Files are same driver; no cmp callback */ if(!f1->cls->cmp) { - if(f1f2) + if(f1>f2) HGOTO_DONE(1) - HGOTO_DONE(0) + HGOTO_DONE(0) } ret_value = (f1->cls->cmp)(f1, f2); @@ -1194,17 +1194,17 @@ done: FUNC_LEAVE_NOAPI(ret_value) } - + /*------------------------------------------------------------------------- - * Function: H5FDquery + * Function: H5FDquery * - * Purpose: Query a VFL driver for its feature flags. (listed in H5FDpublic.h) + * Purpose: Query a VFL driver for its feature flags. (listed in H5FDpublic.h) * - * Return: Success: non-negative + * Return: Success: non-negative * - * Failure: negative + * Failure: negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Friday, August 25, 2000 * * Modifications: @@ -1214,7 +1214,7 @@ done: int H5FDquery(const H5FD_t *f, unsigned long *flags/*out*/) { - int ret_value; + int ret_value; FUNC_ENTER_API(FAIL) H5TRACE2("Is", "*xx", f, flags); @@ -1228,17 +1228,17 @@ done: FUNC_LEAVE_API(ret_value) } - + /*------------------------------------------------------------------------- - * Function: H5FD_query + * Function: H5FD_query * - * Purpose: Private version of H5FDquery() + * Purpose: Private version of H5FDquery() * - * Return: Success: non-negative + * Return: Success: non-negative * - * Failure: negative + * Failure: negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Friday, August 25, 2000 * *------------------------------------------------------------------------- @@ -1246,7 +1246,7 @@ done: static int H5FD_query(const H5FD_t *f, unsigned long *flags/*out*/) { - int ret_value = 0; /* Return value */ + int ret_value = 0; /* Return value */ FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -1262,7 +1262,7 @@ H5FD_query(const H5FD_t *f, unsigned long *flags/*out*/) FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_query() */ - + /*------------------------------------------------------------------------- * Function: H5FD_driver_query * @@ -1292,48 +1292,48 @@ H5FD_driver_query(const H5FD_class_t *driver, unsigned long *flags/*out*/) /* Check for the driver to query and then query it */ if(driver->query) ret_value = (driver->query)(NULL, flags); - else + else *flags = 0; FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_driver_query() */ - + /*------------------------------------------------------------------------- - * Function: H5FDalloc + * Function: H5FDalloc * - * Purpose: Allocates SIZE bytes of memory from the FILE. The memory will - * be used according to the allocation class TYPE. First we try - * to satisfy the request from one of the free lists, according - * to the free list map provided by the driver. The free list - * array has one entry for each request type and the value of - * that array element can be one of four possibilities: + * Purpose: Allocates SIZE bytes of memory from the FILE. The memory will + * be used according to the allocation class TYPE. First we try + * to satisfy the request from one of the free lists, according + * to the free list map provided by the driver. The free list + * array has one entry for each request type and the value of + * that array element can be one of four possibilities: * - * It can be the constant H5FD_MEM_DEFAULT (or zero) which - * indicates that the identity mapping is used. In other - * words, the request type maps to its own free list. + * It can be the constant H5FD_MEM_DEFAULT (or zero) which + * indicates that the identity mapping is used. In other + * words, the request type maps to its own free list. * - * It can be the request type itself, which has the same - * effect as the H5FD_MEM_DEFAULT value above. + * It can be the request type itself, which has the same + * effect as the H5FD_MEM_DEFAULT value above. * - * It can be the ID for another request type, which - * indicates that the free list for the specified type - * should be used instead. + * It can be the ID for another request type, which + * indicates that the free list for the specified type + * should be used instead. * - * It can be the constant H5FD_MEM_NOLIST which means that - * no free list should be used for this type of request. + * It can be the constant H5FD_MEM_NOLIST which means that + * no free list should be used for this type of request. * - * If the request cannot be satisfied from a free list then - * either the driver's `alloc' callback is invoked (if one was - * supplied) or the end-of-address marker is extended. The - * `alloc' callback is always called with the same arguments as - * the H5FDalloc(). + * If the request cannot be satisfied from a free list then + * either the driver's `alloc' callback is invoked (if one was + * supplied) or the end-of-address marker is extended. The + * `alloc' callback is always called with the same arguments as + * the H5FDalloc(). * - * Return: Success: The format address of the new file memory. + * Return: Success: The format address of the new file memory. * - * Failure: The undefined address HADDR_UNDEF + * Failure: The undefined address HADDR_UNDEF * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Tuesday, July 27, 1999 * *------------------------------------------------------------------------- @@ -1341,18 +1341,18 @@ H5FD_driver_query(const H5FD_class_t *driver, unsigned long *flags/*out*/) haddr_t H5FDalloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size) { - haddr_t ret_value = HADDR_UNDEF; + haddr_t ret_value = HADDR_UNDEF; FUNC_ENTER_API(HADDR_UNDEF) H5TRACE4("a", "*xMtih", file, type, dxpl_id, size); /* Check args */ if(!file || !file->cls) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer") if(type < H5FD_MEM_DEFAULT || type >= H5FD_MEM_NTYPES) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid request type") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid request type") if(size == 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "zero-size request") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "zero-size request") if(H5P_DEFAULT == dxpl_id) dxpl_id = H5P_DATASET_XFER_DEFAULT; else @@ -1361,7 +1361,7 @@ H5FDalloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size) /* Do the real work */ if(HADDR_UNDEF == (ret_value = H5FD_alloc_real(file, dxpl_id, type, size, NULL, NULL))) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "unable to allocate file memory") + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "unable to allocate file memory") /* (Note compensating for base address subtraction in internal routine) */ ret_value += file->base_addr; @@ -1370,22 +1370,22 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5FDalloc() */ - + /*------------------------------------------------------------------------- - * Function: H5FDfree + * Function: H5FDfree * - * Purpose: Frees format addresses starting with ADDR and continuing for - * SIZE bytes in the file FILE. The type of space being freed is - * specified by TYPE, which is mapped to a free list as - * described for the H5FDalloc() function above. If the request - * doesn't map to a free list then either the application `free' - * callback is invoked (if defined) or the memory is leaked. + * Purpose: Frees format addresses starting with ADDR and continuing for + * SIZE bytes in the file FILE. The type of space being freed is + * specified by TYPE, which is mapped to a free list as + * described for the H5FDalloc() function above. If the request + * doesn't map to a free list then either the application `free' + * callback is invoked (if defined) or the memory is leaked. * - * Return: Success: Non-negative + * Return: Success: Non-negative * - * Failure: Negative + * Failure: Negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Wednesday, July 28, 1999 * * Modifications: @@ -1420,17 +1420,17 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5FDfree() */ - + /*------------------------------------------------------------------------- - * Function: H5FDget_eoa + * Function: H5FDget_eoa * - * Purpose: Returns the address of the first byte after the last - * allocated memory in the file. + * Purpose: Returns the address of the first byte after the last + * allocated memory in the file. * - * Return: Success: First byte after allocated memory. - * Failure: HADDR_UNDEF + * Return: Success: First byte after allocated memory. + * Failure: HADDR_UNDEF * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Friday, July 30, 1999 * *------------------------------------------------------------------------- @@ -1438,20 +1438,20 @@ done: haddr_t H5FDget_eoa(H5FD_t *file, H5FD_mem_t type) { - haddr_t ret_value; + haddr_t ret_value; FUNC_ENTER_API(HADDR_UNDEF) H5TRACE2("a", "*xMt", file, type); /* Check args */ if(!file || !file->cls) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer") if(type < H5FD_MEM_DEFAULT || type >= H5FD_MEM_NTYPES) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file type") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file type") /* The real work */ if(HADDR_UNDEF == (ret_value = H5FD_get_eoa(file, type))) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eoa request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eoa request failed") /* (Note compensating for base address subtraction in internal routine) */ ret_value += file->base_addr; @@ -1460,27 +1460,27 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5FDget_eoa() */ - + /*------------------------------------------------------------------------- - * Function: H5FDset_eoa + * Function: H5FDset_eoa * - * Purpose: Set the end-of-address marker for the file. The ADDR is the - * address of the first byte past the last allocated byte of the - * file. This function is called from two places: + * Purpose: Set the end-of-address marker for the file. The ADDR is the + * address of the first byte past the last allocated byte of the + * file. This function is called from two places: * - * It is called after an existing file is opened in order to - * "allocate" enough space to read the superblock and then - * to "allocate" the entire hdf5 file based on the contents - * of the superblock. + * It is called after an existing file is opened in order to + * "allocate" enough space to read the superblock and then + * to "allocate" the entire hdf5 file based on the contents + * of the superblock. * - * It is called during file memory allocation if the - * allocation request cannot be satisfied from the free list - * and the driver didn't supply an allocation callback. + * It is called during file memory allocation if the + * allocation request cannot be satisfied from the free list + * and the driver didn't supply an allocation callback. * - * Return: Success: Non-negative - * Failure: Negative, no side effect + * Return: Success: Non-negative + * Failure: Negative, no side effect * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Friday, July 30, 1999 * *------------------------------------------------------------------------- @@ -1495,42 +1495,42 @@ H5FDset_eoa(H5FD_t *file, H5FD_mem_t type, haddr_t addr) /* Check args */ if(!file || !file->cls) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer") if(type < H5FD_MEM_DEFAULT || type >= H5FD_MEM_NTYPES) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file type") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file type") if(!H5F_addr_defined(addr) || addr > file->maxaddr) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid end-of-address value") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid end-of-address value") /* The real work */ /* (Note compensating for base address addition in internal routine) */ if(H5FD_set_eoa(file, type, addr - file->base_addr) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file set eoa request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file set eoa request failed") done: FUNC_LEAVE_API(ret_value) } /* end H5FDset_eoa() */ - + /*------------------------------------------------------------------------- - * Function: H5FDget_eof + * Function: H5FDget_eof * - * Purpose: Returns the end-of-file address, which is the greater of the - * end-of-format address and the actual EOF marker. This - * function is called after an existing file is opened in order - * for the library to learn the true size of the underlying file - * and to determine whether the hdf5 data has been truncated. + * Purpose: Returns the end-of-file address, which is the greater of the + * end-of-format address and the actual EOF marker. This + * function is called after an existing file is opened in order + * for the library to learn the true size of the underlying file + * and to determine whether the hdf5 data has been truncated. * - * It is also used when a file is first opened to learn whether - * the file is empty or not. + * It is also used when a file is first opened to learn whether + * the file is empty or not. * - * It is permissible for the driver to return the maximum address - * for the file size if the file is not empty. + * It is permissible for the driver to return the maximum address + * for the file size if the file is not empty. * - * Return: Success: The EOF address. + * Return: Success: The EOF address. * - * Failure: HADDR_UNDEF + * Failure: HADDR_UNDEF * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, July 29, 1999 * * Modifications: @@ -1540,18 +1540,18 @@ done: haddr_t H5FDget_eof(H5FD_t *file) { - haddr_t ret_value; + haddr_t ret_value; FUNC_ENTER_API(HADDR_UNDEF) H5TRACE1("a", "*x", file); /* Check arguments */ if(!file || !file->cls) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer") /* The real work */ if(HADDR_UNDEF == (ret_value = H5FD_get_eof(file))) - HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eof request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eof request failed") /* (Note compensating for base address subtraction in internal routine) */ ret_value += file->base_addr; @@ -1560,16 +1560,16 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5FDget_eof() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_get_maxaddr + * Function: H5FD_get_maxaddr * - * Purpose: Private version of H5FDget_eof() + * Purpose: Private version of H5FDget_eof() * - * Return: Success: The maximum address allowed in the file. - * Failure: HADDR_UNDEF + * Return: Success: The maximum address allowed in the file. + * Failure: HADDR_UNDEF * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, January 3, 2008 * *------------------------------------------------------------------------- @@ -1590,16 +1590,16 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_get_maxaddr() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_get_feature_flags + * Function: H5FD_get_feature_flags * - * Purpose: Retrieve the feature flags for the VFD + * Purpose: Retrieve the feature flags for the VFD * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Tuesday, January 8, 2008 * *------------------------------------------------------------------------- @@ -1618,16 +1618,16 @@ H5FD_get_feature_flags(const H5FD_t *file, unsigned long *feature_flags) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5FD_get_feature_flags() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_get_fs_type_map + * Function: H5FD_get_fs_type_map * - * Purpose: Retrieve the free space type mapping for the VFD + * Purpose: Retrieve the free space type mapping for the VFD * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, January 17, 2008 * *------------------------------------------------------------------------- @@ -1657,29 +1657,29 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_get_fs_type_map() */ - + /*------------------------------------------------------------------------- - * Function: H5FDread + * Function: H5FDread * - * Purpose: Reads SIZE bytes from FILE beginning at address ADDR - * according to the data transfer property list DXPL_ID (which may - * be the constant H5P_DEFAULT). The result is written into the - * buffer BUF. + * Purpose: Reads SIZE bytes from FILE beginning at address ADDR + * according to the data transfer property list DXPL_ID (which may + * be the constant H5P_DEFAULT). The result is written into the + * buffer BUF. * - * Return: Success: Non-negative. The read result is written into - * the BUF buffer which should be allocated by - * the caller. + * Return: Success: Non-negative. The read result is written into + * the BUF buffer which should be allocated by + * the caller. * - * Failure: Negative. The contents of BUF is undefined. + * Failure: Negative. The contents of BUF is undefined. * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, July 29, 1999 * *------------------------------------------------------------------------- */ herr_t H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, - void *buf/*out*/) + void *buf/*out*/) { H5P_genplist_t *dxpl; /* DXPL object */ herr_t ret_value = SUCCEED; /* Return value */ @@ -1689,7 +1689,7 @@ H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size /* Check args */ if(!file || !file->cls) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer") /* Get the default dataset transfer property list if the user didn't provide one */ if(H5P_DEFAULT == dxpl_id) @@ -1698,7 +1698,7 @@ H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size if(TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list") if(!buf) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null result buffer") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null result buffer") /* Get the DXPL plist object for DXPL ID */ if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) @@ -1707,33 +1707,33 @@ H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size /* Do the real work */ /* (Note compensating for base address addition in internal routine) */ if(H5FD_read(file, dxpl, type, addr - file->base_addr, size, buf) < 0) - HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file read request failed") + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file read request failed") done: FUNC_LEAVE_API(ret_value) } /* end H5FDread() */ - + /*------------------------------------------------------------------------- - * Function: H5FDwrite + * Function: H5FDwrite * - * Purpose: Writes SIZE bytes to FILE beginning at address ADDR according - * to the data transfer property list DXPL_ID (which may be the - * constant H5P_DEFAULT). The bytes to be written come from the - * buffer BUF. + * Purpose: Writes SIZE bytes to FILE beginning at address ADDR according + * to the data transfer property list DXPL_ID (which may be the + * constant H5P_DEFAULT). The bytes to be written come from the + * buffer BUF. * - * Return: Success: Non-negative + * Return: Success: Non-negative * - * Failure: Negative + * Failure: Negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, July 29, 1999 * *------------------------------------------------------------------------- */ herr_t H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, - const void *buf) + const void *buf) { H5P_genplist_t *dxpl; /* DXPL object */ herr_t ret_value = SUCCEED; /* Return value */ @@ -1743,7 +1743,7 @@ H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t siz /* Check args */ if(!file || !file->cls) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer") /* Get the default dataset transfer property list if the user didn't provide one */ if(H5P_DEFAULT == dxpl_id) dxpl_id = H5P_DATASET_XFER_DEFAULT; @@ -1751,7 +1751,7 @@ H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t siz if(TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER)) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list") if(!buf) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null buffer") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null buffer") /* Get the DXPL plist object for DXPL ID */ if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) @@ -1760,24 +1760,24 @@ H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t siz /* The real work */ /* (Note compensating for base address addition in internal routine) */ if(H5FD_write(file, dxpl, type, addr - file->base_addr, size, buf) < 0) - HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "file write request failed") + HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "file write request failed") done: FUNC_LEAVE_API(ret_value) } /* end H5FDwrite() */ - + /*------------------------------------------------------------------------- - * Function: H5FDflush + * Function: H5FDflush * - * Purpose: Notify driver to flush all cached data. If the driver has no - * flush method then nothing happens. + * Purpose: Notify driver to flush all cached data. If the driver has no + * flush method then nothing happens. * - * Return: Success: Non-negative + * Return: Success: Non-negative * - * Failure: Negative + * Failure: Negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, July 29, 1999 * * Modifications: @@ -1796,7 +1796,7 @@ H5FDflush(H5FD_t *file, hid_t dxpl_id, unsigned closing) /* Check args */ if(!file || !file->cls) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer") if(H5P_DEFAULT == dxpl_id) dxpl_id = H5P_DATASET_XFER_DEFAULT; else @@ -1805,22 +1805,22 @@ H5FDflush(H5FD_t *file, hid_t dxpl_id, unsigned closing) /* Do the real work */ if(H5FD_flush(file, dxpl_id, closing) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTFLUSH, FAIL, "file flush request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTFLUSH, FAIL, "file flush request failed") done: FUNC_LEAVE_API(ret_value) } - + /*------------------------------------------------------------------------- - * Function: H5FD_flush + * Function: H5FD_flush * - * Purpose: Private version of H5FDflush() + * Purpose: Private version of H5FDflush() * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Wednesday, August 4, 1999 * *------------------------------------------------------------------------- @@ -1841,16 +1841,16 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_flush() */ - + /*------------------------------------------------------------------------- - * Function: H5FDtruncate + * Function: H5FDtruncate * - * Purpose: Notify driver to truncate the file back to the allocated size. + * Purpose: Notify driver to truncate the file back to the allocated size. * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, January 31, 2008 * *------------------------------------------------------------------------- @@ -1865,7 +1865,7 @@ H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing) /* Check args */ if(!file || !file->cls) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer") + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer") if(H5P_DEFAULT == dxpl_id) dxpl_id = H5P_DATASET_XFER_DEFAULT; else @@ -1874,22 +1874,22 @@ H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing) /* Do the real work */ if(H5FD_truncate(file, dxpl_id, closing) < 0) - HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "file flush request failed") + HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "file flush request failed") done: FUNC_LEAVE_API(ret_value) } - + /*------------------------------------------------------------------------- - * Function: H5FD_truncate + * Function: H5FD_truncate * - * Purpose: Private version of H5FDtruncate() + * Purpose: Private version of H5FDtruncate() * - * Return: Success: Non-negative - * Failure: Negative + * Return: Success: Non-negative + * Failure: Negative * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, January 31, 2008 * *------------------------------------------------------------------------- @@ -1910,18 +1910,18 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_truncate() */ - + /*------------------------------------------------------------------------- - * Function: H5FD_get_fileno + * Function: H5FD_get_fileno * - * Purpose: Quick and dirty routine to retrieve the file's 'fileno' value + * Purpose: Quick and dirty routine to retrieve the file's 'fileno' value * (Mainly added to stop non-file routines from poking about in the * H5FD_t data structure) * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * - * Programmer: Quincey Koziol - * March 27, 2002 + * Programmer: Quincey Koziol + * March 27, 2002 * *------------------------------------------------------------------------- */ @@ -1939,7 +1939,7 @@ H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5FD_get_fileno() */ - + /*-------------------------------------------------------------------------- * Function: H5FDget_vfd_handle * @@ -1973,7 +1973,7 @@ done: FUNC_LEAVE_API(ret_value) } /* end H5FDget_vfd_handle() */ - + /*-------------------------------------------------------------------------- * Function: H5FD_get_vfd_handle * @@ -1998,7 +1998,7 @@ H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void **file_handle) HDassert(file_handle); if(NULL == file->cls->get_handle) - HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "file driver has no `get_vfd_handle' method") + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "file driver has no `get_vfd_handle' method") if((file->cls->get_handle)(file, fapl, file_handle) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get file handle for file driver") @@ -2006,7 +2006,7 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FD_get_vfd_handle() */ - + /*-------------------------------------------------------------------------- * Function: H5FD_set_base_addr * @@ -2033,14 +2033,14 @@ H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5FD_set_base_addr() */ - + /*-------------------------------------------------------------------------- * Function: H5FD_get_base_addr * * Purpose: Get the base address for the file * - * Return: Success: The absolute base address of the file - * Failure: The undefined address (HADDR_UNDEF) + * Return: Success: The absolute base address of the file + * Failure: The undefined address (HADDR_UNDEF) * * Programmer: Quincey Koziol * Sept. 10, 2009 @@ -2058,3 +2058,37 @@ H5FD_get_base_addr(const H5FD_t *file) FUNC_LEAVE_NOAPI(file->base_addr) } /* end H5FD_get_base_addr() */ +/*------------------------------------------------------------------------- +* Function: H5FDdriver_query +* +* Purpose: Similar to H5FD_query(), but intended for cases when we don't +* have a file available (e.g. before one is opened). Since we +* can't use the file to get the driver, the driver ID is passed +* in as a parameter. +* +* Return: SUCCEED/FAIL +* +*------------------------------------------------------------------------- +*/ +herr_t +H5FDdriver_query(hid_t driver_id, unsigned long *flags/*out*/) +{ + H5FD_class_t *driver = NULL; /* Pointer to VFD class struct */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ix", driver_id, flags); + + /* Check arguments */ + if(NULL == flags) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "flags parameter cannot be NULL") + + /* Check for the driver to query and then query it */ + if (NULL == (driver = (H5FD_class_t *)H5I_object_verify(driver_id, H5I_VFL))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "not a VFL ID") + if (H5FD_driver_query(driver, flags) < 0) + HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "driver flag query failed") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5FDdriver_query() */ diff --git a/src/H5FDcore.h b/src/H5FDcore.h index 3ee61b6..f2b0fb0 100644 --- a/src/H5FDcore.h +++ b/src/H5FDcore.h @@ -15,7 +15,7 @@ * Programmer: Robb Matzke * Monday, August 2, 1999 * - * Purpose: The public header file for the sec2 driver. + * Purpose: The public header file for the core driver. */ #ifndef H5FDcore_H #define H5FDcore_H diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c new file mode 100644 index 0000000..59d954f --- /dev/null +++ b/src/H5FDhdfs.c @@ -0,0 +1,1876 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Read-Only HDFS Virtual File Driver (VFD) * + * Copyright (c) 2018, The HDF Group. * + * * + * All rights reserved. * + * * + * NOTICE: * + * All information contained herein is, and remains, the property of The HDF * + * Group. The intellectual and technical concepts contained herein are * + * proprietary to The HDF Group. Dissemination of this information or * + * reproduction of this material is strictly forbidden unless prior written * + * permission is obtained from The HDF Group. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Jacob Smith + * 2018-04-23 + * + * Purpose: Provide read-only access to files on the Hadoop Distributed + * File System (HDFS). + */ + +/* Interface initialization */ +#define H5_INTERFACE_INIT_FUNC H5FD_hdfs_init_interface + + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDhdfs.h" /* hdfs file driver */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Pprivate.h" /* Property lists */ + +#ifdef H5_HAVE_LIBHDFS +#include "hdfs.h" +#endif + +/* toggle function call prints: 1 turns on */ +#define HDFS_DEBUG 0 + +/* toggle stats collection and reporting */ +#define HDFS_STATS 0 + +/* The driver identification number, initialized at runtime */ +static hid_t H5FD_HDFS_g = 0; + +#if HDFS_STATS + +/* arbitrarily large value, such that any reasonable size read will be "less" + * than this value and set a true minimum + * not 0 because that may be a valid recorded minimum in degenerate cases + */ +#define HDFS_STATS_STARTING_MIN 0xfffffffful + +/* Configuration definitions for stats collection and breakdown + * + * 2^10 = 1024 + * Reads up to 1024 bytes (1 kB) fall in bin 0 + * 2^(10+(1*16)) = 2^26 = 64MB + * Reads of 64MB or greater fall in "overflow" bin[BIN_COUNT] + */ +#define HDFS_STATS_BASE 2 +#define HDFS_STATS_INTERVAL 1 +#define HDFS_STATS_START_POWER 10 +#define HDFS_STATS_BIN_COUNT 16 /* MUST BE GREATER THAN 0 */ + + +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Calculate `BASE ^ (START_POWER + (INTERVAL * bin_i))` + * Stores result at `(unsigned long long *) out_ptr`. + * Used in computing boundaries between stats bins. + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + */ +#define HDFS_STATS_POW(bin_i, out_ptr) { \ + unsigned long long donotshadowresult = 1; \ + unsigned donotshadowindex = 0; \ + for (donotshadowindex = 0; \ + donotshadowindex < (((bin_i) * HDFS_STATS_INTERVAL) + \ + HDFS_STATS_START_POWER); \ + donotshadowindex++) \ + { \ + donotshadowresult *= HDFS_STATS_BASE; \ + } \ + *(out_ptr) = donotshadowresult; \ +} + +/* array to hold pre-computed boundaries for stats bins */ +static unsigned long long hdfs_stats_boundaries[HDFS_STATS_BIN_COUNT]; + + +/*************************************************************************** + * + * Structure: hdfs_statsbin + * + * Purpose: + * + * Structure for storing per-file hdfs VFD usage statistics. + * + * + * + * `count` (unsigned long long) + * + * Number of reads with size in this bin's range. + * + * `bytes` (unsigned long long) + * + * Total number of bytes read through this bin. + * + * `min` (unsigned long long) + * + * Smallest read size in this bin. + * + * `max` (unsigned long long) + * + * Largest read size in this bin. + * + * + * + * Programmer: Jacob Smith + * + * Changes: None + * + ***************************************************************************/ +typedef struct { + unsigned long long count; + unsigned long long bytes; + unsigned long long min; + unsigned long long max; +} hdfs_statsbin; + +#endif /* HDFS_STATS */ + +#ifdef H5_HAVE_LIBHDFS + +/* "unique" identifier for `hdfs_t` structures. + * Randomly generated by unweighted dice rolls. + */ +#define HDFS_HDFST_MAGIC 0x1AD5DE84 + + +/*************************************************************************** + * + * Structure: hdfs_t + * + * Purpose: + * + * Contain/retain information associated with a file hosted on Hadoop + * Distributed File System (HDFS). Instantiated and populated via + * `H5FD_hdfs_handle_open()` and cleaned up via `H5FD_hdfs_handle_close()`. + * + * + * + * `magic` (unisgned long) + * + * Number to indicate that this structure is of the promised + * type and should still be valid; should be HDFS_HDFST_MAGIC throughout + * the lifespan of the structure. Upon deletion of the structure, the + * programmer should set magic to anything but HDFS_HDFST_MAGIC, to + * indicate that the structure is to no longer be trusted. + * + * `filesystem` (hdfsFS) + * + * A libhdfs file system handle. + * + * `fileinfo` (hdfsFileInfo*) + * + * A pointer to a libhdfs file info structure. + * + * `file` (hdfsFile) + * + * A libhdfs file handle. + * + * + * + * Programmer: Jacob Smith + * May 2018 + * + * Changes: None + * + *************************************************************************** + */ +typedef struct { + unsigned long magic; + hdfsFS filesystem; + hdfsFileInfo *fileinfo; + hdfsFile file; +} hdfs_t; + + +/*-------------------------------------------------------------------------- + * Function: H5FD_hdfs_handle_open + * + * Purpose: Create a HDFS file handle, 'opening' the target file. + * + * Return: Success: Pointer to HDFS container/handle of opened file. + * Failure: NULL + * + * Programmer: Gerd Herber + * May 2018 + * + * Changes: None. + *-------------------------------------------------------------------------- + */ +static hdfs_t * +H5FD_hdfs_handle_open( + const char *path, + const char *namenode_name, + const int32_t namenode_port, + const char *user_name, + const char *kerberos_ticket_cache, + const int32_t stream_buffer_size) +{ + struct hdfsBuilder *builder = NULL; + hdfs_t *handle = NULL; + hdfs_t *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "called H5FD_hdfs_handle_open.\n"); +#endif + + if (path == NULL || path[0] == '\0') { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "path cannot be null.\n") + } + if (namenode_name == NULL /* || namenode_name[0] == '\0' */ ) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "namenode name cannot be null.\n") + } + if (namenode_port < 0 || namenode_port > 65535) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "namenode port must be non-negative and <= 65535.\n") + } + if (stream_buffer_size < 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "buffer size must non-negative.\n") + } + + handle = (hdfs_t *)H5MM_malloc(sizeof(hdfs_t)); + if (handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, + "could not malloc space for handle.\n") + } + + handle->magic = (unsigned long)HDFS_HDFST_MAGIC; + handle->filesystem = NULL; /* TODO: not a pointer; NULL may cause bug */ + handle->fileinfo = NULL; + handle->file = NULL; /* TODO: not a pointer; NULL may cause bug */ + + builder = hdfsNewBuilder(); + if (!builder) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "(hdfs) failed to create builder") + } + hdfsBuilderSetNameNode(builder, namenode_name); + hdfsBuilderSetNameNodePort(builder, (tPort)namenode_port); + if (user_name != NULL && user_name[0] != '\0') { + hdfsBuilderSetUserName(builder, user_name); + } + if (kerberos_ticket_cache != NULL && kerberos_ticket_cache[0] != '\0') { + hdfsBuilderSetKerbTicketCachePath(builder, kerberos_ticket_cache); + } + /* Call to `hdfsBuilderConnect` releases builder, regardless of success. */ + handle->filesystem = hdfsBuilderConnect(builder); + if (!handle->filesystem) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "(hdfs) could not connect to default namenode") + } + handle->fileinfo = hdfsGetPathInfo(handle->filesystem, path); + if (!handle->fileinfo) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "hdfsGetPathInfo failed") + } + handle->file = hdfsOpenFile( + handle->filesystem, + path, + O_RDONLY, + stream_buffer_size, + 0, + 0); + if (!handle->file) { + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, + "(hdfs) could not open") + } + + ret_value = handle; + +done: + if (ret_value == NULL && handle != NULL) { + /* error; clean up */ + HDassert(handle->magic == HDFS_HDFST_MAGIC); + handle->magic++; + if (handle->file != NULL) { + if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + "unable to close hdfs file handle") + } + } + if (handle->fileinfo != NULL) { + hdfsFreeFileInfo(handle->fileinfo, 1); + } + if (handle->filesystem != NULL) { + if (FAIL == (hdfsDisconnect(handle->filesystem))) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, + "unable to disconnect from hdfs") + } + } + H5MM_xfree(handle); + } + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_hdfs_handle_open() */ + + +/*-------------------------------------------------------------------------- + * Function: H5FD_hdfs_handle_close + * + * Purpose: 'Close' an HDFS file container/handle, releasing underlying + * resources. + * + * Return: Success: `SUCCEED` (0) + * Failure: `FAIL` (-1) + * + * Programmer: Gerd Herber + * May 2018 + * + * Changes: None. + *-------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_handle_close(hdfs_t *handle) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "called H5FD_hdfs_close.\n"); +#endif + + if (handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle cannot be null.\n") + } + if (handle->magic != HDFS_HDFST_MAGIC) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has invalid magic.\n") + } + + handle->magic++; + if (handle->file != NULL) { + if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "unable to close hdfs file handle") + } + } + if (handle->fileinfo != NULL) { + hdfsFreeFileInfo(handle->fileinfo, 1); + } + if (handle->filesystem != NULL) { + if (FAIL == (hdfsDisconnect(handle->filesystem))) { + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "unable to disconnect hdfs file system") + } + } + + H5MM_xfree(handle); + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_hdfs_close() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*************************************************************************** + * + * Structure: H5FD_hdfs_t + * + * Purpose: + * + * H5FD_hdfs_t is a structure used to store all information needed to + * maintain R/O access to a single HDF5 file in an HDFS file system. + * This structure is created when such a file is "opened" and + * discarded when it is "closed". + * + * + * `pub` (H5FD_t) + * + * Instance of H5FD_t which contains all fields common to all VFDs. + * It must be the first item in this structure, since at higher levels, + * this structure will be treated as an instance of H5FD_t. + * + * `fa` (H5FD_hdfs_fapl_t) + * + * Instance of `H5FD_hdfs_fapl_t` containing the HDFS configuration data + * needed to "open" the HDF5 file. + * + * `eoa` (haddr_t) + * + * End of addressed space in file. After open, it should always + * equal the file size. + * + * `hdfs_handle` (hdfs_t *) + * + * Instance of HDFS Request handle associated with the target resource. + * Responsible for communicating with remote host and presenting file + * contents as indistinguishable from a file on the local filesystem. + * + * *** present only if HDFS_SATS is flagged to enable stats collection *** + * + * `meta` (hdfs_statsbin[]) + * `raw` (hdfs_statsbin[]) + * + * Only present if hdfs stats collection is enabled. + * + * Arrays of `hdfs_statsbin` structures to record raw- and metadata reads. + * + * Records count and size of reads performed by the VFD, and is used to + * print formatted usage statistics to stdout upon VFD shutdown. + * + * Reads of each raw- and metadata type are recorded in an individual bin + * determined by the size of the read. The last bin of each type is + * reserved for "big" reads, with no defined upper bound. + * + * *** end HDFS_STATS *** + * + * + * + * Programmer: Jacob Smith + * + * Changes: None. + * + *************************************************************************** + */ +typedef struct H5FD_hdfs_t { + H5FD_t pub; + H5FD_hdfs_fapl_t fa; + haddr_t eoa; +#ifdef H5_HAVE_LIBHDFS + hdfs_t *hdfs_handle; +#endif +#if HDFS_STATS + hdfs_statsbin meta[HDFS_STATS_BIN_COUNT + 1]; + hdfs_statsbin raw[HDFS_STATS_BIN_COUNT + 1]; +#endif +} H5FD_hdfs_t; + +/* + * These macros check for overflow of various quantities. These macros + * assume that HDoff_t is signed and haddr_t and size_t are unsigned. + * + * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' + * is too large to be represented by the second argument + * of the file seek function. + * Only included if HDFS code should compile. + * + */ +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) + +/* Prototypes */ +static void *H5FD_hdfs_fapl_get(H5FD_t *_file); +static void *H5FD_hdfs_fapl_copy(const void *_old_fa); +static herr_t H5FD_hdfs_fapl_free(void *_fa); +static H5FD_t *H5FD_hdfs_open(const char *name, unsigned flags, hid_t fapl_id, + haddr_t maxaddr); +static herr_t H5FD_hdfs_close(H5FD_t *_file); +static int H5FD_hdfs_cmp(const H5FD_t *_f1, const H5FD_t *_f2); +static herr_t H5FD_hdfs_query(const H5FD_t *_f1, unsigned long *flags); +static haddr_t H5FD_hdfs_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD_hdfs_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD_hdfs_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD_hdfs_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle); +static herr_t H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, + size_t size, void *buf); +static herr_t H5FD_hdfs_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr, + size_t size, const void *buf); +static herr_t H5FD_hdfs_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD_hdfs_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD_hdfs_unlock(H5FD_t *_file); +static herr_t H5FD_hdfs_validate_config(const H5FD_hdfs_fapl_t * fa); + +static const H5FD_class_t H5FD_hdfs_g = { + "hdfs", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + sizeof(H5FD_hdfs_fapl_t), /* fapl_size */ + H5FD_hdfs_fapl_get, /* fapl_get */ + H5FD_hdfs_fapl_copy, /* fapl_copy */ + H5FD_hdfs_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD_hdfs_open, /* open */ + H5FD_hdfs_close, /* close */ + H5FD_hdfs_cmp, /* cmp */ + H5FD_hdfs_query, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD_hdfs_get_eoa, /* get_eoa */ + H5FD_hdfs_set_eoa, /* set_eoa */ + H5FD_hdfs_get_eof, /* get_eof */ + H5FD_hdfs_get_handle, /* get_handle */ + H5FD_hdfs_read, /* read */ + H5FD_hdfs_write, /* write */ + NULL, /* flush */ + H5FD_hdfs_truncate, /* truncate */ + H5FD_hdfs_lock, /* lock */ + H5FD_hdfs_unlock, /* unlock */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +#ifdef H5_HAVE_LIBHDFS +/* Declare a free list to manage the H5FD_hdfs_t struct */ +H5FL_DEFINE_STATIC(H5FD_hdfs_t); +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_init_interface + * + * Purpose: Initializes any interface-specific data or routines. + * + * Return: Success: The driver ID for the hdfs driver. + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_init_interface(void) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + FUNC_LEAVE_NOAPI(H5FD_hdfs_init()) +} /* H5FD_hdfs_init_interface() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_init + * + * Purpose: Initialize this driver by registering the driver with the + * library. + * + * Return: Success: The driver ID for the hdfs driver. + * Failure: Negative + * + * Programmer: Robb Matzke + * Thursday, July 29, 1999 + * + *------------------------------------------------------------------------- + */ +hid_t +H5FD_hdfs_init(void) +{ + hid_t ret_value = H5I_INVALID_HID; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_init() called.\n"); +#endif + + if (H5I_VFL != H5I_get_type(H5FD_HDFS_g)) + H5FD_HDFS_g = H5FD_register( &H5FD_hdfs_g, sizeof(H5FD_class_t), FALSE); + +#if HDFS_STATS + /* pre-compute statsbin boundaries + */ + for (unsigned bin_i = 0; bin_i < HDFS_STATS_BIN_COUNT; bin_i++) { + unsigned long long value = 0; + HDFS_STATS_POW(bin_i, &value) + hdfs_stats_boundaries[bin_i] = value; + } +#endif + + ret_value = H5FD_HDFS_g; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_init() */ + + +/*--------------------------------------------------------------------------- + * Function: H5FD_log_term + * + * Purpose: Shut down the VFD + * + * Returns: + * + * Programmer: Jacob Smith 2018 + * + *--------------------------------------------------------------------------- + */ +void +H5FD_hdfs_term(void) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_term() called.\n"); +#endif + + /* Reset VFL ID */ + H5FD_HDFS_g = 0; + + FUNC_LEAVE_NOAPI_VOID +} /* end H5FD_hdfs_term() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_hdfs + * + * Purpose: Modify the file access property list to use the H5FD_HDFS + * driver defined in this source file. All driver specfic + * properties are passed in as a pointer to a suitably + * initialized instance of H5FD_hdfs_fapl_t + * + * Return: SUCCEED/FAIL + * + * Programmer: Jacob Smith 2018 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa) +{ + H5P_genplist_t *plist = NULL; /* Property list pointer */ + herr_t ret_value = FAIL; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa); + + HDassert(fa != NULL); + +#if HDFS_DEBUG + HDfprintf(stdout, "H5Pset_fapl_hdfs() called.\n"); +#endif + + if(NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + + if (FAIL == H5FD_hdfs_validate_config(fa)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,"invalid hdfs config") + + ret_value = H5P_set_driver(plist, H5FD_HDFS, (void *)fa); + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_fapl_hdfs() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_open() + * + * Purpose: Create and/or opens a file as an HDF5 file. + * + * Any flag except H5F_ACC_RDONLY will cause an error. + * + * Return: Success: A pointer to a new file data structure. + * The public fields will be initialized by the caller, which is + * always H5FD_open(). + * Failure: NULL + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS +static H5FD_t * +H5FD_hdfs_open(const char *path, unsigned flags, hid_t fapl_id, haddr_t maxaddr) +{ + H5FD_hdfs_t *file = NULL; + hdfs_t *handle = NULL; + H5FD_hdfs_fapl_t fa; + H5FD_t *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_open() called.\n"); +#endif /* HDFS_DEBUG */ + + /* Sanity check on file offsets */ + HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t)); + + /* Check arguments */ + if(!path || !*path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name") + if(0 == maxaddr || HADDR_UNDEF == maxaddr) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr") + if(ADDR_OVERFLOW(maxaddr)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr") + + if(flags != H5F_ACC_RDONLY) + HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, "only Read-Only access allowed") + if(fapl_id == H5P_DEFAULT || fapl_id == H5P_FILE_ACCESS_DEFAULT) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "fapl cannot be H5P_DEFAULT") + if(FAIL == H5Pget_fapl_hdfs(fapl_id, &fa)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't get property list") + + handle = H5FD_hdfs_handle_open(path, fa.namenode_name, fa.namenode_port, fa.user_name, fa.kerberos_ticket_cache, fa.stream_buffer_size); + + if(handle == NULL) + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "could not open") + + HDassert(handle->magic == HDFS_HDFST_MAGIC); + + /* Create the new file struct */ + if(NULL == (file = H5FL_CALLOC(H5FD_hdfs_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct") + + file->hdfs_handle = handle; + HDmemcpy(&(file->fa), &fa, sizeof(H5FD_hdfs_fapl_t)); + +#if HDFS_STATS + if (FAIL == hdfs_reset_stats(file)) + HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics") +#endif /* HDFS_STATS */ + + ret_value = (H5FD_t*)file; + +done: + if(NULL == ret_value) { + if (handle != NULL) { + if (FAIL == H5FD_hdfs_handle_close(handle)) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close HDFS file handle") + } + if (file != NULL) { + file = H5FL_FREE(H5FD_hdfs_t, file); + } + } /* end if null return value (error) */ + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_hdfs_open() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static H5FD_t * +H5FD_hdfs_open(const char H5_ATTR_UNUSED *path, unsigned H5_ATTR_UNUSED flags, hid_t H5_ATTR_UNUSED fapl_id, haddr_t H5_ATTR_UNUSED maxaddr) +{ + H5FD_t *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "Illegal open of unsupported virtual file (hdfs)"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_open() */ +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_close + * + * Purpose: Closes an HDF5 file. + * + * Return: Success: SUCCEED + * Failure: FAIL, file not closed. + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static herr_t +H5FD_hdfs_close(H5FD_t *_file) +{ + H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_close() called.\n"); +#endif + + /* Sanity checks */ + HDassert(file != NULL); + HDassert(file->hdfs_handle != NULL); + HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC); + + /* Close the underlying request handle */ + if (file->hdfs_handle != NULL) + if (FAIL == H5FD_hdfs_handle_close(file->hdfs_handle)) { + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close HDFS file handle") + } + +#if HDFS_STATS + /* TODO: mechanism to re-target stats printout */ + if (FAIL == hdfs_fprint_stats(stdout, file)) + HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics") +#endif /* HDFS_STATS */ + + /* Release the file info */ + file = H5FL_FREE(H5FD_hdfs_t, file); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_close() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_close(H5FD_t H5_ATTR_UNUSED *_file) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "Illegal close of unsupported Virtual File (hdfs)") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_close() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_cmp() + * + * Purpose: Compares two files using this driver by their HDFS-provided file info, + * field-by-field. + * + * Return: Equivalent: 0 + * Not Equivalent: -1 + * + * Programmer: Gerd Herber + * May 2018 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static int +H5FD_hdfs_cmp(const H5FD_t *_f1, const H5FD_t *_f2) +{ + const H5FD_hdfs_t *f1 = (const H5FD_hdfs_t *)_f1; + const H5FD_hdfs_t *f2 = (const H5FD_hdfs_t *)_f2; + hdfsFileInfo *finfo1 = NULL; + hdfsFileInfo *finfo2 = NULL; + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_cmp() called.\n"); +#endif /* HDFS_DEBUG */ + + HDassert(f1->hdfs_handle != NULL); + HDassert(f2->hdfs_handle != NULL); + HDassert(f1->hdfs_handle->magic == HDFS_HDFST_MAGIC); + HDassert(f2->hdfs_handle->magic == HDFS_HDFST_MAGIC); + + finfo1 = f1->hdfs_handle->fileinfo; + finfo2 = f2->hdfs_handle->fileinfo; + HDassert(finfo1 != NULL); + HDassert(finfo2 != NULL); + + if (finfo1->mKind != finfo2->mKind) { HGOTO_DONE(-1); } + if (finfo1->mName != finfo2->mName) { HGOTO_DONE(-1); } + if (finfo1->mLastMod != finfo2->mLastMod) { HGOTO_DONE(-1); } + if (finfo1->mSize != finfo2->mSize) { HGOTO_DONE(-1); } + if (finfo1->mReplication != finfo2->mReplication) { HGOTO_DONE(-1); } + if (finfo1->mBlockSize != finfo2->mBlockSize) { HGOTO_DONE(-1); } + if (strcmp(finfo1->mOwner, finfo2->mOwner)) { HGOTO_DONE(-1); } + if (strcmp(finfo1->mGroup, finfo2->mGroup)) { HGOTO_DONE(-1); } + if (finfo1->mPermissions != finfo2->mPermissions) { HGOTO_DONE(-1); } + if (finfo1->mLastAccess != finfo2->mLastAccess) { HGOTO_DONE(-1); } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_cmp() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static int +H5FD_hdfs_cmp( + const H5FD_t H5_ATTR_UNUSED *_f1, + const H5FD_t H5_ATTR_UNUSED *_f2) +{ + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_cmp() */ + +#endif /* H5_HAVE_LIBHDFS */ + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_query + * + * Purpose: Set the flags that this VFL driver is capable of supporting. + * (listed in H5FDpublic.h) + * + * Note that since the HDFS VFD is read only, most flags + * are irrelevant. + * + * The term "set" is highly misleading... + * stores/copies the supported flags in the out-pointer `flags`. + * + * Return: SUCCEED (Can't fail) + * + * Programmer: John Mainzer + * 9/11/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags) /* out variable */ +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_query() called.\n"); +#endif + + if(flags) { + *flags = 0; + *flags |= H5FD_FEAT_DATA_SIEVE; + } + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5FD_hdfs_query() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_get_eoa() + * + * Purpose: Gets the end-of-address marker for the file. The EOA marker + * is the first address past the last byte allocated in the + * format address space. + * + * Return: The end-of-address marker. + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static haddr_t +H5FD_hdfs_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_hdfs_t *file = (const H5FD_hdfs_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(file->eoa) +} /* end H5FD_hdfs_get_eoa() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static haddr_t +H5FD_hdfs_get_eoa( const H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(0) +} /* end H5FD_hdfs_get_eoa() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_set_eoa() + * + * Purpose: Set the end-of-address marker for the file. + * + * Return: SUCCEED (Can't fail) + * + * Programmer: Jacob Smith + * 2017-11-03 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static herr_t +H5FD_hdfs_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr) +{ + H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n"); +#endif + + file->eoa = addr; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5FD_hdfs_set_eoa() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_set_eoa(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t H5_ATTR_UNUSED addr) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(FAIL) +} /* H5FD_hdfs_set_eoa() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_get_eof() + * + * Purpose: Returns the end-of-file marker. + * + * Return: EOF: the first address past the end of the "file", either the + * filesystem file or the HDF5 file. + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static haddr_t +H5FD_hdfs_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_hdfs_t *file = (const H5FD_hdfs_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n"); +#endif + + HDassert(file->hdfs_handle != NULL); + HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC); + + FUNC_LEAVE_NOAPI((size_t) file->hdfs_handle->fileinfo->mSize) +} /* end H5FD_hdfs_get_eof() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static haddr_t +H5FD_hdfs_get_eof(const H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n"); +#endif + + FUNC_LEAVE_NOAPI((size_t)0) +} /* end H5FD_hdfs_get_eof() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_get_handle + * + * Purpose: Returns the HDFS handle (hdfs_t) of hdfs file driver. + * + * Returns: SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static herr_t +H5FD_hdfs_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void **file_handle) +{ + H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n"); +#endif /* HDFS_DEBUG */ + + if(!file_handle) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid") + + *file_handle = file->hdfs_handle; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_get_handle() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_get_handle(H5FD_t H5_ATTR_UNUSED *_file, hid_t H5_ATTR_UNUSED fapl, void H5_ATTR_UNUSED **file_handle) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n"); +#endif /* HDFS_DEBUG */ + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "Illegal get-handle of unsupported virtual file (hdfs)"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_get_handle() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_read + * + * Purpose: Reads SIZE bytes of data from FILE beginning at address ADDR + * into buffer BUF according to data transfer properties in DXPL_ID. + * + * Return: Success: SUCCEED + * - Result is stored in caller-supplied buffer BUF. + * Failure: FAIL + * - Unable to complete read. + * - Contents of buffer `buf` are undefined. + * + * Programmer: Jacob Smith + * 2017-11-?? + * + *------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_LIBHDFS + +static herr_t +H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t addr, size_t size, void *buf /*out*/) +{ + H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file; + herr_t ret_value = SUCCEED; + size_t filesize = 0; +#if HDFS_STATS + /* working variables for storing stats */ + hdfs_statsbin *bin = NULL; + unsigned bin_i = 0; +#endif /* HDFS_STATS */ + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_read() called.\n"); +#endif /* HDFS_DEBUG */ + + HDassert(file != NULL); + HDassert(file->hdfs_handle != NULL); + HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC); + HDassert(buf != NULL); + + filesize = (size_t) file->hdfs_handle->fileinfo->mSize; + + if ((addr > filesize) || ((addr + size) > filesize)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address") + + if (FAIL == hdfsPread(file->hdfs_handle->filesystem, file->hdfs_handle->file, (tOffset)addr, buf, (tSize)size)) + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read") + +#if HDFS_STATS + + /* Find which "bin" this read fits in. Can be "overflow" bin. + */ + for (bin_i = 0; bin_i < HDFS_STATS_BIN_COUNT; bin_i++) { + if ((unsigned long long)size < hdfs_stats_boundaries[bin_i]) { + break; + } + } + bin = (type == H5FD_MEM_DRAW) + ? &file->raw[bin_i] + : &file->meta[bin_i]; + + /* Store collected stats in appropriate bin */ + if (bin->count == 0) { + bin->min = size; + bin->max = size; + } + else { + if (size < bin->min) { bin->min = size; } + if (size > bin->max) { bin->max = size; } + } + bin->count++; + bin->bytes += (unsigned long long)size; + +#endif /* HDFS_STATS */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_read() */ + +#else /* H5_HAVE_LIBHDFS not defined */ + +static herr_t +H5FD_hdfs_read(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED size, void H5_ATTR_UNUSED *buf) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_read() called.\n"); +#endif /* HDFS_DEBUG */ + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL,"Illegal get-handle of unsupported virtual file (hdfs)"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_read() */ + +#endif /* H5_HAVE_LIBHDFS */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_write + * + * Purpose: Write bytes to file. + * UNSUPPORTED IN READ-ONLY HDFS VFD. + * + * Return: FAIL (Not possible with Read-Only S3 file.) + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_write(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED size, const void H5_ATTR_UNUSED *buf) +{ + herr_t ret_value = FAIL; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_write() called.\n"); +#endif + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "cannot write to read-only file.") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_truncate + * + * Purpose: Makes sure that the true file size is the same (or larger) + * than the end-of-address. + * + * NOT POSSIBLE ON READ-ONLY S3 FILES. + * + * Return: FAIL (Not possible on Read-Only S3 files.) + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_truncate(H5FD_t H5_ATTR_UNUSED *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDfprintf(stdout, "H5FD_hdfs_truncate() called.\n"); +#endif + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "cannot truncate read-only file.") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_hdfs_truncate() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_lock + * + * Purpose: Place an advisory lock on a file. + * No effect on Read-Only S3 file. + * + * Suggestion: remove lock/unlock from class + * would result in error at H5FD_[un]lock() (H5FD.c) + * + * Return: SUCCEED (No-op always succeeds) + * + * Programmer: Jacob Smith + * 2017-11-03 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_lock( + H5FD_t H5_ATTR_UNUSED *_file, + hbool_t H5_ATTR_UNUSED rw) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_hdfs_lock() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_unlock + * + * Purpose: Remove the existing lock on the file. + * No effect on Read-Only S3 file. + * + * Return: SUCCEED (No-op always succeeds) + * + * Programmer: Jacob Smith + * 2017-11-03 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_unlock(H5FD_t H5_ATTR_UNUSED *_file) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_hdfs_unlock() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_validate_config() + * + * Purpose: Test to see if the supplied instance of H5FD_hdfs_fapl_t + * contains internally consistant data. Return SUCCEED if so, + * and FAIL otherwise. + * + * Note the difference between internally consistant and + * correct. As we will have to try to access the target + * object to determine whether the supplied data is correct, + * we will settle for internal consistancy at this point + * + * Return: SUCCEED if instance of H5FD_hdfs_fapl_t contains internally + * consistant data, FAIL otherwise. + * + * Programmer: Jacob Smith + * 9/10/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_validate_config(const H5FD_hdfs_fapl_t * fa) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(fa != NULL); + + if(fa->version != H5FD__CURR_HDFS_FAPL_T_VERSION) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Unknown H5FD_hdfs_fapl_t version"); + + if(fa->namenode_port > 65535) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Invalid namenode port number"); + if(fa->namenode_port < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Invalid namenode port number"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_validate_config() */ + +/*------------------------------------------------------------------------- + * Function: H5Pget_fapl_hdfs + * + * Purpose: Returns information about the hdfs file access property + * list though the function arguments. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: John Mainzer + * 9/10/17 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out) +{ + const H5FD_hdfs_fapl_t *fa = NULL; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa_out); + +#if HDFS_DEBUG + HDfprintf(stdout, "H5Pget_fapl_hdfs() called.\n"); +#endif + + if (fa_out == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL") + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if (plist == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") + if (H5FD_HDFS != H5P_get_driver(plist)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + + fa = (const H5FD_hdfs_fapl_t *)H5P_get_driver_info(plist); + if (fa == NULL) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") + + /* Copy the hdfs fapl data out */ + HDmemcpy(fa_out, fa, sizeof(H5FD_hdfs_fapl_t)); + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pget_fapl_hdfs() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_fapl_get + * + * Purpose: Gets a file access property list which could be used to + * create an identical file. + * + * Return: Success: Ptr to new file access property list value. + * Failure: NULL + * + * Programmer: John Mainzer + * 9/8/17 + * + *------------------------------------------------------------------------- + */ +static void * +H5FD_hdfs_fapl_get(H5FD_t *_file) +{ + H5FD_hdfs_t *file = (H5FD_hdfs_t*)_file; + H5FD_hdfs_fapl_t *fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + fa = (H5FD_hdfs_fapl_t *)H5MM_calloc(sizeof(H5FD_hdfs_fapl_t)); + if(fa == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + + /* Copy the fields of the structure */ + HDmemcpy(fa, &(file->fa), sizeof(H5FD_hdfs_fapl_t)); + + ret_value = fa; + +done: + if (ret_value == NULL && fa != NULL) { + H5MM_xfree(fa); /* clean up on error */ + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_fapl_get() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_fapl_copy + * + * Purpose: Copies the hdfs-specific file access properties. + * + * Return: Success: Ptr to a new property list + * Failure: NULL + * + * Programmer: John Mainzer + * 9/8/17 + * + *------------------------------------------------------------------------- + */ +static void * +H5FD_hdfs_fapl_copy(const void *_old_fa) +{ + const H5FD_hdfs_fapl_t *old_fa = (const H5FD_hdfs_fapl_t*)_old_fa; + H5FD_hdfs_fapl_t *new_fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + new_fa = (H5FD_hdfs_fapl_t *)H5MM_malloc(sizeof(H5FD_hdfs_fapl_t)); + if(new_fa == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + + HDmemcpy(new_fa, old_fa, sizeof(H5FD_hdfs_fapl_t)); + ret_value = new_fa; + +done: + if (ret_value == NULL && new_fa != NULL) + H5MM_xfree(new_fa); /* clean up on error */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_hdfs_fapl_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_hdfs_fapl_free + * + * Purpose: Frees the hdfs-specific file access properties. + * + * Return: SUCCEED (cannot fail) + * + * Programmer: John Mainzer + * 9/8/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_hdfs_fapl_free(void *_fa) +{ + H5FD_hdfs_fapl_t *fa = (H5FD_hdfs_fapl_t*)_fa; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(fa != NULL); /* sanity check */ + + H5MM_xfree(fa); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5FD_hdfs_fapl_free() */ + + +#if HDFS_STATS + +/*---------------------------------------------------------------------------- + * Function: hdfs_reset_stats() + * + * Purpose: Reset the stats collection elements in this virtual file structure. + * Clears any set data in stats bins; initializes/zeroes values. + * + * Return: - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - Occurs if the file is invalid somehow + * + * Programmer: Jacob Smith + * 2017-12-08 + * + *---------------------------------------------------------------------------- + */ +static herr_t +hdfs_reset_stats(H5FD_hdfs_t *file) +{ + unsigned i = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if HDFS_DEBUG + HDprintf("hdfs_reset_stats() called\n"); +#endif + + if (file == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file was null") + + for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { + file->raw[i].bytes = 0; + file->raw[i].count = 0; + file->raw[i].min = (unsigned long long)HDFS_STATS_STARTING_MIN; + file->raw[i].max = 0; + + file->meta[i].bytes = 0; + file->meta[i].count = 0; + file->meta[i].min = (unsigned long long)HDFS_STATS_STARTING_MIN; + file->meta[i].max = 0; + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* hdfs_reset_stats */ +#endif /* HDFS_STATS */ + + +#if HDFS_STATS + +/*---------------------------------------------------------------------------- + * Function: hdfs_fprint_stats() + * + * Purpose: Tabulate and pretty-print statistics for this virtual file. + * + * Should be called upon file close. + * + * Shows number of reads and bytes read, broken down by + * "raw" (H5FD_MEM_DRAW) + * or "meta" (any other flag) + * + * Prints filename and listing of total number of reads and bytes read, + * both as a grand total and separate meta- and rawdata reads. + * + * If any reads were done, prints out two tables: + * + * 1. overview of raw- and metadata reads + * - min (smallest size read) + * - average of size read + * - k,M,G suffixes by powers of 1024 (2^10) + * - max (largest size read) + * 2. tabulation of "bins", sepraring reads into exponentially-larger + * ranges of size. + * - columns for number of reads, total bytes, and average size, with + * separate sub-colums for raw- and metadata reads. + * - each row represents one bin, identified by the top of its range + * + * Bin ranges can be modified with pound-defines at the top of this file. + * + * Bins without any reads in their bounds are not printed. + * + * An "overflow" bin is also present, to catch "big" reads. + * + * Output for all bins (and range ceiling and average size report) + * is divied by powers of 1024. By corollary, four digits before the decimal + * is valid. + * + * - 41080 bytes is represented by 40.177k, not 41.080k + * - 1004.831M represents approx. 1052642000 bytes + * + * Return: - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - occurs if the file passed in is invalid + * - TODO: if stream is invalid? how can we check this? + * + * Programmer: Jacob Smith + * + *---------------------------------------------------------------------------- + */ +static herr_t +hdfs_fprint_stats(FILE *stream, const H5FD_hdfs_t *file) +{ + herr_t ret_value = SUCCEED; + parsed_url_t *purl = NULL; + unsigned i = 0; + unsigned long count_meta = 0; + unsigned long count_raw = 0; + double average_meta = 0.0; + double average_raw = 0.0; + unsigned long long min_meta = (unsigned long long)HDFS_STATS_STARTING_MIN; + unsigned long long min_raw = (unsigned long long)HDFS_STATS_STARTING_MIN; + unsigned long long max_meta = 0; + unsigned long long max_raw = 0; + unsigned long long bytes_raw = 0; + unsigned long long bytes_meta = 0; + double re_dub = 0.0; /* re-usable double variable */ + unsigned suffix_i = 0; + const char suffixes[] = { ' ', 'K', 'M', 'G', 'T', 'P' }; + + FUNC_ENTER_NOAPI_NOINIT + + if (stream == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file stream cannot be null" ) + if (file == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file cannot be null") + if (file->hdfs_handle == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hdfs handle cannot be null") + if (file->hdfs_handle->magic != HDFS_HDFST_MAGIC) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hdfs handle has invalid magic") + + /******************* + * AGGREGATE STATS * + *******************/ + + for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { + const hdfs_statsbin *r = &file->raw[i]; + const hdfs_statsbin *m = &file->meta[i]; + + if (m->min < min_meta) { min_meta = m->min; } + if (r->min < min_raw) { min_raw = r->min; } + if (m->max > max_meta) { max_meta = m->max; } + if (r->max > max_raw) { max_raw = r->max; } + + count_raw += r->count; + count_meta += m->count; + bytes_raw += r->bytes; + bytes_meta += m->bytes; + } + if (count_raw > 0) { + average_raw = (double)bytes_raw / (double)count_raw; + } + if (count_meta > 0) { + average_meta = (double)bytes_meta / (double)count_meta; + } + + /****************** + * PRINT OVERVIEW * + ******************/ + + HDfprintf(stream, "TOTAL READS: %llu (%llu meta, %llu raw)\n", + count_raw + count_meta, count_meta, count_raw); + HDfprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n", + bytes_raw + bytes_meta, bytes_meta, bytes_raw); + + if (count_raw + count_meta == 0) { + goto done; + } + + /************************* + * PRINT AGGREGATE STATS * + *************************/ + + HDfprintf(stream, "SIZES meta raw\n"); + HDfprintf(stream, " min "); + if (count_meta == 0) { + HDfprintf(stream, " 0.000 "); + } + else { + re_dub = (double)min_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + } + + if (count_raw == 0) { + HDfprintf(stream, " 0.000 \n"); + } + else { + re_dub = (double)min_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + } + + HDfprintf(stream, " avg "); + re_dub = (double)average_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + + re_dub = (double)average_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + + HDfprintf(stream, " max "); + re_dub = (double)max_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + + re_dub = (double)max_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + + /****************************** + * PRINT INDIVIDUAL BIN STATS * + ******************************/ + + HDfprintf(stream, + "BINS # of reads total bytes average size\n"); + HDfprintf(stream, + " up-to meta raw meta raw meta raw\n"); + + for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) { + const hdfs_statsbin *m; + const hdfs_statsbin *r; + unsigned long long range_end = 0; + char bm_suffix = ' '; /* bytes-meta */ + double bm_val = 0.0; + char br_suffix = ' '; /* bytes-raw */ + double br_val = 0.0; + char am_suffix = ' '; /* average-meta */ + double am_val = 0.0; + char ar_suffix = ' '; /* average-raw */ + double ar_val = 0.0; + + m = &file->meta[i]; + r = &file->raw[i]; + if (r->count == 0 && m->count == 0) { + continue; + } + + range_end = hdfs_stats_boundaries[i]; + + if (i == HDFS_STATS_BIN_COUNT) { + range_end = hdfs_stats_boundaries[i-1]; + HDfprintf(stream, ">"); + } else { + HDfprintf(stream, " "); + } + + bm_val = (double)m->bytes; + for (suffix_i = 0; bm_val >= 1024.0; suffix_i++) { + bm_val /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + bm_suffix = suffixes[suffix_i]; + + br_val = (double)r->bytes; + for (suffix_i = 0; br_val >= 1024.0; suffix_i++) { + br_val /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + br_suffix = suffixes[suffix_i]; + + if (m->count > 0) { + am_val = (double)(m->bytes) / (double)(m->count); + } + for (suffix_i = 0; am_val >= 1024.0; suffix_i++) { + am_val /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + am_suffix = suffixes[suffix_i]; + + if (r->count > 0) { + ar_val = (double)(r->bytes) / (double)(r->count); + } + for (suffix_i = 0; ar_val >= 1024.0; suffix_i++) { + ar_val /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + ar_suffix = suffixes[suffix_i]; + + re_dub = (double)range_end; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) { + re_dub /= 1024.0; + } + HDassert(suffix_i < sizeof(suffixes)); + + HDfprintf( + stream, + " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", + re_dub, suffixes[suffix_i], /* bin ceiling */ + m->count, /* metadata reads */ + r->count, /* rawdata reads */ + bm_val, bm_suffix, /* metadata bytes */ + br_val, br_suffix, /* rawdata bytes */ + am_val, am_suffix, /* metadata average */ + ar_val, ar_suffix); /* rawdata average */ + fflush(stream); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* hdfs_fprint_stats */ +#endif /* HDFS_STATS */ + diff --git a/src/H5FDhdfs.h b/src/H5FDhdfs.h new file mode 100644 index 0000000..328d36a --- /dev/null +++ b/src/H5FDhdfs.h @@ -0,0 +1,123 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Read-Only HDFS Virtual File Driver (VFD) * + * Copyright (c) 2018, The HDF Group. * + * * + * All rights reserved. * + * * + * NOTICE: * + * All information contained herein is, and remains, the property of The HDF * + * Group. The intellectual and technical concepts contained herein are * + * proprietary to The HDF Group. Dissemination of this information or * + * reproduction of this material is strictly forbidden unless prior written * + * permission is obtained from The HDF Group. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Jacob Smith + * 2018-04-23 + * + * Purpose: The public header file for the hdfs driver. + */ + +#ifndef H5FDhdfs_H +#define H5FDhdfs_H + +#define H5FD_HDFS (H5FD_hdfs_init()) + +#ifdef __cplusplus +extern "C" { +#endif + +/**************************************************************************** + * + * Structure: H5FD_hdfs_fapl_t + * + * Purpose: + * + * H5FD_hdfs_fapl_t is a public structure that is used to pass + * configuration information to the appropriate HDFS VFD via the FAPL. + * A pointer to an instance of this structure is a parameter to + * H5Pset_fapl_hdfs() and H5Pget_fapl_hdfs(). + * + * + * + * `version` (int32_t) + * + * Version number of the `H5FD_hdfs_fapl_t` structure. Any instance passed + * to the above calls must have a recognized version number, or an error + * will be flagged. + * + * This field should be set to `H5FD__CURR_HDFS_FAPL_T_VERSION`. + * + * `namenode_name` (const char[]) + * + * Name of "Name Node" to access as the HDFS server. + * + * Must not be longer than `H5FD__HDFS_NODE_NAME_SPACE`. + * + * TBD: Can be NULL. + * + * `namenode_port` (int32_t) TBD + * + * Port number to use to connect with Name Node. + * + * TBD: If 0, uses a default port. + * + * `kerberos_ticket_cache` (const char[]) + * + * Path to the location of the Kerberos authentication cache. + * + * Must not be longer than `H5FD__HDFS_KERB_CACHE_PATH_SPACE`. + * + * TBD: Can be NULL. + * + * `user_name` (const char[]) + * + * Username to use when accessing file. + * + * Must not be longer than `H5FD__HDFS_USER_NAME_SPACE`. + * + * TBD: Can be NULL. + * + * `stream_buffer_size` (int32_t) + * + * Size (in bytes) of the file read stream buffer. + * + * TBD: If -1, relies on a default value. + * + * + * + * Programmer: Jacob Smith + * 2018-04-23 + * + * Changes: None + * + ****************************************************************************/ + +#define H5FD__CURR_HDFS_FAPL_T_VERSION 1 + +#define H5FD__HDFS_NODE_NAME_SPACE 128 +#define H5FD__HDFS_USER_NAME_SPACE 128 +#define H5FD__HDFS_KERB_CACHE_PATH_SPACE 128 + +typedef struct H5FD_hdfs_fapl_t { + int32_t version; + char namenode_name[H5FD__HDFS_NODE_NAME_SPACE + 1]; + int32_t namenode_port; + char user_name[H5FD__HDFS_USER_NAME_SPACE + 1]; + char kerberos_ticket_cache[H5FD__HDFS_KERB_CACHE_PATH_SPACE + 1]; + int32_t stream_buffer_size; +} H5FD_hdfs_fapl_t; + +H5_DLL hid_t H5FD_hdfs_init(void); +H5_DLL void H5FD_hdfs_term(void); +H5_DLL herr_t H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out); +H5_DLL herr_t H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa); + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef H5FDhdfs_H */ + + diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h index 9568f46..8c7f414 100644 --- a/src/H5FDprivate.h +++ b/src/H5FDprivate.h @@ -22,13 +22,13 @@ #include "H5FDpublic.h" /* Private headers needed by this file */ -#include "H5Pprivate.h" /* Property lists */ +#include "H5Pprivate.h" /* Property lists */ /* * The MPI drivers are needed because there are * places where we check for things that aren't handled by these drivers. */ -#include "H5FDmpi.h" /* MPI-based file drivers */ +#include "H5FDmpi.h" /* MPI-based file drivers */ /**************************/ @@ -88,6 +88,12 @@ typedef struct { } \ } +/* Define structure to hold driver ID & info for FAPLs */ +typedef struct { + hid_t driver_id; /* Driver's ID */ + const void *driver_info; /* Driver info, for open callbacks */ +} H5FD_driver_prop_t; + /*****************************/ /* Library Private Variables */ @@ -113,7 +119,7 @@ H5_DLL herr_t H5FD_fapl_open(struct H5P_genplist_t *plist, hid_t driver_id, cons H5_DLL herr_t H5FD_fapl_close(hid_t driver_id, void *fapl); H5_DLL hid_t H5FD_register(const void *cls, size_t size, hbool_t app_ref); H5_DLL H5FD_t *H5FD_open(const char *name, unsigned flags, hid_t fapl_id, - haddr_t maxaddr); + haddr_t maxaddr); H5_DLL herr_t H5FD_close(H5FD_t *file); H5_DLL int H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2); H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f, @@ -134,6 +140,8 @@ H5_DLL herr_t H5FD_write(H5FD_t *file, const H5P_genplist_t *dxpl, H5FD_mem_t ty haddr_t addr, size_t size, const void *buf); H5_DLL herr_t H5FD_flush(H5FD_t *file, hid_t dxpl_id, unsigned closing); H5_DLL herr_t H5FD_truncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing); +H5_DLL herr_t H5FD_lock(H5FD_t *file, hbool_t rw); +H5_DLL herr_t H5FD_unlock(H5FD_t *file); H5_DLL herr_t H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum); H5_DLL herr_t H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void** file_handle); H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr); @@ -145,7 +153,7 @@ H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file); H5_DLL haddr_t H5FD_mpi_MPIOff_to_haddr(MPI_Offset mpi_off); H5_DLL herr_t H5FD_mpi_haddr_to_MPIOff(haddr_t addr, MPI_Offset *mpi_off/*out*/); H5_DLL herr_t H5FD_mpi_comm_info_dup(MPI_Comm comm, MPI_Info info, - MPI_Comm *comm_new, MPI_Info *info_new); + MPI_Comm *comm_new, MPI_Info *info_new); H5_DLL herr_t H5FD_mpi_comm_info_free(MPI_Comm *comm, MPI_Info *info); #ifdef NOT_YET H5_DLL herr_t H5FD_mpio_wait_for_left_neighbor(H5FD_t *file); diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index a52c57a..3faaf33 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -19,13 +19,13 @@ #define _H5FDpublic_H #include "H5public.h" -#include "H5Fpublic.h" /*for H5F_close_degree_t */ +#include "H5Fpublic.h" /*for H5F_close_degree_t */ #define H5_HAVE_VFL 1 /*define a convenient app feature test*/ #define H5FD_VFD_DEFAULT 0 /* Default VFL driver value */ /* Types of allocation requests: see H5Fpublic.h */ -typedef enum H5F_mem_t H5FD_mem_t; +typedef enum H5F_mem_t H5FD_mem_t; /* Map "fractal heap" header blocks to 'ohdr' type file memory, since its * a fair amount of work to add a new kind of file memory and they are similar @@ -79,42 +79,42 @@ typedef enum H5F_mem_t H5FD_mem_t; * want to make most efficient reuse of freed memory. The use of the * H5FD_MEM_SUPER free list is arbitrary. */ -#define H5FD_FLMAP_SINGLE { \ - H5FD_MEM_SUPER, /*default*/ \ - H5FD_MEM_SUPER, /*super*/ \ - H5FD_MEM_SUPER, /*btree*/ \ - H5FD_MEM_SUPER, /*draw*/ \ - H5FD_MEM_SUPER, /*gheap*/ \ - H5FD_MEM_SUPER, /*lheap*/ \ - H5FD_MEM_SUPER /*ohdr*/ \ +#define H5FD_FLMAP_SINGLE { \ + H5FD_MEM_SUPER, /*default*/ \ + H5FD_MEM_SUPER, /*super*/ \ + H5FD_MEM_SUPER, /*btree*/ \ + H5FD_MEM_SUPER, /*draw*/ \ + H5FD_MEM_SUPER, /*gheap*/ \ + H5FD_MEM_SUPER, /*lheap*/ \ + H5FD_MEM_SUPER /*ohdr*/ \ } /* * A free-list map which segregates requests into `raw' or `meta' data * pools. */ -#define H5FD_FLMAP_DICHOTOMY { \ - H5FD_MEM_SUPER, /*default*/ \ - H5FD_MEM_SUPER, /*super*/ \ - H5FD_MEM_SUPER, /*btree*/ \ - H5FD_MEM_DRAW, /*draw*/ \ - H5FD_MEM_DRAW, /*gheap*/ \ - H5FD_MEM_SUPER, /*lheap*/ \ - H5FD_MEM_SUPER /*ohdr*/ \ +#define H5FD_FLMAP_DICHOTOMY { \ + H5FD_MEM_SUPER, /*default*/ \ + H5FD_MEM_SUPER, /*super*/ \ + H5FD_MEM_SUPER, /*btree*/ \ + H5FD_MEM_DRAW, /*draw*/ \ + H5FD_MEM_DRAW, /*gheap*/ \ + H5FD_MEM_SUPER, /*lheap*/ \ + H5FD_MEM_SUPER /*ohdr*/ \ } /* * The default free list map which causes each request type to use it's own * free-list. */ -#define H5FD_FLMAP_DEFAULT { \ - H5FD_MEM_DEFAULT, /*default*/ \ - H5FD_MEM_DEFAULT, /*super*/ \ - H5FD_MEM_DEFAULT, /*btree*/ \ - H5FD_MEM_DEFAULT, /*draw*/ \ - H5FD_MEM_DEFAULT, /*gheap*/ \ - H5FD_MEM_DEFAULT, /*lheap*/ \ - H5FD_MEM_DEFAULT /*ohdr*/ \ +#define H5FD_FLMAP_DEFAULT { \ + H5FD_MEM_DEFAULT, /*default*/ \ + H5FD_MEM_DEFAULT, /*super*/ \ + H5FD_MEM_DEFAULT, /*btree*/ \ + H5FD_MEM_DEFAULT, /*draw*/ \ + H5FD_MEM_DEFAULT, /*gheap*/ \ + H5FD_MEM_DEFAULT, /*lheap*/ \ + H5FD_MEM_DEFAULT /*ohdr*/ \ } @@ -187,7 +187,7 @@ typedef enum H5F_mem_t H5FD_mem_t; * instead of the default H5D_ALLOC_TIME_LATE */ #define H5FD_FEAT_ALLOCATE_EARLY 0x00000200 - /* + /* * Defining the H5FD_FEAT_ALLOW_FILE_IMAGE for a VFL driver means that * the driver is able to use a file image in the fapl as the initial * contents of a file. @@ -245,9 +245,9 @@ typedef struct H5FD_class_t { /* A free list is a singly-linked list of address/size pairs. */ typedef struct H5FD_free_t { - haddr_t addr; - hsize_t size; - struct H5FD_free_t *next; + haddr_t addr; + hsize_t size; + struct H5FD_free_t *next; } H5FD_free_t; /* @@ -270,7 +270,7 @@ struct H5FD_t { /* Define enum for the source of file image callbacks */ typedef enum { H5FD_FILE_IMAGE_OP_NO_OP, - H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET, + H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET, H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY, H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET, H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE, @@ -281,13 +281,13 @@ typedef enum { /* Define structure to hold file image callbacks */ typedef struct { - void *(*image_malloc)(size_t size, H5FD_file_image_op_t file_image_op, + void *(*image_malloc)(size_t size, H5FD_file_image_op_t file_image_op, void *udata); void *(*image_memcpy)(void *dest, const void *src, size_t size, H5FD_file_image_op_t file_image_op, void *udata); - void *(*image_realloc)(void *ptr, size_t size, + void *(*image_realloc)(void *ptr, size_t size, H5FD_file_image_op_t file_image_op, void *udata); - herr_t (*image_free)(void *ptr, H5FD_file_image_op_t file_image_op, + herr_t (*image_free)(void *ptr, H5FD_file_image_op_t file_image_op, void *udata); void *(*udata_copy)(void *udata); herr_t (*udata_free)(void *udata); @@ -319,6 +319,11 @@ H5_DLL herr_t H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size, const void *buf); H5_DLL herr_t H5FDflush(H5FD_t *file, hid_t dxpl_id, unsigned closing); H5_DLL herr_t H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing); +/* H5_DLL herr_t H5FDlock(H5FD_t *file, hbool_t rw); +H5_DLL herr_t H5FDunlock(H5FD_t *file); */ +/* Allows querying a VFD ID for features before the file is opened */ +H5_DLL herr_t H5FDdriver_query(hid_t driver_id, unsigned long *flags/*out*/); + #ifdef __cplusplus } diff --git a/src/H5FDros3.c b/src/H5FDros3.c new file mode 100644 index 0000000..5456a1b --- /dev/null +++ b/src/H5FDros3.c @@ -0,0 +1,1787 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Read-Only S3 Virtual File Driver (VFD) + * + * Programmer: Jacob Smith + * 2017-10-13 + * + * Purpose: Provide read-only access to files hosted on Amazon's S3 service. + * Relies on "s3comms" utility layer to implement the AWS REST API. + */ + +/* Interface initialization */ +#define H5_INTERFACE_INIT_FUNC H5FD_ros3_init_interface + + +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* File access */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5FDros3.h" /* ros3 file driver */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Pprivate.h" /* Property lists */ +#include "H5FDs3comms.h" /* S3 Communications */ + +#ifdef H5_HAVE_ROS3_VFD + +/* toggle function call prints: 1 turns on + */ +#define ROS3_DEBUG 0 + +/* toggle stats collection and reporting + */ +#define ROS3_STATS 0 + +/* The driver identification number, initialized at runtime + */ +static hid_t H5FD_ROS3_g = 0; + +#if ROS3_STATS + +/* arbitrarily large value, such that any reasonable size read will be "less" + * than this value and set a true minimum + * not 0 because that may be a valid recorded minimum in degenerate cases + */ +#define ROS3_STATS_STARTING_MIN 0xfffffffful + +/* Configuration definitions for stats collection and breakdown + * + * 2^10 = 1024 + * Reads up to 1024 bytes (1 kB) fall in bin 0 + * 2^(10+(1*16)) = 2^26 = 64MB + * Reads of 64MB or greater fall in "overflow" bin[BIN_COUNT] + */ +#define ROS3_STATS_BASE 2 +#define ROS3_STATS_INTERVAL 1 +#define ROS3_STATS_START_POWER 10 +#define ROS3_STATS_BIN_COUNT 16 /* MUST BE GREATER THAN 0 */ + + +/* + * Calculate `BASE ^ (START_POWER + (INTERVAL * bin_i))` + * Stores result at `(unsigned long long *) out_ptr`. + * Used in computing boundaries between stats bins. + */ +#define ROS3_STATS_POW(bin_i, out_ptr) { \ + unsigned long long donotshadowresult = 1; \ + unsigned donotshadowindex = 0; \ + for (donotshadowindex = 0; \ + donotshadowindex < (((bin_i) * ROS3_STATS_INTERVAL) + \ + ROS3_STATS_START_POWER); \ + donotshadowindex++) \ + { \ + donotshadowresult *= ROS3_STATS_BASE; \ + } \ + *(out_ptr) = donotshadowresult; \ +} + +/* array to hold pre-computed boundaries for stats bins + */ +static unsigned long long ros3_stats_boundaries[ROS3_STATS_BIN_COUNT]; + +/*************************************************************************** + * + * Structure: ros3_statsbin + * + * Purpose: + * + * Structure for storing per-file ros3 VFD usage statistics. + * + * + * + * `count` (unsigned long long) + * + * Number of reads with size in this bin's range. + * + * `bytes` (unsigned long long) + * + * Total number of bytes read through this bin. + * + * `min` (unsigned long long) + * + * Smallest read size in this bin. + * + * `max` (unsigned long long) + * + * Largest read size in this bin. + * + * + * + * Programmer: Jacob Smith + * + ***************************************************************************/ +typedef struct { + unsigned long long count; + unsigned long long bytes; + unsigned long long min; + unsigned long long max; +} ros3_statsbin; + +#endif /* ROS3_STATS */ + +/*************************************************************************** + * + * Structure: H5FD_ros3_t + * + * Purpose: + * + * H5FD_ros3_t is a structure used to store all information needed to + * maintain R/O access to a single HDF5 file that has been stored as a + * S3 object. This structure is created when such a file is "opened" and + * discarded when it is "closed". + * + * Presents an S3 object as a file to the HDF5 library. + * + * + * + * `pub` (H5FD_t) + * + * Instance of H5FD_t which contains all fields common to all VFDs. + * It must be the first item in this structure, since at higher levels, + * this structure will be treated as an instance of H5FD_t. + * + * `fa` (H5FD_ros3_fapl_t) + * + * Instance of `H5FD_ros3_fapl_t` containing the S3 configuration data + * needed to "open" the HDF5 file. + * + * `eoa` (haddr_t) + * + * End of addressed space in file. After open, it should always + * equal the file size. + * + * `s3r_handle` (s3r_t *) + * + * Instance of S3 Request handle associated with the target resource. + * Responsible for communicating with remote host and presenting file + * contents as indistinguishable from a file on the local filesystem. + * + * *** present only if ROS3_SATS is flagged to enable stats collection *** + * + * `meta` (ros3_statsbin[]) + * `raw` (ros3_statsbin[]) + * + * Only present if ros3 stats collection is enabled. + * + * Arrays of `ros3_statsbin` structures to record raw- and metadata reads. + * + * Records count and size of reads performed by the VFD, and is used to + * print formatted usage statistics to stdout upon VFD shutdown. + * + * Reads of each raw- and metadata type are recorded in an individual bin + * determined by the size of the read. The last bin of each type is + * reserved for "big" reads, with no defined upper bound. + * + * *** end ROS3_STATS *** + * + * + * + * Programmer: Jacob Smith + * + ***************************************************************************/ +typedef struct H5FD_ros3_t { + H5FD_t pub; + H5FD_ros3_fapl_t fa; + haddr_t eoa; + s3r_t *s3r_handle; +#if ROS3_STATS + ros3_statsbin meta[ROS3_STATS_BIN_COUNT + 1]; + ros3_statsbin raw[ROS3_STATS_BIN_COUNT + 1]; +#endif +} H5FD_ros3_t; + +/* + * These macros check for overflow of various quantities. These macros + * assume that HDoff_t is signed and haddr_t and size_t are unsigned. + * + * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t' + * is too large to be represented by the second argument + * of the file seek function. + * Only included if it may be used -- ROS3 VFD is enabled. + * + */ +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR)) + +/* Prototypes */ +static H5FD_t *H5FD_ros3_open(const char *name, unsigned flags, hid_t fapl_id, + haddr_t maxaddr); +static herr_t H5FD_ros3_close(H5FD_t *_file); +static int H5FD_ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2); +static herr_t H5FD_ros3_query(const H5FD_t *_f1, unsigned long *flags); +static haddr_t H5FD_ros3_get_eoa(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD_ros3_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr); +static haddr_t H5FD_ros3_get_eof(const H5FD_t *_file, H5FD_mem_t type); +static herr_t H5FD_ros3_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle); +static herr_t H5FD_ros3_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, void *buf); +static herr_t H5FD_ros3_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, + haddr_t addr, size_t size, const void *buf); +static herr_t H5FD_ros3_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing); +static herr_t H5FD_ros3_lock(H5FD_t *_file, hbool_t rw); +static herr_t H5FD_ros3_unlock(H5FD_t *_file); +static herr_t H5FD_ros3_validate_config(const H5FD_ros3_fapl_t * fa); +static void *H5FD_ros3_fapl_get(H5FD_t *_file); +static void *H5FD_ros3_fapl_copy(const void *_old_fa); +static herr_t H5FD_ros3_fapl_free(void *_fa); + +static const H5FD_class_t H5FD_ros3_g = { + "ros3", /* name */ + MAXADDR, /* maxaddr */ + H5F_CLOSE_WEAK, /* fc_degree */ + H5FD_ros3_term, /* terminate */ + NULL, /* sb_size */ + NULL, /* sb_encode */ + NULL, /* sb_decode */ + sizeof(H5FD_ros3_fapl_t), /* fapl_size */ + H5FD_ros3_fapl_get, /* fapl_get */ + H5FD_ros3_fapl_copy, /* fapl_copy */ + H5FD_ros3_fapl_free, /* fapl_free */ + 0, /* dxpl_size */ + NULL, /* dxpl_copy */ + NULL, /* dxpl_free */ + H5FD_ros3_open, /* open */ + H5FD_ros3_close, /* close */ + H5FD_ros3_cmp, /* cmp */ + H5FD_ros3_query, /* query */ + NULL, /* get_type_map */ + NULL, /* alloc */ + NULL, /* free */ + H5FD_ros3_get_eoa, /* get_eoa */ + H5FD_ros3_set_eoa, /* set_eoa */ + H5FD_ros3_get_eof, /* get_eof */ + H5FD_ros3_get_handle, /* get_handle */ + H5FD_ros3_read, /* read */ + H5FD_ros3_write, /* write */ + NULL, /* flush */ + H5FD_ros3_truncate, /* truncate */ + H5FD_ros3_lock, /* lock */ + H5FD_ros3_unlock, /* unlock */ + H5FD_FLMAP_DICHOTOMY /* fl_map */ +}; + +/* Declare a free list to manage the H5FD_ros3_t struct */ +H5FL_DEFINE_STATIC(H5FD_ros3_t); + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_init_interface + * + * Purpose: Initializes any interface-specific data or routines. + * + * Return: Success: The driver ID for the ros3 driver. + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_init_interface(void) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + FUNC_LEAVE_NOAPI(H5FD_ros3_init()) +} /* H5FD_ros3_init_interface() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_init + * + * Purpose: Initialize this driver by registering the driver with the + * library. + * + * Return: Success: The driver ID for the ros3 driver. + * Failure: Negative + * + * Programmer: Jacob Smith 2017 + * + *------------------------------------------------------------------------- + */ +hid_t +H5FD_ros3_init(void) +{ + hid_t ret_value = H5I_INVALID_HID; + + FUNC_ENTER_NOAPI(FAIL) + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_init() called.\n"); +#endif + + if (H5I_VFL != H5I_get_type(H5FD_ROS3_g)) + H5FD_ROS3_g = H5FD_register(&H5FD_ros3_g, sizeof(H5FD_class_t), FALSE); + +#if ROS3_STATS + /* pre-compute statsbin boundaries + */ + for (unsigned bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) { + unsigned long long value = 0; + ROS3_STATS_POW(bin_i, &value) + ros3_stats_boundaries[bin_i] = value; + } +#endif + + /* Set return value */ + ret_value = H5FD_ROS3_g; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_ros3_init() */ + + +/*--------------------------------------------------------------------------- + * Function: H5FD_ros3_term + * + * Purpose: Shut down the VFD + * + * Returns: SUCCEED (Can't fail) + * + * Programmer: Jacob Smith 2017 + * + *--------------------------------------------------------------------------- + */ +void +H5FD_ros3_term(void) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_term() called.\n"); +#endif + + /* Reset VFL ID */ + H5FD_ROS3_g = 0; + + FUNC_LEAVE_NOAPI_VOID +} /* end H5FD_ros3_term() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_fapl_ros3 + * + * Purpose: Modify the file access property list to use the H5FD_ROS3 + * driver defined in this source file. All driver specfic + * properties are passed in as a pointer to a suitably + * initialized instance of H5FD_ros3_fapl_t + * + * Return: SUCCEED/FAIL + * + * Programmer: John Mainzer + * 9/10/17 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa) +{ + H5P_genplist_t *plist = NULL; /* Property list pointer */ + herr_t ret_value = FAIL; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa); + + HDassert(fa != NULL); + +#if ROS3_DEBUG + HDfprintf(stdout, "H5Pset_fapl_ros3() called.\n"); +#endif + + if(NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list") + + if (FAIL == H5FD_ros3_validate_config(fa)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid ros3 config") + + ret_value = H5P_set_driver(plist, H5FD_ROS3, (void *)fa); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_fapl_ros3() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_open() + * + * Purpose: Create and/or opens a file as an HDF5 file. + * Any flag except H5F_ACC_RDONLY will cause an error. + * + * Name (as received from `H5FD_open()`) must conform to web url: + * NAME :: HTTP "://" DOMAIN [PORT] ["/" [URI] [QUERY] ] + * HTTP :: "http" [ "s" ] + * DOMAIN :: e.g., "mybucket.host.org" + * PORT :: ":" (e.g., ":9000" ) + * URI :: (e.g., "path/to/resource.hd5" ) + * QUERY :: "?" (e.g., "arg1=param1&arg2=param2") + * + * Return: Success: A pointer to a new file data structure. + * The public fields will be initialized by the caller, which is + * always H5FD_open(). + * Failure: NULL + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +static H5FD_t * +H5FD_ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr) +{ + H5FD_ros3_t *file = NULL; + struct tm *now = NULL; + char iso8601now[ISO8601_SIZE]; + unsigned char signing_key[SHA256_DIGEST_LENGTH]; + s3r_t *handle = NULL; + H5FD_ros3_fapl_t fa; + H5FD_t *ret_value = NULL; + + + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_open() called.\n"); +#endif + + /* Sanity check on file offsets */ + HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t)); + + /* Check arguments */ + if(!url || !*url) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name") + if(0 == maxaddr || HADDR_UNDEF == maxaddr) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr") + if(ADDR_OVERFLOW(maxaddr)) + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr") + if(flags != H5F_ACC_RDONLY) + HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, "only Read-Only access allowed") + + if (FAIL == H5Pget_fapl_ros3(fapl_id, &fa)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't get property list") + + if (CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "unable to initialize curl global (placeholder flags)") + + /* open file; procedure depends on whether or not the fapl instructs to + * authenticate requests or not. + */ + if (fa.authenticate == TRUE) { + /* compute signing key (part of AWS/S3 REST API) + * can be re-used by user/key for 7 days after creation. + * find way to re-use/share + */ + now = gmnow(); + HDassert( now != NULL ); + if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while writing iso8601 timestamp") + if (FAIL == H5FD_s3comms_signing_key(signing_key, + (const char *)fa.secret_key, + (const char *)fa.aws_region, + (const char *)iso8601now) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while computing signing key") + } + + handle = H5FD_s3comms_s3r_open( + url, + (const char *)fa.aws_region, + (const char *)fa.secret_id, + (const unsigned char *)signing_key); + } + else { + handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL); + } /* if/else should authenticate */ + + if (handle == NULL) { + /* If we want to check CURL's say on the matter in a controlled + * fashion, this is the place to do it, but would need to make a + * few minor changes to s3comms `s3r_t` and `s3r_read()`. + */ + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "could not open"); + } + + /* create new file struct + */ + file = H5FL_CALLOC(H5FD_ros3_t); + if (file == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct") + + file->s3r_handle = handle; + HDmemcpy(&(file->fa), &fa, sizeof(H5FD_ros3_fapl_t)); + +#if ROS3_STATS + if (FAIL == ros3_reset_stats(file)) + HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics") +#endif /* ROS3_STATS */ + + ret_value = (H5FD_t*)file; + +done: + if (ret_value == NULL) { + if (handle != NULL) { + if (FAIL == H5FD_s3comms_s3r_close(handle)) + HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle") + } + if (file != NULL) { + file = H5FL_FREE(H5FD_ros3_t, file); + } + curl_global_cleanup(); /* early cleanup because open failed */ + } /* end if null return value (error) */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_ros3_open() */ + + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_close() + * + * Purpose: + * + * Close an HDF5 file. + * + * Return: + * + * SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_close(H5FD_t H5_ATTR_UNUSED *_file) +{ + H5FD_ros3_t *file = (H5FD_ros3_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_close() called.\n"); +#endif + + /* Sanity checks + */ + HDassert(file != NULL); + HDassert(file->s3r_handle != NULL); + + /* Close the underlying request handle + */ + if (FAIL == H5FD_s3comms_s3r_close(file->s3r_handle)) { + HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, + "unable to close S3 request handle") + } + +#if ROS3_STATS + /* TODO: mechanism to re-target stats printout */ + if (ros3_fprint_stats(stdout, file) == FAIL) { + HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, + "problem while writing file statistics") + } +#endif /* ROS3_STATS */ + + /* Release the file info + */ + file = H5FL_FREE(H5FD_ros3_t, file); + +done: + curl_global_cleanup(); /* cleanup to answer init on open */ + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_close() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_cmp() + * + * Purpose: + * + * Compares two files belonging to this driver using an arbitrary + * (but consistent) ordering: + * + * + url scheme + * + url host + * + url port + * + url path + * + url query + * + fapl aws_region + * + fapl secret_id + * + fapl secret_key + * + * tl;dr -> check URL, check crentials + * + * Return: + * + * - Equivalent: 0 + * - Not Equivalent: -1 + * + * Programmer: Jacob Smith + * 2017-11-06 + * + *------------------------------------------------------------------------- + */ +static int +H5FD_ros3_cmp( + const H5FD_t *_f1, + const H5FD_t *_f2) +{ + const H5FD_ros3_t *f1 = (const H5FD_ros3_t *)_f1; + const H5FD_ros3_t *f2 = (const H5FD_ros3_t *)_f2; + const parsed_url_t *purl1 = NULL; + const parsed_url_t *purl2 = NULL; + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_cmp() called.\n"); +#endif + + HDassert(f1->s3r_handle != NULL); + HDassert(f2->s3r_handle != NULL); + + purl1 = (const parsed_url_t *)f1->s3r_handle->purl; + purl2 = (const parsed_url_t *)f2->s3r_handle->purl; + HDassert(purl1 != NULL); + HDassert(purl2 != NULL); + HDassert(purl1->scheme != NULL); + HDassert(purl2->scheme != NULL); + HDassert(purl1->host != NULL); + HDassert(purl2->host != NULL); + + /* URL: SCHEME */ + if (HDstrcmp(purl1->scheme, purl2->scheme)) { + HGOTO_DONE(-1); + } + + /* URL: HOST */ + if (HDstrcmp(purl1->host, purl2->host)) { + HGOTO_DONE(-1); + } + + /* URL: PORT */ + if (purl1->port && purl2->port) { + if (HDstrcmp(purl1->port, purl2->port)) { + HGOTO_DONE(-1); + } + } + else + if (purl1->port) { + HGOTO_DONE(-1); + } + else + if (purl2->port) { + HGOTO_DONE(-1); + } + + /* URL: PATH */ + if (purl1->path && purl2->path) { + if (HDstrcmp(purl1->path, purl2->path)) { + HGOTO_DONE(-1); + } + } + else + if (purl1->path && !purl2->path) { + HGOTO_DONE(-1); + } + else + if (purl2->path && !purl1->path) { + HGOTO_DONE(-1); + } + + /* URL: QUERY */ + if (purl1->query && purl2->query) { + if (HDstrcmp(purl1->query, purl2->query)) { + HGOTO_DONE(-1); + } + } + else + if (purl1->query && !purl2->query) { + HGOTO_DONE(-1); + } + else + if (purl2->query && !purl1->query) { + HGOTO_DONE(-1); + } + + /* FAPL: AWS_REGION */ + if (f1->fa.aws_region[0] != '\0' && f1->fa.aws_region[0] != '\0') { + if (HDstrcmp(f1->fa.aws_region, f2->fa.aws_region)) { + HGOTO_DONE(-1); + } + } + else + if (f1->fa.aws_region[0] != '\0') { + HGOTO_DONE(-1); + } + else + if (f2->fa.aws_region[0] != '\0') { + HGOTO_DONE(-1); + } + + /* FAPL: SECRET_ID */ + if (f1->fa.secret_id[0] != '\0' && f1->fa.secret_id[0] != '\0') { + if (HDstrcmp(f1->fa.secret_id, f2->fa.secret_id)) { + HGOTO_DONE(-1); + } + } + else + if (f1->fa.secret_id[0] != '\0') { + HGOTO_DONE(-1); + } + else + if (f2->fa.secret_id[0] != '\0') { + HGOTO_DONE(-1); + } + + /* FAPL: SECRET_KEY */ + if (f1->fa.secret_key[0] != '\0' && f1->fa.secret_key[0] != '\0') { + if (HDstrcmp(f1->fa.secret_key, f2->fa.secret_key)) { + HGOTO_DONE(-1); + } + } + else + if (f1->fa.secret_key[0] != '\0') { + HGOTO_DONE(-1); + } + else + if (f2->fa.secret_key[0] != '\0') { + HGOTO_DONE(-1); + } + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_ros3_cmp() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_query + * + * Purpose: Set the flags that this VFL driver is capable of supporting. + * (listed in H5FDpublic.h) + * + * Note that since the ROS3 VFD is read only, most flags + * are irrelevant. + * + * The term "set" is highly misleading... + * stores/copies the supported flags in the out-pointer `flags`. + * + * Return: SUCCEED (Can't fail) + * + * Programmer: John Mainzer + * 9/11/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file, + unsigned long *flags /* out */) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_query() called.\n"); +#endif + + /* Set the VFL feature flags that this driver supports */ + if (flags) { + *flags = 0; + /* OK to perform data sieving for faster raw data reads & writes */ + *flags |= H5FD_FEAT_DATA_SIEVE; + } /* end if */ + + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* H5FD_ros3_query() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_get_eoa() + * + * Purpose: + * + * Gets the end-of-address marker for the file. The EOA marker + * is the first address past the last byte allocated in the + * format address space. + * + * Return: + * + * The end-of-address marker. + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD_ros3_get_eoa(const H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_ros3_t *file = (const H5FD_ros3_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_get_eoa() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(file->eoa) + +} /* end H5FD_ros3_get_eoa() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_set_eoa() + * + * Purpose: + * + * Set the end-of-address marker for the file. + * + * Return: + * + * SUCCEED (can't fail) + * + * Programmer: Jacob Smith + * 2017-11-03 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_set_eoa(H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + haddr_t addr) +{ + H5FD_ros3_t *file = (H5FD_ros3_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_set_eoa() called.\n"); +#endif + + file->eoa = addr; + + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* H5FD_ros3_set_eoa() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_get_eof() + * + * Purpose: + * + * Returns the end-of-file marker. + * + * Return: + * + * EOF: the first address past the end of the "file", either the + * filesystem file or the HDF5 file. + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +static haddr_t +H5FD_ros3_get_eof(const H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type) +{ + const H5FD_ros3_t *file = (const H5FD_ros3_t *)_file; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_get_eof() called.\n"); +#endif + + FUNC_LEAVE_NOAPI(H5FD_s3comms_s3r_get_filesize(file->s3r_handle)) + +} /* end H5FD_ros3_get_eof() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_get_handle() + * + * Purpose: + * + * Returns the S3 Request handle (s3r_t) of ros3 file driver. + * + * Returns: + * + * SUCCEED/FAIL + * + * Programmer: Jacob Smith + * 2017-11-02 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_get_handle(H5FD_t *_file, + hid_t H5_ATTR_UNUSED fapl, + void **file_handle) +{ + H5FD_ros3_t *file = (H5FD_ros3_t *)_file; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_get_handle() called.\n"); +#endif + + if (!file_handle) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid") + } + + *file_handle = file->s3r_handle; + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_get_handle() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_read() + * + * Purpose + * + * Reads SIZE bytes of data from FILE beginning at address ADDR + * into buffer BUF according to data transfer properties in DXPL_ID. + * + * Return: + * + * Success: `SUCCEED` + * - Result is stored in caller-supplied buffer BUF. + * Failure: `FAIL` + * - Unable to complete read. + * - Contents of buffer `buf` are undefined. + * + * Programmer: Jacob Smith + * 2017-11-?? + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_read(H5FD_t *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t addr, /* start offset */ + size_t size, /* length of read */ + void *buf) /* out */ +{ + H5FD_ros3_t *file = (H5FD_ros3_t *)_file; + size_t filesize = 0; + herr_t ret_value = SUCCEED; +#if ROS3_STATS + /* working variables for storing stats */ + ros3_statsbin *bin = NULL; + unsigned bin_i = 0; +#endif /* ROS3_STATS */ + + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_read() called.\n"); +#endif + + HDassert(file != NULL); + HDassert(file->s3r_handle != NULL); + HDassert(buf != NULL); + + filesize = H5FD_s3comms_s3r_get_filesize(file->s3r_handle); + + if ((addr > filesize) || ((addr + size) > filesize)) { + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address") + } + + if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL) { + HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read") + } + +#if ROS3_STATS + + /* Find which "bin" this read fits in. Can be "overflow" bin. + */ + for (bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) { + if ((unsigned long long)size < ros3_stats_boundaries[bin_i]) { + break; + } + } + bin = (type == H5FD_MEM_DRAW) + ? &file->raw[bin_i] + : &file->meta[bin_i]; + + /* Store collected stats in appropriate bin + */ + if (bin->count == 0) { + bin->min = size; + bin->max = size; + } + else { + if (size < bin->min) { + bin->min = size; + } + if (size > bin->max) { + bin->max = size; + } + } + bin->count++; + bin->bytes += (unsigned long long)size; + +#endif /* ROS3_STATS */ + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_read() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_write() + * + * Purpose: + * + * Write bytes to file. + * UNSUPPORTED IN READ-ONLY ROS3 VFD. + * + * Return: + * + * FAIL (Not possible with Read-Only S3 file.) + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_write(H5FD_t H5_ATTR_UNUSED *_file, + H5FD_mem_t H5_ATTR_UNUSED type, + hid_t H5_ATTR_UNUSED dxpl_id, + haddr_t H5_ATTR_UNUSED addr, + size_t H5_ATTR_UNUSED size, + const void H5_ATTR_UNUSED *buf) +{ + herr_t ret_value = FAIL; + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_write() called.\n"); +#endif + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, + "cannot write to read-only file.") + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_ros3_write() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_truncate() + * + * Purpose: + * + * Makes sure that the true file size is the same (or larger) + * than the end-of-address. + * + * NOT POSSIBLE ON READ-ONLY S3 FILES. + * + * Return: + * + * FAIL (Not possible on Read-Only S3 files.) + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_truncate(H5FD_t H5_ATTR_UNUSED *_file, + hid_t H5_ATTR_UNUSED dxpl_id, + hbool_t H5_ATTR_UNUSED closing) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDfprintf(stdout, "H5FD_ros3_truncate() called.\n"); +#endif + + HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, + "cannot truncate read-only file.") + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_truncate() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_lock() + * + * Purpose: + * + * Place an advisory lock on a file. + * No effect on Read-Only S3 file. + * + * Suggestion: remove lock/unlock from class + * > would result in error at H5FD_[un]lock() (H5FD.c) + * + * Return: + * + * SUCCEED (No-op always succeeds) + * + * Programmer: Jacob Smith + * 2017-11-03 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_lock(H5FD_t H5_ATTR_UNUSED *_file, + hbool_t H5_ATTR_UNUSED rw) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* end H5FD_ros3_lock() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5FD_ros3_unlock() + * + * Purpose: + * + * Remove the existing lock on the file. + * No effect on Read-Only S3 file. + * + * Return: + * + * SUCCEED (No-op always succeeds) + * + * Programmer: Jacob Smith + * 2017-11-03 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_unlock(H5FD_t H5_ATTR_UNUSED *_file) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* end H5FD_ros3_unlock() */ + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_validate_config() + * + * Purpose: Test to see if the supplied instance of H5FD_ros3_fapl_t + * contains internally consistant data. Return SUCCEED if so, + * and FAIL otherwise. + * + * Note the difference between internally consistant and + * correct. As we will have to try to access the target + * object to determine whether the supplied data is correct, + * we will settle for internal consistancy at this point + * + * Return: SUCCEED if instance of H5FD_ros3_fapl_t contains internally + * consistant data, FAIL otherwise. + * + * Programmer: Jacob Smith + * 9/10/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_validate_config(const H5FD_ros3_fapl_t * fa) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(fa != NULL); + + if ( fa->version != H5FD_CURR_ROS3_FAPL_T_VERSION ) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Unknown H5FD_ros3_fapl_t version"); + } + + /* if set to authenticate, region and id cannot be empty strings + */ + if (fa->authenticate == TRUE) { + if ((fa->aws_region[0] == '\0') || + (fa->secret_id[0] == '\0')) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Inconsistent authentication information"); + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_validate_config() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_fapl_ros3 + * + * Purpose: Returns information about the ros3 file access property + * list though the function arguments. + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: John Mainzer + * 9/10/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_fapl_ros3(hid_t fapl_id, + H5FD_ros3_fapl_t *fa_out) +{ + const H5FD_ros3_fapl_t *fa = NULL; + H5P_genplist_t *plist = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*x", fapl_id, fa_out); + +#if ROS3_DEBUG + HDfprintf(stdout, "H5Pget_fapl_ros3() called.\n"); +#endif + + if (fa_out == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL") + } + + plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS); + if (plist == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list") + } + + if (H5FD_ROS3 != H5P_get_driver(plist)) { + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver") + } + + fa = (const H5FD_ros3_fapl_t *)H5P_get_driver_info(plist); + if (fa == NULL) { + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info") + } + + /* Copy the ros3 fapl data out */ + HDmemcpy(fa_out, fa, sizeof(H5FD_ros3_fapl_t)); + +done: + FUNC_LEAVE_API(ret_value) + +} /* end H5Pget_fapl_ros3() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_fapl_get + * + * Purpose: Gets a file access property list which could be used to + * create an identical file. + * + * Return: Success: Ptr to new file access property list value. + * + * Failure: NULL + * + * Programmer: John Mainzer + * 9/8/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void * +H5FD_ros3_fapl_get(H5FD_t *_file) +{ + H5FD_ros3_t *file = (H5FD_ros3_t*)_file; + H5FD_ros3_fapl_t *fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + fa = (H5FD_ros3_fapl_t *)H5MM_calloc(sizeof(H5FD_ros3_fapl_t)); + if (fa == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "memory allocation failed") + } + + /* Copy the fields of the structure */ + HDmemcpy(fa, &(file->fa), sizeof(H5FD_ros3_fapl_t)); + + /* Set return value */ + ret_value = fa; + +done: + if (ret_value == NULL) { + if (fa != NULL) { + H5MM_xfree(fa); + } + } + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_fapl_get() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_fapl_copy + * + * Purpose: Copies the ros3-specific file access properties. + * + * Return: Success: Ptr to a new property list + * + * Failure: NULL + * + * Programmer: John Mainzer + * 9/8/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void * +H5FD_ros3_fapl_copy(const void *_old_fa) +{ + const H5FD_ros3_fapl_t *old_fa = (const H5FD_ros3_fapl_t*)_old_fa; + H5FD_ros3_fapl_t *new_fa = NULL; + void *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + + new_fa = (H5FD_ros3_fapl_t *)H5MM_malloc(sizeof(H5FD_ros3_fapl_t)); + if (new_fa == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "memory allocation failed"); + } + + HDmemcpy(new_fa, old_fa, sizeof(H5FD_ros3_fapl_t)); + ret_value = new_fa; + +done: + if (ret_value == NULL) { + if (new_fa != NULL) { + H5MM_xfree(new_fa); + } + } + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5FD_ros3_fapl_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5FD_ros3_fapl_free + * + * Purpose: Frees the ros3-specific file access properties. + * + * Return: SUCCEED (cannot fail) + * + * Programmer: John Mainzer + * 9/8/17 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5FD_ros3_fapl_free(void *_fa) +{ + H5FD_ros3_fapl_t *fa = (H5FD_ros3_fapl_t*)_fa; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(fa != NULL); /* sanity check */ + + H5MM_xfree(fa); + + FUNC_LEAVE_NOAPI(SUCCEED) + +} /* end H5FD_ros3_fapl_free() */ + +#if ROS3_STATS + +/*---------------------------------------------------------------------------- + * + * Function: ros3_reset_stats() + * + * Purpose: + * + * Reset the stats collection elements in this virtual file structure. + * + * Clears any set data in stats bins; initializes/zeroes values. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - Occurs if the file is invalid somehow + * + * Programmer: Jacob Smith + * 2017-12-08 + * + *---------------------------------------------------------------------------- + */ +static herr_t +ros3_reset_stats(H5FD_ros3_t *file) +{ + unsigned i = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if ROS3_DEBUG + HDprintf("ros3_reset_stats() called\n"); +#endif + + if (file == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "file was null"); + } + + for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { + file->raw[i].bytes = 0; + file->raw[i].count = 0; + file->raw[i].min = (unsigned long long)ROS3_STATS_STARTING_MIN; + file->raw[i].max = 0; + + file->meta[i].bytes = 0; + file->meta[i].count = 0; + file->meta[i].min = (unsigned long long)ROS3_STATS_STARTING_MIN; + file->meta[i].max = 0; + } + +done: + FUNC_LEAVE_NOAPI(ret_value); + +} /* end ros3_reset_stats() */ + +#endif /* ROS3_STATS */ +#if ROS3_STATS + +/*---------------------------------------------------------------------------- + * + * Function: ros3_fprint_stats() + * + * Purpose: + * + * Tabulate and pretty-print statistics for this virtual file. + * + * Should be called upon file close. + * + * Shows number of reads and bytes read, broken down by + * "raw" (H5FD_MEM_DRAW) + * or "meta" (any other flag) + * + * Prints filename and listing of total number of reads and bytes read, + * both as a grand total and separate meta- and rawdata reads. + * + * If any reads were done, prints out two tables: + * + * 1. overview of raw- and metadata reads + * - min (smallest size read) + * - average of size read + * - k,M,G suffixes by powers of 1024 (2^10) + * - max (largest size read) + * 2. tabulation of "bins", sepraring reads into exponentially-larger + * ranges of size. + * - columns for number of reads, total bytes, and average size, with + * separate sub-colums for raw- and metadata reads. + * - each row represents one bin, identified by the top of its range + * + * Bin ranges can be modified with pound-defines at the top of this file. + * + * Bins without any reads in their bounds are not printed. + * + * An "overflow" bin is also present, to catch "big" reads. + * + * Output for all bins (and range ceiling and average size report) + * is divied by powers of 1024. By corollary, four digits before the decimal + * is valid. + * + * - 41080 bytes is represented by 40.177k, not 41.080k + * - 1004.831M represents approx. 1052642000 bytes + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - occurs if the file passed in is invalid + * - TODO: if stream is invalid? how can we check this? + * + * Programmer: Jacob Smith + * + *---------------------------------------------------------------------------- + */ +static herr_t +ros3_fprint_stats(FILE *stream, + const H5FD_ros3_t *file) +{ + herr_t ret_value = SUCCEED; + parsed_url_t *purl = NULL; + unsigned i = 0; + unsigned long count_meta = 0; + unsigned long count_raw = 0; + double average_meta = 0.0; + double average_raw = 0.0; + unsigned long long min_meta = (unsigned long long)ROS3_STATS_STARTING_MIN; + unsigned long long min_raw = (unsigned long long)ROS3_STATS_STARTING_MIN; + unsigned long long max_meta = 0; + unsigned long long max_raw = 0; + unsigned long long bytes_raw = 0; + unsigned long long bytes_meta = 0; + double re_dub = 0.0; /* re-usable double variable */ + unsigned suffix_i = 0; + const char suffixes[] = { ' ', 'K', 'M', 'G', 'T', 'P' }; + + + + FUNC_ENTER_NOAPI_NOINIT + + if (stream == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "file stream cannot be null" ); + } + if (file == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "file cannot be null"); + } + if (file->s3r_handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "s3 request handle cannot be null"); + } + if (file->s3r_handle->purl == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "parsed url structure cannot be null"); + } + purl = file->s3r_handle->purl; + + /****************** + * PRINT FILENAME * + ******************/ + + HDfprintf(stream, "stats for %s://%s", purl->scheme, purl->host); + if (purl->port != NULL && purl->port[0] != '\0') + HDfprintf(stream, ":%s", purl->port); + if (purl->query != NULL && purl->query[0] != '\0') { + if (purl->path != NULL && purl->path[0] != '\0') + HDfprintf(stream, "/%s", purl->path); + else + HDfprintf(stream, "/"); + HDfprintf(stream, "?%s", purl->query); + } else if (purl->path != NULL && purl->path[0] != '\0') { + HDfprintf(stream, "/%s", purl->path); + } + HDfprintf(stream, "\n"); + + /******************* + * AGGREGATE STATS * + *******************/ + + for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { + const ros3_statsbin *r = &file->raw[i]; + const ros3_statsbin *m = &file->meta[i]; + + if (m->min < min_meta) min_meta = m->min; + if (r->min < min_raw) min_raw = r->min; + if (m->max > max_meta) max_meta = m->max; + if (r->max > max_raw) max_raw = r->max; + + count_raw += r->count; + count_meta += m->count; + bytes_raw += r->bytes; + bytes_meta += m->bytes; + } + if (count_raw > 0) + average_raw = (double)bytes_raw / (double)count_raw; + if (count_meta > 0) + average_meta = (double)bytes_meta / (double)count_meta; + + /****************** + * PRINT OVERVIEW * + ******************/ + + HDfprintf(stream, "TOTAL READS: %llu (%llu meta, %llu raw)\n", + count_raw + count_meta, count_meta, count_raw); + HDfprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n", + bytes_raw + bytes_meta, bytes_meta, bytes_raw); + + if (count_raw + count_meta == 0) + goto done; + + /************************* + * PRINT AGGREGATE STATS * + *************************/ + + HDfprintf(stream, "SIZES meta raw\n"); + HDfprintf(stream, " min "); + if (count_meta == 0) { + HDfprintf(stream, " 0.000 "); + } else { + re_dub = (double)min_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + } + + if (count_raw == 0) { + HDfprintf(stream, " 0.000 \n"); + } else { + re_dub = (double)min_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + } + + HDfprintf(stream, " avg "); + re_dub = (double)average_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + + re_dub = (double)average_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + + HDfprintf(stream, " max "); + re_dub = (double)max_meta; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]); + + re_dub = (double)max_raw; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]); + + /****************************** + * PRINT INDIVIDUAL BIN STATS * + ******************************/ + + HDfprintf(stream, + "BINS # of reads total bytes average size\n"); + HDfprintf(stream, + " up-to meta raw meta raw meta raw\n"); + + for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) { + const ros3_statsbin *m; + const ros3_statsbin *r; + unsigned long long range_end = 0; + char bm_suffix = ' '; /* bytes-meta */ + double bm_val = 0.0; + char br_suffix = ' '; /* bytes-raw */ + double br_val = 0.0; + char am_suffix = ' '; /* average-meta */ + double am_val = 0.0; + char ar_suffix = ' '; /* average-raw */ + double ar_val = 0.0; + + m = &file->meta[i]; + r = &file->raw[i]; + if (r->count == 0 && m->count == 0) + continue; + + range_end = ros3_stats_boundaries[i]; + + if (i == ROS3_STATS_BIN_COUNT) { + range_end = ros3_stats_boundaries[i-1]; + HDfprintf(stream, ">"); + } else { + HDfprintf(stream, " "); + } + + bm_val = (double)m->bytes; + for (suffix_i = 0; bm_val >= 1024.0; suffix_i++) + bm_val /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + bm_suffix = suffixes[suffix_i]; + + br_val = (double)r->bytes; + for (suffix_i = 0; br_val >= 1024.0; suffix_i++) + br_val /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + br_suffix = suffixes[suffix_i]; + + if (m->count > 0) + am_val = (double)(m->bytes) / (double)(m->count); + for (suffix_i = 0; am_val >= 1024.0; suffix_i++) + am_val /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + am_suffix = suffixes[suffix_i]; + + if (r->count > 0) + ar_val = (double)(r->bytes) / (double)(r->count); + for (suffix_i = 0; ar_val >= 1024.0; suffix_i++) + ar_val /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + ar_suffix = suffixes[suffix_i]; + + re_dub = (double)range_end; + for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) + re_dub /= 1024.0; + HDassert(suffix_i < sizeof(suffixes)); + + HDfprintf(stream, + " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n", + re_dub, suffixes[suffix_i], /* bin ceiling */ + m->count, /* metadata reads */ + r->count, /* rawdata reads */ + bm_val, bm_suffix, /* metadata bytes */ + br_val, br_suffix, /* rawdata bytes */ + am_val, am_suffix, /* metadata average */ + ar_val, ar_suffix); /* rawdata average */ + + fflush(stream); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); + +} /* ros3_fprint_stats */ +#endif /* ROS3_STATS */ + +#endif /* H5_HAVE_ROS3_VFD */ + diff --git a/src/H5FDros3.h b/src/H5FDros3.h new file mode 100644 index 0000000..8ecd7f0 --- /dev/null +++ b/src/H5FDros3.h @@ -0,0 +1,106 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Read-Only S3 Virtual File Driver (VFD) + * + * Programmer: John Mainzer + * 2017-10-10 + * + * Purpose: The public header file for the ros3 driver. + */ +#ifndef H5FDros3_H +#define H5FDros3_H + +#ifdef H5_HAVE_ROS3_VFD +#define H5FD_ROS3 (H5FD_ros3_init()) +#else +#define H5FD_ROS3 (H5I_INVALID_HID) +#endif /* H5_HAVE_ROS3_VFD */ + +#ifdef H5_HAVE_ROS3_VFD + +/**************************************************************************** + * + * Structure: H5FD_ros3_fapl_t + * + * Purpose: + * + * H5FD_ros3_fapl_t is a public structure that is used to pass S3 + * authentication data to the appropriate S3 VFD via the FAPL. A pointer + * to an instance of this structure is a parameter to H5Pset_fapl_ros3() + * and H5Pget_fapl_ros3(). + * + * + * + * `version` (int32_t) + * + * Version number of the H5FD_ros3_fapl_t structure. Any instance passed + * to the above calls must have a recognized version number, or an error + * will be flagged. + * + * This field should be set to H5FD_CURR_ROS3_FAPL_T_VERSION. + * + * `authenticate` (hbool_t) + * + * Flag TRUE or FALSE whether or not requests are to be authenticated + * with the AWS4 algorithm. + * If TRUE, `aws_region`, `secret_id`, and `secret_key` must be populated. + * If FALSE, those three components are unused. + * + * `aws_region` (char[]) + * + * String: name of the AWS "region" of the host, e.g. "us-east-1". + * + * `secret_id` (char[]) + * + * String: "Access ID" for the resource. + * + * `secret_key` (char[]) + * + * String: "Secret Access Key" associated with the ID and resource. + * + ****************************************************************************/ + +#define H5FD_CURR_ROS3_FAPL_T_VERSION 1 + +#define H5FD_ROS3_MAX_REGION_LEN 32 +#define H5FD_ROS3_MAX_SECRET_ID_LEN 128 +#define H5FD_ROS3_MAX_SECRET_KEY_LEN 128 + +typedef struct H5FD_ros3_fapl_t { + int32_t version; + hbool_t authenticate; + char aws_region[H5FD_ROS3_MAX_REGION_LEN + 1]; + char secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1]; + char secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1]; +} H5FD_ros3_fapl_t; + + +#ifdef __cplusplus +extern "C" { +#endif + +H5_DLL hid_t H5FD_ros3_init(void); +H5_DLL void H5FD_ros3_term(void); +H5_DLL herr_t H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out); +H5_DLL herr_t H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa); + +#ifdef __cplusplus +} +#endif + +#endif /* H5_HAVE_ROS3_VFD */ + +#endif /* ifndef H5FDros3_H */ + + diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c new file mode 100644 index 0000000..f08e9d5 --- /dev/null +++ b/src/H5FDs3comms.c @@ -0,0 +1,3593 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/***************************************************************************** + * Read-Only S3 Virtual File Driver (VFD) + * + * Source for S3 Communications module + * + * ***NOT A FILE DRIVER*** + * + * Provide functions and structures required for interfacing with Amazon + * Simple Storage Service (S3). + * + * Provide S3 object access as if it were a local file. + * + * Connect to remote host, send and receive HTTP requests and responses + * as part of the AWS REST API, authenticating requests as appropriate. + * + * Programmer: Jacob Smith + * 2017-11-30 + * + *****************************************************************************/ + +/****************/ +/* Module Setup */ +/****************/ + +/***********/ +/* Headers */ +/***********/ + +#include "H5private.h" /* generic functions */ +#include "H5Eprivate.h" /* error handling */ +#include "H5MMprivate.h" /* memory management */ +#include "H5FDs3comms.h" /* S3 Communications */ + +/****************/ +/* Local Macros */ +/****************/ + +#ifdef H5_HAVE_ROS3_VFD + +/* toggle debugging (enable with 1) + */ +#define S3COMMS_DEBUG 0 + +/* manipulate verbosity of CURL output + * operates separately from S3COMMS_DEBUG + * + * 0 -> no explicit curl output + * 1 -> on error, print failure info to stderr + * 2 -> in addition to above, print information for all performs; sets all + * curl handles with CURLOPT_VERBOSE + */ +#define S3COMMS_CURL_VERBOSITY 0 + +/* size to allocate for "bytes=[-]" HTTP Range value + */ +#define S3COMMS_MAX_RANGE_STRING_SIZE 128 + + +/******************/ +/* Local Typedefs */ +/******************/ + +/********************/ +/* Local Structures */ +/********************/ + +/* struct s3r_datastruct + * Structure passed to curl write callback + * pointer to data region and record of bytes written (offset) + */ +struct s3r_datastruct { + unsigned long magic; + char *data; + size_t size; +}; +#define S3COMMS_CALLBACK_DATASTRUCT_MAGIC 0x28c2b2ul + +/********************/ +/* Local Prototypes */ +/********************/ + +size_t curlwritecallback(char *ptr, + size_t size, + size_t nmemb, + void *userdata); + +herr_t H5FD_s3comms_s3r_getsize(s3r_t *handle); + +/*********************/ +/* Package Variables */ +/*********************/ + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + +/*******************/ +/* Local Variables */ +/*******************/ + +/*************/ +/* Functions */ +/*************/ + + +/*---------------------------------------------------------------------------- + * + * Function: curlwritecallback() + * + * Purpose: + * + * Function called by CURL to write received data. + * + * Writes bytes to `userdata`. + * + * Internally manages number of bytes processed. + * + * Return: + * + * - Number of bytes processed. + * - Should equal number of bytes passed to callback. + * - Failure will result in curl error: CURLE_WRITE_ERROR. + * + * Programmer: Jacob Smith + * 2017-08-17 + * + *---------------------------------------------------------------------------- + */ +size_t +curlwritecallback(char *ptr, + size_t size, + size_t nmemb, + void *userdata) +{ + struct s3r_datastruct *sds = (struct s3r_datastruct *)userdata; + size_t product = (size * nmemb); + size_t written = 0; + + if (sds->magic != S3COMMS_CALLBACK_DATASTRUCT_MAGIC) { + return written; + } + + if (size > 0) { + HDmemcpy(&(sds->data[sds->size]), ptr, product); + sds->size += product; + written = product; + } + + return written; + +} /* end curlwritecallback() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_hrb_node_set() + * + * Purpose: + * + * Create, insert, modify, and remove elements in a field node list. + * + * `name` cannot be null; will return FAIL and list will be unaltered. + * + * Entries are accessed via the lowercase representation of their name: + * "Host", "host", and "hOSt" would all access the same node, + * but name's case is relevant in HTTP request output. + * + * List pointer `L` must always point to either of : + * - header node with lowest alphabetical order (by lowername) + * - NULL, if list is empty + * + * Types of operations: + * + * - CREATE + * - If `L` is NULL and `name` and `value` are not NULL, + * a new node is created at `L`, starting a list. + * - MODIFY + * - If a node is found with a matching lowercase name and `value` + * is not NULL, the existing name, value, and cat values are released + * and replaced with the new data. + * - No modifications are made to the list pointers. + * - REMOVE + * - If `value` is NULL, will attempt to remove node with matching + * lowercase name. + * - If no match found, returns FAIL and list is not modified. + * - When removing a node, all its resources is released. + * - If removing the last node in the list, list pointer is set to NULL. + * - INSERT + * - If no nodes exists with matching lowercase name and `value` + * is not NULL, a new node is created, inserted into list + * alphabetically by lowercase name. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - List was successfully modified + * - FAILURE: `FAIL` + * - Unable to perform operation + * - Forbidden (attempting to remove absent node, e.g.) + * - Internal error + * + * Programmer: Jacob Smith + * 2017-09-22 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_hrb_node_set( + hrb_node_t **L, + const char *name, + const char *value) +{ + size_t i = 0; + char *valuecpy = NULL; + char *namecpy = NULL; + size_t namelen = 0; + char *lowername = NULL; + char *nvcat = NULL; + hrb_node_t *node_ptr = NULL; + hrb_node_t *new_node = NULL; + hbool_t is_looking = TRUE; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_hrb_node_set.\n"); + HDprintf("NAME: %s\n", name); + HDprintf("VALUE: %s\n", value); + HDprintf("LIST:\n->"); + for (node_ptr = (*L); node_ptr != NULL; node_ptr = node_ptr->next) { + HDfprintf(stdout, "{%s}\n->", node_ptr->cat); + } + HDprintf("(null)\n"); + fflush(stdout); + node_ptr = NULL; +#endif + + if (name == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to operate on null name.\n"); + } + namelen = HDstrlen(name); + + /*********************** + * PREPARE ALL STRINGS * + **********************/ + + /* copy and lowercase name + */ + lowername = (char *)H5MM_malloc(sizeof(char) * (namelen + 1)); + if (lowername == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "cannot make space for lowercase name copy.\n"); + } + for (i = 0; i < namelen; i++) { + lowername[i] = (char)tolower((int)name[i]); + } + lowername[namelen] = 0; + + /* If value supplied, copy name, value, and concatenated "name: value". + * If NULL, we will be removing a node or doing nothing, so no need for + * copies + */ + if (value != NULL) { + int ret = 0; + size_t valuelen = HDstrlen(value); + size_t catlen = namelen + valuelen + 2; /* +2 from ": " */ + size_t catwrite = catlen + 3; /* 3 not 1 to quiet compiler warning */ + + + namecpy = (char *)H5MM_malloc(sizeof(char) * (namelen + 1)); + if (namecpy == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "cannot make space for name copy.\n"); + } + HDmemcpy(namecpy, name, (namelen + 1)); + + valuecpy = (char *)H5MM_malloc(sizeof(char) * (valuelen + 1)); + if (valuecpy == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "cannot make space for value copy.\n"); + } + HDmemcpy(valuecpy, value, (valuelen + 1)); + + nvcat = (char *)H5MM_malloc(sizeof(char) * catwrite); + if (nvcat == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "cannot make space for concatenated string.\n"); + } + ret = HDsnprintf(nvcat, catwrite, "%s: %s", name, value); + if (ret < 0 || (size_t)ret > catlen) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot concatenate `%s: %s", name, value); + } + HDassert( catlen == HDstrlen(nvcat) ); + + /* create new_node, should we need it + */ + new_node = (hrb_node_t *)H5MM_malloc(sizeof(hrb_node_t)); + if (new_node == NULL) { + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, + "cannot make space for new set.\n"); + } + + new_node->magic = S3COMMS_HRB_NODE_MAGIC; + new_node->name = NULL; + new_node->value = NULL; + new_node->cat = NULL; + new_node->lowername = NULL; + new_node->next = NULL; + } + + /*************** + * ACT ON LIST * + ***************/ + + if (*L == NULL) { + if (value == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "trying to remove node from empty list"); + } + else { +#if S3COMMS_DEBUG +HDprintf("CREATE NEW\n"); fflush(stdout); +#endif + /******************* + * CREATE NEW LIST * + *******************/ + + new_node->cat = nvcat; + new_node->name = namecpy; + new_node->lowername = lowername; + new_node->value = valuecpy; + + *L = new_node; + goto done; /* bypass further seeking */ + } + } + + /* sanity-check pointer passed in + */ + HDassert( (*L) != NULL ); + HDassert( (*L)->magic == S3COMMS_HRB_NODE_MAGIC ); + node_ptr = (*L); + + /* Check whether to modify/remove first node in list + */ + if (strcmp(lowername, node_ptr->lowername) == 0) { + + is_looking = FALSE; + + if (value == NULL) { +#if S3COMMS_DEBUG +HDprintf("REMOVE HEAD\n"); fflush(stdout); +#endif + /*************** + * REMOVE HEAD * + ***************/ + + *L = node_ptr->next; + +#if S3COMMS_DEBUG +HDprintf("FREEING CAT (node)\n"); fflush(stdout); +#endif + H5MM_xfree(node_ptr->cat); +#if S3COMMS_DEBUG +HDprintf("FREEING LOWERNAME (node)\n"); fflush(stdout); +#endif + H5MM_xfree(node_ptr->lowername); +#if S3COMMS_DEBUG +HDprintf("FREEING NAME (node)\n"); fflush(stdout); +#endif + H5MM_xfree(node_ptr->name); +#if S3COMMS_DEBUG +HDprintf("FREEING VALUE (node)\n"); fflush(stdout); +#endif + H5MM_xfree(node_ptr->value); +#if S3COMMS_DEBUG +HDprintf("MAGIC OK? %s\n", + (node_ptr->magic == S3COMMS_HRB_NODE_MAGIC) ? "YES" : "NO"); +fflush(stdout); +#endif + HDassert( node_ptr->magic == S3COMMS_HRB_NODE_MAGIC ); + node_ptr->magic += 1ul; +#if S3COMMS_DEBUG +HDprintf("FREEING POINTER\n"); fflush(stdout); +#endif + H5MM_xfree(node_ptr); + +#if S3COMMS_DEBUG +HDprintf("FREEING WORKING LOWERNAME\n"); fflush(stdout); +#endif + H5MM_xfree(lowername); lowername = NULL; + } + else { +#if S3COMMS_DEBUG +HDprintf("MODIFY HEAD\n"); fflush(stdout); +#endif + /*************** + * MODIFY HEAD * + ***************/ + + H5MM_xfree(node_ptr->cat); + H5MM_xfree(node_ptr->name); + H5MM_xfree(node_ptr->value); + + node_ptr->name = namecpy; + node_ptr->value = valuecpy; + node_ptr->cat = nvcat; + + H5MM_xfree(lowername); + lowername = NULL; + new_node->magic += 1ul; + H5MM_xfree(new_node); + new_node = NULL; + } + } + else + if (strcmp(lowername, node_ptr->lowername) < 0) { + + is_looking = FALSE; + + if (value == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "trying to remove a node 'before' head"); + } + else { +#if S3COMMS_DEBUG +HDprintf("PREPEND NEW HEAD\n"); fflush(stdout); +#endif + /******************* + * INSERT NEW HEAD * + *******************/ + + new_node->name = namecpy; + new_node->value = valuecpy; + new_node->lowername = lowername; + new_node->cat = nvcat; + new_node->next = node_ptr; + *L = new_node; + } + } + + /*************** + * SEARCH LIST * + ***************/ + + while (is_looking) { + if (node_ptr->next == NULL) { + + is_looking = FALSE; + + if (value == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "trying to remove absent node"); + } + else { +#if S3COMMS_DEBUG +HDprintf("APPEND A NODE\n"); fflush(stdout); +#endif + /******************* + * APPEND NEW NODE * + *******************/ + + HDassert( strcmp(lowername, node_ptr->lowername) > 0 ); + new_node->name = namecpy; + new_node->value = valuecpy; + new_node->lowername = lowername; + new_node->cat = nvcat; + node_ptr->next = new_node; + } + } + else + if (strcmp(lowername, node_ptr->next->lowername) < 0) { + + is_looking = FALSE; + + if (value == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "trying to remove absent node"); + } + else { +#if S3COMMS_DEBUG +HDprintf("INSERT A NODE\n"); fflush(stdout); +#endif + /******************* + * INSERT NEW NODE * + *******************/ + + HDassert( strcmp(lowername, node_ptr->lowername) > 0 ); + new_node->name = namecpy; + new_node->value = valuecpy; + new_node->lowername = lowername; + new_node->cat = nvcat; + new_node->next = node_ptr->next; + node_ptr->next = new_node; + } + } + else + if (strcmp(lowername, node_ptr->next->lowername) == 0) { + + is_looking = FALSE; + + if (value == NULL) { + /***************** + * REMOVE A NODE * + *****************/ + + hrb_node_t *tmp = node_ptr->next; + node_ptr->next = tmp->next; + +#if S3COMMS_DEBUG +HDprintf("REMOVE A NODE\n"); fflush(stdout); +#endif + H5MM_xfree(tmp->cat); + H5MM_xfree(tmp->lowername); + H5MM_xfree(tmp->name); + H5MM_xfree(tmp->value); + + HDassert( tmp->magic == S3COMMS_HRB_NODE_MAGIC ); + tmp->magic += 1ul; + H5MM_xfree(tmp); + + H5MM_xfree(lowername); + lowername = NULL; + } + else { +#if S3COMMS_DEBUG +HDprintf("MODIFY A NODE\n"); fflush(stdout); +#endif + /***************** + * MODIFY A NODE * + *****************/ + + node_ptr = node_ptr->next; + H5MM_xfree(node_ptr->name); + H5MM_xfree(node_ptr->value); + H5MM_xfree(node_ptr->cat); + + HDassert( new_node->magic == S3COMMS_HRB_NODE_MAGIC ); + new_node->magic += 1ul; + H5MM_xfree(new_node); + H5MM_xfree(lowername); + new_node = NULL; + lowername = NULL; + + node_ptr->name = namecpy; + node_ptr->value = valuecpy; + node_ptr->cat = nvcat; + } + } + else { + /**************** + * KEEP LOOKING * + ****************/ + + node_ptr = node_ptr->next; + } + } /* end while is_looking */ + +done: + if (ret_value == FAIL) { + /* clean up + */ + if (nvcat != NULL) { H5MM_xfree(nvcat); } + if (namecpy != NULL) { H5MM_xfree(namecpy); } + if (lowername != NULL) { H5MM_xfree(lowername); } + if (valuecpy != NULL) { H5MM_xfree(valuecpy); } + if (new_node != NULL) { + HDassert( new_node->magic == S3COMMS_HRB_NODE_MAGIC ); + new_node->magic += 1ul; + H5MM_xfree(new_node); + } + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_hrb_node_set() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_hrb_destroy() + * + * Purpose: + * + * Destroy and free resources _directly_ associated with an HTTP Buffer. + * + * Takes a pointer to pointer to the buffer structure. + * This allows for the pointer itself to be NULLed from within the call. + * + * If buffer or buffer pointer is NULL, there is no effect. + * + * Headers list at `first_header` is not touched. + * + * - Programmer should re-use or destroy `first_header` pointer + * (hrb_node_t *) as suits their purposes. + * - Recommend fetching prior to destroy() + * e.g., `reuse_node = hrb_to_die->first_header; destroy(hrb_to_die);` + * or maintaining an external reference. + * - Destroy node/list separately as appropriate + * - Failure to account for this will result in a memory leak. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - successfully released buffer resources + * - if `buf` is NULL or `*buf` is NULL, no effect + * - FAILURE: `FAIL` + * - `buf->magic != S3COMMS_HRB_MAGIC` + * + * Programmer: Jacob Smith + * 2017-07-21 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_hrb_destroy(hrb_t **_buf) +{ + hrb_t *buf = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_hrb_destroy.\n"); +#endif + + if (_buf != NULL && *_buf != NULL) { + buf = *_buf; + if (buf->magic != S3COMMS_HRB_MAGIC) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "pointer's magic does not match.\n"); + } + + H5MM_xfree(buf->verb); + H5MM_xfree(buf->version); + H5MM_xfree(buf->resource); + buf->magic += 1ul; + H5MM_xfree(buf); + *_buf = NULL; + } /* end if `_buf` has some value */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_hrb_destroy() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_hrb_init_request() + * + * Purpose: + * + * Create a new HTTP Request Buffer + * + * All non-null arguments should be null-terminated strings. + * + * If `verb` is NULL, defaults to "GET". + * If `http_version` is NULL, defaults to "HTTP/1.1". + * + * `resource` cannot be NULL; should be string beginning with slash + * character ('/'). + * + * All strings are copied into the structure, making them safe from + * modification in source strings. + * + * Return: + * + * - SUCCESS: pointer to new `hrb_t` + * - FAILURE: `NULL` + * + * Programmer: Jacob Smith + * 2017-07-21 + * + *---------------------------------------------------------------------------- + */ +hrb_t * +H5FD_s3comms_hrb_init_request(const char *_verb, + const char *_resource, + const char *_http_version) +{ + hrb_t *request = NULL; + char *res = NULL; + size_t reslen = 0; + hrb_t *ret_value = NULL; + char *verb = NULL; + size_t verblen = 0; + char *vrsn = NULL; + size_t vrsnlen = 0; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_hrb_init_request.\n"); +#endif + + if (_resource == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "resource string cannot be null.\n"); + } + + /* populate valid NULLs with defaults + */ + if (_verb == NULL) { + _verb = "GET"; + } + + if (_http_version == NULL) { + _http_version = "HTTP/1.1"; + } + + /* malloc space for and prepare structure + */ + request = (hrb_t *)H5MM_malloc(sizeof(hrb_t)); + if (request == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, + "no space for request structure"); + } + request->magic = S3COMMS_HRB_MAGIC; + request->body = NULL; + request->body_len = 0; + request->first_header = NULL; + + + + /* malloc and copy strings for the structure + */ + reslen = HDstrlen(_resource); + + if (_resource[0] == '/') { + res = (char *)H5MM_malloc(sizeof(char) * (reslen+1)); + if (res == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, + "no space for resource string"); + } + HDmemcpy(res, _resource, (reslen+1)); + } + else { + res = (char *)H5MM_malloc(sizeof(char) * (reslen+2)); + if (res == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, + "no space for resource string"); + } + *res = '/'; + HDmemcpy((&res[1]), _resource, (reslen+1)); + HDassert( (reslen+1) == HDstrlen(res) ); + } /* end if (else resource string not starting with '/') */ + + verblen = HDstrlen(_verb) + 1; + verb = (char *)H5MM_malloc(sizeof(char) * verblen); + if (verb == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "no space for verb string"); + } + HDstrncpy(verb, _verb, verblen); + + vrsnlen = HDstrlen(_http_version) + 1; + vrsn = (char *)H5MM_malloc(sizeof(char) * vrsnlen); + if (vrsn == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "no space for http-version string"); + } + HDstrncpy(vrsn, _http_version, vrsnlen); + + + + /* place new copies into structure + */ + request->resource = res; + request->verb = verb; + request->version = vrsn; + + ret_value = request; + +done: + /* if there is an error, clean up after ourselves + */ + if (ret_value == NULL) { + if (request != NULL) H5MM_xfree(request); + if (vrsn != NULL) H5MM_xfree(vrsn); + if (verb != NULL) H5MM_xfree(verb); + if (res != NULL) H5MM_xfree(res); + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_hrb_init_request() */ + + +/**************************************************************************** + * S3R FUNCTIONS + ****************************************************************************/ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_s3r_close() + * + * Purpose: + * + * Close communications through given S3 Request Handle (`s3r_t`) + * and clean up associated resources. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - fails if handle is null or has invalid magic number + * + * + * Programmer: Jacob Smith + * 2017-08-31 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_s3r_close(s3r_t *handle) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_s3r_close.\n"); +#endif + + if (handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle cannot be null.\n"); + } + if (handle->magic != S3COMMS_S3R_MAGIC) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has invalid magic.\n"); + } + + curl_easy_cleanup(handle->curlhandle); + + H5MM_xfree(handle->secret_id); + H5MM_xfree(handle->region); + H5MM_xfree(handle->signing_key); + + HDassert( handle->httpverb != NULL ); + H5MM_xfree(handle->httpverb); + + if (FAIL == H5FD_s3comms_free_purl(handle->purl)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to release parsed url structure") + } + + H5MM_xfree(handle); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_s3comms_s3r_close */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_s3r_get_filesize() + * + * Purpose: + * + * Retrieve the filesize of an open request handle. + * + * Wrapper "getter" to hide implementation details. + * + * + * Return: + * + * - SUCCESS: size of file, in bytes, if handle is valid. + * - FAILURE: 0, if handle is NULL or undefined. + * + * Programmer: Jacob Smith 2017-01-14 + * + *---------------------------------------------------------------------------- + */ +size_t +H5FD_s3comms_s3r_get_filesize(s3r_t *handle) +{ + size_t ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if (handle != NULL) { + ret_value = handle->filesize; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_s3comms_s3r_get_filesize */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_s3r_getsize() + * + * Purpose: + * + * Get the number of bytes of handle's target resource. + * + * Sets handle and curlhandle with to enact an HTTP HEAD request on file, + * and parses received headers to extract "Content-Length" from response + * headers, storing file size at `handle->filesize`. + * + * Critical step in opening (initiating) an `s3r_t` handle. + * + * Wraps `s3r_read()`. + * Sets curlhandle to write headers to a temporary buffer (using extant + * write callback) and provides no buffer for body. + * + * Upon exit, unsets HTTP HEAD settings from curl handle, returning to + * initial state. In event of error, curl handle state is undefined and is + * not to be trusted. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * + * Programmer: Jacob Smith + * 2017-08-23 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_s3r_getsize(s3r_t *handle) +{ + uintmax_t content_length = 0; + CURL *curlh = NULL; + char *end = NULL; + char *headerresponse = NULL; + struct s3r_datastruct sds = { + S3COMMS_CALLBACK_DATASTRUCT_MAGIC, + NULL, + 0 }; + char *start = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_s3r_getsize.\n"); +#endif + + if (handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle cannot be null.\n"); + } + if (handle->magic != S3COMMS_S3R_MAGIC) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has invalid magic.\n"); + } + if (handle->curlhandle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has bad (null) curlhandle.\n") + } + + /******************** + * PREPARE FOR HEAD * + ********************/ + + curlh = handle->curlhandle; + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_NOBODY, + 1L) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "error while setting CURL option (CURLOPT_NOBODY). " + "(placeholder flags)"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_HEADERDATA, + &sds) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "error while setting CURL option (CURLOPT_HEADERDATA). " + "(placeholder flags)"); + } + + HDassert( handle->httpverb == NULL ); + handle->httpverb = (char *)H5MM_malloc(sizeof(char) * 16); + if (handle->httpverb == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "unable to allocate space for S3 request HTTP verb"); + } + HDmemcpy(handle->httpverb, "HEAD", 5); + + headerresponse = (char *)H5MM_malloc(sizeof(char) * CURL_MAX_HTTP_HEADER); + if (headerresponse == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "unable to allocate space for curl header response"); + } + sds.data = headerresponse; + + /******************* + * PERFORM REQUEST * + *******************/ + + /* these parameters fetch the entire file, + * but, with a NULL destination and NOBODY and HEADERDATA supplied above, + * only http metadata will be sent by server and recorded by s3comms + */ + if (FAIL == + H5FD_s3comms_s3r_read(handle, 0, 0, NULL) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem in reading during getsize.\n"); + } + + if (sds.size > CURL_MAX_HTTP_HEADER) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "HTTP metadata buffer overrun\n"); + } else if (sds.size == 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "No HTTP metadata\n"); +#if S3COMMS_DEBUG + } else { + HDfprintf(stderr, "GETSIZE: OK\n"); +#endif + } + + + /****************** + * PARSE RESPONSE * + ******************/ + + start = strstr(headerresponse, + "\r\nContent-Length: "); + if (start == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not find \"Content-Length\" in response.\n"); + } + + /* move "start" to beginning of value in line; find end of line + */ + start = start + HDstrlen("\r\nContent-Length: "); + end = strstr(start, "\r\n"); + if (end == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not find end of content length line"); + } + + /* place null terminator at end of numbers + */ + *end = '\0'; + + content_length = strtoumax((const char *)start, + NULL, + 0); + + if (UINTMAX_MAX > SIZE_MAX && content_length > SIZE_MAX) { + HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "content_length overflows size_t\n"); + } + + if (content_length == 0 || + errno == ERANGE) /* errno set by strtoumax*/ + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not convert found \"Content-Length\" response (\"%s\")", + start); /* range is null-terminated, remember */ + } + + handle->filesize = (size_t)content_length; + + /********************** + * UNDO HEAD SETTINGS * + **********************/ + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_NOBODY, + NULL) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "error while setting CURL option (CURLOPT_NOBODY). " + "(placeholder flags)"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_HEADERDATA, + NULL) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "error while setting CURL option (CURLOPT_HEADERDATA). " + "(placeholder flags)"); + } + +done: + H5MM_xfree(headerresponse); + sds.magic += 1; /* set to bad magic */ + + FUNC_LEAVE_NOAPI(ret_value); + +} /* H5FD_s3comms_s3r_getsize */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_s3r_open() + * + * Purpose: + * + * Logically 'open' a file hosted on S3. + * + * - create new Request Handle + * - copy supplied url + * - copy authentication info if supplied + * - create CURL handle + * - fetch size of file + * - connect with server and execute HEAD request + * - return request handle ready for reads + * + * To use 'default' port to connect, `port` should be 0. + * + * To prevent AWS4 authentication, pass null pointer to `region`, `id`, + * and `signing_key`. + * + * Uses `H5FD_s3comms_parse_url()` to validate and parse url input. + * + * Return: + * + * - SUCCESS: Pointer to new request handle. + * - FAILURE: NULL + * - occurs if: + * - authentication strings are inconsistent + * - must _all_ be null, or have at least `region` and `id` + * - url is NULL (no filename) + * - unable to parse url (malformed?) + * - error while performing `getsize()` + * + * Programmer: Jacob Smith + * 2017-09-01 + * + *---------------------------------------------------------------------------- + */ +s3r_t * +H5FD_s3comms_s3r_open(const char *url, + const char *region, + const char *id, + const unsigned char *signing_key) +{ + size_t tmplen = 0; + CURL *curlh = NULL; + s3r_t *handle = NULL; + parsed_url_t *purl = NULL; + s3r_t *ret_value = NULL; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_s3r_open.\n"); +#endif + + + + if (url == NULL || url[0] == '\0') { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "url cannot be null.\n"); + } + + if (FAIL == H5FD_s3comms_parse_url(url, &purl)) { + /* probably a malformed url, but could be internal error */ + HGOTO_ERROR(H5E_ARGS, H5E_CANTCREATE, NULL, + "unable to create parsed url structure"); + } + HDassert( purl != NULL ); /* if above passes, this must be true */ + HDassert( purl->magic == S3COMMS_PARSED_URL_MAGIC ); + + handle = (s3r_t *)H5MM_malloc(sizeof(s3r_t)); + if (handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL, + "could not malloc space for handle.\n"); + } + + handle->magic = S3COMMS_S3R_MAGIC; + handle->purl = purl; + handle->filesize = 0; + handle->region = NULL; + handle->secret_id = NULL; + handle->signing_key = NULL; + handle->httpverb = NULL; + + /************************************* + * RECORD AUTHENTICATION INFORMATION * + *************************************/ + + if ((region != NULL && *region != '\0') || + (id != NULL && *id != '\0') || + (signing_key != NULL && *signing_key != '\0')) + { + /* if one exists, all three must exist + */ + if (region == NULL || region[0] == '\0') { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "region cannot be null.\n"); + } + if (id == NULL || id[0] == '\0') { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "secret id cannot be null.\n"); + } + if (signing_key == NULL || signing_key[0] == '\0') { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "signing key cannot be null.\n"); + } + + /* copy strings + */ + tmplen = HDstrlen(region) + 1; + handle->region = (char *)H5MM_malloc(sizeof(char) * tmplen); + if (handle->region == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "could not malloc space for handle region copy.\n"); + } + HDmemcpy(handle->region, region, tmplen); + + tmplen = HDstrlen(id) + 1; + handle->secret_id = (char *)H5MM_malloc(sizeof(char) * tmplen); + if (handle->secret_id == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "could not malloc space for handle ID copy.\n"); + } + HDmemcpy(handle->secret_id, id, tmplen); + + tmplen = SHA256_DIGEST_LENGTH; + handle->signing_key = + (unsigned char *)H5MM_malloc(sizeof(unsigned char) * tmplen); + if (handle->signing_key == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "could not malloc space for handle key copy.\n"); + } + HDmemcpy(handle->signing_key, signing_key, tmplen); + } /* if authentication information provided */ + + /************************ + * INITIATE CURL HANDLE * + ************************/ + + curlh = curl_easy_init(); + + if (curlh == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "problem creating curl easy handle!\n"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_HTTPGET, + 1L) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "error while setting CURL option (CURLOPT_HTTPGET). " + "(placeholder flags)"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_HTTP_VERSION, + CURL_HTTP_VERSION_1_1) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "error while setting CURL option (CURLOPT_HTTP_VERSION). " + "(placeholder flags)"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_FAILONERROR, + 1L) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "error while setting CURL option (CURLOPT_FAILONERROR). " + "(placeholder flags)"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_WRITEFUNCTION, + curlwritecallback) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "error while setting CURL option (CURLOPT_WRITEFUNCTION). " + "(placeholder flags)"); + } + + if ( CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_URL, + url) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "error while setting CURL option (CURLOPT_URL). " + "(placeholder flags)"); + } + +#if S3COMMS_CURL_VERBOSITY > 1 + /* CURL will print (to stdout) information for each operation + */ + curl_easy_setopt(curlh, CURLOPT_VERBOSE, 1L); +#endif + + handle->curlhandle = curlh; + + /******************* + * OPEN CONNECTION * + * * * * * * * * * * + * GET FILE SIZE * + *******************/ + + if (FAIL == + H5FD_s3comms_s3r_getsize(handle) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "problem in H5FD_s3comms_s3r_getsize.\n"); + } + + /********************* + * FINAL PREPARATION * + *********************/ + + HDassert( handle->httpverb != NULL ); + HDmemcpy(handle->httpverb, "GET", 4); + + ret_value = handle; + +done: + if (ret_value == NULL) { + if (curlh != NULL) { + curl_easy_cleanup(curlh); + } + if (FAIL == H5FD_s3comms_free_purl(purl)) { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, + "unable to free parsed url structure") + } + if (handle != NULL) { + H5MM_xfree(handle->region); + H5MM_xfree(handle->secret_id); + H5MM_xfree(handle->signing_key); + if (handle->httpverb != NULL) { + H5MM_xfree(handle->httpverb); + } + H5MM_xfree(handle); + } + } + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5FD_s3comms_s3r_open */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_s3r_read() + * + * Purpose: + * + * Read file pointed to by request handle, writing specified + * `offset` .. `offset + len` bytes to buffer `dest`. + * + * If `len` is 0, reads entirety of file starting at `offset`. + * If `offset` and `len` are both 0, reads entire file. + * + * If `offset` or `offset+len` is greater than the file size, read is + * aborted and returns `FAIL`. + * + * Uses configured "curl easy handle" to perform request. + * + * In event of error, buffer should remain unaltered. + * + * If handle is set to authorize a request, creates a new (temporary) + * HTTP Request object (hrb_t) for generating requisite headers, + * which is then translated to a `curl slist` and set in the curl handle + * for the request. + * + * `dest` _may_ be NULL, but no body data will be recorded. + * + * - In general practice, NULL should never be passed in as `dest`. + * - NULL `dest` passed in by internal function `s3r_getsize()`, in + * conjunction with CURLOPT_NOBODY to preempt transmission of file data + * from server. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * + * Programmer: Jacob Smith + * 2017-08-22 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_s3r_read(s3r_t *handle, + haddr_t offset, + size_t len, + void *dest) +{ + CURL *curlh = NULL; + CURLcode p_status = CURLE_OK; + struct curl_slist *curlheaders = NULL; + hrb_node_t *headers = NULL; + hrb_node_t *node = NULL; + struct tm *now = NULL; + char *rangebytesstr = NULL; + hrb_t *request = NULL; + int ret = 0; /* working variable to check */ + /* return value of HDsnprintf */ + struct s3r_datastruct *sds = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_s3r_read.\n"); +#endif + + /************************************** + * ABSOLUTELY NECESSARY SANITY-CHECKS * + **************************************/ + + if (handle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle cannot be null.\n"); + } + if (handle->magic != S3COMMS_S3R_MAGIC) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has invalid magic.\n"); + } + if (handle->curlhandle == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has bad (null) curlhandle.\n") + } + if (handle->purl == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle has bad (null) url.\n") + } + HDassert( handle->purl->magic == S3COMMS_PARSED_URL_MAGIC ); + if (offset > handle->filesize || (len + offset) > handle->filesize) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to read past EoF") + } + + curlh = handle->curlhandle; + + /********************* + * PREPARE WRITEDATA * + *********************/ + + if (dest != NULL) { + sds = (struct s3r_datastruct *)H5MM_malloc( + sizeof(struct s3r_datastruct)); + if (sds == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "could not malloc destination datastructure.\n"); + } + + sds->magic = S3COMMS_CALLBACK_DATASTRUCT_MAGIC; + sds->data = (char *)dest; + sds->size = 0; + if (CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_WRITEDATA, + sds) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, + "error while setting CURL option (CURLOPT_WRITEDATA). " + "(placeholder flags)"); + } + } + + /********************* + * FORMAT HTTP RANGE * + *********************/ + + if (len > 0) { + rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \ + (S3COMMS_MAX_RANGE_STRING_SIZE+1) ); + if (rangebytesstr == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "could not malloc range format string.\n"); + } + ret = HDsnprintf(rangebytesstr, + (S3COMMS_MAX_RANGE_STRING_SIZE), + "bytes="H5_PRINTF_HADDR_FMT"-"H5_PRINTF_HADDR_FMT, + offset, + offset + len - 1); + if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to format HTTP Range value"); + } else if (offset > 0) { + rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \ + (S3COMMS_MAX_RANGE_STRING_SIZE+1)); + if (rangebytesstr == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "could not malloc range format string.\n"); + } + ret = HDsnprintf(rangebytesstr, + (S3COMMS_MAX_RANGE_STRING_SIZE), + "bytes="H5_PRINTF_HADDR_FMT"-", + offset); + if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to format HTTP Range value"); + } + + /******************* + * COMPILE REQUEST * + *******************/ + + if (handle->signing_key == NULL) { + /* Do not authenticate. + */ + if (rangebytesstr != NULL) { + /* Pass in range directly + */ + char *bytesrange_ptr = NULL; /* pointer past "bytes=" portion */ + + bytesrange_ptr = strchr(rangebytesstr, '='); + HDassert( bytesrange_ptr != NULL ); + bytesrange_ptr++; /* move to first char past '=' */ + HDassert( *bytesrange_ptr != '\0' ); + + if (CURLE_OK != + curl_easy_setopt(curlh, + CURLOPT_RANGE, + bytesrange_ptr) ) + { + HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL, + "error while setting CURL option (CURLOPT_RANGE). "); + } + } + } else { + /* authenticate request + */ + char authorization[512+1]; + /* 512 := approximate max length... + * 67 + * + 8 + * + 64 + * + 128 + * + 20 + * + 128 + */ + char buffer1[512+1]; /* -> Canonical Request -> Signature */ + char buffer2[256+1]; /* -> String To Sign -> Credential */ + char iso8601now[ISO8601_SIZE]; + char signed_headers[48+1]; + /* should be large enough for nominal listing: + * "host;range;x-amz-content-sha256;x-amz-date" + * + '\0', with "range;" possibly absent + */ + + /* zero start of strings */ + authorization[0] = 0; + buffer1[0] = 0; + buffer2[0] = 0; + iso8601now[0] = 0; + signed_headers[0] = 0; + + /**** VERIFY INFORMATION EXISTS ****/ + + if (handle->region == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle must have non-null region.\n"); + } + if (handle->secret_id == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle must have non-null secret_id.\n"); + } + if (handle->signing_key == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle must have non-null signing_key.\n"); + } + if (handle->httpverb == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle must have non-null httpverb.\n"); + } + if (handle->purl->host == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle must have non-null host.\n"); + } + if (handle->purl->path == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "handle must have non-null resource.\n"); + } + + /**** CREATE HTTP REQUEST STRUCTURE (hrb_t) ****/ + + request = H5FD_s3comms_hrb_init_request( + (const char *)handle->httpverb, + (const char *)handle->purl->path, + "HTTP/1.1"); + if (request == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not allocate hrb_t request.\n"); + } + HDassert( request->magic == S3COMMS_HRB_MAGIC ); + + now = gmnow(); + if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not format ISO8601 time.\n"); + } + + if (FAIL == + H5FD_s3comms_hrb_node_set( + &headers, + "x-amz-date", + (const char *)iso8601now) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to set x-amz-date header") + } + if (headers == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem building headers list. " + "(placeholder flags)\n"); + } + HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC ); + + if (FAIL == + H5FD_s3comms_hrb_node_set( + &headers, + "x-amz-content-sha256", + (const char *)EMPTY_SHA256) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to set x-amz-content-sha256 header") + } + if (headers == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem building headers list. " + "(placeholder flags)\n"); + } + HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC ); + + if (rangebytesstr != NULL) { + if (FAIL == + H5FD_s3comms_hrb_node_set( + &headers, + "Range", + (const char *)rangebytesstr) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to set range header") + } + if (headers == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem building headers list. " + "(placeholder flags)\n"); + } + HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC ); + } + + if (FAIL == + H5FD_s3comms_hrb_node_set( + &headers, + "Host", + (const char *)handle->purl->host) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to set host header") + } + if (headers == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem building headers list. " + "(placeholder flags)\n"); + } + HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC ); + + request->first_header = headers; + + /**** COMPUTE AUTHORIZATION ****/ + + if (FAIL == /* buffer1 -> canonical request */ + H5FD_s3comms_aws_canonical_request( + buffer1, + 512, + signed_headers, + 48, + request) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "(placeholder flags)\n"); + } + if ( FAIL == /* buffer2->string-to-sign */ + H5FD_s3comms_tostringtosign(buffer2, + buffer1, + iso8601now, + handle->region) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "(placeholder flags)\n"); + } + if (FAIL == /* buffer1 -> signature */ + H5FD_s3comms_HMAC_SHA256(handle->signing_key, + SHA256_DIGEST_LENGTH, + buffer2, + HDstrlen(buffer2), + buffer1) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "(placeholder flags)\n"); + } + + iso8601now[8] = 0; /* trim to yyyyMMDD */ + ret = S3COMMS_FORMAT_CREDENTIAL(buffer2, + handle->secret_id, + iso8601now, + handle->region, + "s3"); + if (ret == 0 || ret >= S3COMMS_MAX_CREDENTIAL_SIZE) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to format aws4 credential string"); + + ret = HDsnprintf( + authorization, + 512, + "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s", + buffer2, + signed_headers, + buffer1); + if (ret <= 0 || ret >= 512) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to format aws4 authorization string"); + } + + /* append authorization header to http request buffer + */ + if (H5FD_s3comms_hrb_node_set( + &headers, + "Authorization", + (const char *)authorization) + == FAIL) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to set Authorization header") + } + if (headers == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem building headers list. " + "(placeholder flags)\n"); + } + + /* update hrb's "first header" pointer + */ + request->first_header = headers; + + /**** SET CURLHANDLE HTTP HEADERS FROM GENERATED DATA ****/ + + node = request->first_header; + while (node != NULL) { + HDassert( node->magic == S3COMMS_HRB_NODE_MAGIC ); + curlheaders = curl_slist_append(curlheaders, + (const char *)node->cat); + if (curlheaders == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not append header to curl slist. " + "(placeholder flags)\n"); + } + node = node->next; + } + + /* sanity-check + */ + if (curlheaders == NULL) { + /* above loop was probably never run */ + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "curlheaders was never populated.\n"); + } + + /* finally, set http headers in curl handle + */ + if (curl_easy_setopt( + curlh, + CURLOPT_HTTPHEADER, + curlheaders) + != CURLE_OK) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "error while setting CURL option " + "(CURLOPT_HTTPHEADER). (placeholder flags)"); + } + + } /* end if should authenticate (info provided) */ + + /******************* + * PERFORM REQUEST * + *******************/ + +#if S3COMMS_CURL_VERBOSITY > 0 + /* In event of error, print detailed information to stderr + * This is not the default behavior. + */ + { + long int httpcode = 0; + char curlerrbuf[CURL_ERROR_SIZE]; + curlerrbuf[0] = '\0'; + + if (CURLE_OK != + curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, curlerrbuf) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem setting error buffer") + } + + p_status = curl_easy_perform(curlh); + + if (p_status != CURLE_OK) { + if (CURLE_OK != + curl_easy_getinfo(curlh, CURLINFO_RESPONSE_CODE, &httpcode) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem getting response code") + } + HDfprintf(stderr, "CURL ERROR CODE: %d\nHTTP CODE: %d\n", + p_status, httpcode); + HDfprintf(stderr, "%s\n", curl_easy_strerror(p_status)); + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, + "problem while performing request.\n"); + } + if (CURLE_OK != + curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, NULL) ) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem unsetting error buffer") + } + } /* verbose error reporting */ +#else + p_status = curl_easy_perform(curlh); + + if (p_status != CURLE_OK) { + HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL, + "curl cannot perform request\n") + } +#endif + +#if S3COMMS_DEBUG + if (dest != NULL) { + HDfprintf(stderr, "len: %d\n", (int)len); + HDfprintf(stderr, "CHECKING FOR BUFFER OVERFLOW\n"); + if (sds == NULL) { + HDfprintf(stderr, "sds is NULL!\n"); + } + else { + HDfprintf(stderr, "sds: 0x%lx\n", (long long)sds); + HDfprintf(stderr, "sds->size: %d\n", (int)sds->size); + if (len > sds->size) { + HDfprintf(stderr, "buffer overwrite\n"); + } + } + } + else { + HDfprintf(stderr, "performed on entire file\n"); + } +#endif + +done: + /* clean any malloc'd resources + */ + if (curlheaders != NULL) { + curl_slist_free_all(curlheaders); + curlheaders = NULL; + } + if (rangebytesstr != NULL) { + H5MM_xfree(rangebytesstr); + rangebytesstr = NULL; + } + if (sds != NULL) { + H5MM_xfree(sds); + sds = NULL; + } + if (request != NULL) { + while (headers != NULL) + if (FAIL == + H5FD_s3comms_hrb_node_set(&headers, headers->name, NULL)) + { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot release header node") + } + HDassert( NULL == headers ); + if (FAIL == H5FD_s3comms_hrb_destroy(&request)) { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot release header request structure") + } + HDassert( NULL == request ); + } + + if (curlh != NULL) { + /* clear any Range */ + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, NULL) ) { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot unset CURLOPT_RANGE") + } + + /* clear headers */ + if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, NULL) ) { + HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot unset CURLOPT_HTTPHEADER") + } + } + + FUNC_LEAVE_NOAPI(ret_value); +} /* H5FD_s3comms_s3r_read */ + + +/**************************************************************************** + * MISCELLANEOUS FUNCTIONS + ****************************************************************************/ + + +/*---------------------------------------------------------------------------- + * + * Function: gmnow() + * + * Purpose: + * + * Get the output of `time.h`'s `gmtime()` call while minimizing setup + * clutter where important. + * + * Return: + * + * Pointer to resulting `struct tm`,as created by gmtime(time_t * T). + * + * Programmer: Jacob Smith + * 2017-07-12 + * + *---------------------------------------------------------------------------- + */ +struct tm * +gmnow(void) +{ + time_t now; + time_t *now_ptr = &now; + struct tm *ret_value = NULL; + + /* Doctor assert, checks against error in time() */ + if ( (time_t)(-1) != time(now_ptr) ) { + ret_value = gmtime(now_ptr); + } + + HDassert( ret_value != NULL ); + + return ret_value; +} /* end gmnow() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_aws_canonical_request() + * + * Purpose: + * + * Compose AWS "Canonical Request" (and signed headers string) + * as defined in the REST API documentation. + * + * Both destination strings are null-terminated. + * + * Destination string arguments must be provided with adequate space. + * + * Canonical Request format: + * + * "\n" + * "\n" + * "\n" + * "\n" (`lowercase(name)`":"`trim(value)`) + * "\n" + * ... (headers sorted by name) + * "\n" + * "\n" + * "\n" (`lowercase(header 1 name)`";"`header 2 name`;...) + * ("e3b0c4429...", e.g.) + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - writes canonical request to respective `...dest` strings + * - FAILURE: `FAIL` + * - one or more input argument was NULL + * - internal error + * + * Programmer: Jacob Smith + * 2017-10-04 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_aws_canonical_request( + char *canonical_request_dest, + int _cr_size, + char *signed_headers_dest, + int _sh_size, + hrb_t *http_request) +{ + hrb_node_t *node = NULL; + const char *query_params = ""; /* unused at present */ + herr_t ret_value = SUCCEED; + int ret = 0; + size_t cr_size = (size_t)_cr_size; + size_t sh_size = (size_t)_sh_size; + size_t cr_len = 0; /* working length of canonical request str */ + size_t sh_len = 0; /* working length of signed headers str */ + char tmpstr[256+1]; + tmpstr[256] = 0; /* terminating NULL */ + + /* "query params" refers to the optional element in the URL, e.g. + * http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J + * ^-----------------^ + * + * Not handled/implemented as of 2017-10-xx. + * Element introduced as empty placeholder and reminder. + * Further research to be done if this is ever relevant for the + * VFD use-cases. + */ + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_aws_canonical_request.\n"); +#endif + + if (http_request == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "hrb object cannot be null.\n"); + } + HDassert( http_request->magic == S3COMMS_HRB_MAGIC ); + + if (canonical_request_dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "canonical request destination cannot be null.\n"); + } + + if (signed_headers_dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "signed headers destination cannot be null.\n"); + } + + /* HTTP verb, resource path, and query string lines + */ + cr_len = (HDstrlen(http_request->verb) + + HDstrlen(http_request->resource) + + HDstrlen(query_params) + + (size_t)3); /* three newline chars */ + if (cr_len >= cr_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "not enough space in canonical request"); + } + ret = HDsnprintf( /* TODO: compiler warning */ + canonical_request_dest, + (cr_size-1), + "%s\n%s\n%s\n", + http_request->verb, + http_request->resource, + query_params); + if (ret < 0 || (size_t)ret >= cr_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to compose canonical request first line"); + } + + /* write in canonical headers, building signed headers concurrently + */ + node = http_request->first_header; /* assumed sorted */ + while (node != NULL) { + + HDassert(node->magic == S3COMMS_HRB_NODE_MAGIC); + + ret = HDsnprintf( + tmpstr, + 256, + "%s:%s\n", + node->lowername, + node->value); + if (ret < 0 || ret >= 256) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to concatenate HTTP header %s:%s", + node->lowername, + node->value); + } + cr_len += HDstrlen(tmpstr); + if (cr_len + 1 > cr_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "not enough space in canonical request"); + } + strcat(canonical_request_dest, tmpstr); + + ret = HDsnprintf( + tmpstr, + 256, + "%s;", + node->lowername); + if (ret < 0 || ret >= 256) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to append semicolon to lowername %s", + node->lowername); + } + sh_len += HDstrlen(tmpstr); + if (sh_len + 1 > sh_size) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "not enough space in signed headers"); + } + strcat(signed_headers_dest, tmpstr); + + node = node->next; + } /* end while node is not NULL */ + + /* remove tailing ';' from signed headers sequence + */ + signed_headers_dest[HDstrlen(signed_headers_dest) - 1] = '\0'; + + /* append signed headers and payload hash + * NOTE: at present, no HTTP body is handled, per the nature of + * requests/range-gets + */ + strcat(canonical_request_dest, "\n"); + strcat(canonical_request_dest, signed_headers_dest); + strcat(canonical_request_dest, "\n"); + strcat(canonical_request_dest, EMPTY_SHA256); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_aws_canonical_request() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_bytes_to_hex() + * + * Purpose: + * + * Produce human-readable hex string [0-9A-F] from sequence of bytes. + * + * For each byte (char), writes two-character hexadecimal representation. + * + * No null-terminator appended. + * + * Assumes `dest` is allocated to enough size (msg_len * 2). + * + * Fails if either `dest` or `msg` are null. + * + * `msg_len` message length of 0 has no effect. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - hex string written to `dest` (not null-terminated) + * - FAILURE: `FAIL` + * - `dest == NULL` + * - `msg == NULL` + * + * Programmer: Jacob Smith + * 2017-07-12 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_bytes_to_hex( + char *dest, + const unsigned char *msg, + size_t msg_len, + hbool_t lowercase) +{ + size_t i = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_bytes_to_hex.\n"); +#endif + + if (dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "hex destination cannot be null.\n") + } + if (msg == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "bytes sequence cannot be null.\n") + } + + for (i = 0; i < msg_len; i++) { + int chars_written = HDsnprintf(&(dest[i * 2]), + 3, /* 'X', 'X', '\n' */ + (lowercase == TRUE) ? "%02x" : "%02X", + msg[i]); + if (chars_written != 2) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem while writing hex chars for %c", + msg[i]); + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value); + +} /* end H5FD_s3comms_bytes_to_hex() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_free_purl() + * + * Purpose: + * + * Release resources from a parsed_url_t pointer. + * + * If pointer is null, nothing happens. + * + * Return: + * + * `SUCCEED` (never fails) + * + * Programmer: Jacob Smith + * 2017-11-01 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_free_purl(parsed_url_t *purl) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#if S3COMMS_DEBUG + HDprintf("called H5FD_s3comms_free_purl.\n"); +#endif + + if (purl != NULL) { + HDassert( purl->magic == S3COMMS_PARSED_URL_MAGIC ); + if (purl->scheme != NULL) { H5MM_xfree(purl->scheme); } + if (purl->host != NULL) { H5MM_xfree(purl->host); } + if (purl->port != NULL) { H5MM_xfree(purl->port); } + if (purl->path != NULL) { H5MM_xfree(purl->path); } + if (purl->query != NULL) { H5MM_xfree(purl->query); } + purl->magic += 1ul; + H5MM_xfree(purl); + } + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5FD_s3comms_free_purl() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_HMAC_SHA256() + * + * Purpose: + * + * Generate Hash-based Message Authentication Checksum using the SHA-256 + * hashing algorithm. + * + * Given a key, message, and respective lengths (to accommodate null + * characters in either), generate _hex string_ of authentication checksum + * and write to `dest`. + * + * `dest` must be at least `SHA256_DIGEST_LENGTH * 2` characters in size. + * Not enforceable by this function. + * `dest` will _not_ be null-terminated by this function. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - hex string written to `dest` (not null-terminated) + * - FAILURE: `FAIL` + * - `dest == NULL` + * - error while generating hex string output + * + * Programmer: Jacob Smith + * 2017-07-?? + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_HMAC_SHA256( + const unsigned char *key, + size_t key_len, + const char *msg, + size_t msg_len, + char *dest) +{ + unsigned char md[SHA256_DIGEST_LENGTH]; + unsigned int md_len = SHA256_DIGEST_LENGTH; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_HMAC_SHA256.\n"); +#endif + + if (dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "destination cannot be null."); + } + + HMAC(EVP_sha256(), + key, + (int)key_len, + (const unsigned char *)msg, + msg_len, + md, + &md_len); + + if (H5FD_s3comms_bytes_to_hex( + dest, + (const unsigned char *)md, + (size_t)md_len, + true) + == FAIL) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not convert to hex string."); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* H5FD_s3comms_HMAC_SHA256 */ + + +/*----------------------------------------------------------------------------- + * + * Function: H5FD__s3comms_load_aws_creds_from_file() + * + * Purpose: + * + * Extract AWS configuration information from a target file. + * + * Given a file and a profile name, e.g. "ros3_vfd_test", attempt to locate + * that region in the file. If not found, returns in error and output + * pointers are not modified. + * + * If the profile label is found, attempts to locate and parse configuration + * data, stopping at the first line where: + * + reached end of file + * + line does not start with a recognized setting name + * + * Following AWS documentation, looks for any of: + * + aws_access_key_id + * + aws_secret_access_key + * + region + * + * To be valid, the setting must begin the line with one of the keywords, + * followed immediately by an equals sign '=', and have some data before + * newline at end of line. + * + `spam=eggs` would be INVALID because name is unrecognized + * + `region = us-east-2` would be INVALID because of spaces + * + `region=` would be INVALID because no data. + * + * Upon successful parsing of a setting line, will store the result in the + * corresponding output pointer. If the output pointer is NULL, will skip + * any matching setting line while parsing -- useful to prevent overwrite + * when reading from multiple files. + * + * Return: + * + * + SUCCESS: `SUCCEED` + * + no error. settings may or may not have been loaded. + * + FAILURE: `FAIL` + * + internal error occurred. + * + -1 :: unable to format profile label + * + -2 :: profile name/label not found in file + * + -3 :: some other error + * + * Programmer: Jacob Smith + * 2018-02-27 + * + *----------------------------------------------------------------------------- + */ +static herr_t +H5FD__s3comms_load_aws_creds_from_file( + FILE *file, + const char *profile_name, + char *key_id, + char *access_key, + char *aws_region) +{ + char profile_line[32]; + char buffer[128]; + const char *setting_names[] = { + "region", + "aws_access_key_id", + "aws_secret_access_key", + }; + char * const setting_pointers[] = { + aws_region, + key_id, + access_key, + }; + unsigned setting_count = 3; + herr_t ret_value = SUCCEED; + unsigned buffer_i = 0; + unsigned setting_i = 0; + int found_setting = 0; + char *line_buffer = &(buffer[0]); + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called load_aws_creds_from_file.\n"); +#endif + + /* format target line for start of profile */ + if (32 < HDsnprintf(profile_line, 32, "[%s]", profile_name)) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to format profile label") + } + + /* look for start of profile */ + do { + /* clear buffer */ + for (buffer_i=0; buffer_i < 128; buffer_i++) { + buffer[buffer_i] = 0; + } + + line_buffer = fgets(line_buffer, 128, file); + if (line_buffer == NULL) { /* reached end of file */ + goto done; + } + } while (strncmp(line_buffer, profile_line, HDstrlen(profile_line))); + + /* extract credentials from lines */ + do { + /* clear buffer */ + for (buffer_i=0; buffer_i < 128; buffer_i++) { + buffer[buffer_i] = 0; + } + + /* collect a line from file */ + line_buffer = fgets(line_buffer, 128, file); + if (line_buffer == NULL) { + goto done; /* end of file */ + } + + /* loop over names to see if line looks like assignment */ + for (setting_i = 0; setting_i < setting_count; setting_i++) { + size_t setting_name_len = 0; + const char *setting_name = NULL; + char line_prefix[128]; + + setting_name = setting_names[setting_i]; + setting_name_len = HDstrlen(setting_name); + if (HDsnprintf(line_prefix, 128, "%s=", setting_name) < 0) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to format line prefix") + } + + /* found a matching name? */ + if (!HDstrncmp(line_buffer, line_prefix, setting_name_len + 1)) { + found_setting = 1; + + /* skip NULL destination buffer */ + if (setting_pointers[setting_i] == NULL) { + break; + } + + /* advance to end of name in string */ + do { + line_buffer++; + } while (*line_buffer != 0 && *line_buffer != '='); + + if (*line_buffer == 0 || *(line_buffer+1) == 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "incomplete assignment in file") + } + line_buffer++; /* was pointing at '='; advance */ + + /* copy line buffer into out pointer */ + if (HDstrncpy( + setting_pointers[setting_i], + (const char *)line_buffer, + HDstrlen(line_buffer)) + == NULL) + { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to copy line into pointer") + } + + /* "trim" tailing whitespace by replacing with null terminator*/ + buffer_i = 0; + while (!isspace(setting_pointers[setting_i][buffer_i])) { + buffer_i++; + } + setting_pointers[setting_i][buffer_i] = '\0'; + + break; /* have read setting; don't compare with others */ + } /* end if possible name match */ + } /* end for each setting name */ + } while (found_setting); + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD__s3comms_load_aws_creds_from_file() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_load_aws_profile() + * + * Purpose : + * + * Read aws profile elements from standard location on system and store + * settings in memory. + * + * Looks for both `~/.aws/config` and `~/.aws/credentials`, the standard + * files for AWS tools. If a file exists (can be opened), looks for the + * given profile name and reads the settings into the relevant buffer. + * + * Any setting duplicated in both files will be set to that from + * `credentials`. + * + * Settings are stored in the supplied buffers as null-terminated strings. + * + * Return: + * + * + SUCCESS: `SUCCEED` (0) + * + no error occurred and all settings were populated + * + FAILURE: `FAIL` (-1) + * + internal error occurred + * + unable to locate profile + * + region, key id, and secret key were not all found and set + * + * Programmer: Jacob Smith + * 2018-02-27 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_load_aws_profile(const char *profile_name, + char *key_id_out, + char *secret_access_key_out, + char *aws_region_out) +{ + herr_t ret_value = SUCCEED; + FILE *credfile = NULL; + char awspath[117]; + char filepath[128]; + int ret = 0; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_load_aws_profile.\n"); +#endif + +#ifdef H5_HAVE_WIN32_API + ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("USERPROFILE")) ; +#else + ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("HOME")) ; +#endif + if (ret < 0 || (size_t)ret >= 117) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to format home-aws path") + } + ret = HDsnprintf(filepath, 128, "%s%s", awspath, "credentials"); + if (ret < 0 || (size_t)ret >= 128) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to format credentials path") + } + + credfile = fopen(filepath, "r"); + if (credfile != NULL) { + if (H5FD__s3comms_load_aws_creds_from_file( + credfile, + profile_name, + key_id_out, + secret_access_key_out, + aws_region_out) + == FAIL) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to load from aws credentials") + } + if (fclose(credfile) == EOF) { + HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, + "unable to close credentials file") + } + credfile = NULL; + } /* end if credential file opened */ + + ret = HDsnprintf(filepath, 128, "%s%s", awspath, "config"); + if (ret < 0 || (size_t)ret >= 128) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL, + "unable to format config path") + } + credfile = fopen(filepath, "r"); + if (credfile != NULL) { + if (H5FD__s3comms_load_aws_creds_from_file( + credfile, + profile_name, + (*key_id_out == 0) ? key_id_out : NULL, + (*secret_access_key_out == 0) ? secret_access_key_out : NULL, + (*aws_region_out == 0) ? aws_region_out : NULL) + == FAIL) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to load from aws config") + } + if (fclose(credfile) == EOF) { + HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, + "unable to close config file") + } + credfile = NULL; + } /* end if credential file opened */ + + /* fail if not all three settings were loaded */ + if (*key_id_out == 0 || + *secret_access_key_out == 0 || + *aws_region_out == 0) + { + ret_value = FAIL; + } + +done: + if (credfile != NULL) { + if (fclose(credfile) == EOF) { + HDONE_ERROR(H5E_ARGS, H5E_ARGS, FAIL, + "problem error-closing aws configuration file") + } + } + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_load_aws_profile() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_nlowercase() + * + * Purpose: + * + * From string starting at `s`, write `len` characters to `dest`, + * converting all to lowercase. + * + * Behavior is undefined if `s` is NULL or `len` overruns the allocated + * space of either `s` or `dest`. + * + * Provided as convenience. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - upon completion, `dest` is populated + * - FAILURE: `FAIL` + * - `dest == NULL` + * + * Programmer: Jacob Smith + * 2017-09-18 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_nlowercase( + char *dest, + const char *s, + size_t len) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_nlowercase.\n"); +#endif + + if (dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "destination cannot be null.\n"); + } + + if (len > 0) { + HDmemcpy(dest, s, len); + do { + len--; + dest[len] = (char)tolower( (int)dest[len] ); + } while (len > 0); + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_nlowercase() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_parse_url() + * + * Purpose: + * + * Parse URL-like string and stuff URL components into + * `parsed_url` structure, if possible. + * + * Expects null-terminated string of format: + * SCHEME "://" HOST [":" PORT ] ["/" [ PATH ] ] ["?" QUERY] + * where SCHEME :: "[a-zA-Z/.-]+" + * PORT :: "[0-9]" + * + * Stores resulting structure in argument pointer `purl`, if successful, + * creating and populating new `parsed_url_t` structure pointer. + * Empty or absent elements are NULL in new purl structure. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - `purl` pointer is populated + * - FAILURE: `FAIL` + * - unable to parse + * - `purl` is unaltered (probably NULL) + * + * Programmer: Jacob Smith + * 2017-10-30 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_parse_url( + const char *str, + parsed_url_t **_purl) +{ + parsed_url_t *purl = NULL; /* pointer to new structure */ + const char *tmpstr = NULL; /* working pointer in string */ + const char *curstr = str; /* "start" pointer in string */ + long int len = 0; /* substring length */ + long int urllen = 0; /* length of passed-in url string */ + unsigned int i = 0; + herr_t ret_value = FAIL; + + FUNC_ENTER_NOAPI_NOINIT; + +#if S3COMMS_DEBUG + HDprintf("called H5FD_s3comms_parse_url.\n"); +#endif + + if (str == NULL || *str == '\0') { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid url string"); + } + + urllen = (long int)HDstrlen(str); + + purl = (parsed_url_t *)H5MM_malloc(sizeof(parsed_url_t)); + if (purl == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for parsed_url_t"); + } + purl->magic = S3COMMS_PARSED_URL_MAGIC; + purl->scheme = NULL; + purl->host = NULL; + purl->port = NULL; + purl->path = NULL; + purl->query = NULL; + + /*************** + * READ SCHEME * + ***************/ + + tmpstr = HDstrchr(curstr, ':'); + if (tmpstr == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid SCHEME construction: probably not URL"); + } + len = tmpstr - curstr; + HDassert( (0 <= len) && (len < urllen) ); + + /* check for restrictions + */ + for (i = 0; i < len; i++) { + /* scheme = [a-zA-Z+-.]+ (terminated by ":") */ + if (!isalpha(curstr[i]) && + '+' != curstr[i] && + '-' != curstr[i] && + '.' != curstr[i]) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "invalid SCHEME construction"); + } + } + /* copy lowercased scheme to structure + */ + purl->scheme = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1)); + if (purl->scheme == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for SCHEME"); + } + (void)HDstrncpy(purl->scheme, curstr, (size_t)len); + purl->scheme[len] = '\0'; + for ( i = 0; i < len; i++ ) { + purl->scheme[i] = (char)tolower(purl->scheme[i]); + } + + /* Skip "://" */ + tmpstr += 3; + curstr = tmpstr; + + /************* + * READ HOST * + *************/ + + if (*curstr == '[') { + /* IPv6 */ + while (']' != *tmpstr) { + if (tmpstr == 0) { /* end of string reached! */ + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "reached end of URL: incomplete IPv6 HOST"); + } + tmpstr++; + } + tmpstr++; + } /* end if (IPv6) */ + else { + while (0 != *tmpstr) { + if (':' == *tmpstr || + '/' == *tmpstr || + '?' == *tmpstr) + { + break; + } + tmpstr++; + } + } /* end else (IPv4) */ + len = tmpstr - curstr; + if (len == 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "HOST substring cannot be empty"); + } + else + if (len > urllen) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem with length of HOST substring"); + } + + /* copy host + */ + purl->host = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1)); + if (purl->host == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for HOST"); + } + (void)HDstrncpy(purl->host, curstr, (size_t)len); + purl->host[len] = 0; + + /************* + * READ PORT * + *************/ + + if (':' == *tmpstr) { + tmpstr += 1; /* advance past ':' */ + curstr = tmpstr; + while ((0 != *tmpstr) && ('/' != *tmpstr) && ('?' != *tmpstr)) { + tmpstr++; + } + len = tmpstr - curstr; + if (len == 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "PORT element cannot be empty"); + } + else + if (len > urllen) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem with length of PORT substring"); + } + for (i = 0; i < len; i ++) { + if (!isdigit(curstr[i])) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "PORT is not a decimal string"); + } + } + + /* copy port + */ + purl->port = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1)); + if (purl->port == NULL) { /* cannot malloc */ + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for PORT"); + } + (void)HDstrncpy(purl->port, curstr, (size_t)len); + purl->port[len] = 0; + } /* end if PORT element */ + + /************* + * READ PATH * + *************/ + + if ('/' == *tmpstr) { + /* advance past '/' */ + tmpstr += 1; + curstr = tmpstr; + + /* seek end of PATH + */ + while ((0 != *tmpstr) && ('?' != *tmpstr)) { + tmpstr++; + } + len = tmpstr - curstr; + if (len > urllen) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem with length of PATH substring"); + } + if (len > 0) { + purl->path = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1)); + if (purl->path == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for PATH"); + } /* cannot malloc path pointer */ + (void)HDstrncpy(purl->path, curstr, (size_t)len); + purl->path[len] = 0; + } + } /* end if PATH element */ + + /************** + * READ QUERY * + **************/ + + if ('?' == *tmpstr) { + tmpstr += 1; + curstr = tmpstr; + while (0 != *tmpstr) { + tmpstr++; + } + len = tmpstr - curstr; + if (len == 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "QUERY cannot be empty"); + } else if (len > urllen) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem with length of QUERY substring"); + } + purl->query = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1)); + if (purl->query == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL, + "can't allocate space for QUERY"); + } /* cannot malloc path pointer */ + (void)HDstrncpy(purl->query, curstr, (size_t)len); + purl->query[len] = 0; + } /* end if QUERY exists */ + + *_purl = purl; + ret_value = SUCCEED; + +done: + if (ret_value == FAIL) { + H5FD_s3comms_free_purl(purl); + } + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_parse_url() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_percent_encode_char() + * + * Purpose: + * + * "Percent-encode" utf-8 character `c`, e.g., + * '$' -> "%24" + * '¢' -> "%C2%A2" + * + * `c` cannot be null. + * + * Does not (currently) accept multi-byte characters... + * limit to (?) u+00ff, well below upper bound for two-byte utf-8 encoding + * (u+0080..u+07ff). + * + * Writes output to `repr`. + * `repr` cannot be null. + * Assumes adequate space i `repr`... + * >>> char[4] or [7] for most characters, + * >>> [13] as theoretical maximum. + * + * Representation `repr` is null-terminated. + * + * Stores length of representation (without null terminator) at pointer + * `repr_len`. + * + * Return : SUCCEED/FAIL + * + * - SUCCESS: `SUCCEED` + * - percent-encoded representation written to `repr` + * - 'repr' is null-terminated + * - FAILURE: `FAIL` + * - `c` or `repr` was NULL + * + * Programmer: Jacob Smith + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_percent_encode_char( + char *repr, + const unsigned char c, + size_t *repr_len) +{ + unsigned int i = 0; + int chars_written = 0; + herr_t ret_value = SUCCEED; +#if S3COMMS_DEBUG + unsigned char s[2] = {c, 0}; + unsigned char hex[3] = {0, 0, 0}; +#endif + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_percent_encode_char.\n"); +#endif + + if (repr == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no destination `repr`.\n") + } + +#if S3COMMS_DEBUG + H5FD_s3comms_bytes_to_hex((char *)hex, s, 1, FALSE); + HDfprintf(stdout, " CHAR: \'%s\'\n", s); + HDfprintf(stdout, " CHAR-HEX: \"%s\"\n", hex); +#endif + + if (c <= (unsigned char)0x7f) { + /* character represented in a single "byte" + * and single percent-code + */ +#if S3COMMS_DEBUG + HDfprintf(stdout, " SINGLE-BYTE\n"); +#endif + *repr_len = 3; + chars_written = HDsnprintf(repr, 4, "%%%02X", c); + if (chars_written < 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot write char %c", + c); + } + } /* end if single-byte unicode char */ + else { + /* multi-byte, multi-percent representation + */ + unsigned int acc = 0; /* byte accumulator */ + unsigned int k = 0; /* uint character representation */ + unsigned int stack_size = 0; + unsigned char stack[4] = {0, 0, 0, 0}; +#if S3COMMS_DEBUG + HDfprintf(stdout, " MULTI-BYTE\n"); +#endif + stack_size = 0; + k = (unsigned int)c; + *repr_len = 0; + do { + /* push number onto stack in six-bit slices + */ + acc = k; + acc >>= 6; /* cull least */ + acc <<= 6; /* six bits */ + stack[stack_size++] = (unsigned char)(k - acc); + k = acc >> 6; + } while (k > 0); + + /* `stack` now has two to four six-bit 'numbers' to be put into + * UTF-8 byte fields. + */ + +#if S3COMMS_DEBUG + HDfprintf(stdout, " STACK:\n {\n"); + for (i = 0; i < stack_size; i++) { + H5FD_s3comms_bytes_to_hex( + (char *)hex, + (&stack[i]), + 1, + FALSE); + hex[2] = 0; + HDfprintf(stdout, " %s,\n", hex); + } + HDfprintf(stdout, " }\n"); +#endif + + /**************** + * leading byte * + ****************/ + + /* prepend 11[1[1]]0 to first byte */ + /* 110xxxxx, 1110xxxx, or 11110xxx */ + acc = 0xC0; /* 0x11000000 */ + acc += (stack_size > 2) ? 0x20 : 0; /* 0x00100000 */ + acc += (stack_size > 3) ? 0x10 : 0; /* 0x00010000 */ + stack_size--; + chars_written = HDsnprintf( + repr, + 4, + "%%%02X", + (unsigned char)(acc + stack[stack_size])); + if (chars_written < 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot write char %c", + c); + } + *repr_len += 3; + + /************************ + * continuation byte(s) * + ************************/ + + /* 10xxxxxx */ + for (i = 0; i < stack_size; i++) { + chars_written = HDsnprintf( + &repr[i*3 + 3], + 4, + "%%%02X", + (unsigned char)(0x80 + stack[stack_size - 1 - i])); + if (chars_written < 0) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "cannot write char %c", + c); + } + *repr_len += 3; + } /* end for each continuation byte */ + } /* end else (multi-byte) */ + + *(repr + *repr_len) = '\0'; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* H5FD_s3comms_percent_encode_char */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_signing_key() + * + * Purpose: + * + * Create AWS4 "Signing Key" from secret key, AWS region, and timestamp. + * + * Sequentially runs HMAC_SHA256 on strings in specified order, + * generating re-usable checksum (according to documentation, valid for + * 7 days from time given). + * + * `secret` is `access key id` for targeted service/bucket/resource. + * + * `iso8601now` must conform to format, yyyyMMDD'T'hhmmss'Z' + * e.g. "19690720T201740Z". + * + * `region` should be one of AWS service region names, e.g. "us-east-1". + * + * Hard-coded "service" algorithm requirement to "s3". + * + * Inputs must be null-terminated strings. + * + * Writes to `md` the raw byte data, length of `SHA256_DIGEST_LENGTH`. + * Programmer must ensure that `md` is appropriately allocated. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - raw byte data of signing key written to `md` + * - FAILURE: `FAIL` + * - if any input arguments was NULL + * + * Programmer: Jacob Smith + * 2017-07-13 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_signing_key( + unsigned char *md, + const char *secret, + const char *region, + const char *iso8601now) +{ + char *AWS4_secret = NULL; + size_t AWS4_secret_len = 0; + unsigned char datekey[SHA256_DIGEST_LENGTH]; + unsigned char dateregionkey[SHA256_DIGEST_LENGTH]; + unsigned char dateregionservicekey[SHA256_DIGEST_LENGTH]; + int ret = 0; /* return value of HDsnprintf */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_signing_key.\n"); +#endif + + if (md == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Destination `md` cannot be NULL.\n") + } + if (secret == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "`secret` cannot be NULL.\n") + } + if (region == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "`region` cannot be NULL.\n") + } + if (iso8601now == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "`iso8601now` cannot be NULL.\n") + } + + AWS4_secret_len = 4 + HDstrlen(secret) + 1; + AWS4_secret = (char*)H5MM_malloc(sizeof(char *) * AWS4_secret_len); + if (AWS4_secret == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Could not allocate space.\n") + } + + /* prepend "AWS4" to start of the secret key + */ + ret = HDsnprintf(AWS4_secret, AWS4_secret_len,"%s%s", "AWS4", secret); + if ((size_t)ret != (AWS4_secret_len - 1)) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem writing AWS4+secret `%s`", + secret); + } + + /* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest + * we know digest length, so ignore via NULL + */ + HMAC(EVP_sha256(), + (const unsigned char *)AWS4_secret, + (int)HDstrlen(AWS4_secret), + (const unsigned char*)iso8601now, + 8, /* 8 --> length of 8 --> "yyyyMMDD" */ + datekey, + NULL); + HMAC(EVP_sha256(), + (const unsigned char *)datekey, + SHA256_DIGEST_LENGTH, + (const unsigned char *)region, + HDstrlen(region), + dateregionkey, + NULL); + HMAC(EVP_sha256(), + (const unsigned char *)dateregionkey, + SHA256_DIGEST_LENGTH, + (const unsigned char *)"s3", + 2, + dateregionservicekey, + NULL); + HMAC(EVP_sha256(), + (const unsigned char *)dateregionservicekey, + SHA256_DIGEST_LENGTH, + (const unsigned char *)"aws4_request", + 12, + md, + NULL); + +done: + H5MM_xfree(AWS4_secret); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5FD_s3comms_signing_key() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_tostringtosign() + * + * Purpose: + * + * Get AWS "String to Sign" from Canonical Request, timestamp, + * and AWS "region". + * + * Common between single request and "chunked upload", + * conforms to: + * "AWS4-HMAC-SHA256\n" + + * + "\n" + // yyyyMMDD'T'hhmmss'Z' + * + "/" + + "/s3/aws4-request\n" + + * hex(SHA256()) + * + * Inputs `creq` (canonical request string), `now` (ISO8601 format), + * and `region` (s3 region designator string) must all be + * null-terminated strings. + * + * Result is written to `dest` with null-terminator. + * It is left to programmer to ensure `dest` has adequate space. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - "string to sign" written to `dest` and null-terminated + * - FAILURE: `FAIL` + * - if any of the inputs are NULL + * - if an error is encountered while computing checksum + * + * Programmer: Jacob Smith + * 2017-07-?? + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_tostringtosign( + char *dest, + const char *req, + const char *now, + const char *region) +{ + unsigned char checksum[SHA256_DIGEST_LENGTH * 2 + 1]; + size_t d = 0; + char day[9]; + char hexsum[SHA256_DIGEST_LENGTH * 2 + 1]; + size_t i = 0; + int ret = 0; /* HDsnprintf return value */ + herr_t ret_value = SUCCEED; + char tmp[128]; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_tostringtosign.\n"); +#endif + + if (dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "destination buffer cannot be null.\n") + } + if (req == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "canonical request cannot be null.\n") + } + if (now == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Timestring cannot be NULL.\n") + } + if (region == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "Region cannot be NULL.\n") + } + + for (i = 0; i < 128; i++) { + tmp[i] = '\0'; + } + for (i = 0; i < SHA256_DIGEST_LENGTH * 2 + 1; i++) { + checksum[i] = '\0'; + hexsum[i] = '\0'; + } + HDstrncpy(day, now, 8); + day[8] = '\0'; + ret = HDsnprintf(tmp, 127, "%s/%s/s3/aws4_request", day, region); + if (ret <= 0 || ret >= 127) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "problem adding day and region to string") + } + + HDmemcpy((dest + d), "AWS4-HMAC-SHA256\n", 17); + d = 17; + + HDmemcpy((dest+d), now, HDstrlen(now)); + d += HDstrlen(now); + dest[d++] = '\n'; + + HDmemcpy((dest + d), tmp, HDstrlen(tmp)); + d += HDstrlen(tmp); + dest[d++] = '\n'; + + SHA256((const unsigned char *)req, + HDstrlen(req), + checksum); + + if (H5FD_s3comms_bytes_to_hex( + hexsum, + (const unsigned char *)checksum, + SHA256_DIGEST_LENGTH, + true) + == FAIL) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "could not create hex string"); + } + + for (i = 0; i < SHA256_DIGEST_LENGTH * 2; i++) { + dest[d++] = hexsum[i]; + } + + dest[d] = '\0'; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5ros3_tostringtosign() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_trim() + * + * Purpose: + * + * Remove all whitespace characters from start and end of a string `s` + * of length `s_len`, writing trimmed string copy to `dest`. + * Stores number of characters remaining at `n_written`. + * + * Destination for trimmed copy `dest` cannot be null. + * `dest` must have adequate space allocated for trimmed copy. + * If inadequate space, behavior is undefined, possibly resulting + * in segfault or overwrite of other data. + * + * If `s` is NULL or all whitespace, `dest` is untouched and `n_written` + * is set to 0. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - `dest == NULL` + * + * Programmer: Jacob Smith + * 2017-09-18 + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_trim(char *dest, + char *s, + size_t s_len, + size_t *n_written) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "called H5FD_s3comms_trim.\n"); +#endif + + if (dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "destination cannot be null.") + } + if (s == NULL) { + s_len = 0; + } + + if (s_len > 0) { + /* Find first non-whitespace character from start; + * reduce total length per character. + */ + while ((s_len > 0) && + isspace((unsigned char)s[0]) && s_len > 0) + { + s++; + s_len--; + } + + /* Find first non-whitespace character from tail; + * reduce length per-character. + * If length is 0 already, there is no non-whitespace character. + */ + if (s_len > 0) { + do { + s_len--; + } while( isspace((unsigned char)s[s_len]) ); + s_len++; + + /* write output into dest + */ + HDmemcpy(dest, s, s_len); + } + } + + *n_written = s_len; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FD_s3comms_trim() */ + + +/*---------------------------------------------------------------------------- + * + * Function: H5FD_s3comms_uriencode() + * + * Purpose: + * + * URIencode (percent-encode) every byte except "[a-zA-Z0-9]-._~". + * + * For each character in source string `_s` from `s[0]` to `s[s_len-1]`, + * writes to `dest` either the raw character or its percent-encoded + * equivalent. + * + * See `H5FD_s3comms_bytes_to_hex` for information on percent-encoding. + * + * Space (' ') character encoded as "%20" (not "+") + * + * Forward-slash ('/') encoded as "%2F" only when `encode_slash == true`. + * + * Records number of characters written at `n_written`. + * + * Assumes that `dest` has been allocated with enough space. + * + * Neither `dest` nor `s` can be NULL. + * + * `s_len == 0` will have no effect. + * + * Return: + * + * - SUCCESS: `SUCCEED` + * - FAILURE: `FAIL` + * - source strings `s` or destination `dest` are NULL + * - error while attempting to percent-encode a character + * + * Programmer: Jacob Smith + * 2017-07-?? + * + *---------------------------------------------------------------------------- + */ +herr_t +H5FD_s3comms_uriencode( + char *dest, + const char *s, + size_t s_len, + hbool_t encode_slash, + size_t *n_written) +{ + char c = 0; + size_t dest_off = 0; + char hex_buffer[13]; + size_t hex_off = 0; + size_t hex_len = 0; + herr_t ret_value = SUCCEED; + size_t s_off = 0; + + FUNC_ENTER_NOAPI_NOINIT + +#if S3COMMS_DEBUG + HDfprintf(stdout, "H5FD_s3comms_uriencode called.\n"); +#endif + + if (s == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "source string cannot be NULL"); + } + if (dest == NULL) { + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "destination cannot be NULL"); + } + + /* Write characters to destination, converting to percent-encoded + * "hex-utf-8" strings if necessary. + * e.g., '$' -> "%24" + */ + for (s_off = 0; s_off < s_len; s_off++) { + c = s[s_off]; + if (isalnum(c) || + c == '.' || + c == '-' || + c == '_' || + c == '~' || + (c == '/' && encode_slash == FALSE)) + { + dest[dest_off++] = c; + } + else { + hex_off = 0; + if (H5FD_s3comms_percent_encode_char( + hex_buffer, + (const unsigned char)c, + &hex_len) + == FAIL) + { + hex_buffer[0] = c; + hex_buffer[1] = 0; + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "unable to percent-encode character \'%s\' " + "at %d in \"%s\"", hex_buffer, (int)s_off, s); + } + + for (hex_off = 0; hex_off < hex_len; hex_off++) { + dest[dest_off++] = hex_buffer[hex_off]; + } + } /* end else (not a regular character) */ + } /* end for each character */ + + if (dest_off < s_len) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, + "buffer overflow"); + + *n_written = dest_off; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5FD_s3comms_uriencode */ + +#endif /* H5_HAVE_ROS3_VFD */ + diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h new file mode 100644 index 0000000..c9d6c9d --- /dev/null +++ b/src/H5FDs3comms.h @@ -0,0 +1,610 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/***************************************************************************** + * Read-Only S3 Virtual File Driver (VFD) + * + * This is the header for the S3 Communications module + * + * ***NOT A FILE DRIVER*** + * + * Purpose: + * + * - Provide structures and functions related to communicating with + * Amazon S3 (Simple Storage Service). + * - Abstract away the REST API (HTTP, + * networked communications) behind a series of uniform function calls. + * - Handle AWS4 authentication, if appropriate. + * - Fail predictably in event of errors. + * - Eventually, support more S3 operations, such as creating, writing to, + * and removing Objects remotely. + * + * translates: + * `read(some_file, bytes_offset, bytes_length, &dest_buffer);` + * to: + * ``` + * GET myfile HTTP/1.1 + * Host: somewhere.me + * Range: bytes=4096-5115 + * ``` + * and places received bytes from HTTP response... + * ``` + * HTTP/1.1 206 Partial-Content + * Content-Range: 4096-5115/63239 + * + * + * ``` + * ...in destination buffer. + * + * TODO: put documentation in a consistent place and point to it from here. + * + * Programmer: Jacob Smith + * 2017-11-30 + * + *****************************************************************************/ + +#ifndef H5FDs3comms_H +#define H5FDs3comms_H + + +#include "H5private.h" /* Generic Functions */ + +#ifdef H5_HAVE_ROS3_VFD + +/* Necessary S3 headers */ +#include +#include +#include +#include + +/***************** + * PUBLIC MACROS * + *****************/ + +/* hexadecimal string of pre-computed sha256 checksum of the empty string + * hex(sha256sum("")) + */ +#define EMPTY_SHA256 \ +"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + +/* string length (plus null terminator) + * example ISO8601-format string: "20170713T145903Z" (YYYYmmdd'T'HHMMSS'_') + */ +#define ISO8601_SIZE 17 + +/* string length (plus null terminator) + * example RFC7231-format string: "Fri, 30 Jun 2017 20:41:55 GMT" + */ +#define RFC7231_SIZE 30 + +/*--------------------------------------------------------------------------- + * + * Macro: ISO8601NOW() + * + * Purpose: + * + * write "YYYYmmdd'T'HHMMSS'Z'" (less single-quotes) to dest + * e.g., "20170630T204155Z" + * + * wrapper for strftime() + * + * It is left to the programmer to check return value of + * ISO8601NOW (should equal ISO8601_SIZE - 1). + * + *--------------------------------------------------------------------------- + */ +#define ISO8601NOW(dest, now_gm) \ +strftime((dest), ISO8601_SIZE, "%Y%m%dT%H%M%SZ", (now_gm)) + +/*--------------------------------------------------------------------------- + * + * Macro: RFC7231NOW() + * + * Purpose: + * + * write "Day, dd Mmm YYYY HH:MM:SS GMT" to dest + * e.g., "Fri, 30 Jun 2017 20:41:55 GMT" + * + * wrapper for strftime() + * + * It is left to the programmer to check return value of + * RFC7231NOW (should equal RFC7231_SIZE - 1). + * + *--------------------------------------------------------------------------- + */ +#define RFC7231NOW(dest, now_gm) \ +strftime((dest), RFC7231_SIZE, "%a, %d %b %Y %H:%M:%S GMT", (now_gm)) + + +/* Reasonable maximum length of a credential string. + * Provided for error-checking S3COMMS_FORMAT_CREDENTIAL (below). + * 17 <- "////aws4_request\0" + * 2 < "s3" (service) + * 8 <- "YYYYmmdd" (date) + * 128 <- (access_id) + * 155 :: sum + */ +#define S3COMMS_MAX_CREDENTIAL_SIZE 155 + + +/*--------------------------------------------------------------------------- + * + * Macro: H5FD_S3COMMS_FORMAT_CREDENTIAL() + * + * Purpose: + * + * Format "S3 Credential" string from inputs, for AWS4. + * + * Wrapper for HDsnprintf(). + * + * _HAS NO ERROR-CHECKING FACILITIES_ + * It is left to programmer to ensure that return value confers success. + * e.g., + * ``` + * assert( S3COMMS_MAX_CREDENTIAL_SIZE >= + * S3COMMS_FORMAT_CREDENTIAL(...) ); + * ``` + * + * "////aws4_request" + * assuming that `dest` has adequate space. + * + * ALL inputs must be null-terminated strings. + * + * `access` should be the user's access key ID. + * `date` must be of format "YYYYmmdd". + * `region` should be relevant AWS region, i.e. "us-east-1". + * `service` should be "s3". + * + *--------------------------------------------------------------------------- + */ +#define S3COMMS_FORMAT_CREDENTIAL(dest, access, iso8601_date, region, service) \ +HDsnprintf((dest), S3COMMS_MAX_CREDENTIAL_SIZE, \ + "%s/%s/%s/%s/aws4_request", \ + (access), (iso8601_date), (region), (service)) + +/********************* + * PUBLIC STRUCTURES * + *********************/ + + +/*---------------------------------------------------------------------------- + * + * Structure: hrb_node_t + * + * HTTP Header Field Node + * + * + * + * Maintain a ordered (linked) list of HTTP Header fields. + * + * Provides efficient access and manipulation of a logical sequence of + * HTTP header fields, of particular use when composing an + * "S3 Canonical Request" for authentication. + * + * - The creation of a Canoncial Request involves: + * - convert field names to lower case + * - sort by this lower-case name + * - convert ": " name-value separator in HTTP string to ":" + * - get sorted lowercase names without field or separator + * + * As HTTP headers allow headers in any order (excepting the case of multiple + * headers with the same name), the list ordering can be optimized for Canonical + * Request creation, suggesting alphabtical order. For more expedient insertion + * and removal of elements in the list, linked list seems preferable to a + * dynamically-expanding array. The usually-smaller number of entries (5 or + * fewer) makes performance overhead of traversing the list trivial. + * + * The above requirements of creating at Canonical Request suggests a reasonable + * trade-off of speed for space with the option to compute elements as needed + * or to have the various elements prepared and stored in the structure + * (e.g. name, value, lowername, concatenated name:value) + * The structure currently is implemented to pre-compute. + * + * At all times, the "first" node of the list should be the least, + * alphabetically. For all nodes, the `next` node should be either NULL or + * of greater alphabetical value. + * + * Each node contains its own header field information, plus a pointer to the + * next node. + * + * It is not allowed to have multiple nodes with the same _lowercase_ `name`s + * in the same list + * (i.e., name is case-insensitive for access and modification.) + * + * All data (`name`, `value`, `lowername`, and `cat`) are null-terminated + * strings allocated specifically for their node. + * + * + * + * `magic` (unsigned long) + * + * "unique" idenfier number for the structure type + * + * `name` (char *) + * + * Case-meaningful name of the HTTP field. + * Given case is how it is supplied to networking code. + * e.g., "Range" + * + * `lowername` (char *) + * + * Lowercase copy of name. + * e.g., "range" + * + * `value` (char *) + * + * Case-meaningful value of HTTP field. + * e.g., "bytes=0-9" + * + * `cat` (char *) + * + * Concatenated, null-terminated string of HTTP header line, + * as the field would appear in an HTTP request. + * e.g., "Range: bytes=0-9" + * + * `next` (hrb_node_t *) + * + * Pointers to next node in the list, or NULL sentinel as end of list. + * Next node must have a greater `lowername` as determined by strcmp(). + * + *---------------------------------------------------------------------------- + */ +typedef struct hrb_node_t { + unsigned long magic; + char *name; + char *value; + char *cat; + char *lowername; + struct hrb_node_t *next; +} hrb_node_t; +#define S3COMMS_HRB_NODE_MAGIC 0x7F5757UL + + +/*---------------------------------------------------------------------------- + * + * Structure: hrb_t + * + * HTTP Request Buffer structure + * + * + * + * Logically represent an HTTP request + * + * GET /myplace/myfile.h5 HTTP/1.1 + * Host: over.rainbow.oz + * Date: Fri, 01 Dec 2017 12:35:04 CST + * + * + * + * ...with fast, efficient access to and modification of primary and field + * elements. + * + * Structure for building HTTP requests while hiding much of the string + * processing required "under the hood." + * + * Information about the request target -- the first line -- and the body text, + * if any, are managed directly with this structure. All header fields, e.g., + * "Host" and "Date" above, are created with a linked list of `hrb_node_t` and + * included in the request by a pointer to the head of the list. + * + * + * + * `magic` (unsigned long) + * + * "Magic" number confirming that this is an hrb_t structure and + * what operations are valid for it. + * + * Must be S3COMMS_HRB_MAGIC to be valid. + * + * `body` (char *) : + * + * Pointer to start of HTTP body. + * + * Can be NULL, in which case it is treated as the empty string, "". + * + * `body_len` (size_t) : + * + * Number of bytes (characters) in `body`. 0 if empty or NULL `body`. + * + * `first_header` (hrb_node_t *) : + * + * Pointer to first SORTED header node, if any. + * It is left to the programmer to ensure that this node and associated + * list is destroyed when done. + * + * `resource` (char *) : + * + * Pointer to resource URL string, e.g., "/folder/page.xhtml". + * + * `verb` (char *) : + * + * Pointer to HTTP verb string, e.g., "GET". + * + * `version` (char *) : + * + * Pointer to HTTP version string, e.g., "HTTP/1.1". + * + *---------------------------------------------------------------------------- + */ +typedef struct { + unsigned long magic; + char *body; + size_t body_len; + hrb_node_t *first_header; + char *resource; + char *verb; + char *version; +} hrb_t; +#define S3COMMS_HRB_MAGIC 0x6DCC84UL + + +/*---------------------------------------------------------------------------- + * + * Structure: parsed_url_t + * + * + * Represent a URL with easily-accessed pointers to logical elements within. + * These elements (components) are stored as null-terminated strings (or just + * NULLs). These components should be allocated for the structure, making the + * data as safe as possible from modification. If a component is NULL, it is + * either implicit in or absent from the URL. + * + * "http://mybucket.s3.amazonaws.com:8080/somefile.h5?param=value&arg=value" + * ^--^ ^-----------------------^ ^--^ ^---------^ ^-------------------^ + * Scheme Host Port Resource Query/-ies + * + * + * + * `magic` (unsigned long) + * + * Structure identification and validation identifier. + * Identifies as `parsed_url_t` type. + * + * `scheme` (char *) + * + * String representing which protocol is to be expected. + * _Must_ be present. + * "http", "https", "ftp", e.g. + * + * `host` (char *) + * + * String of host, either domain name, IPv4, or IPv6 format. + * _Must_ be present. + * "over.rainbow.oz", "192.168.0.1", "[0000:0000:0000:0001]" + * + * `port` (char *) + * + * String representation of specified port. Must resolve to a valid unsigned + * integer. + * "9000", "80" + * + * `path` (char *) + * + * Path to resource on host. If not specified, assumes root "/". + * "lollipop_guild.wav", "characters/witches/white.dat" + * + * `query` (char *) + * + * Single string of all query parameters in url (if any). + * "arg1=value1&arg2=value2" + * + *---------------------------------------------------------------------------- + */ +typedef struct { + unsigned long magic; + char *scheme; /* required */ + char *host; /* required */ + char *port; + char *path; + char *query; +} parsed_url_t; +#define S3COMMS_PARSED_URL_MAGIC 0x21D0DFUL + + +/*---------------------------------------------------------------------------- + * + * Structure: s3r_t + * + * + * + * S3 request structure "handle". + * + * Holds persistent information for Amazon S3 requests. + * + * Instantiated through `H5FD_s3comms_s3r_open()`, copies data into self. + * + * Intended to be re-used for operations on a remote object. + * + * Cleaned up through `H5FD_s3comms_s3r_close()`. + * + * _DO NOT_ share handle between threads: curl easy handle `curlhandle` has + * undefined behavior if called to perform in multiple threads. + * + * + * + * `magic` (unsigned long) + * + * "magic" number identifying this structure as unique type. + * MUST equal `S3R_MAGIC` to be valid. + * + * `curlhandle` (CURL) + * + * Pointer to the curl_easy handle generated for the request. + * + * `httpverb` (char *) + * + * Pointer to NULL-terminated string. HTTP verb, + * e.g. "GET", "HEAD", "PUT", etc. + * + * Default is NULL, resulting in a "GET" request. + * + * `purl` (parsed_url_t *) + * + * Pointer to structure holding the elements of URL for file open. + * + * e.g., "http://bucket.aws.com:8080/myfile.dat?q1=v1&q2=v2" + * parsed into... + * { scheme: "http" + * host: "bucket.aws.com" + * port: "8080" + * path: "myfile.dat" + * query: "q1=v1&q2=v2" + * } + * + * Cannot be NULL. + * + * `region` (char *) + * + * Pointer to NULL-terminated string, specifying S3 "region", + * e.g., "us-east-1". + * + * Required to authenticate. + * + * `secret_id` (char *) + * + * Pointer to NULL-terminated string for "secret" access id to S3 resource. + * + * Requred to authenticate. + * + * `signing_key` (unsigned char *) + * + * Pointer to `SHA256_DIGEST_LENGTH`-long string for "re-usable" signing + * key, generated via + * `HMAC-SHA256(HMAC-SHA256(HMAC-SHA256(HMAC-SHA256("AWS4", + * ""), ""), "aws4_request")` + * which may be re-used for several (up to seven (7)) days from creation? + * Computed once upon file open. + * + * Requred to authenticate. + * + *---------------------------------------------------------------------------- + */ +typedef struct { + unsigned long magic; + CURL *curlhandle; + size_t filesize; + char *httpverb; + parsed_url_t *purl; + char *region; + char *secret_id; + unsigned char *signing_key; +} s3r_t; + +#define S3COMMS_S3R_MAGIC 0x44d8d79 + +#ifdef __cplusplus +extern "C" { +#endif + +/******************************************* + * DECLARATION OF HTTP FIELD LIST ROUTINES * + *******************************************/ + +H5_DLL herr_t H5FD_s3comms_hrb_node_set(hrb_node_t **L, + const char *name, + const char *value); + +/*********************************************** + * DECLARATION OF HTTP REQUEST BUFFER ROUTINES * + ***********************************************/ + +H5_DLL herr_t H5FD_s3comms_hrb_destroy(hrb_t **buf); + +H5_DLL hrb_t * H5FD_s3comms_hrb_init_request(const char *verb, + const char *resource, + const char *host); + +/************************************* + * DECLARATION OF S3REQUEST ROUTINES * + *************************************/ + +H5_DLL herr_t H5FD_s3comms_s3r_close(s3r_t *handle); + +H5_DLL size_t H5FD_s3comms_s3r_get_filesize(s3r_t *handle); + +H5_DLL s3r_t * H5FD_s3comms_s3r_open(const char url[], + const char region[], + const char id[], + const unsigned char signing_key[]); + +H5_DLL herr_t H5FD_s3comms_s3r_read(s3r_t *handle, + haddr_t offset, + size_t len, + void *dest); + +/********************************* + * DECLARATION OF OTHER ROUTINES * + *********************************/ + +H5_DLL struct tm * gmnow(void); + +H5_DLL herr_t H5FD_s3comms_aws_canonical_request(char *canonical_request_dest, + int cr_size, + char *signed_headers_dest, + int sh_size, + hrb_t *http_request); + +H5_DLL herr_t H5FD_s3comms_bytes_to_hex(char *dest, + const unsigned char *msg, + size_t msg_len, + hbool_t lowercase); + +H5_DLL herr_t H5FD_s3comms_free_purl(parsed_url_t *purl); + +H5_DLL herr_t H5FD_s3comms_HMAC_SHA256(const unsigned char *key, + size_t key_len, + const char *msg, + size_t msg_len, + char *dest); + +H5_DLL herr_t H5FD_s3comms_load_aws_profile(const char *name, + char *key_id_out, + char *secret_access_key_out, + char *aws_region_out); + +H5_DLL herr_t H5FD_s3comms_nlowercase(char *dest, + const char *s, + size_t len); + +H5_DLL herr_t H5FD_s3comms_parse_url(const char *str, + parsed_url_t **purl); + +H5_DLL herr_t H5FD_s3comms_percent_encode_char(char *repr, + const unsigned char c, + size_t *repr_len); + +H5_DLL herr_t H5FD_s3comms_signing_key(unsigned char *md, + const char *secret, + const char *region, + const char *iso8601now); + +H5_DLL herr_t H5FD_s3comms_tostringtosign(char *dest, + const char *req_str, + const char *now, + const char *region); + +H5_DLL herr_t H5FD_s3comms_trim(char *dest, + char *s, + size_t s_len, + size_t *n_written); + +H5_DLL herr_t H5FD_s3comms_uriencode(char *dest, const char *s, size_t s_len, + hbool_t encode_slash, size_t *n_written); + +#ifdef __cplusplus +} +#endif + +#endif /* H5_HAVE_ROS3_VFD */ + +#endif /* ifndef H5FDs3comms_H */ + diff --git a/src/Makefile.am b/src/Makefile.am index 93a90be..4774ae2 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -53,7 +53,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Fmount.c H5Fmpi.c H5Fquery.c \ H5Fsfile.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \ H5FD.c H5FDcore.c \ - H5FDdirect.c H5FDfamily.c H5FDint.c H5FDlog.c H5FDmpi.c H5FDmpio.c \ + H5FDdirect.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c H5FDmpi.c H5FDmpio.c \ H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c \ H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSsection.c H5FSstat.c H5FStest.c \ H5G.c H5Gbtree2.c H5Gcache.c \ @@ -98,9 +98,13 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Topaque.c \ H5Torder.c \ H5Tpad.c H5Tprecis.c H5Tstrpad.c H5Tvisit.c H5Tvlen.c H5TS.c H5VM.c H5WB.c H5Z.c \ - H5Zdeflate.c H5Zfletcher32.c H5Znbit.c H5Zshuffle.c H5Zszip.c \ - H5Zscaleoffset.c H5Ztrans.c + H5Zdeflate.c H5Zfletcher32.c H5Znbit.c H5Zshuffle.c \ + H5Zscaleoffset.c H5Zszip.c H5Ztrans.c +# Only compile the read-only S3 VFD if necessary +if ROS3_VFD_CONDITIONAL + libhdf5_la_SOURCES += H5FDros3.c H5FDs3comms.c +endif # Public headers include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5version.h \ @@ -108,8 +112,8 @@ include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5vers H5Cpublic.h H5Dpublic.h \ H5Epubgen.h H5Epublic.h H5Fpublic.h \ H5FDpublic.h H5FDcore.h H5FDdirect.h \ - H5FDfamily.h H5FDlog.h H5FDmpi.h H5FDmpio.h \ - H5FDmulti.h H5FDsec2.h H5FDstdio.h \ + H5FDfamily.h H5FDhdfs.h H5FDlog.h H5FDmpi.h H5FDmpio.h \ + H5FDmulti.h H5FDros3.h H5FDsec2.h H5FDstdio.h \ H5Gpublic.h H5Ipublic.h H5Lpublic.h \ H5MMpublic.h H5Opublic.h H5Ppublic.h \ H5PLextern.h H5PLpublic.h \ diff --git a/src/Makefile.in b/src/Makefile.in index 97bfbc7..8284041 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -163,8 +163,9 @@ am_libhdf5_la_OBJECTS = H5.lo H5checksum.lo H5dbg.lo H5system.lo \ H5Faccum.lo H5Fcwfs.lo H5Fdbg.lo H5Fefc.lo H5Ffake.lo H5Fio.lo \ H5Fmount.lo H5Fmpi.lo H5Fquery.lo H5Fsfile.lo H5Fsuper.lo \ H5Fsuper_cache.lo H5Ftest.lo H5FD.lo H5FDcore.lo H5FDdirect.lo \ - H5FDfamily.lo H5FDint.lo H5FDlog.lo H5FDmpi.lo H5FDmpio.lo \ - H5FDmulti.lo H5FDsec2.lo H5FDspace.lo H5FDstdio.lo H5FL.lo \ + H5FDfamily.lo H5FDhdfs.lo H5FDint.lo H5FDlog.lo H5FDs3comms.lo \ + H5FDmpi.lo H5FDmpio.lo H5FDmulti.lo H5FDros3.lo H5FDsec2.lo \ + H5FDspace.lo H5FDstdio.lo H5FL.lo \ H5FO.lo H5FS.lo H5FScache.lo H5FSdbg.lo H5FSsection.lo \ H5FSstat.lo H5FStest.lo H5G.lo H5Gbtree2.lo H5Gcache.lo \ H5Gcompact.lo H5Gdense.lo H5Gdeprec.lo H5Gent.lo H5Gint.lo \ @@ -530,6 +531,7 @@ H5_CPPFLAGS = @H5_CPPFLAGS@ H5_CXXFLAGS = @H5_CXXFLAGS@ H5_FCFLAGS = @H5_FCFLAGS@ H5_FORTRAN_SHARED = @H5_FORTRAN_SHARED@ +HAVE_LIBHDFS = @HAVE_LIBHDFS@ H5_LDFLAGS = @H5_LDFLAGS@ H5_VERSION = @H5_VERSION@ HADDR_T = @HADDR_T@ @@ -590,6 +592,7 @@ PATH_SEPARATOR = @PATH_SEPARATOR@ PERL = @PERL@ RANLIB = @RANLIB@ ROOT = @ROOT@ +ROS3_VFD = @ROS3_VFD@ RUNPARALLEL = @RUNPARALLEL@ RUNSERIAL = @RUNSERIAL@ R_INTEGER = @R_INTEGER@ @@ -781,8 +784,8 @@ libhdf5_la_SOURCES = H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Fmount.c H5Fmpi.c H5Fquery.c \ H5Fsfile.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \ H5FD.c H5FDcore.c \ - H5FDdirect.c H5FDfamily.c H5FDint.c H5FDlog.c H5FDmpi.c H5FDmpio.c \ - H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c \ + H5FDdirect.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c H5FDmpi.c H5FDmpio.c H5FDs3comms.c \ + H5FDmulti.c H5FDros3.c H5FDsec2.c H5FDspace.c H5FDstdio.c \ H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSsection.c H5FSstat.c H5FStest.c \ H5G.c H5Gbtree2.c H5Gcache.c \ H5Gcompact.c H5Gdense.c H5Gdeprec.c H5Gent.c \ diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in index 5cb0930..5b65d77 100644 --- a/src/libhdf5.settings.in +++ b/src/libhdf5.settings.in @@ -3,37 +3,47 @@ General Information: ------------------- - HDF5 Version: @H5_VERSION@ - Configured on: @CONFIG_DATE@ - Configured by: @CONFIG_USER@ - Configure mode: @CONFIG_MODE@ - Host system: @host_cpu@-@host_vendor@-@host_os@ - Uname information: @UNAME_INFO@ - Byte sex: @BYTESEX@ - Libraries: @STATIC_SHARED@ - Installation point: @prefix@ + HDF5 Version: @H5_VERSION@ + Configured on: @CONFIG_DATE@ + Configured by: @CONFIG_USER@ + Configure mode: @CONFIG_MODE@ + Host system: @host_cpu@-@host_vendor@-@host_os@ + Uname information: @UNAME_INFO@ + Byte sex: @BYTESEX@ + Installation point: @prefix@ Compiling Options: ------------------ - Compilation Mode: @CONFIG_MODE@ + Build Mode: @BUILD_MODE@ + Debugging Symbols: @SYMBOLS@ + Asserts: @ASSERTS@ + Profiling: @PROFILING@ + Optimization Level: @OPTIMIZATION@ + +Linking Options: +---------------- + Libraries: @STATIC_SHARED@ + Statically Linked Executables: @LT_STATIC_EXEC@ + LDFLAGS: @LDFLAGS@ + H5_LDFLAGS: @H5_LDFLAGS@ + AM_LDFLAGS: @AM_LDFLAGS@ + Extra libraries: @LIBS@ + Archiver: @AR@ + AR_FLAGS: @AR_FLAGS@ + Ranlib: @RANLIB@ + +Languages: +---------- + C: yes C Compiler: @CC_VERSION@ - CFLAGS: @CFLAGS@ - H5_CFLAGS: @H5_CFLAGS@ - AM_CFLAGS: @AM_CFLAGS@ CPPFLAGS: @CPPFLAGS@ H5_CPPFLAGS: @H5_CPPFLAGS@ AM_CPPFLAGS: @AM_CPPFLAGS@ + C Flags: @CFLAGS@ + H5 C Flags: @H5_CFLAGS@ + AM C Flags: @AM_CFLAGS@ Shared C Library: @enable_shared@ Static C Library: @enable_static@ - Statically Linked Executables: @STATIC_EXEC@ - LDFLAGS: @LDFLAGS@ - H5_LDFLAGS: @H5_LDFLAGS@ - AM_LDFLAGS: @AM_LDFLAGS@ - Extra libraries: @LIBS@ - Archiver: @AR@ - Ranlib: @RANLIB@ - Debugged Packages: @DEBUG_PKG@ - API Tracing: @TRACE_API@ Languages: ---------- @@ -56,19 +66,21 @@ Languages: Features: --------- - Parallel HDF5: @PARALLEL@ - High Level library: @HDF5_HL@ - Build HDF5 Tests: @HDF5_TESTS@ - Build HDF5 Tools: @HDF5_TOOLS@ - Threadsafety: @THREADSAFE@ - Default API Mapping: @DEFAULT_API_VERSION@ - With Deprecated Public Symbols: @DEPRECATED_SYMBOLS@ - I/O filters (external): @EXTERNAL_FILTERS@ - MPE: @MPE@ - Direct VFD: @DIRECT_VFD@ - dmalloc: @HAVE_DMALLOC@ -Clear file buffers before write: @CLEARFILEBUF@ - Using memory checker: @USINGMEMCHECKER@ - Function Stack Tracing: @CODESTACK@ - Strict File Format Checks: @STRICT_FORMAT_CHECKS@ - Optimization Instrumentation: @INSTRUMENT@ + Parallel HDF5: @PARALLEL@ + High-level library: @HDF5_HL@ + Build HDF5 Tests: @HDF5_TESTS@ + Build HDF5 Tools: @HDF5_TOOLS@ + Threadsafety: @THREADSAFE@ + Default API mapping: @DEFAULT_API_VERSION@ + With deprecated public symbols: @DEPRECATED_SYMBOLS@ + I/O filters (external): @EXTERNAL_FILTERS@ + MPE: @MPE@ + Direct VFD: @DIRECT_VFD@ + (Read-Only) S3 VFD: @ROS3_VFD@ + (Read-Only) HDFS VFD: @HAVE_LIBHDFS@ + dmalloc: @HAVE_DMALLOC@ + Clear file buffers before write: @CLEARFILEBUF@ + Using memory checker: @USINGMEMCHECKER@ + Function stack tracing: @CODESTACK@ + Strict file format checks: @STRICT_FORMAT_CHECKS@ + Optimization instrumentation: @INSTRUMENT@ diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 22cdb90..a67755b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -207,6 +207,9 @@ set (H5_TESTS ttsafe # multiple source getname vfd + ros3 + s3comms + hdfs ntypes dangle dtransform @@ -250,6 +253,7 @@ endforeach () ### M U L T I P L E S O U R C E T E S T S ### ############################################################################## + ######### Special handling for multiple sources ############# #-- Adding test for testhdf5 add_executable (testhdf5 ${testhdf5_SOURCES}) diff --git a/test/Makefile.am b/test/Makefile.am index fd7f9b5..2e8f85a 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -43,7 +43,7 @@ TEST_PROG=testhdf5 lheap ohdr stab gheap cache cache_api \ dtypes dsets cmpd_dset filter_fail extend external efc objcopy links unlink \ big mtime fillval mount flush1 flush2 app_ref enum \ set_extent ttsafe \ - getname vfd ntypes dangle dtransform reserved cross_read \ + getname vfd ros3 s3comms hdfs ntypes dangle dtransform reserved cross_read \ freespace mf btree2 fheap file_image unregister # List programs to be built when testing here. error_test and err_compat are @@ -157,8 +157,8 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offse dtransform.h5 test_filters.h5 get_file_name.h5 tstint[1-2].h5 \ unlink_chunked.h5 btree2.h5 objcopy_src.h5 objcopy_dst.h5 \ objcopy_ext.dat trefer1.h5 trefer2.h5 app_ref.h5 tcheck_version_*.out \ - tcheck_version_*.err efc[0-5].h5 log_vfd_out.log \ - new_multi_file_v16-r.h5 new_multi_file_v16-s.h5 \ + tcheck_version_*.err efc[0-5].h5 log_vfd_out.log log_ros3_out.log \ + log_s3comms_out.log new_multi_file_v16-r.h5 new_multi_file_v16-s.h5 \ split_get_file_image_test-m.h5 split_get_file_image_test-r.h5 \ file_image_core_test.h5.copy unregister_filter_1.h5 unregister_filter_2.h5 diff --git a/test/dtypes.c b/test/dtypes.c index 9747af7..27ad8e7 100644 --- a/test/dtypes.c +++ b/test/dtypes.c @@ -23,7 +23,7 @@ #include "H5Iprivate.h" /* For checking that datatype id's don't leak */ /* Number of elements in each test */ -#define NTESTELEM 100000 +#define NTESTELEM 100000 /* For test_compound_8 and test_compound_10 */ #define ARRAY_DIM 4 @@ -32,7 +32,7 @@ * Offset from alinged memory returned by malloc(). This can be used to test * that type conversions handle non-aligned buffers correctly. */ -#define ALIGNMENT 1 +#define ALIGNMENT 1 /* * Define if you want to test alignment code on a machine that doesn't @@ -118,22 +118,22 @@ static int num_opaque_conversions_g = 0; static int opaque_check(int tag_it); static herr_t convert_opaque(hid_t st, hid_t dt, H5T_cdata_t *cdata, - size_t nelmts, size_t buf_stride, + size_t nelmts, size_t buf_stride, size_t bkg_stride, void *_buf, - void *bkg, hid_t dset_xfer_plid); + void *bkg, hid_t dset_xfer_plid); static int opaque_long(void); static int opaque_funcs(void); - + /*------------------------------------------------------------------------- - * Function: reset_hdf5 + * Function: reset_hdf5 * - * Purpose: Reset the hdf5 library. This causes statistics to be printed - * and counters to be reset. + * Purpose: Reset the hdf5 library. This causes statistics to be printed + * and counters to be reset. * - * Return: void + * Return: void * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, November 16, 1998 * * Modifications: @@ -164,7 +164,7 @@ reset_hdf5(void) } - + /*------------------------------------------------------------------------- * Function: test_classes * @@ -265,7 +265,7 @@ test_classes(void) return 1; } - + /*------------------------------------------------------------------------- * Function: test_copy * @@ -286,7 +286,7 @@ static int test_copy(void) { hid_t a_copy; - herr_t status; + herr_t status; TESTING("H5Tcopy()"); @@ -295,12 +295,12 @@ test_copy(void) /* We should not be able to close a built-in byte */ H5E_BEGIN_TRY { - status = H5Tclose (H5T_NATIVE_SCHAR); + status = H5Tclose (H5T_NATIVE_SCHAR); } H5E_END_TRY; if (status>=0) { - H5_FAILED(); - HDputs (" Should not be able to close a predefined type!"); - goto error; + H5_FAILED(); + HDputs (" Should not be able to close a predefined type!"); + goto error; } PASSED(); @@ -310,7 +310,7 @@ test_copy(void) return 1; } - + /*------------------------------------------------------------------------- * Function: test_detect * @@ -487,7 +487,7 @@ error: return 1; } - + /*------------------------------------------------------------------------- * Function: test_compound_1 * @@ -651,19 +651,19 @@ error: return retval; } - + /*------------------------------------------------------------------------- - * Function: test_compound_2 + * Function: test_compound_2 * - * Purpose: Tests a compound type conversion where the source and - * destination are the same except for the order of the - * elements. + * Purpose: Tests a compound type conversion where the source and + * destination are the same except for the order of the + * elements. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, June 17, 1999 * * Modifications: @@ -674,19 +674,19 @@ static int test_compound_2(void) { struct st { - int a, b, c[4], d, e; + int a, b, c[4], d, e; } *s_ptr; struct dt { - int e, d, c[4], b, a; + int e, d, c[4], b, a; } *d_ptr; - const size_t nelmts = NTESTELEM; - const hsize_t four = 4; - unsigned char *buf=NULL, *orig=NULL, *bkg=NULL; - hid_t st=-1, dt=-1; + const size_t nelmts = NTESTELEM; + const hsize_t four = 4; + unsigned char *buf=NULL, *orig=NULL, *bkg=NULL; + hid_t st=-1, dt=-1; hid_t array_dt; - int64_t nmembs; - int i; + int64_t nmembs; + int i; TESTING("compound element reordering"); @@ -698,15 +698,15 @@ test_compound_2(void) bkg = (unsigned char*)HDmalloc(nelmts * sizeof(struct dt)); orig = (unsigned char*)HDmalloc(nelmts * sizeof(struct st)); for (i=0; i<(int)nelmts; i++) { - s_ptr = ((struct st*)((void *)orig)) + i; - s_ptr->a = i*8+0; - s_ptr->b = i*8+1; - s_ptr->c[0] = i*8+2; - s_ptr->c[1] = i*8+3; - s_ptr->c[2] = i*8+4; - s_ptr->c[3] = i*8+5; - s_ptr->d = i*8+6; - s_ptr->e = i*8+7; + s_ptr = ((struct st*)((void *)orig)) + i; + s_ptr->a = i*8+0; + s_ptr->b = i*8+1; + s_ptr->c[0] = i*8+2; + s_ptr->c[1] = i*8+3; + s_ptr->c[2] = i*8+4; + s_ptr->c[3] = i*8+5; + s_ptr->d = i*8+6; + s_ptr->e = i*8+7; } HDmemcpy(buf, orig, nelmts*sizeof(struct st)); @@ -736,26 +736,26 @@ test_compound_2(void) /* Compare results */ for (i=0; i<(int)nelmts; i++) { - s_ptr = ((struct st*)((void *)orig)) + i; - d_ptr = ((struct dt*)((void *)buf)) + i; - if (s_ptr->a != d_ptr->a || - s_ptr->b != d_ptr->b || - s_ptr->c[0] != d_ptr->c[0] || - s_ptr->c[1] != d_ptr->c[1] || - s_ptr->c[2] != d_ptr->c[2] || - s_ptr->c[3] != d_ptr->c[3] || - s_ptr->d != d_ptr->d || - s_ptr->e != d_ptr->e) { - H5_FAILED(); - HDprintf(" i=%d\n", i); - HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", - s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2], - s_ptr->c[3], s_ptr->d, s_ptr->e); - HDprintf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", - d_ptr->a, d_ptr->b, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2], - d_ptr->c[3], d_ptr->d, d_ptr->e); - goto error; - } + s_ptr = ((struct st*)((void *)orig)) + i; + d_ptr = ((struct dt*)((void *)buf)) + i; + if (s_ptr->a != d_ptr->a || + s_ptr->b != d_ptr->b || + s_ptr->c[0] != d_ptr->c[0] || + s_ptr->c[1] != d_ptr->c[1] || + s_ptr->c[2] != d_ptr->c[2] || + s_ptr->c[3] != d_ptr->c[3] || + s_ptr->d != d_ptr->d || + s_ptr->e != d_ptr->e) { + H5_FAILED(); + HDprintf(" i=%d\n", i); + HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", + s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2], + s_ptr->c[3], s_ptr->d, s_ptr->e); + HDprintf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", + d_ptr->a, d_ptr->b, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2], + d_ptr->c[3], d_ptr->d, d_ptr->e); + goto error; + } } /* Release resources */ @@ -782,19 +782,19 @@ error: return 1; } - + /*------------------------------------------------------------------------- - * Function: test_compound_3 + * Function: test_compound_3 * - * Purpose: Tests compound conversions where the source and destination - * are the same except the destination is missing a couple - * members which appear in the source. + * Purpose: Tests compound conversions where the source and destination + * are the same except the destination is missing a couple + * members which appear in the source. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, June 17, 1999 * * Modifications: @@ -805,19 +805,19 @@ static int test_compound_3(void) { struct st { - int a, b, c[4], d, e; + int a, b, c[4], d, e; } *s_ptr; struct dt { - int a, c[4], e; + int a, c[4], e; } *d_ptr; - const size_t nelmts = NTESTELEM; - const hsize_t four = 4; - unsigned char *buf=NULL, *orig=NULL, *bkg=NULL; - hid_t st=-1, dt=-1; + const size_t nelmts = NTESTELEM; + const hsize_t four = 4; + unsigned char *buf=NULL, *orig=NULL, *bkg=NULL; + hid_t st=-1, dt=-1; hid_t array_dt; - int64_t nmembs; - int i; + int64_t nmembs; + int i; TESTING("compound subset conversions"); @@ -866,24 +866,24 @@ test_compound_3(void) /* Compare results */ for (i=0; i<(int)nelmts; i++) { - s_ptr = ((struct st*)((void *)orig)) + i; - d_ptr = ((struct dt*)((void *)buf)) + i; - if (s_ptr->a != d_ptr->a || - s_ptr->c[0] != d_ptr->c[0] || - s_ptr->c[1] != d_ptr->c[1] || - s_ptr->c[2] != d_ptr->c[2] || - s_ptr->c[3] != d_ptr->c[3] || - s_ptr->e != d_ptr->e) { - H5_FAILED(); - HDprintf(" i=%d\n", i); - HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", - s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2], - s_ptr->c[3], s_ptr->d, s_ptr->e); - HDprintf(" dst={a=%d, c=[%d,%d,%d,%d], e=%d\n", - d_ptr->a, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2], - d_ptr->c[3], d_ptr->e); - goto error; - } + s_ptr = ((struct st*)((void *)orig)) + i; + d_ptr = ((struct dt*)((void *)buf)) + i; + if (s_ptr->a != d_ptr->a || + s_ptr->c[0] != d_ptr->c[0] || + s_ptr->c[1] != d_ptr->c[1] || + s_ptr->c[2] != d_ptr->c[2] || + s_ptr->c[3] != d_ptr->c[3] || + s_ptr->e != d_ptr->e) { + H5_FAILED(); + HDprintf(" i=%d\n", i); + HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", + s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2], + s_ptr->c[3], s_ptr->d, s_ptr->e); + HDprintf(" dst={a=%d, c=[%d,%d,%d,%d], e=%d\n", + d_ptr->a, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2], + d_ptr->c[3], d_ptr->e); + goto error; + } } /* Release resources */ @@ -909,19 +909,19 @@ error: return 1; } - + /*------------------------------------------------------------------------- - * Function: test_compound_4 + * Function: test_compound_4 * - * Purpose: Tests compound conversions when the destination has the same - * fields as the source but one or more of the fields are - * smaller. + * Purpose: Tests compound conversions when the destination has the same + * fields as the source but one or more of the fields are + * smaller. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, June 17, 1999 * * Modifications: @@ -933,22 +933,22 @@ test_compound_4(void) { struct st { - int a, b, c[4], d, e; + int a, b, c[4], d, e; } *s_ptr; struct dt { - short b; - int a, c[4]; - short d; - int e; + short b; + int a, c[4]; + short d; + int e; } *d_ptr; - const size_t nelmts = NTESTELEM; - const hsize_t four = 4; - unsigned char *buf=NULL, *orig=NULL, *bkg=NULL; - hid_t st=-1, dt=-1; + const size_t nelmts = NTESTELEM; + const hsize_t four = 4; + unsigned char *buf=NULL, *orig=NULL, *bkg=NULL; + hid_t st=-1, dt=-1; hid_t array_dt; - int64_t nmembs; - int i; + int64_t nmembs; + int i; TESTING("compound element shrinking & reordering"); @@ -999,26 +999,26 @@ test_compound_4(void) /* Compare results */ for (i=0; i<(int)nelmts; i++) { - s_ptr = ((struct st*)((void *)orig)) + i; - d_ptr = ((struct dt*)((void *)buf)) + i; - if (s_ptr->a != d_ptr->a || - s_ptr->b != d_ptr->b || - s_ptr->c[0] != d_ptr->c[0] || - s_ptr->c[1] != d_ptr->c[1] || - s_ptr->c[2] != d_ptr->c[2] || - s_ptr->c[3] != d_ptr->c[3] || - s_ptr->d != d_ptr->d || - s_ptr->e != d_ptr->e) { - H5_FAILED(); - HDprintf(" i=%d\n", i); - HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", - s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2], - s_ptr->c[3], s_ptr->d, s_ptr->e); - HDprintf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", - d_ptr->a, d_ptr->b, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2], - d_ptr->c[3], d_ptr->d, d_ptr->e); - goto error; - } + s_ptr = ((struct st*)((void *)orig)) + i; + d_ptr = ((struct dt*)((void *)buf)) + i; + if (s_ptr->a != d_ptr->a || + s_ptr->b != d_ptr->b || + s_ptr->c[0] != d_ptr->c[0] || + s_ptr->c[1] != d_ptr->c[1] || + s_ptr->c[2] != d_ptr->c[2] || + s_ptr->c[3] != d_ptr->c[3] || + s_ptr->d != d_ptr->d || + s_ptr->e != d_ptr->e) { + H5_FAILED(); + HDprintf(" i=%d\n", i); + HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", + s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2], + s_ptr->c[3], s_ptr->d, s_ptr->e); + HDprintf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n", + d_ptr->a, d_ptr->b, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2], + d_ptr->c[3], d_ptr->d, d_ptr->e); + goto error; + } } /* Release resources */ @@ -1044,20 +1044,20 @@ error: return 1; } - + /*------------------------------------------------------------------------- - * Function: test_compound_5 + * Function: test_compound_5 * - * Purpose: Many versions of HDF5 have a bug in the optimized compound + * Purpose: Many versions of HDF5 have a bug in the optimized compound * datatype conversion function, H5T_conv_struct_opt(), which * is triggered when the top-level type contains a struct * which must undergo a conversion. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, June 17, 1999 * * Modifications: @@ -1149,19 +1149,19 @@ test_compound_5(void) return retval; } - + /*------------------------------------------------------------------------- - * Function: test_compound_6 + * Function: test_compound_6 * - * Purpose: Tests compound conversions when the destination has the same - * fields as the source but one or more of the fields are - * larger. + * Purpose: Tests compound conversions when the destination has the same + * fields as the source but one or more of the fields are + * larger. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Wednesday, December 13, 2000 * * Modifications: @@ -1181,11 +1181,11 @@ test_compound_6(void) long d; } *d_ptr; - const size_t nelmts = NTESTELEM; - unsigned char *buf=NULL, *orig=NULL, *bkg=NULL; - hid_t st=-1, dt=-1; - int64_t nmembs; - int i; + const size_t nelmts = NTESTELEM; + unsigned char *buf=NULL, *orig=NULL, *bkg=NULL; + hid_t st=-1, dt=-1; + int64_t nmembs; + int i; TESTING("compound element growing"); @@ -1226,18 +1226,18 @@ test_compound_6(void) /* Compare results */ for (i=0; i<(int)nelmts; i++) { - s_ptr = ((struct st*)((void *)orig)) + i; - d_ptr = ((struct dt*)((void *)buf)) + i; - if (s_ptr->b != d_ptr->b || - s_ptr->d != d_ptr->d) { - H5_FAILED(); - HDprintf(" i=%d\n", i); - HDprintf(" src={b=%d, d=%d\n", + s_ptr = ((struct st*)((void *)orig)) + i; + d_ptr = ((struct dt*)((void *)buf)) + i; + if (s_ptr->b != d_ptr->b || + s_ptr->d != d_ptr->d) { + H5_FAILED(); + HDprintf(" i=%d\n", i); + HDprintf(" src={b=%d, d=%d\n", (int)s_ptr->b, (int)s_ptr->d); - HDprintf(" dst={b=%ld, d=%ld\n", + HDprintf(" dst={b=%ld, d=%ld\n", d_ptr->b, d_ptr->d); - goto error; - } + goto error; + } } /* Release resources */ @@ -1264,17 +1264,17 @@ error: } /*------------------------------------------------------------------------- - * Function: test_compound_7 + * Function: test_compound_7 * - * Purpose: Tests inserting fields into compound datatypes when the field + * Purpose: Tests inserting fields into compound datatypes when the field * overlaps the end of the compound datatype. Also, tests * increasing compound type size. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Tuesday, December 18, 2001 * * Modifications: @@ -1410,7 +1410,7 @@ error: return 1; } - + /*------------------------------------------------------------------------- * Function: test_compound_8 * @@ -1656,7 +1656,7 @@ test_compound_8(void) return 1; } - + /*------------------------------------------------------------------------- * Function: test_compound_9 * @@ -1676,14 +1676,20 @@ test_compound_8(void) static int test_compound_9(void) { - typedef struct cmpd_struct { + typedef struct cmpd_struct_w { int i1; const char* str; int i2; - } cmpd_struct; + } cmpd_struct_w; - cmpd_struct wdata = {11, "variable-length string", 22}; - cmpd_struct rdata; + typedef struct cmpd_struct_r { + int i1; + char* str; + int i2; + } cmpd_struct_r; + + cmpd_struct_w wdata = {11, "variable-length string", 22}; + cmpd_struct_r rdata; hid_t file; hid_t cmpd_tid, str_id, dup_tid; hid_t space_id; @@ -1702,13 +1708,13 @@ test_compound_9(void) } /* end if */ /* Create first compound datatype */ - if((cmpd_tid = H5Tcreate( H5T_COMPOUND, sizeof(struct cmpd_struct))) < 0) { + if((cmpd_tid = H5Tcreate( H5T_COMPOUND, sizeof(struct cmpd_struct_w))) < 0) { H5_FAILED(); AT(); HDprintf("Can't create datatype!\n"); goto error; } /* end if */ - if(H5Tinsert(cmpd_tid,"i1",HOFFSET(struct cmpd_struct,i1),H5T_NATIVE_INT) < 0) { + if(H5Tinsert(cmpd_tid, "i1", HOFFSET(struct cmpd_struct_w, i1), H5T_NATIVE_INT) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'i1'\n"); goto error; @@ -1721,13 +1727,13 @@ test_compound_9(void) goto error; } /* end if */ - if(H5Tinsert(cmpd_tid, "vl_string", HOFFSET(cmpd_struct, str), str_id) < 0) { + if(H5Tinsert(cmpd_tid, "vl_string", HOFFSET(cmpd_struct_w, str), str_id) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'i1'\n"); goto error; } /* end if */ - if(H5Tinsert(cmpd_tid, "i2", HOFFSET(struct cmpd_struct, i2), H5T_NATIVE_INT) < 0) { + if(H5Tinsert(cmpd_tid, "i2", HOFFSET(struct cmpd_struct_w, i2), H5T_NATIVE_INT) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'i2'\n"); goto error; @@ -1878,7 +1884,7 @@ test_compound_9(void) return 1; } - + /*------------------------------------------------------------------------- * Function: test_compound_10 * @@ -2072,7 +2078,7 @@ test_compound_10(void) return 1; } - + /*------------------------------------------------------------------------- * Function: test_compound_11 * @@ -2315,7 +2321,7 @@ error: return retval; } - + /*------------------------------------------------------------------------- * Function: test_compound_12 * @@ -2357,14 +2363,14 @@ test_compound_12(void) size+=tmp_size; if (H5Tset_size(complex_id, size) < 0) goto error; if (H5Tinsert(complex_id, "real", offset, - H5T_NATIVE_DOUBLE) < 0) goto error; + H5T_NATIVE_DOUBLE) < 0) goto error; offset = size; if((tmp_size=H5Tget_size(H5T_NATIVE_DOUBLE))==0) goto error; size+=tmp_size; if (H5Tset_size(complex_id, size) < 0) goto error; if (H5Tinsert(complex_id, "imaginary", offset, - H5T_NATIVE_DOUBLE) < 0) goto error; + H5T_NATIVE_DOUBLE) < 0) goto error; /* Increase and decrease the size. */ if((tmp_size=H5Tget_size(H5T_NATIVE_DOUBLE))==0) goto error; @@ -2398,7 +2404,7 @@ test_compound_12(void) return 1; } - + /*------------------------------------------------------------------------- * Function: test_compound_13 * @@ -2500,7 +2506,7 @@ error: return 1; } /* end test_compound_13() */ - + /*------------------------------------------------------------------------- * Function: test_compound_14 * @@ -2523,27 +2529,42 @@ error: static int test_compound_14(void) { - typedef struct cmpd_struct_1 { + typedef struct cmpd_struct_1_w { char c1; char c2; const char* str; - } cmpd_struct_1; + } cmpd_struct_1_w; - typedef struct cmpd_struct_2 { + typedef struct cmpd_struct_1_r { char c1; char c2; char* str; + } cmpd_struct_1_r; + + typedef struct cmpd_struct_2_w { + char c1; + char c2; + const char* str; long l1; long l2; long l3; long l4; - } cmpd_struct_2; + } cmpd_struct_2_w; - cmpd_struct_1 wdata1 = {'A', 'B', "variable-length string"}; + typedef struct cmpd_struct_2_r { + char c1; + char c2; + char* str; + long l1; + long l2; + long l3; + long l4; + } cmpd_struct_2_r; - cmpd_struct_1 rdata1; - cmpd_struct_2 wdata2 = {'C', 'D', "another vlen!", 1, 2, -1, 9001}; - cmpd_struct_2 rdata2; + cmpd_struct_1_w wdata1 = {'A', 'B', "variable-length string"}; + cmpd_struct_1_r rdata1; + cmpd_struct_2_w wdata2 = {'C', 'D', "another vlen!", 1, 2, -1, 9001}; + cmpd_struct_2_r rdata2; hid_t file; hid_t cmpd_m1_tid, cmpd_f1_tid, cmpd_m2_tid, cmpd_f2_tid, str_id; hid_t space_id; @@ -2562,19 +2583,19 @@ test_compound_14(void) } /* end if */ /* Create memory compound datatype 1 */ - if((cmpd_m1_tid = H5Tcreate( H5T_COMPOUND, sizeof(struct cmpd_struct_1))) < 0) { + if((cmpd_m1_tid = H5Tcreate( H5T_COMPOUND, sizeof(struct cmpd_struct_1_w))) < 0) { H5_FAILED(); AT(); HDprintf("Can't create datatype!\n"); goto error; } /* end if */ - if(H5Tinsert(cmpd_m1_tid,"c1",HOFFSET(struct cmpd_struct_1,c1),H5T_NATIVE_CHAR) < 0) { + if(H5Tinsert(cmpd_m1_tid,"c1",HOFFSET(struct cmpd_struct_1_w, c1), H5T_NATIVE_CHAR) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'c1'\n"); goto error; } /* end if */ - if(H5Tinsert(cmpd_m1_tid,"c2",HOFFSET(struct cmpd_struct_1,c2),H5T_NATIVE_CHAR) < 0) { + if(H5Tinsert(cmpd_m1_tid,"c2",HOFFSET(struct cmpd_struct_1_w, c2), H5T_NATIVE_CHAR) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'c2'\n"); goto error; @@ -2587,7 +2608,7 @@ test_compound_14(void) goto error; } /* end if */ - if(H5Tinsert(cmpd_m1_tid, "vl_string", HOFFSET(cmpd_struct_1, str), str_id) < 0) { + if(H5Tinsert(cmpd_m1_tid, "vl_string", HOFFSET(cmpd_struct_1_w, str), str_id) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'vl_string'\n"); goto error; @@ -2619,49 +2640,49 @@ test_compound_14(void) } /* end if */ /* Create memory compound datatype 2 */ - if((cmpd_m2_tid = H5Tcreate( H5T_COMPOUND, sizeof(struct cmpd_struct_2))) < 0) { + if((cmpd_m2_tid = H5Tcreate( H5T_COMPOUND, sizeof(struct cmpd_struct_2_w))) < 0) { H5_FAILED(); AT(); HDprintf("Can't create datatype!\n"); goto error; } /* end if */ - if(H5Tinsert(cmpd_m2_tid,"c1",HOFFSET(struct cmpd_struct_2,c1),H5T_NATIVE_CHAR) < 0) { + if(H5Tinsert(cmpd_m2_tid,"c1",HOFFSET(struct cmpd_struct_2_w, c1), H5T_NATIVE_CHAR) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'c1'\n"); goto error; } /* end if */ - if(H5Tinsert(cmpd_m2_tid,"c2",HOFFSET(struct cmpd_struct_2,c2),H5T_NATIVE_CHAR) < 0) { + if(H5Tinsert(cmpd_m2_tid,"c2",HOFFSET(struct cmpd_struct_2_w, c2), H5T_NATIVE_CHAR) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'c2'\n"); goto error; } /* end if */ - if(H5Tinsert(cmpd_m2_tid, "vl_string", HOFFSET(cmpd_struct_2, str), str_id) < 0) { + if(H5Tinsert(cmpd_m2_tid, "vl_string", HOFFSET(cmpd_struct_2_w, str), str_id) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'vl_string'\n"); goto error; } /* end if */ - if(H5Tinsert(cmpd_m2_tid,"l1",HOFFSET(struct cmpd_struct_2,l1),H5T_NATIVE_LONG) < 0) { + if(H5Tinsert(cmpd_m2_tid,"l1",HOFFSET(struct cmpd_struct_2_w, l1), H5T_NATIVE_LONG) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'l1'\n"); goto error; } /* end if */ - if(H5Tinsert(cmpd_m2_tid,"l2",HOFFSET(struct cmpd_struct_2,l2),H5T_NATIVE_LONG) < 0) { + if(H5Tinsert(cmpd_m2_tid,"l2",HOFFSET(struct cmpd_struct_2_w, l2), H5T_NATIVE_LONG) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'l2'\n"); goto error; } /* end if */ - if(H5Tinsert(cmpd_m2_tid,"l3",HOFFSET(struct cmpd_struct_2,l3),H5T_NATIVE_LONG) < 0) { + if(H5Tinsert(cmpd_m2_tid,"l3",HOFFSET(struct cmpd_struct_2_w, l3), H5T_NATIVE_LONG) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'l3'\n"); goto error; } /* end if */ - if(H5Tinsert(cmpd_m2_tid,"l4",HOFFSET(struct cmpd_struct_2,l4),H5T_NATIVE_LONG) < 0) { + if(H5Tinsert(cmpd_m2_tid,"l4",HOFFSET(struct cmpd_struct_2_w, l4), H5T_NATIVE_LONG) < 0) { H5_FAILED(); AT(); HDprintf("Can't insert field 'l4'\n"); goto error; @@ -2892,7 +2913,7 @@ test_compound_14(void) return 1; } /* end test_compound_14() */ - + /*------------------------------------------------------------------------- * Function: test_compound_15 * @@ -3051,7 +3072,7 @@ test_compound_15(void) return 1; } /* end test_compound_15() */ - + /*------------------------------------------------------------------------- * Function: test_compound_16 * @@ -3143,7 +3164,7 @@ error: return 1; } /* end test_compound_16() */ - + /*------------------------------------------------------------------------- * Function: test_compound_17 * @@ -3266,7 +3287,7 @@ error: return 1; } /* end test_compound_17() */ - + /*------------------------------------------------------------------------- * Function: test_compound_18 * @@ -3332,7 +3353,7 @@ test_compound_18(void) FAIL_PUTS_ERROR("created attribute with bad compound datatype") } /* end if */ - /* Commit the datatype */ + /* Commit the datatype */ H5E_BEGIN_TRY { ret = H5Tcommit2(file, "cmpnd", tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); } H5E_END_TRY; @@ -3392,7 +3413,7 @@ error: return 1; } /* end test_compound_18() */ - + /*------------------------------------------------------------------------- * Function: test_query * @@ -3638,17 +3659,17 @@ test_query(void) return 1; } - + /*------------------------------------------------------------------------- - * Function: test_transient + * Function: test_transient * - * Purpose: Tests transient datatypes. + * Purpose: Tests transient datatypes. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, June 4, 1998 * * Modifications: @@ -3658,36 +3679,36 @@ test_query(void) static int test_transient (hid_t fapl) { - static hsize_t ds_size[2] = {10, 20}; - hid_t file=-1, type=-1, space=-1, dset=-1, t2=-1; - char filename[1024]; - hid_t ret_id; /* Generic hid_t return value */ - herr_t status; + static hsize_t ds_size[2] = {10, 20}; + hid_t file=-1, type=-1, space=-1, dset=-1, t2=-1; + char filename[1024]; + hid_t ret_id; /* Generic hid_t return value */ + herr_t status; TESTING("transient datatypes"); h5_fixname(FILENAME[0], fapl, filename, sizeof filename); if ((file=H5Fcreate (filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) { - goto error; + goto error; } if ((space = H5Screate_simple (2, ds_size, ds_size)) < 0) goto error; /* Predefined types cannot be modified or closed */ H5E_BEGIN_TRY { - status = H5Tset_precision(H5T_NATIVE_INT, (size_t)256); + status = H5Tset_precision(H5T_NATIVE_INT, (size_t)256); } H5E_END_TRY; if (status>=0) { - H5_FAILED(); - HDputs (" Predefined types should not be modifiable!"); - goto error; + H5_FAILED(); + HDputs (" Predefined types should not be modifiable!"); + goto error; } H5E_BEGIN_TRY { - status = H5Tclose (H5T_NATIVE_INT); + status = H5Tclose (H5T_NATIVE_INT); } H5E_END_TRY; if (status>=0) { - H5_FAILED(); - HDputs (" Predefined types should not be closable!"); - goto error; + H5_FAILED(); + HDputs (" Predefined types should not be closable!"); + goto error; } /* Copying a predefined type results in a modifiable copy */ @@ -3696,29 +3717,29 @@ test_transient (hid_t fapl) /* It should not be possible to create an attribute for a transient type */ H5E_BEGIN_TRY { - ret_id = H5Acreate2(type, "attr1", H5T_NATIVE_INT, space, H5P_DEFAULT, H5P_DEFAULT); + ret_id = H5Acreate2(type, "attr1", H5T_NATIVE_INT, space, H5P_DEFAULT, H5P_DEFAULT); } H5E_END_TRY; if (ret_id>=0) { - H5_FAILED(); - HDputs (" Attributes should not be allowed for transient types!"); - goto error; + H5_FAILED(); + HDputs (" Attributes should not be allowed for transient types!"); + goto error; } /* Create a dataset from a transient datatype */ if(H5Tclose(type) < 0) goto error; if((type = H5Tcopy(H5T_NATIVE_INT)) < 0) goto error; if((dset = H5Dcreate2(file, "dset1", type, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) - goto error; + goto error; /* The type returned from a dataset should not be modifiable */ if((t2 = H5Dget_type(dset)) < 0) goto error; H5E_BEGIN_TRY { - status = H5Tset_precision(t2, (size_t)256); + status = H5Tset_precision(t2, (size_t)256); } H5E_END_TRY; if(status >= 0) { - H5_FAILED(); - HDputs (" Dataset datatypes should not be modifiable!"); - goto error; + H5_FAILED(); + HDputs (" Dataset datatypes should not be modifiable!"); + goto error; } if(H5Tclose(t2) < 0) goto error; @@ -3730,12 +3751,12 @@ test_transient (hid_t fapl) if((dset = H5Dopen2(file, "dset1", H5P_DEFAULT)) < 0) goto error; if((t2 = H5Dget_type(dset)) < 0) goto error; H5E_BEGIN_TRY { - status = H5Tset_precision(t2, (size_t)256); + status = H5Tset_precision(t2, (size_t)256); } H5E_END_TRY; if(status >= 0) { - H5_FAILED(); - HDputs (" Dataset datatypes should not be modifiable!"); - goto error; + H5_FAILED(); + HDputs (" Dataset datatypes should not be modifiable!"); + goto error; } if(H5Tclose(t2) < 0) goto error; @@ -3758,26 +3779,26 @@ test_transient (hid_t fapl) error: H5E_BEGIN_TRY { - H5Tclose (t2); - H5Tclose (type); - H5Sclose (space); - H5Dclose (dset); - H5Fclose (file); + H5Tclose (t2); + H5Tclose (type); + H5Sclose (space); + H5Dclose (dset); + H5Fclose (file); } H5E_END_TRY; return 1; } - + /*------------------------------------------------------------------------- - * Function: test_named + * Function: test_named * - * Purpose: Tests named datatypes. + * Purpose: Tests named datatypes. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, June 1, 1998 * * Modifications: @@ -3787,29 +3808,29 @@ test_transient (hid_t fapl) static int test_named (hid_t fapl) { - hid_t file=-1, type=-1, space=-1, dset=-1, t2=-1, t3=-1, attr1=-1; - herr_t status; - static hsize_t ds_size[2] = {10, 20}; - size_t i,j; - unsigned attr_data[10][20]; - char filename[1024]; + hid_t file=-1, type=-1, space=-1, dset=-1, t2=-1, t3=-1, attr1=-1; + herr_t status; + static hsize_t ds_size[2] = {10, 20}; + size_t i,j; + unsigned attr_data[10][20]; + char filename[1024]; TESTING("named datatypes"); h5_fixname(FILENAME[1], fapl, filename, sizeof filename); if ((file=H5Fcreate (filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) { - goto error; + goto error; } if ((space = H5Screate_simple (2, ds_size, ds_size)) < 0) goto error; /* Predefined types cannot be committed */ H5E_BEGIN_TRY { - status = H5Tcommit2(file, "test_named_1 (should not exist)", H5T_NATIVE_INT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + status = H5Tcommit2(file, "test_named_1 (should not exist)", H5T_NATIVE_INT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); } H5E_END_TRY; if(status >= 0) { - H5_FAILED(); - HDputs (" Predefined types should not be committable!"); - goto error; + H5_FAILED(); + HDputs (" Predefined types should not be committable!"); + goto error; } /* Copy a predefined datatype and commit the copy */ @@ -3817,34 +3838,34 @@ test_named (hid_t fapl) if(H5Tcommit2(file, "native-int", type, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) goto error; if((status = H5Tcommitted(type)) < 0) goto error; if(0 == status) { - H5_FAILED(); - HDputs (" H5Tcommitted() returned false!"); - goto error; + H5_FAILED(); + HDputs (" H5Tcommitted() returned false!"); + goto error; } /* We should not be able to modify a type after it has been committed. */ H5E_BEGIN_TRY { - status = H5Tset_precision (type, (size_t)256); + status = H5Tset_precision (type, (size_t)256); } H5E_END_TRY; if (status>=0) { - H5_FAILED(); - HDputs (" Committed type is not constant!"); - goto error; + H5_FAILED(); + HDputs (" Committed type is not constant!"); + goto error; } /* We should not be able to re-commit a committed type */ H5E_BEGIN_TRY { - status = H5Tcommit2(file, "test_named_2 (should not exist)", type, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + status = H5Tcommit2(file, "test_named_2 (should not exist)", type, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); } H5E_END_TRY; if(status >= 0) { - H5_FAILED(); - HDputs (" Committed types should not be recommitted!"); - goto error; + H5_FAILED(); + HDputs (" Committed types should not be recommitted!"); + goto error; } /* It should be possible to define an attribute for the named type */ if((attr1 = H5Acreate2(type, "attr1", H5T_NATIVE_UCHAR, space, - H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; + H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; for(i = 0; i < (size_t)ds_size[0]; i++) for(j = 0; j < (size_t)ds_size[1]; j++) attr_data[i][j] = (unsigned)(i * ds_size[1] + j); @@ -3858,9 +3879,9 @@ test_named (hid_t fapl) if((t2 = H5Tcopy(type)) < 0) goto error; if((status = H5Tcommitted(t2)) < 0) goto error; if(status) { - H5_FAILED(); - HDputs (" Copying a named type should result in a transient type!"); - goto error; + H5_FAILED(); + HDputs (" Copying a named type should result in a transient type!"); + goto error; } if(H5Tset_precision(t2, (size_t)256) < 0) goto error; if(H5Tclose(t2) < 0) goto error; @@ -3873,22 +3894,22 @@ test_named (hid_t fapl) FAIL_STACK_ERROR if((status = H5Tcommitted(type)) < 0) goto error; if(!status) { - H5_FAILED(); - HDputs (" Opened named types should be named types!"); - goto error; + H5_FAILED(); + HDputs (" Opened named types should be named types!"); + goto error; } /* Create a dataset that uses the named type */ if((dset = H5Dcreate2(file, "dset1", type, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) - goto error; + goto error; /* Get the dataset's datatype and make sure it's a named type */ if((t2 = H5Dget_type(dset)) < 0) goto error; if((status = H5Tcommitted(t2)) < 0) goto error; if(!status) { - H5_FAILED(); - HDputs (" Dataset type should be a named type!"); - goto error; + H5_FAILED(); + HDputs (" Dataset type should be a named type!"); + goto error; } /* Close the dataset, then close its type, then reopen the dataset */ @@ -3900,9 +3921,9 @@ test_named (hid_t fapl) if((t2 = H5Dget_type(dset)) < 0) goto error; if((status = H5Tcommitted(t2)) < 0) goto error; if(!status) { - H5_FAILED(); - HDputs (" Dataset type should be a named type!"); - goto error; + H5_FAILED(); + HDputs (" Dataset type should be a named type!"); + goto error; } /* @@ -3919,9 +3940,9 @@ test_named (hid_t fapl) if((t2 = H5Dget_type(dset)) < 0) goto error; if((status = H5Tcommitted(t2)) < 0) goto error; if(!status) { - H5_FAILED(); - HDputs (" Dataset type should be a named type!"); - goto error; + H5_FAILED(); + HDputs (" Dataset type should be a named type!"); + goto error; } if(H5Tclose(t2) < 0) goto error; @@ -3940,17 +3961,17 @@ test_named (hid_t fapl) if((t2 = H5Tcopy(type)) < 0) goto error; if((status = H5Tcommitted(t2)) < 0) goto error; if(status) { - H5_FAILED(); - HDputs (" Copied type should not be a named type!"); - goto error; + H5_FAILED(); + HDputs (" Copied type should not be a named type!"); + goto error; } if((dset = H5Dcreate2(file, "dset3", t2, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error; if((t3 = H5Dget_type(dset)) < 0) goto error; if((status = H5Tcommitted(t3)) < 0) goto error; if(status) { - H5_FAILED(); - HDputs (" Datatype from dataset using copied type should not be a named type!"); - goto error; + H5_FAILED(); + HDputs (" Datatype from dataset using copied type should not be a named type!"); + goto error; } if(H5Tclose(t3) < 0) goto error; if(H5Dclose(dset) < 0) goto error; @@ -3995,26 +4016,26 @@ test_named (hid_t fapl) error: H5E_BEGIN_TRY { - H5Tclose(t3); - H5Tclose(t2); - H5Tclose(type); - H5Sclose(space); - H5Dclose(dset); - H5Fclose(file); + H5Tclose(t3); + H5Tclose(t2); + H5Tclose(type); + H5Sclose(space); + H5Dclose(dset); + H5Fclose(file); } H5E_END_TRY; return 1; } - + /*------------------------------------------------------------------------- - * Function: mkstr + * Function: mkstr * - * Purpose: Create a new string datatype + * Purpose: Create a new string datatype * - * Return: Success: New type - * Failure: -1 + * Return: Success: New type + * Failure: -1 * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, August 10, 1998 * *------------------------------------------------------------------------- @@ -4022,7 +4043,7 @@ error: static hid_t mkstr(size_t len, H5T_str_t strpad) { - hid_t t; + hid_t t; if((t = H5Tcopy(H5T_C_S1)) < 0) return -1; @@ -4034,16 +4055,16 @@ mkstr(size_t len, H5T_str_t strpad) return t; } - + /*------------------------------------------------------------------------- - * Function: test_str_create + * Function: test_str_create * - * Purpose: Test string type creation using H5Tcreate + * Purpose: Test string type creation using H5Tcreate * - * Return: Success: 0 - * Failure: number of errors + * Return: Success: 0 + * Failure: number of errors * - * Programmer: Raymond Lu + * Programmer: Raymond Lu * 19 May 2011 * *------------------------------------------------------------------------- @@ -4084,10 +4105,10 @@ test_str_create(void) if(!H5Tequal(vlen_str1, vlen_str2)) goto error; if((is_vl_str = H5Tis_variable_str(vlen_str1)) < 0) goto error; - if(!is_vl_str) goto error; + if(!is_vl_str) goto error; if((is_vl_str = H5Tis_variable_str(vlen_str2)) < 0) goto error; - if(!is_vl_str) goto error; + if(!is_vl_str) goto error; if(H5Tclose(vlen_str1) < 0) goto error; if(H5Tclose(vlen_str2) < 0) goto error; @@ -4100,16 +4121,16 @@ error: return 1; } - + /*------------------------------------------------------------------------- - * Function: test_conv_str_1 + * Function: test_conv_str_1 * - * Purpose: Test string conversions + * Purpose: Test string conversions * - * Return: Success: 0 - * Failure: number of errors + * Return: Success: 0 + * Failure: number of errors * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, August 10, 1998 * *------------------------------------------------------------------------- @@ -4117,8 +4138,8 @@ error: static int test_conv_str_1(void) { - char *buf = NULL; - hid_t src_type = -1; + char *buf = NULL; + hid_t src_type = -1; hid_t dst_type = -1; TESTING("string conversions"); @@ -4133,15 +4154,15 @@ test_conv_str_1(void) HDmemcpy(buf, "abcdefghi\0abcdefghi\0", (size_t)20); if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcd\0abcd\0abcdefghi\0", (size_t)20)) { - H5_FAILED(); - HDputs(" Truncated C-string test failed"); - goto error; + H5_FAILED(); + HDputs(" Truncated C-string test failed"); + goto error; } if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcd\0\0\0\0\0\0abcd\0\0\0\0\0\0", (size_t)20)) { - H5_FAILED(); - HDputs(" Extended C-string test failed"); - goto error; + H5_FAILED(); + HDputs(" Extended C-string test failed"); + goto error; } HDfree(buf); buf = NULL; @@ -4157,15 +4178,15 @@ test_conv_str_1(void) HDmemcpy(buf, "abcdefghijabcdefghij", (size_t)20); if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcdeabcdeabcdefghij", (size_t)20)) { - H5_FAILED(); - HDputs(" Truncated C buffer test failed"); - goto error; + H5_FAILED(); + HDputs(" Truncated C buffer test failed"); + goto error; } if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcde\0\0\0\0\0abcde\0\0\0\0\0", (size_t)20)) { - H5_FAILED(); - HDputs(" Extended C buffer test failed"); - goto error; + H5_FAILED(); + HDputs(" Extended C buffer test failed"); + goto error; } HDfree(buf); buf = NULL; @@ -4181,15 +4202,15 @@ test_conv_str_1(void) HDmemcpy(buf, "abcdefghijabcdefghij", (size_t)20); if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcdeabcdeabcdefghij", (size_t)20)) { - H5_FAILED(); - HDputs(" Truncated Fortran-string test failed"); - goto error; + H5_FAILED(); + HDputs(" Truncated Fortran-string test failed"); + goto error; } if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcde abcde ", (size_t)20)) { - H5_FAILED(); - HDputs(" Extended Fortran-string test failed"); - goto error; + H5_FAILED(); + HDputs(" Extended Fortran-string test failed"); + goto error; } HDfree(buf); buf = NULL; @@ -4208,25 +4229,25 @@ test_conv_str_1(void) HDmemcpy(buf, "abcdefghijabcdefghij", (size_t)20); if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcdefghijabcdefghij", (size_t)20)) { - H5_FAILED(); - HDputs(" Non-terminated string test 1"); - goto error; + H5_FAILED(); + HDputs(" Non-terminated string test 1"); + goto error; } H5Tclose(dst_type); if((dst_type = mkstr((size_t)5, H5T_STR_NULLTERM)) < 0) goto error; HDmemcpy(buf, "abcdefghijabcdefghij", (size_t)20); if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcd\0abcd\0abcdefghij", (size_t)20)) { - H5_FAILED(); - HDputs(" Non-terminated string test 2"); - goto error; + H5_FAILED(); + HDputs(" Non-terminated string test 2"); + goto error; } HDmemcpy(buf, "abcdeabcdexxxxxxxxxx", (size_t)20); if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcde\0\0\0\0\0abcde\0\0\0\0\0", (size_t)20)) { - H5_FAILED(); - HDputs(" Non-terminated string test 2"); - goto error; + H5_FAILED(); + HDputs(" Non-terminated string test 2"); + goto error; } HDfree(buf); buf = NULL; @@ -4242,30 +4263,30 @@ test_conv_str_1(void) HDmemcpy(buf, "abcdefghi\0abcdefghi\0", (size_t)20); if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcdefghi abcdefghi ", (size_t)20)) { - H5_FAILED(); - HDputs(" C string to Fortran test 1"); - goto error; + H5_FAILED(); + HDputs(" C string to Fortran test 1"); + goto error; } if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcdefghi\0abcdefghi\0", (size_t)20)) { - H5_FAILED(); - HDputs(" Fortran to C string test 1"); - goto error; + H5_FAILED(); + HDputs(" Fortran to C string test 1"); + goto error; } if (H5Tclose(dst_type) < 0) goto error; if((dst_type = mkstr((size_t)5, H5T_STR_SPACEPAD)) < 0) goto error; HDmemcpy(buf, "abcdefgh\0\0abcdefgh\0\0", (size_t)20); if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcdeabcdeabcdefgh\0\0", (size_t)20)) { - H5_FAILED(); - HDputs(" C string to Fortran test 2"); - goto error; + H5_FAILED(); + HDputs(" C string to Fortran test 2"); + goto error; } if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcde\0\0\0\0\0abcde\0\0\0\0\0", (size_t)20)) { - H5_FAILED(); - HDputs(" Fortran to C string test 2"); - goto error; + H5_FAILED(); + HDputs(" Fortran to C string test 2"); + goto error; } if (H5Tclose(src_type) < 0) goto error; if (H5Tclose(dst_type) < 0) goto error; @@ -4274,15 +4295,15 @@ test_conv_str_1(void) HDmemcpy(buf, "abcd\0abcd\0xxxxxxxxxx", (size_t)20); if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcd abcd ", (size_t)20)) { - H5_FAILED(); - HDputs(" C string to Fortran test 3"); - goto error; + H5_FAILED(); + HDputs(" C string to Fortran test 3"); + goto error; } if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcd\0abcd\0abcd ", (size_t)20)) { - H5_FAILED(); - HDputs(" Fortran to C string test 3"); - goto error; + H5_FAILED(); + HDputs(" Fortran to C string test 3"); + goto error; } HDfree(buf); buf = NULL; @@ -4298,30 +4319,30 @@ test_conv_str_1(void) HDmemcpy(buf, "abcdefghijabcdefghij", (size_t)20); if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcdefghijabcdefghij", (size_t)20)) { - H5_FAILED(); - HDputs(" C buffer to Fortran test 1"); - goto error; + H5_FAILED(); + HDputs(" C buffer to Fortran test 1"); + goto error; } if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcdefghijabcdefghij", (size_t)20)) { - H5_FAILED(); - HDputs(" Fortran to C buffer test 1"); - goto error; + H5_FAILED(); + HDputs(" Fortran to C buffer test 1"); + goto error; } if (H5Tclose(dst_type) < 0) goto error; if((dst_type = mkstr((size_t)5, H5T_STR_SPACEPAD)) < 0) goto error; HDmemcpy(buf, "abcdefgh\0\0abcdefgh\0\0", (size_t)20); if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcdeabcdeabcdefgh\0\0", (size_t)20)) { - H5_FAILED(); - HDputs(" C buffer to Fortran test 2"); - goto error; + H5_FAILED(); + HDputs(" C buffer to Fortran test 2"); + goto error; } if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcde\0\0\0\0\0abcde\0\0\0\0\0", (size_t)20)) { - H5_FAILED(); - HDputs(" Fortran to C buffer test 2"); - goto error; + H5_FAILED(); + HDputs(" Fortran to C buffer test 2"); + goto error; } if (H5Tclose(src_type) < 0) goto error; if (H5Tclose(dst_type) < 0) goto error; @@ -4330,15 +4351,15 @@ test_conv_str_1(void) HDmemcpy(buf, "abcd\0abcd\0xxxxxxxxxx", (size_t)20); if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcd abcd ", (size_t)20)) { - H5_FAILED(); - HDputs(" C buffer to Fortran test 3"); - goto error; + H5_FAILED(); + HDputs(" C buffer to Fortran test 3"); + goto error; } if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error; if (HDmemcmp(buf, "abcd\0abcd\0abcd ", (size_t)20)) { - H5_FAILED(); - HDputs(" Fortran to C buffer test 3"); - goto error; + H5_FAILED(); + HDputs(" Fortran to C buffer test 3"); + goto error; } if(H5Tclose(src_type) < 0) goto error; if(H5Tclose(dst_type) < 0) goto error; @@ -4354,7 +4375,7 @@ test_conv_str_1(void) error: H5E_BEGIN_TRY { - H5Tclose(src_type); + H5Tclose(src_type); H5Tclose(dst_type); } H5E_END_TRY; @@ -4369,16 +4390,16 @@ error: return 1; } - + /*------------------------------------------------------------------------- - * Function: test_conv_str_2 + * Function: test_conv_str_2 * - * Purpose: Tests C-to-Fortran and Fortran-to-C string conversion speed. + * Purpose: Tests C-to-Fortran and Fortran-to-C string conversion speed. * - * Return: Success: 0 - * Failure: number of errors + * Return: Success: 0 + * Failure: number of errors * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Monday, August 10, 1998 * *------------------------------------------------------------------------- @@ -4386,12 +4407,12 @@ error: static int test_conv_str_2(void) { - char *buf = NULL, s[80]; - hid_t c_type = -1; - hid_t f_type = -1; - const size_t nelmts = NTESTELEM; - size_t i, j, nchars; - int ret_value = 1; + char *buf = NULL, s[80]; + hid_t c_type = -1; + hid_t f_type = -1; + const size_t nelmts = NTESTELEM; + size_t i, j, nchars; + int ret_value = 1; /* * Initialize types and buffer. @@ -4400,10 +4421,10 @@ test_conv_str_2(void) if((f_type = mkstr((size_t)8, H5T_STR_SPACEPAD)) < 0) goto error; if(NULL == (buf = (char*)HDcalloc(nelmts, (size_t)8))) goto error; for(i = 0; i < nelmts; i++) { - nchars = (size_t)(HDrand() % 8); - for(j = 0; j < nchars; j++) - buf[i * 8 + j] = (char)('a' + HDrand() % 26); - while(j < nchars) + nchars = (size_t)(HDrand() % 8); + for(j = 0; j < nchars; j++) + buf[i * 8 + j] = (char)('a' + HDrand() % 26); + while(j < nchars) buf[i * 8 + j++] = '\0'; } /* end for */ @@ -4436,17 +4457,17 @@ error: return ret_value; } - + /*------------------------------------------------------------------------- - * Function: test_conv_str_3 + * Function: test_conv_str_3 * - * Purpose: Tests some functions that are or aren't supposed to work + * Purpose: Tests some functions that are or aren't supposed to work * for string type. * - * Return: Success: 0 - * Failure: number of errors + * Return: Success: 0 + * Failure: number of errors * - * Programmer: Raymond Lu + * Programmer: Raymond Lu * Tuesday, April 4, 2006 * *------------------------------------------------------------------------- @@ -4454,12 +4475,12 @@ error: static int test_conv_str_3(void) { - char *buf=NULL; - hid_t type = -1; - hid_t super = -1; - const size_t nelmts = NTESTELEM; - size_t i, j, nchars; - int ret_value = 1; + char *buf=NULL; + hid_t type = -1; + hid_t super = -1; + const size_t nelmts = NTESTELEM; + size_t i, j, nchars; + int ret_value = 1; size_t size; H5T_pad_t inpad; H5T_sign_t sign; @@ -4475,10 +4496,10 @@ test_conv_str_3(void) if(NULL == (buf = (char*)HDcalloc(nelmts, (size_t)8))) FAIL_PUTS_ERROR("Allocation failed."); for(i = 0; i < nelmts; i++) { - nchars = (size_t)(HDrand() % 8); - for(j = 0; j < nchars; j++) - buf[i * 8 + j] = (char)('a' + HDrand() % 26); - while(j < nchars) + nchars = (size_t)(HDrand() % 8); + for(j = 0; j < nchars; j++) + buf[i * 8 + j] = (char)('a' + HDrand() % 26); + while(j < nchars) buf[i * 8 + j++] = '\0'; } /* end for */ @@ -4541,7 +4562,7 @@ error: H5Tclose(super); } H5E_END_TRY; - if(buf) + if(buf) HDfree(buf); if(tag) H5free_memory(tag); /* Technically allocated by API call */ @@ -4554,17 +4575,17 @@ error: return ret_value; /* Number of errors */ } - + /*------------------------------------------------------------------------- - * Function: test_conv_enum_1 + * Function: test_conv_enum_1 * - * Purpose: Test conversion speed for enum datatypes + * Purpose: Test conversion speed for enum datatypes * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Tuesday, January 5, 1999 * * Modifications: @@ -4575,11 +4596,11 @@ static int test_conv_enum_1(void) { const size_t nelmts=NTESTELEM; - int i, val, *buf=NULL; - hid_t t1 = -1; - hid_t t2 = -1; - char s[80]; - int ret_value = 1; + int i, val, *buf=NULL; + hid_t t1 = -1; + hid_t t2 = -1; + char s[80]; + int ret_value = 1; size_t u; /* Build the datatypes */ @@ -4587,9 +4608,9 @@ test_conv_enum_1(void) if((t2 = H5Tenum_create(H5T_NATIVE_INT)) < 0) goto error; s[1] = '\0'; for(i = 0; i < 26; i++) { - s[0] = (char)('A' + i); - H5Tenum_insert(t1, s, &i); - H5Tenum_insert(t2, s, (val = i * 1000 + i, &val)); + s[0] = (char)('A' + i); + H5Tenum_insert(t1, s, &i); + H5Tenum_insert(t2, s, (val = i * 1000 + i, &val)); } /* end for */ /* Initialize the buffer */ @@ -4630,7 +4651,7 @@ error: return ret_value; } - + /*------------------------------------------------------------------------- * Function: test_conv_enum_2 * @@ -4656,7 +4677,7 @@ test_conv_enum_2(void) "PURPLE", "ORANGE", "WHITE" }; - + TESTING("non-native enumeration type conversion"); /* Source enum type */ @@ -4715,17 +4736,17 @@ test_conv_enum_2(void) return 0; } - + /*------------------------------------------------------------------------- - * Function: test_conv_bitfield + * Function: test_conv_bitfield * - * Purpose: Test bitfield conversions. + * Purpose: Test bitfield conversions. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, May 20, 1999 * * Modifications: @@ -4735,8 +4756,8 @@ test_conv_enum_2(void) static int test_conv_bitfield(void) { - unsigned char buf[4]; - hid_t st=-1, dt=-1; + unsigned char buf[4]; + hid_t st=-1, dt=-1; TESTING("bitfield conversions"); @@ -4751,10 +4772,10 @@ test_conv_bitfield(void) buf[2] = buf[3] = 0x55; /*irrelevant*/ if (H5Tconvert(st, dt, (size_t)1, buf, NULL, H5P_DEFAULT) < 0) goto error; if (buf[0]!=0xAA || buf[1]!=0xAA || buf[2]!=0 || buf[3]!=0) { - H5_FAILED(); - HDprintf(" s=0xaaaa, d=0x%02x%02x%02x%02x (test 1)\n", - buf[3], buf[2], buf[1], buf[0]); - goto error; + H5_FAILED(); + HDprintf(" s=0xaaaa, d=0x%02x%02x%02x%02x (test 1)\n", + buf[3], buf[2], buf[1], buf[0]); + goto error; } /* @@ -4770,10 +4791,10 @@ test_conv_bitfield(void) buf[0] = 0xA8; buf[1] = 0x2A; buf[2] = buf[3] = 0; if (H5Tconvert(st, dt, (size_t)1, buf, NULL, H5P_DEFAULT) < 0) goto error; if (buf[0]!=0 || buf[1]!=0xA8 || buf[2]!=0x2A || buf[3]!=0) { - H5_FAILED(); - HDprintf(" s=0x2AA8 d=0x%02x%02x%02x%02x (test 2)\n", - buf[3], buf[2], buf[1], buf[0]); - goto error; + H5_FAILED(); + HDprintf(" s=0x2AA8 d=0x%02x%02x%02x%02x (test 2)\n", + buf[3], buf[2], buf[1], buf[0]); + goto error; } /* @@ -4784,10 +4805,10 @@ test_conv_bitfield(void) buf[0] = 0xA8; buf[1] = 0x2A; buf[2] = buf[3] = 0; if (H5Tconvert(st, dt, (size_t)1, buf, NULL, H5P_DEFAULT) < 0) goto error; if (buf[0]!=0xff || buf[1]!=0xAB || buf[2]!=0xEA || buf[3]!=0xff) { - H5_FAILED(); - HDprintf(" s=0x2AA8 d=0x%02x%02x%02x%02x (test 3)\n", - buf[3], buf[2], buf[1], buf[0]); - goto error; + H5_FAILED(); + HDprintf(" s=0x2AA8 d=0x%02x%02x%02x%02x (test 3)\n", + buf[3], buf[2], buf[1], buf[0]); + goto error; } H5Tclose(st); @@ -4812,18 +4833,18 @@ error: return 1; } - + /*------------------------------------------------------------------------- - * Function: test_bitfield_funcs + * Function: test_bitfield_funcs * - * Purpose: Test some datatype functions that are and aren't supposed + * Purpose: Test some datatype functions that are and aren't supposed * work for bitfield type. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Raymond Lu + * Programmer: Raymond Lu * Wednesday, April 5, 2006 * * Modifications: @@ -4833,7 +4854,7 @@ error: static int test_bitfield_funcs(void) { - hid_t type=-1, ntype=-1, super=-1; + hid_t type=-1, ntype=-1, super=-1; size_t size; char* tag=0; H5T_pad_t inpad; @@ -4941,17 +4962,17 @@ error: return retval; } - + /*------------------------------------------------------------------------- - * Function: convert_opaque + * Function: convert_opaque * - * Purpose: A fake opaque conversion functions + * Purpose: A fake opaque conversion functions * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: -1 + * Failure: -1 * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Friday, June 4, 1999 * * Modifications: @@ -4960,25 +4981,25 @@ error: */ static herr_t convert_opaque(hid_t H5_ATTR_UNUSED st, hid_t H5_ATTR_UNUSED dt, H5T_cdata_t *cdata, - size_t H5_ATTR_UNUSED nelmts, size_t H5_ATTR_UNUSED buf_stride, + size_t H5_ATTR_UNUSED nelmts, size_t H5_ATTR_UNUSED buf_stride, size_t H5_ATTR_UNUSED bkg_stride, void H5_ATTR_UNUSED *_buf, - void H5_ATTR_UNUSED *bkg, hid_t H5_ATTR_UNUSED dset_xfer_plid) + void H5_ATTR_UNUSED *bkg, hid_t H5_ATTR_UNUSED dset_xfer_plid) { if (H5T_CONV_CONV==cdata->command) num_opaque_conversions_g++; return 0; } - + /*------------------------------------------------------------------------- - * Function: test_opaque + * Function: test_opaque * - * Purpose: Driver function to test opaque datatypes + * Purpose: Driver function to test opaque datatypes * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Raymond Lu + * Programmer: Raymond Lu * June 2, 2004 * * Modifications: @@ -5011,17 +5032,17 @@ test_opaque(void) return num_errors; } - + /*------------------------------------------------------------------------- - * Function: opaque_check + * Function: opaque_check * - * Purpose: Test opaque datatypes + * Purpose: Test opaque datatypes * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, May 20, 1999 * *------------------------------------------------------------------------- @@ -5030,10 +5051,10 @@ static int opaque_check(int tag_it) { #define OPAQUE_NELMTS 1000 - hid_t st=-1, dt=-1; - herr_t status; - char buf[1]; /*not really used*/ - int saved; + hid_t st=-1, dt=-1; + herr_t status; + char buf[1]; /*not really used*/ + int saved; saved = num_opaque_conversions_g = 0; @@ -5049,29 +5070,29 @@ opaque_check(int tag_it) /* Make sure that we can't convert between the types yet */ H5E_BEGIN_TRY { - status = H5Tconvert(st, dt, (size_t)OPAQUE_NELMTS, buf, NULL, H5P_DEFAULT); + status = H5Tconvert(st, dt, (size_t)OPAQUE_NELMTS, buf, NULL, H5P_DEFAULT); } H5E_END_TRY; if (status>=0) { - H5_FAILED(); - HDprintf(" opaque conversion should have failed but succeeded\n"); - goto error; + H5_FAILED(); + HDprintf(" opaque conversion should have failed but succeeded\n"); + goto error; } /* Register a conversion function */ if (H5Tregister(H5T_PERS_HARD, "o_test", st, dt, convert_opaque) < 0) - goto error; + goto error; /* Try the conversion again, this time it should work */ if (H5Tconvert(st, dt, (size_t)OPAQUE_NELMTS, buf, NULL, H5P_DEFAULT) < 0) goto error; if (saved+1 != num_opaque_conversions_g) { - H5_FAILED(); - HDprintf(" unexpected number of opaque conversions\n"); - goto error; + H5_FAILED(); + HDprintf(" unexpected number of opaque conversions\n"); + goto error; } /* Unregister conversion function */ if (H5Tunregister(H5T_PERS_HARD, "o_test", st, dt, convert_opaque) < 0) - goto error; + goto error; H5Tclose(st); H5Tclose(dt); @@ -5084,16 +5105,16 @@ opaque_check(int tag_it) return 1; } - + /*------------------------------------------------------------------------- - * Function: opaque_long + * Function: opaque_long * - * Purpose: Test named (committed) opaque datatypes w/very long tags + * Purpose: Test named (committed) opaque datatypes w/very long tags * - * Return: Success: 0 - * Failure: number of errors + * Return: Success: 0 + * Failure: number of errors * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Tuesday, June 14, 2005 * *------------------------------------------------------------------------- @@ -5101,8 +5122,8 @@ opaque_check(int tag_it) static int opaque_long(void) { - char *long_tag = NULL; - hid_t dt = -1; + char *long_tag = NULL; + hid_t dt = -1; herr_t ret; /* Build opaque type */ @@ -5115,7 +5136,7 @@ opaque_long(void) /* Set opaque type's tag */ H5E_BEGIN_TRY { - ret = H5Tset_tag(dt, long_tag); + ret = H5Tset_tag(dt, long_tag); } H5E_END_TRY; if(ret != FAIL) TEST_ERROR @@ -5136,18 +5157,18 @@ error: return 1; } - + /*------------------------------------------------------------------------- - * Function: opaque_funcs + * Function: opaque_funcs * - * Purpose: Test some type functions that are and aren't supposed to + * Purpose: Test some type functions that are and aren't supposed to * work with opaque type. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Raymond Lu + * Programmer: Raymond Lu * Wednesday, April 5, 2006 * * Modifications: @@ -5157,7 +5178,7 @@ error: static int opaque_funcs(void) { - hid_t type = -1, super=-1; + hid_t type = -1, super=-1; size_t size; H5T_pad_t inpad; H5T_cset_t cset; @@ -5252,7 +5273,7 @@ opaque_funcs(void) return 1; } - + /*------------------------------------------------------------------------- * Function: test_encode * @@ -5293,7 +5314,7 @@ test_encode(void) size_t enum_buf_size = 0; size_t vlstr_buf_size = 0; unsigned char *cmpd_buf=NULL, *enum_buf=NULL, *vlstr_buf=NULL; - hid_t ret_id; + hid_t ret_id; herr_t ret; TESTING("functions of encoding and decoding datatypes"); @@ -5394,7 +5415,7 @@ test_encode(void) /* Try decoding bogus buffer */ H5E_BEGIN_TRY { - ret_id = H5Tdecode(cmpd_buf); + ret_id = H5Tdecode(cmpd_buf); } H5E_END_TRY; if(ret_id!=FAIL) { H5_FAILED(); @@ -5706,39 +5727,39 @@ test_encode(void) */ /* Make sure the reference counts for the decoded datatypes are one. */ - if(H5Iget_ref(decoded_tid1) != 1) { + if(H5Iget_ref(decoded_tid1) != 1) { H5_FAILED(); HDprintf("Decoded datatype has incorrect reference count\n"); goto error; } /* end if */ - if(H5Iget_ref(decoded_tid2) != 1) { + if(H5Iget_ref(decoded_tid2) != 1) { H5_FAILED(); HDprintf("Decoded datatype has incorrect reference count\n"); goto error; } /* end if */ - if(H5Iget_ref(decoded_tid3) != 1) { + if(H5Iget_ref(decoded_tid3) != 1) { H5_FAILED(); HDprintf("Decoded datatype has incorrect reference count\n"); goto error; } /* end if */ - /* Make sure the reference counts for the decoded datatypes can be + /* Make sure the reference counts for the decoded datatypes can be * decremented and the datatypes are closed. */ - if(H5Idec_ref(decoded_tid1) != 0) { + if(H5Idec_ref(decoded_tid1) != 0) { H5_FAILED(); HDprintf("Decoded datatype can't close\n"); goto error; } /* end if */ - if(H5Idec_ref(decoded_tid2) != 0) { + if(H5Idec_ref(decoded_tid2) != 0) { H5_FAILED(); HDprintf("Decoded datatype can't close\n"); goto error; } /* end if */ - if(H5Idec_ref(decoded_tid3) != 0) { + if(H5Idec_ref(decoded_tid3) != 0) { H5_FAILED(); HDprintf("Decoded datatype can't close\n"); goto error; @@ -5746,7 +5767,7 @@ test_encode(void) /* Make sure the decoded datatypes are already closed. */ H5E_BEGIN_TRY { - ret = H5Tclose(decoded_tid1); + ret = H5Tclose(decoded_tid1); } H5E_END_TRY; if(ret!=FAIL) { H5_FAILED(); @@ -5755,7 +5776,7 @@ test_encode(void) } H5E_BEGIN_TRY { - ret = H5Tclose(decoded_tid2); + ret = H5Tclose(decoded_tid2); } H5E_END_TRY; if(ret!=FAIL) { H5_FAILED(); @@ -5764,7 +5785,7 @@ test_encode(void) } H5E_BEGIN_TRY { - ret = H5Tclose(decoded_tid3); + ret = H5Tclose(decoded_tid3); } H5E_END_TRY; if(ret!=FAIL) { H5_FAILED(); @@ -5818,7 +5839,7 @@ test_encode(void) return 1; } - + /*------------------------------------------------------------------------- * Function: test_latest * @@ -5845,7 +5866,7 @@ test_latest(void) hid_t file = (-1); /* File ID */ hid_t tid1 = (-1), tid2 = (-1); /* Datatype ID */ hid_t fapl = (-1); /* File access property list */ - H5O_info_t oi; /* Stat buffer for committed datatype */ + H5O_info_t oi; /* Stat buffer for committed datatype */ hsize_t old_dtype_oh_size; /* Size of object header with "old" format */ hsize_t new_dtype_oh_size; /* Size of object header with "new" format */ char filename[1024]; /* Buffer for filename */ @@ -6036,7 +6057,7 @@ conv_except(H5T_conv_except_t except_type, hid_t H5_ATTR_UNUSED src_id, hid_t H5 return(H5T_CONV_UNHANDLED); } - + /*------------------------------------------------------------------------- * Function: test_int_float_except * @@ -6179,7 +6200,7 @@ error: #endif /* H5_SIZEOF_INT==4 && H5_SIZEOF_FLOAT==4 */ } /* end test_int_float_except() */ - + /*------------------------------------------------------------------------- * Function: test_set_order * @@ -6340,11 +6361,11 @@ error: return 1; } /* end test_set_order() */ - + /*------------------------------------------------------------------------- * Function: test_set_order_compound * - * Purpose: Tests H5Tset_order/H5Tget_order for complicated compound + * Purpose: Tests H5Tset_order/H5Tget_order for complicated compound * type. * * Return: Success: 0 @@ -6376,13 +6397,13 @@ test_set_order_compound(hid_t fapl) hid_t cmpd = -1, memb_cmpd = -1, memb_array1 = -1, memb_array2 = -1, cmpd_array = -1; hid_t vl_id = -1; hsize_t dims[2] = {3, 4}; /* Array dimenstions */ - char filename[1024]; + char filename[1024]; herr_t ret; /* Generic return value */ TESTING("H5Tset/get_order for compound type"); if((memb_cmpd = H5Tcreate(H5T_COMPOUND, sizeof(atomic_cmpd))) < 0) FAIL_STACK_ERROR - if(H5Tinsert(memb_cmpd, "i", HOFFSET(atomic_cmpd, i), H5T_NATIVE_INT) < 0) FAIL_STACK_ERROR + if(H5Tinsert(memb_cmpd, "i", HOFFSET(atomic_cmpd, i), H5T_NATIVE_INT) < 0) FAIL_STACK_ERROR if(H5Tinsert(memb_cmpd, "c", HOFFSET(atomic_cmpd, c), H5T_NATIVE_CHAR) < 0) FAIL_STACK_ERROR if(H5Tinsert(memb_cmpd, "s", HOFFSET(atomic_cmpd, s), H5T_NATIVE_SHORT) < 0) FAIL_STACK_ERROR if(H5Tinsert(memb_cmpd, "f", HOFFSET(atomic_cmpd, f), H5T_NATIVE_FLOAT) < 0) FAIL_STACK_ERROR @@ -6402,19 +6423,19 @@ test_set_order_compound(hid_t fapl) /* Create a compound type using the types above. */ if((cmpd = H5Tcreate(H5T_COMPOUND, sizeof(complex_cmpd))) < 0) FAIL_STACK_ERROR - if(H5Tinsert(cmpd, "a", HOFFSET(complex_cmpd, a), memb_cmpd) < 0) FAIL_STACK_ERROR - if(H5Tinsert(cmpd, "vl_type", HOFFSET(complex_cmpd, vl), vl_id) < 0) FAIL_STACK_ERROR - if(H5Tinsert(cmpd, "b", HOFFSET(complex_cmpd, b), memb_array1) < 0) FAIL_STACK_ERROR - if(H5Tinsert(cmpd, "d", HOFFSET(complex_cmpd, d), memb_array2) < 0) FAIL_STACK_ERROR + if(H5Tinsert(cmpd, "a", HOFFSET(complex_cmpd, a), memb_cmpd) < 0) FAIL_STACK_ERROR + if(H5Tinsert(cmpd, "vl_type", HOFFSET(complex_cmpd, vl), vl_id) < 0) FAIL_STACK_ERROR + if(H5Tinsert(cmpd, "b", HOFFSET(complex_cmpd, b), memb_array1) < 0) FAIL_STACK_ERROR + if(H5Tinsert(cmpd, "d", HOFFSET(complex_cmpd, d), memb_array2) < 0) FAIL_STACK_ERROR /* The order should be mixed now. */ - if(H5Tget_order(cmpd) != H5T_ORDER_MIXED) FAIL_STACK_ERROR + if(H5Tget_order(cmpd) != H5T_ORDER_MIXED) FAIL_STACK_ERROR /* Create an array of the compound type above */ cmpd_array = H5Tarray_create2(cmpd, 2, dims); /* The order of the array type should be the same as the compound type */ - if(H5Tget_order(cmpd_array) != H5T_ORDER_MIXED) FAIL_STACK_ERROR + if(H5Tget_order(cmpd_array) != H5T_ORDER_MIXED) FAIL_STACK_ERROR /* Verify that the order can't be 'none'. */ H5E_BEGIN_TRY @@ -6432,13 +6453,13 @@ test_set_order_compound(hid_t fapl) if(H5Tset_order(cmpd, H5T_ORDER_BE) < 0) FAIL_STACK_ERROR /* Verify that the order of the compound type is big-endian */ - if(H5Tget_order(cmpd) != H5T_ORDER_BE) FAIL_STACK_ERROR + if(H5Tget_order(cmpd) != H5T_ORDER_BE) FAIL_STACK_ERROR /* Change the order of the array type to little-endian*/ if(H5Tset_order(cmpd_array, H5T_ORDER_LE) < 0) FAIL_STACK_ERROR /* Verify that the order of the array type is little-endian */ - if(H5Tget_order(cmpd_array) != H5T_ORDER_LE) FAIL_STACK_ERROR + if(H5Tget_order(cmpd_array) != H5T_ORDER_LE) FAIL_STACK_ERROR /* Create file */ h5_fixname(FILENAME[1], fapl, filename, sizeof filename); @@ -6473,23 +6494,23 @@ error: H5Tclose(vl_id); H5Tclose(cmpd); H5Tclose(cmpd_array); - H5Fclose(file); + H5Fclose(file); H5E_END_TRY; return 1; } /* end test_set_order_compound() */ - + /*------------------------------------------------------------------------- - * Function: test_named_indirect_reopen + * Function: test_named_indirect_reopen * - * Purpose: Tests that open named datatypes can be reopened indirectly + * Purpose: Tests that open named datatypes can be reopened indirectly * through H5Dget_type without causing problems. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Neil Fortner + * Programmer: Neil Fortner * Thursday, June 4, 2009 * * Modifications: @@ -6499,13 +6520,13 @@ error: static int test_named_indirect_reopen(hid_t fapl) { - hid_t file=-1, type=-1, reopened_type=-1, strtype=-1, dset=-1, space=-1; - static hsize_t dims[1] = {3}; + hid_t file=-1, type=-1, reopened_type=-1, strtype=-1, dset=-1, space=-1; + static hsize_t dims[1] = {3}; size_t dt_size; int enum_value; const char *tag = "opaque_tag"; char *tag_ret = NULL; - char filename[1024]; + char filename[1024]; TESTING("indirectly reopening committed datatypes"); @@ -6669,12 +6690,12 @@ test_named_indirect_reopen(hid_t fapl) error: H5E_BEGIN_TRY { - H5Tclose(type); - H5Tclose(strtype); - H5Tclose(reopened_type); - H5Sclose(space); - H5Dclose(dset); - H5Fclose(file); + H5Tclose(type); + H5Tclose(strtype); + H5Tclose(reopened_type); + H5Sclose(space); + H5Dclose(dset); + H5Fclose(file); } H5E_END_TRY; if(tag_ret) H5free_memory(tag_ret); @@ -6682,18 +6703,18 @@ error: } /* end test_named_indirect_reopen() */ /*------------------------------------------------------------------------- - * Function: test_named_indirect_reopen_file + * Function: test_named_indirect_reopen_file * - * Purpose: Tests that a named compound datatype that refers to a named + * Purpose: Tests that a named compound datatype that refers to a named * string datatype can be reopened indirectly through H5Dget_type, * and shows the correct H5Tcommitted() state, including after the * file has been closed and reopened. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: number of errors + * Failure: number of errors * - * Programmer: Mark Hodson + * Programmer: Mark Hodson * Tuesday, March 17, 2015 * * Modifications: @@ -6756,7 +6777,7 @@ test_named_indirect_reopen_file(hid_t fapl) /* Indirectly reopen string type, verify that they report as committed, and the size doesn't change */ if((reopened_strtype = H5Tget_member_type(reopened_cmptype, 0)) < 0) TEST_ERROR if(strtype_size != H5Tget_size(reopened_strtype)) TEST_ERROR - if(H5Tcommitted(reopened_strtype) != 1) TEST_ERROR + if(H5Tcommitted(reopened_strtype) != 1) TEST_ERROR /* Close types and dataset */ if(H5Tclose(reopened_strtype) < 0) TEST_ERROR @@ -6799,13 +6820,13 @@ test_named_indirect_reopen_file(hid_t fapl) error: H5E_BEGIN_TRY { - H5Tclose(cmptype); - H5Tclose(strtype); - H5Tclose(reopened_cmptype); + H5Tclose(cmptype); + H5Tclose(strtype); + H5Tclose(reopened_cmptype); H5Tclose(reopened_strtype); - H5Sclose(space); - H5Dclose(dset); - H5Fclose(file); + H5Sclose(space); + H5Dclose(dset); + H5Fclose(file); } H5E_END_TRY; return 1; } /* end test_named_indirect_reopen() */ @@ -6910,17 +6931,17 @@ static void create_del_obj_named_test_file(const char *filename, hid_t fapl, HDassert(status >= 0); } /* end create_del_obj_named_test_file() */ - + /*------------------------------------------------------------------------- - * Function: test_delete_obj_named + * Function: test_delete_obj_named * - * Purpose: Tests that delete objects that use named datatypes through + * Purpose: Tests that delete objects that use named datatypes through * different file IDs * - * Return: Success: 0 - * Failure: number of errors + * Return: Success: 0 + * Failure: number of errors * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Monday, July 18, 2011 * *------------------------------------------------------------------------- @@ -6991,27 +7012,27 @@ test_delete_obj_named(hid_t fapl) error: H5E_BEGIN_TRY { - H5Tclose(attr); - H5Dclose(dset); - H5Pclose(fapl2); - H5Fclose(filea1); - H5Fclose(filea2); - H5Fclose(fileb); + H5Tclose(attr); + H5Dclose(dset); + H5Pclose(fapl2); + H5Fclose(filea1); + H5Fclose(filea2); + H5Fclose(fileb); } H5E_END_TRY; return 1; } /* end test_delete_obj_named() */ - + /*------------------------------------------------------------------------- - * Function: test_delete_obj_named_fileid + * Function: test_delete_obj_named_fileid * - * Purpose: Tests that objects that use named datatypes through + * Purpose: Tests that objects that use named datatypes through * different file IDs get the correct file IDs * - * Return: Success: 0 - * Failure: number of errors + * Return: Success: 0 + * Failure: number of errors * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, July 28, 2011 * *------------------------------------------------------------------------- @@ -7147,26 +7168,26 @@ test_delete_obj_named_fileid(hid_t fapl) error: H5E_BEGIN_TRY { - H5Tclose(attr); - H5Dclose(dset); - H5Pclose(fapl2); - H5Fclose(filea1); - H5Fclose(filea2); - H5Fclose(fileb); + H5Tclose(attr); + H5Dclose(dset); + H5Pclose(fapl2); + H5Fclose(filea1); + H5Fclose(filea2); + H5Fclose(fileb); } H5E_END_TRY; return 1; } /* end test_delete_obj_named_fileid() */ - + /*------------------------------------------------------------------------- - * Function: test_deprec + * Function: test_deprec * - * Purpose: Tests deprecated API routines for datatypes. + * Purpose: Tests deprecated API routines for datatypes. * - * Return: Success: 0 - * Failure: number of errors + * Return: Success: 0 + * Failure: number of errors * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, September 27, 2007 * *------------------------------------------------------------------------- @@ -7175,17 +7196,17 @@ error: static int test_deprec(hid_t fapl) { - hid_t file = -1; /* File ID */ - hid_t type = -1; /* Datatype ID */ + hid_t file = -1; /* File ID */ + hid_t type = -1; /* Datatype ID */ unsigned rank = 2; /* Rank for array datatype */ hsize_t dims[2] = {3, 3}; /* Dimensions for array datatype */ int perm[2] = {0, 1}; /* Dimensions permutations for array datatype */ hsize_t rdims[2]= {0, 0}; /* Dimensions for querying array datatype */ int rperm[2] = {-2, -2}; /* Dimensions permutations for array datatype */ hbool_t dim_mismatch; /* Whether any dimensions didn't match */ - char filename[1024]; + char filename[1024]; unsigned u; /* Local index variable */ - herr_t status; /* Generic routine value */ + herr_t status; /* Generic routine value */ TESTING("deprected API routines for datatypes"); @@ -7213,7 +7234,7 @@ test_deprec(hid_t fapl) dim_mismatch = TRUE; } /* end if */ if(dim_mismatch) - FAIL_PUTS_ERROR(" Dimensions didn't match!") + FAIL_PUTS_ERROR(" Dimensions didn't match!") /* Check the array dimension permutations */ dim_mismatch = FALSE; @@ -7223,7 +7244,7 @@ test_deprec(hid_t fapl) dim_mismatch = TRUE; } /* end if */ if(dim_mismatch) - FAIL_PUTS_ERROR(" Dimension permutations modified!") + FAIL_PUTS_ERROR(" Dimension permutations modified!") /* Close the datatype */ if(H5Tclose(type) < 0) @@ -7236,31 +7257,31 @@ test_deprec(hid_t fapl) /* Predefined types cannot be committed */ H5E_BEGIN_TRY { - status = H5Tcommit1(file, "test_named_1 (should not exist)", H5T_NATIVE_INT); + status = H5Tcommit1(file, "test_named_1 (should not exist)", H5T_NATIVE_INT); } H5E_END_TRY; if(status >= 0) - FAIL_PUTS_ERROR(" Predefined types should not be committable!") + FAIL_PUTS_ERROR(" Predefined types should not be committable!") /* Copy a predefined datatype and commit the copy */ if((type = H5Tcopy(H5T_NATIVE_INT)) < 0) FAIL_STACK_ERROR if(H5Tcommit1(file, "native-int", type) < 0) FAIL_STACK_ERROR if((status = H5Tcommitted(type)) < 0) FAIL_STACK_ERROR if(0 == status) - FAIL_PUTS_ERROR(" H5Tcommitted() returned false!") + FAIL_PUTS_ERROR(" H5Tcommitted() returned false!") /* We should not be able to modify a type after it has been committed. */ H5E_BEGIN_TRY { - status = H5Tset_precision(type, (size_t)256); + status = H5Tset_precision(type, (size_t)256); } H5E_END_TRY; if(status >= 0) - FAIL_PUTS_ERROR(" Committed type is not constant!") + FAIL_PUTS_ERROR(" Committed type is not constant!") /* We should not be able to re-commit a committed type */ H5E_BEGIN_TRY { - status = H5Tcommit1(file, "test_named_2 (should not exist)", type); + status = H5Tcommit1(file, "test_named_2 (should not exist)", type); } H5E_END_TRY; if(status >= 0) - FAIL_PUTS_ERROR(" Committed types should not be recommitted!") + FAIL_PUTS_ERROR(" Committed types should not be recommitted!") /* * Close the committed type and reopen it. It should return a named type. @@ -7269,7 +7290,7 @@ test_deprec(hid_t fapl) if((type = H5Topen1(file, "native-int")) < 0) FAIL_STACK_ERROR if((status = H5Tcommitted(type)) < 0) FAIL_STACK_ERROR if(!status) - FAIL_PUTS_ERROR(" Opened named types should be named types!") + FAIL_PUTS_ERROR(" Opened named types should be named types!") /* Close */ if(H5Tclose(type) < 0) FAIL_STACK_ERROR @@ -7299,24 +7320,24 @@ test_deprec(hid_t fapl) error: H5E_BEGIN_TRY { - H5Tclose(type); - H5Fclose(file); + H5Tclose(type); + H5Fclose(file); } H5E_END_TRY; return 1; } /* end test_deprec() */ #endif /* H5_NO_DEPRECATED_SYMBOLS */ - + /*------------------------------------------------------------------------- - * Function: test_utf_ascii_conv + * Function: test_utf_ascii_conv * - * Purpose: Make sure the library doesn't conversion strings between + * Purpose: Make sure the library doesn't conversion strings between * ASCII and UTF8. * - * Return: Success: 0 - * Failure: number of errors + * Return: Success: 0 + * Failure: number of errors * - * Programmer: Raymond Lu + * Programmer: Raymond Lu * 10 November 2011 *------------------------------------------------------------------------- */ @@ -7334,12 +7355,12 @@ test_utf_ascii_conv(void) char *ascii_r = NULL; const char *ascii_w = "bar!"; char *utf8_r = NULL; - char filename[1024]; + char filename[1024]; char ascii2[4], utf8_2[4]; herr_t status; TESTING("string conversion between ASCII and UTF"); - + /************************************************ * Test VL string conversion from UTF8 to ASCII ************************************************/ @@ -7506,20 +7527,20 @@ test_utf_ascii_conv(void) error: H5E_BEGIN_TRY { - H5Tclose(utf8_vtid); - H5Tclose(ascii_vtid); - H5Tclose(utf8_tid); - H5Tclose(ascii_tid); - H5Dclose(did); - H5Sclose(sid); - H5Fclose(fid); + H5Tclose(utf8_vtid); + H5Tclose(ascii_vtid); + H5Tclose(utf8_tid); + H5Tclose(ascii_tid); + H5Dclose(did); + H5Sclose(sid); + H5Fclose(fid); } H5E_END_TRY; return 1; } - + /*------------------------------------------------------------------------- * Function: main * @@ -7539,8 +7560,8 @@ error: int main(void) { - long nerrors = 0; - hid_t fapl = -1; + long nerrors = 0; + hid_t fapl = -1; /* Set the random # seed */ HDsrandom((unsigned)HDtime(NULL)); @@ -7549,7 +7570,7 @@ main(void) fapl = h5_fileaccess(); if(ALIGNMENT) - HDprintf("Testing non-aligned conversions (ALIGNMENT=%d)....\n", ALIGNMENT); + HDprintf("Testing non-aligned conversions (ALIGNMENT=%d)....\n", ALIGNMENT); /* Do the tests */ nerrors += test_classes(); diff --git a/test/extend.c b/test/extend.c index f8c091b..a31ac0e 100644 --- a/test/extend.c +++ b/test/extend.c @@ -219,23 +219,15 @@ error: /*------------------------------------------------------------------------- - * Function: main + * Function: main * - * Purpose: Tests extendible datasets + * Purpose: Tests extendible datasets * - * Return: Success: exit(0) - * - * Failure: exit(non-zero) + * Return: EXIT_SUCCESS/EXIT_FAILURE * * Programmer: Robb Matzke * Friday, January 30, 1998 * - * Modifications: - * Took main data code out into write_data() routine, to allow - * different dataset creation property list settings to be tested. - * Quincey Koziol - * Tuesday, June 10, 2003 - * *------------------------------------------------------------------------- */ int @@ -290,16 +282,16 @@ main (void) if(nerrors) { HDprintf("***** %d FAILURE%s! *****\n", nerrors, (1 == nerrors) ? "" : "S"); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } /* end if */ HDprintf("All extend tests passed.\n"); h5_cleanup(FILENAME, fapl); - return 0; + HDexit(EXIT_SUCCESS); error: HDprintf("*** One or more extend tests failed ***\n"); - return 1; -} + HDexit(EXIT_FAILURE); +} /* end main() */ diff --git a/test/filter_fail.c b/test/filter_fail.c index 2acce01..4be2547 100644 --- a/test/filter_fail.c +++ b/test/filter_fail.c @@ -347,17 +347,15 @@ error: * Purpose: Tests the library's behavior when a mandate filter returns * failure. * - * Return: Success: exit(EXIT_SUCCESS) - * Failure: exit(EXIT_FAILURE) + * Return: EXIT_SUCCESS/EXIT_FAILURE * * Programmer: Raymond Lu * 25 August 2010 * - * Modifications: - * *------------------------------------------------------------------------- */ -int main(void) +int +main(void) { hid_t fapl; int mdc_nelmts = 0; @@ -398,7 +396,7 @@ int main(void) if (nerrors) TEST_ERROR - return 0; + HDexit(EXIT_SUCCESS); error: if (nerrors) { @@ -406,4 +404,4 @@ error: nerrors, 1==nerrors?"":"S"); HDexit(EXIT_FAILURE); } -} +} /* end main() */ diff --git a/test/flush1.c b/test/flush1.c index 1e759d0..1098824 100644 --- a/test/flush1.c +++ b/test/flush1.c @@ -15,11 +15,11 @@ * Programmer: Robb Matzke * Friday, October 23, 1998 * - * Purpose: This is the first half of a two-part test that makes sure - * that a file can be read after an application crashes as long - * as the file was flushed first. We simulate a crash by - * calling _exit(EXIT_SUCCESS) since this doesn't flush HDF5 caches but - * still exits with success. + * Purpose: This is the first half of a two-part test that makes sure + * that a file can be read after an application crashes as long + * as the file was flushed first. We simulate a crash by + * calling _exit(EXIT_SUCCESS) since this doesn't flush HDF5 caches but + * still exits with success. */ #include "h5test.h" @@ -30,57 +30,67 @@ const char *FILENAME[] = { NULL }; -static double the_data[100][100]; +static double the_data[100][100]; /*------------------------------------------------------------------------- * Function: create_file * * Purpose: Creates files and datasets used in part 1 of the test * - * Return: Success: a valid file ID - * Failure: -1 + * Return: Success: a valid file ID + * Failure: -1 * - * Programmer: Leon Arber + * Programmer: Leon Arber * Sept. 26, 2006 * *------------------------------------------------------------------------- */ static hid_t -create_file(char* name, hid_t fapl) +create_file(const char *filename, hid_t fapl_id) { - hid_t file, dcpl, space, dset, groups, grp; - hsize_t ds_size[2] = {100, 100}; - hsize_t ch_size[2] = {5, 5}; - size_t i, j; + hid_t fid = -1; /* file ID */ + hid_t dcpl, space, dset, groups, grp; + hsize_t ds_size[2] = {100, 100}; + hsize_t ch_size[2] = {5, 5}; + size_t i, j; - if((file = H5Fcreate(name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) + STACK_ERROR /* Create a chunked dataset */ - if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) FAIL_STACK_ERROR - if(H5Pset_chunk(dcpl, 2, ch_size) < 0) FAIL_STACK_ERROR - if((space = H5Screate_simple(2, ds_size, NULL)) < 0) FAIL_STACK_ERROR - if((dset = H5Dcreate2(file, "dset", H5T_NATIVE_FLOAT, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR + if(H5Pset_chunk(dcpl, 2, ch_size) < 0) + FAIL_STACK_ERROR + if((space = H5Screate_simple(2, ds_size, NULL)) < 0) + FAIL_STACK_ERROR + if((dset = H5Dcreate2(fid, "dset", H5T_NATIVE_FLOAT, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR /* Write some data */ for(i = 0; i < ds_size[0]; i++) - /* - * The extra cast in the following statement is a bug workaround - * for the Win32 version 5.0 compiler. - * 1998-11-06 ptl - */ + /* + * The extra cast in the following statement is a bug workaround + * for the Win32 version 5.0 compiler. + * 1998-11-06 ptl + */ for(j = 0; j < (size_t)ds_size[1]; j++) - the_data[i][j] = (double)(hssize_t)i/(hssize_t)(j+1); - if(H5Dwrite(dset, H5T_NATIVE_DOUBLE, space, space, H5P_DEFAULT, the_data) < 0) FAIL_STACK_ERROR + the_data[i][j] = (double)(hssize_t)i/(hssize_t)(j+1); + if(H5Dwrite(dset, H5T_NATIVE_DOUBLE, space, space, H5P_DEFAULT, the_data) < 0) + FAIL_STACK_ERROR /* Create some groups */ - if((groups = H5Gcreate2(file, "some_groups", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR + if((groups = H5Gcreate2(fid, "some_groups", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR for(i = 0; i < 100; i++) { - HDsprintf(name, "grp%02u", (unsigned)i); - if((grp = H5Gcreate2(groups, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR - if(H5Gclose(grp) < 0) FAIL_STACK_ERROR + HDsprintf(filename, "grp%02u", (unsigned)i); + if((grp = H5Gcreate2(groups, filename, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + if(H5Gclose(grp) < 0) + FAIL_STACK_ERROR } /* end for */ - return file; + return fid; error: HD_exit(1); @@ -88,15 +98,15 @@ error: /*------------------------------------------------------------------------- - * Function: extend_file + * Function: extend_file * - * Purpose: Add a small dataset to the file. + * Purpose: Add a small dataset to the file. * - * Return: Success: 0 + * Return: Success: 0 * - * Failure: 1 + * Failure: 1 * - * Programmer: Leon Arber + * Programmer: Leon Arber * Oct. 4, 2006 * * Modifications: @@ -106,31 +116,34 @@ error: static hid_t extend_file(hid_t file) { - hid_t dcpl, space, dset; - hsize_t ds_size[2] = {100, 100}; - hsize_t ch_size[2] = {5, 5}; - size_t i, j; + hid_t dcpl, space, dset; + hsize_t ds_size[2] = {100, 100}; + hsize_t ch_size[2] = {5, 5}; + size_t i, j; /* Create a chunked dataset */ - if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error; - if(H5Pset_chunk(dcpl, 2, ch_size) < 0) goto error; - if((space = H5Screate_simple(2, ds_size, NULL)) < 0) goto error; + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto error; + if(H5Pset_chunk(dcpl, 2, ch_size) < 0) + goto error; + if((space = H5Screate_simple(2, ds_size, NULL)) < 0) + goto error; if((dset = H5Dcreate2(file, "dset2", H5T_NATIVE_FLOAT, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) - goto error; + goto error; /* Write some data */ for (i=0; i 0.0001F) { H5_FAILED(); - HDprintf(" dset[%lu][%lu] = %g\n", - (unsigned long)i, (unsigned long)j, the_data[i][j]); - HDprintf(" should be %g\n", - (double)(hssize_t)i/(hssize_t)(j+1)); + HDprintf(" dset[%lu][%lu] = %g\n", (unsigned long)i, (unsigned long)j, the_data[i][j]); + HDprintf(" should be %g\n", (double)(hssize_t)i/(hssize_t)(j+1)); goto error; } /* end if */ } /* end for */ @@ -110,23 +111,31 @@ check_file(char* filename, hid_t fapl, int flag) char name[1024]; int i; - if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) goto error; - if(check_dset(file, "dset")) goto error; + if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) + goto error; + if(check_dset(file, "dset")) + goto error; /* Open some groups */ - if((groups = H5Gopen2(file, "some_groups", H5P_DEFAULT)) < 0) goto error; + if((groups = H5Gopen2(file, "some_groups", H5P_DEFAULT)) < 0) + goto error; for(i = 0; i < 100; i++) { HDsprintf(name, "grp%02u", (unsigned)i); - if((grp = H5Gopen2(groups, name, H5P_DEFAULT)) < 0) goto error; - if(H5Gclose(grp) < 0) goto error; + if((grp = H5Gopen2(groups, name, H5P_DEFAULT)) < 0) + goto error; + if(H5Gclose(grp) < 0) + goto error; } /* end for */ /* Check to see if that last added dataset in the third file is accessible * (it shouldn't be...but it might. Flag an error in case it is for now */ - if(flag && check_dset(file, "dset2")) goto error; + if(flag && check_dset(file, "dset2")) + goto error; - if(H5Gclose(groups) < 0) goto error; - if(H5Fclose(file) < 0) goto error; + if(H5Gclose(groups) < 0) + goto error; + if(H5Fclose(file) < 0) + goto error; return 0; @@ -179,14 +188,13 @@ main(void) h5_fixname(FILENAME[1], fapl, name, sizeof name); if(check_file(name, fapl, FALSE)) PASSED() - else - { + else { #if defined H5_HAVE_WIN32_API && !defined (hdf5_EXPORTS) - SKIPPED(); - puts(" DLL will flush the file even when calling _exit, skip this test temporarily"); + SKIPPED(); + HDputs(" DLL will flush the file even when calling _exit, skip this test temporarily"); #else - H5_FAILED() - goto error; + H5_FAILED() + goto error; #endif } H5Eset_auto2(H5E_DEFAULT, func, NULL); @@ -200,14 +208,13 @@ main(void) h5_fixname(FILENAME[2], fapl, name, sizeof name); if(check_file(name, fapl, TRUE)) PASSED() - else - { + else { #if defined H5_HAVE_WIN32_API && !defined (hdf5_EXPORTS) - SKIPPED(); - puts(" DLL will flush the file even when calling _exit, skip this test temporarily"); + SKIPPED(); + HDputs(" DLL will flush the file even when calling _exit, skip this test temporarily"); #else - H5_FAILED() - goto error; + H5_FAILED() + goto error; #endif } @@ -218,6 +225,6 @@ main(void) return 0; error: - return 1; -} + HDexit(EXIT_FAILURE); +} /* end main() */ diff --git a/test/h5test.c b/test/h5test.c index c2e78e4..64b05f4 100644 --- a/test/h5test.c +++ b/test/h5test.c @@ -113,8 +113,7 @@ static H5E_auto2_t err_func = NULL; static herr_t h5_errors(hid_t estack, void *client_data); static char *h5_fixname_real(const char *base_name, hid_t fapl, const char *suffix, - char *fullname, size_t size); - + char *fullname, size_t size, hbool_t nest_printf); /*------------------------------------------------------------------------- * Function: h5_errors @@ -477,7 +476,7 @@ h5_test_init(void) char * h5_fixname(const char *base_name, hid_t fapl, char *fullname, size_t size) { - return (h5_fixname_real(base_name, fapl, ".h5", fullname, size)); + return (h5_fixname_real(base_name, fapl, ".h5", fullname, size, FALSE)); } @@ -497,7 +496,7 @@ h5_fixname(const char *base_name, hid_t fapl, char *fullname, size_t size) char * h5_fixname_no_suffix(const char *base_name, hid_t fapl, char *fullname, size_t size) { - return (h5_fixname_real(base_name, fapl, NULL, fullname, size)); + return (h5_fixname_real(base_name, fapl, NULL, fullname, size, FALSE)); } @@ -523,7 +522,7 @@ h5_fixname_no_suffix(const char *base_name, hid_t fapl, char *fullname, size_t s char * h5_fixname_printf(const char *base_name, hid_t fapl, char *fullname, size_t size) { - return (h5_fixname_real(base_name, fapl, ".h5", fullname, size)); + return (h5_fixname_real(base_name, fapl, ".h5", fullname, size, TRUE)); } @@ -551,7 +550,7 @@ h5_fixname_printf(const char *base_name, hid_t fapl, char *fullname, size_t size */ static char * h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, - char *fullname, size_t size) + char *fullname, size_t size, hbool_t nest_printf) { const char *prefix = NULL; const char *env = NULL; /* HDF5_DRIVER environment variable */ @@ -572,10 +571,30 @@ h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, return NULL; if(suffix) { - if(H5FD_FAMILY == driver) - suffix = "%05d.h5"; - else if (H5FD_MULTI == driver) - suffix = NULL; + if(H5FD_FAMILY == driver) { + suffix = nest_printf ? "%%05d.h5" : "%05d.h5"; + } + else if (H5FD_MULTI == driver) { + + /* Get the environment variable, if it exists, in case + * we are using the split driver since both of those + * use the multi VFD under the hood. + */ + env = HDgetenv("HDF5_DRIVER"); +#ifdef HDF5_DRIVER + /* Use the environment variable, then the compile-time constant */ + if(!env) + env = HDF5_DRIVER; +#endif + if(env && !HDstrcmp(env, "split")) { + /* split VFD */ + suffix = NULL; + } + else { + /* multi VFD */ + suffix = NULL; + } + } } } diff --git a/test/hdfs.c b/test/hdfs.c new file mode 100644 index 0000000..ab39da6 --- /dev/null +++ b/test/hdfs.c @@ -0,0 +1,1767 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Read-Only HDFS Virtual File Driver (VFD) + * + * Purpose: + * + * Verify behavior for Read-Only HDFS VFD. + * + * Demonstrates basic use cases and fapl interaction. + * + * Programmer: Jacob Smith + * 2018-04-23 + */ + +#include "h5test.h" /* testing utilities */ +#include "H5FDhdfs.h" /* this file driver's utilities */ + + +#ifdef H5_HAVE_LIBHDFS +#define HDFS_TEST_DEBUG 0 +#define HDFS_TEST_MAX_BUF_SIZE 256 +#endif /* H5_HAVE_LIBHDFS */ + +/***************************************************************************** + * + * FILE-LOCAL TESTING MACROS + * + * Purpose: + * + * 1) Upon test failure, goto-jump to single-location teardown in test + * function. E.g., `error:` (consistency with HDF corpus) or + * `failed:` (reflects purpose). + * >>> using "error", in part because `H5E_BEGIN_TRY` expects it. + * 2) Increase clarity and reduce overhead found with `TEST_ERROR`. + * e.g., "if(somefunction(arg, arg2) < 0) TEST_ERROR:" + * requires reading of entire line to know whether this if/call is + * part of the test setup, test operation, or a test unto itself. + * 3) Provide testing macros with optional user-supplied failure message; + * if not supplied (NULL), generate comparison output in the spirit of + * test-driven development. E.g., "expected 5 but was -3" + * User messages clarify test's purpose in code, encouraging description + * without relying on comments. + * 4) Configurable expected-actual order in generated comparison strings. + * Some prefer `VERIFY(expected, actual)`, others + * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch + * to satifsy both parties, assuming one paradigm per test file. + * (One could #undef and redefine the flag through the file as desired, + * but _why_.) + * + * Provided as courtesy, per consideration for inclusion in the library + * proper. + * + * Macros: + * + * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order + * FAIL_IF() - check condition + * FAIL_UNLESS() - check _not_ condition + * JSVERIFY() - long-int equality check; prints reason/comparison + * JSVERIFY_NOT() - long-int inequality check; prints + * JSVERIFY_STR() - string equality check; prints + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *****************************************************************************/ + + +/*---------------------------------------------------------------------------- + * + * ifdef flag: JSVERIFY_EXP_ACT + * + * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason]) + * default, if this is undefined, is (ACTUAL, EXPECTED[, reason]) + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_EXP_ACT 1L + + +/*---------------------------------------------------------------------------- + * + * Macro: JSFAILED_AT() + * + * Purpose: + * + * Preface a test failure by printing "*FAILED*" and location to stdout + * Similar to `H5_FAILED(); AT();` from h5test.h + * + * *FAILED* at somefile.c:12 in function_name()... + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSFAILED_AT() { \ + HDprintf("*FAILED* at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_IF() + * + * Purpose: + * + * Make tests more accessible and less cluttered than + * `if (thing == otherthing()) TEST_ERROR` + * paradigm. + * + * The following lines are roughly equivalent: + * + * `if (myfunc() < 0) TEST_ERROR;` (as seen elsewhere in HDF tests) + * `FAIL_IF(myfunc() < 0)` + * + * Prints a generic "FAILED AT" line to stdout and jumps to `error`, + * similar to `TEST_ERROR` in h5test.h + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *---------------------------------------------------------------------------- + */ +#define FAIL_IF(condition) \ +if (condition) { \ + JSFAILED_AT() \ + goto error; \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_UNLESS() + * + * Purpose: + * + * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests", + * e.g., "a != b". + * + * Opposite of FAIL_IF; fails if the given condition is _not_ true. + * + * `FAIL_IF( 5 != my_op() )` + * is equivalent to + * `FAIL_UNLESS( 5 == my_op() )` + * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer. + * (see JSVERIFY) + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#if 0 /* UNUSED */ +#define FAIL_UNLESS(condition) \ +if (!(condition)) { \ + JSFAILED_AT() \ + goto error; \ +} +#endif /* UNUSED */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_LONG() + * + * Purpose: + * + * Print an failure message for long-int arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:488 in somefunc()... + * forest must be made of trees. + * + * or + * + * *FAILED* at myfile.c:488 in somefunc()... + * ! Expected 425 + * ! Actual 3 + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_LONG(expected, actual, reason) { \ + JSFAILED_AT() \ + if (reason!= NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf(" ! Expected %ld\n ! Actual %ld\n", \ + (long)(expected), (long)(actual)); \ + } \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_STR() + * + * Purpose: + * + * Print an failure message for string arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:421 in myfunc()... + * Blue and Red strings don't match! + * + * or + * + * *FAILED* at myfile.c:421 in myfunc()... + * !!! Expected: + * this is my expected + * string + * !!! Actual: + * not what I expected at all + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_STR(expected, actual, reason) { \ + JSFAILED_AT() \ + if ((reason) != NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf("!!! Expected:\n%s\n!!!Actual:\n%s\n", \ + (expected), (actual)); \ + } \ +} + + + +#ifdef JSVERIFY_EXP_ACT + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY() + * + * Purpose: + * + * Verify that two long integers are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(expected, actual, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_NOT() + * + * Purpose: + * + * Verify that two long integers are _not_ equal. + * If equal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(expected, actual, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_STR() + * + * Purpose: + * + * Verify that two strings are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(expected, actual, reason) \ +if (strcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + + +#else +/* JSVERIFY_EXP_ACT not defined + * + * Repeats macros above, but with actual/expected parameters reversed. + */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY() + * See: JSVERIFY documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(actual, expected, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_NOT() + * See: JSVERIFY_NOT documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(actual, expected, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_STR() + * See: JSVERIFY_STR documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(actual, expected, reason) \ +if (strcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + +#endif /* ifdef/else JSVERIFY_EXP_ACT */ + +/******************************** + * OTHER MACROS AND DEFINITIONS * + ********************************/ + +/* copied from src/hdfs.c + */ +#ifdef H5_HAVE_LIBHDFS +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) +#endif /* H5_HAVE_LIBHDFS */ + +#define HDFS_NAMENODE_NAME_MAX_SIZE 128 + +/******************************* + * FILE-LOCAL GLOBAL VARIABLES * + *******************************/ + +#ifdef H5_HAVE_LIBHDFS +static const char filename_missing[] = "/tmp/missing.txt"; +static const char filename_bard[] = "/tmp/t8.shakespeare.txt"; +static const char filename_raven[] = "/tmp/Poe_Raven.txt"; +static const char filename_example_h5[] = "/tmp/t.h5"; +#endif /* H5_HAVE_LIBHDFS */ + +static H5FD_hdfs_fapl_t default_fa = { + 1, /* fa version */ + "localhost", /* namenode name */ + 0, /* namenode port */ + "", /* user name */ + "", /* kerberos path */ + 1024, /* buffer size */ +}; + +/****************** + * TEST FUNCTIONS * + ******************/ + + +/*--------------------------------------------------------------------------- + * + * Function: test_fapl_config_validation() + * + * Purpose: + * + * Test data consistency of fapl configuration. + * Tests `H5FD_hdfs_validate_config` indirectly through `H5Pset_fapl_hdfs`. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2018-04-25 + * + * Changes: None. + * + *--------------------------------------------------------------------------- + */ +static int +test_fapl_config_validation(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char *msg; + herr_t expected; + H5FD_hdfs_fapl_t config; + }; + + /************************ + * test-local variables * + ************************/ + + hid_t fapl_id = -1; /* file access property list ID */ + H5FD_hdfs_fapl_t config; + H5FD_hdfs_fapl_t fa_fetch; + herr_t success = SUCCEED; + unsigned int i = 0; + unsigned int ncases = 6; /* should equal number of cases */ + struct testcase *case_ptr = NULL; /* dumb work-around for possible */ + /* dynamic cases creation because */ + /* of compiler warnings Wlarger-than */ + struct testcase cases_arr[] = { + { "default config fapl", + SUCCEED, + { 1, /* version */ + "localhost", /* namenode_name */ + 0, /* namenode_port number */ + "some_user", /* user_name */ + "", /* kerberos_ticket_cache path */ + -1, /* stream_buffer_size */ + }, + }, + { "invalid version number (2)", + FAIL, + { 2, /* version */ + "localhost", /* namenode_name */ + 0, /* namenode_port number */ + "some_user", /* user_name */ + "", /* kerberos_ticket_cache path */ + -1, /* stream_buffer_size */ + }, + }, + { "invalid version number (0)", + FAIL, + { 0, /* version */ + "localhost", /* namenode_name */ + 0, /* namenode_port number */ + "some_user", /* user_name */ + "", /* kerberos_ticket_cache path */ + -1, /* stream_buffer_size */ + }, + }, + { "nonsense kerberos path still ok?", + SUCCEED, + { 1, /* version */ + "localhost", /* namenode_name */ + 0, /* namenode_port number */ + "some_user", /* user_name */ + "pathToSomewhere", /* kerberos_ticket_cache path */ + -1, /* stream_buffer_size */ + }, + }, + { "namenode port number too high", + FAIL, + { 1, /* version */ + "localhost", /* namenode_name */ + 88000, /* namenode_port number */ + "some_user", /* user_name */ + "", /* kerberos_ticket_cache path */ + -1, /* stream_buffer_size */ + }, + }, + { "negative namenode port number", + FAIL, + { 1, /* version */ + "localhost", /* namenode_name */ + -1, /* namenode_port number */ + "some_user", /* user_name */ + "", /* kerberos_ticket_cache path */ + -1, /* stream_buffer_size */ + }, + }, + }; + + TESTING("HDFS fapl configuration validation"); + + /********* + * TESTS * + *********/ + + for (i = 0; i < ncases; i++) { + + /*--------------- + * per-test setup + *--------------- + */ + case_ptr = &cases_arr[i]; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) /* sanity-check */ + + /*----------------------------------- + * Actually test -- set fapl. + * Mute stack trace in failure cases. + *----------------------------------- + */ + H5E_BEGIN_TRY { + /* `H5FD_hdfs_validate_config(...)` is static/private + * to src/hdfs.c and cannot (and should not?) be tested directly? + * Instead, validate config through public api. + */ + success = H5Pset_fapl_hdfs(fapl_id, &case_ptr->config); + } H5E_END_TRY; + + JSVERIFY( case_ptr->expected, success, case_ptr->msg ) + + /* Make sure we can get back what we put in. + * Only valid if the fapl configuration does not result in error. + */ + if (success == SUCCEED) { + config = case_ptr->config; + JSVERIFY( SUCCEED, + H5Pget_fapl_hdfs(fapl_id, &fa_fetch), + "unable to get fapl" ) + JSVERIFY( H5FD__CURR_HDFS_FAPL_T_VERSION, + fa_fetch.version, + "invalid version number" ) + JSVERIFY( config.version, + fa_fetch.version, + "version number mismatch" ) + JSVERIFY( config.namenode_port, + fa_fetch.namenode_port, + "namenode port mismatch" ) + JSVERIFY( config.stream_buffer_size, + fa_fetch.stream_buffer_size, + "streambuffer size mismatch" ) + JSVERIFY_STR( config.namenode_name, + fa_fetch.namenode_name, + NULL ) + JSVERIFY_STR( config.user_name, + fa_fetch.user_name, + NULL ) + JSVERIFY_STR( config.kerberos_ticket_cache, + fa_fetch.kerberos_ticket_cache, + NULL ) + } + + /*----------------------------- + * per-test sanitation/teardown + *----------------------------- + */ + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + } /* for each test case */ + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fapl_id < 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + return 1; + +} /* end test_fapl_config_validation() */ + + +/*------------------------------------------------------------------------- + * + * Function: test_hdfs_fapl() + * + * Purpose: Tests the file handle interface for the HDFS driver. + * + * For now, test only fapl & flags. Extend as the + * work on the VFD continues. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Jacob Smith + * 2018-04-25 + * + * Changes: None. + * + *------------------------------------------------------------------------- + */ +static int +test_hdfs_fapl(void) +{ + /************************ + * test-local variables * + ************************/ + + hid_t fapl_id = -1; /* file access property list ID */ + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ + H5FD_hdfs_fapl_t hdfs_fa_0 = { + 1, /* version*/ + "", /* node name */ + 9000, /* node port */ + "", /* username */ + "", /* kerb cache path */ + 1024, /* stream buffer size */ + }; + + TESTING("HDFS fapl "); + + /* Set property list and file name for HDFS driver. + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &hdfs_fa_0) ) + + driver_id = H5Pget_driver(fapl_id); + FAIL_IF( driver_id < 0 ) + + /**************** + * Check that the VFD feature flags are correct + * SPEC MAY CHANGE + ******************/ + + FAIL_IF( H5FDdriver_query(driver_id, &driver_flags) < 0 ) + + JSVERIFY_NOT( 0, (driver_flags & H5FD_FEAT_DATA_SIEVE), + "bit(s) in `driver_flags` must align with " + "H5FD_FEAT_DATA_SIEVE" ) + + JSVERIFY( H5FD_FEAT_DATA_SIEVE, driver_flags, + "H5FD_FEAT_DATA_SIEVE should be the only supported flag") + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + + return 1; + +} /* end test_hdfs_fapl() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_vfd_open() + * + * Purpose: + * + * Demonstrate/specify VFD-level "Open" failure cases + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2018-06-07 + * + *--------------------------------------------------------------------------- + */ +static int +test_vfd_open(void) +{ + +#ifndef H5_HAVE_LIBHDFS + TESTING("HDFS VFD-level open"); + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; + +#else + + /********************* + * test-local macros * + *********************/ + +/* selectors for which fapl to use in testcase */ +#define FAPL_H5P_DEFAULT -2 +#define FAPL_UNCONFIGURED -3 /* H5P_FILE_ACCESS */ +#define FAPL_HDFS -4 + + /************************* + * test-local structures * + *************************/ + + struct test_condition { + const char *message; + const char *url; + unsigned flags; + int which_fapl; + haddr_t maxaddr; + hbool_t might_use_other_driver; + }; + + /************************ + * test-local variables * + ************************/ + + struct test_condition failing_conditions[] = { + { "default property list (H5P_DEFAULT) is invalid", + filename_bard, + H5F_ACC_RDONLY, + FAPL_H5P_DEFAULT, + MAXADDR, + TRUE, + }, + { "generic file access property list is invalid", + filename_bard, + H5F_ACC_RDONLY, + FAPL_UNCONFIGURED, + MAXADDR, + TRUE, + }, + { "filename cannot be null", + NULL, + H5F_ACC_RDONLY, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "filename cannot be empty", + "", + H5F_ACC_RDONLY, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "file at filename must exist", + filename_missing, + H5F_ACC_RDONLY, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "read-write flag not supported", + filename_bard, + H5F_ACC_RDWR, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "truncate flag not supported", + filename_bard, + H5F_ACC_TRUNC, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "create flag not supported", + filename_bard, + H5F_ACC_CREAT, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "EXCL flag not supported", + filename_bard, + H5F_ACC_EXCL, + FAPL_HDFS, + MAXADDR, + FALSE, + }, + { "maxaddr cannot be 0 (caught in `H5FD_open()`)", + filename_bard, + H5F_ACC_RDONLY, + FAPL_HDFS, + 0, + FALSE, + }, + }; + unsigned i = 0; + unsigned failing_conditions_count = 10; + H5FD_t *fd = NULL; + hid_t fapl_hdfs = -1; + hid_t fapl_unconfigured = -1; + + TESTING("HDFS VFD-level open"); + + fapl_unconfigured = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_unconfigured < 0 ) + + fapl_hdfs = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_hdfs < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_hdfs, &default_fa) ) + + /********* + * TESTS * + *********/ + + /* all the test cases that will _not_ open + */ + for (i = 0; i < failing_conditions_count; i++) { + struct test_condition T = failing_conditions[i]; + hid_t fapl_id = H5P_DEFAULT; + + fd = NULL; + + if (T.which_fapl == FAPL_UNCONFIGURED) { + fapl_id = fapl_unconfigured; + } + else + if (T.which_fapl == FAPL_HDFS) { + fapl_id = fapl_hdfs; + } + +#if HDFS_TEST_DEBUG + HDfprintf(stderr, "testing: %s\n", T.message); +#endif /* HDFS_TEST_DEBUG */ + + H5E_BEGIN_TRY { + fd = H5FDopen(T.url, T.flags, fapl_id, T.maxaddr); + } H5E_END_TRY; + if (NULL != fd) { + if (TRUE == T.might_use_other_driver && + H5FD_HDFS != fd->driver_id) + { + HDfprintf(stderr, "\n!!!!! WARNING !!!!!\n" \ + " Successful open of file on local system " \ + "with non-HDFS VFD.\n"); + JSVERIFY(SUCCEED, H5FDclose(fd), + "unable to close errant open"); + fd = NULL; + } + else { + JSVERIFY(1, 0, T.message); /* print message and fail */ + } + } + } + + FAIL_IF( NULL != fd ) /* sanity check */ + +#if HDFS_TEST_DEBUG + HDfprintf(stderr, "nominal open\n"); +#endif /* HDFS_TEST_DEBUG */ + + /* finally, show that a file can be opened + */ + fd = H5FDopen( + filename_bard, + H5F_ACC_RDONLY, + fapl_hdfs, + MAXADDR); + FAIL_IF( NULL == fd ) + + /************ + * TEARDOWN * + ************/ + +#if HDFS_TEST_DEBUG + HDfprintf(stderr, "teardown...\n"); +#endif /* HDFS_TEST_DEBUG */ + + FAIL_IF( FAIL == H5FDclose(fd) ) + fd = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_hdfs) ) + fapl_hdfs = -1; + + FAIL_IF( FAIL == H5Pclose(fapl_unconfigured) ) + fapl_unconfigured = -1; + + PASSED(); + return 0; + +error: + + /*********** + * CLEANUP * + ***********/ + + if (fd) { + (void)H5FDclose(fd); + } + H5E_BEGIN_TRY { + if (fapl_hdfs >= 0) { + (void)H5Pclose(fapl_hdfs); + } + if (fapl_unconfigured >= 0) { + (void)H5Pclose(fapl_unconfigured); + } + } H5E_END_TRY; + + return 1; + +#undef FAPL_H5P_DEFAULT +#undef FAPL_UNCONFIGURED +#undef FAPL_HDFS + +#endif /* H5_HAVE_LIBHDFS */ + +} /* end test_vfd_open() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_eof_eoa() + * + * Purpose: + * + * Demonstrate behavior of get_eof, get_eoa, and set_eoa. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2018-06-07 + * + *--------------------------------------------------------------------------- + */ +static int +test_eof_eoa(void) +{ +#ifndef H5_HAVE_LIBHDFS + TESTING("HDFS eof/eoa gets and sets"); + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; + +#else + + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + + H5FD_t *fd_shakespeare = NULL; + hid_t fapl_id = -1; + + TESTING("HDFS eof/eoa gets and sets"); + + /********* + * SETUP * + *********/ + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( 0 > fapl_id ) + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) ) + + fd_shakespeare = H5FDopen( + filename_bard, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_shakespeare ) + + /********* + * TESTS * + *********/ + + /* verify as found + */ + JSVERIFY( 5458199, H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), NULL ) + JSVERIFY( H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + H5FDget_eof(fd_shakespeare, H5FD_MEM_DRAW), + "mismatch between DEFAULT and RAW memory types" ) + JSVERIFY( 0, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoA should be unset by H5FDopen" ) + + /* set EoA below EoF + */ + JSVERIFY( SUCCEED, + H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 44442202), + "unable to set EoA (lower)" ) + JSVERIFY( 5458199, + H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoF changed" ) + JSVERIFY( 44442202, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoA unchanged" ) + + /* set EoA above EoF + */ + JSVERIFY( SUCCEED, + H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 6789012), + "unable to set EoA (higher)" ) + JSVERIFY( 5458199, + H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoF changed" ) + JSVERIFY( 6789012, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoA unchanged" ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(fd_shakespeare) ) + fd_shakespeare = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; + +error: + + /*********** + * CLEANUP * + ***********/ + + if (fd_shakespeare != NULL) { + (void)H5FDclose(fd_shakespeare); + } + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; + +#endif /* H5_HAVE_LIBHDFS */ + +} /* end test_eof_eoa() */ + + +/*----------------------------------------------------------------------------- + * + * Function: test_H5FDread_without_eoa_set_fails() + * + * Purpose: + * + * Demonstrate a not-obvious constraint by the library, preventing + * file read before EoA is set + * + * Programmer: Jacob Smith + * 2018-06-08 + * + *----------------------------------------------------------------------------- + */ +static int +test_H5FDread_without_eoa_set_fails(void) +{ +#ifndef H5_HAVE_LIBHDFS + TESTING("HDFS VFD read-eoa temporal coupling library limitation"); + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; + +#else + + char buffer[HDFS_TEST_MAX_BUF_SIZE]; + unsigned int i = 0; + H5FD_t *file_shakespeare = NULL; + hid_t fapl_id = -1; + + TESTING("HDFS VFD read-eoa temporal coupling library limitation"); + + /********* + * SETUP * + *********/ + + /* create HDFS fapl + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) ) + + file_shakespeare = H5FDopen( + filename_bard, + H5F_ACC_RDONLY, + fapl_id, + MAXADDR); + FAIL_IF( NULL == file_shakespeare ) + + JSVERIFY( 0, H5FDget_eoa(file_shakespeare, H5FD_MEM_DEFAULT), + "EoA should remain unset by H5FDopen" ) + + /* zero buffer contents */ + for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) { + buffer[i] = 0; + } + + /******** + * TEST * + ********/ + + H5E_BEGIN_TRY { /* mute stack trace on expected failure */ + JSVERIFY( FAIL, + H5FDread(file_shakespeare, + H5FD_MEM_DRAW, + H5P_DEFAULT, + 1200699, + 102, + buffer), + "cannot read before eoa is set" ) + } H5E_END_TRY; + for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) { + JSVERIFY( 0, (unsigned)buffer[i], "buffer was modified by write!" ) + } + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(file_shakespeare) ) + file_shakespeare = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; + +error: + + /*********** + * CLEANUP * + ***********/ + + if (file_shakespeare) { + (void)H5FDclose(file_shakespeare); + } + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; + +#endif /* H5_HAVE_LIBHDFS */ + +} /* end test_H5FDread_without_eoa_set_fails() */ + + + +/*--------------------------------------------------------------------------- + * + * Function: test_read() + * + * Purpose: + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2018-06-08 + * + *--------------------------------------------------------------------------- + */ +static int +test_read(void) +{ +#ifndef H5_HAVE_LIBHDFS + TESTING("HDFS VFD read/range-gets"); + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; + +#else + + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + struct testcase { + const char *message; /* purpose of test case */ + haddr_t eoa_set; /* set file EOA to this prior to read */ + size_t addr; /* offset of read in file */ + size_t len; /* length of read in file */ + herr_t success; /* expected return value of read function */ + const char *expected; /* expected contents of buffer; failure ignores */ + }; + + /************************ + * test-local variables * + ************************/ + struct testcase cases[] = { + { "successful range-get", + 6464, + 5691, + 32, /* fancy quotes are three bytes each(?) */ + SUCCEED, + "Quoth the Raven “Nevermore.”", + }, + { "read past EOA fails (EOA < EOF < addr)", + 3000, + 4000, + 100, + FAIL, + NULL, + }, + { "read overlapping EOA fails (EOA < addr < EOF < (addr+len))", + 3000, + 8000, + 100, + FAIL, + NULL, + }, + { "read past EOA/EOF fails ((EOA==EOF) < addr)", + 6464, + 7000, + 100, + FAIL, + NULL, + }, + { "read overlapping EOA/EOF fails (addr < (EOA==EOF) < (addr+len))", + 6464, + 6400, + 100, + FAIL, + NULL, + }, + { "read between EOF and EOA fails (EOF < addr < (addr+len) < EOA)", + 8000, + 7000, + 100, + FAIL, + NULL, + }, + }; + unsigned testcase_count = 6; + unsigned test_i = 0; + struct testcase test; + herr_t open_return = FAIL; + char buffer[HDFS_TEST_MAX_BUF_SIZE]; + unsigned int i = 0; + H5FD_t *file_raven = NULL; + hid_t fapl_id = -1; + + TESTING("HDFS VFD read/range-gets"); + + /********* + * SETUP * + *********/ + + /* create HDFS fapl + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) ) + + /* zero buffer contents */ + for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) { + buffer[i] = 0; + } + + /* open file + */ + file_raven = H5FDopen( + filename_raven, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); /* Demonstrate success with "automatic" value */ + FAIL_IF( NULL == file_raven ) + + JSVERIFY( 6464, H5FDget_eof(file_raven, H5FD_MEM_DEFAULT), NULL ) + + /********* + * TESTS * + *********/ + + for (test_i = 0; test_i < testcase_count; test_i++) { + + /* -------------- * + * per-test setup * + * -------------- */ + + test = cases[test_i]; + open_return = FAIL; + + FAIL_IF( HDFS_TEST_MAX_BUF_SIZE < test.len ) /* buffer too small! */ + + FAIL_IF( FAIL == + H5FDset_eoa( file_raven, H5FD_MEM_DEFAULT, test.eoa_set) ) + + /* zero buffer contents */ + for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) { + buffer[i] = 0; + } + + /* ------------ * + * conduct test * + * ------------ */ + + H5E_BEGIN_TRY { + open_return = H5FDread( + file_raven, + H5FD_MEM_DRAW, + H5P_DEFAULT, + test.addr, + test.len, + buffer); + } H5E_END_TRY; + + JSVERIFY( test.success, + open_return, + test.message ) + + if (open_return == SUCCEED) { + JSVERIFY_STR( test.expected, buffer, NULL ) + } + + } /* for each testcase */ + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(file_raven) ) + file_raven = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; + +error: + + /*********** + * CLEANUP * + ***********/ + + if (file_raven != 0) { + (void)H5FDclose(file_raven); + } + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; + +#endif /* H5_HAVE_LIBHDFS */ + +} /* end test_read() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_noops_and_autofails() + * + * Purpose: + * + * Demonstrate the unavailable and do-nothing routines unique to + * Read-Only VFD. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-06 + * + * Changes: + * + modify from S3VFD codebase to HDFS; Minor changes, mostly. + * + Jacob Smith 2018-06-08 + * + *--------------------------------------------------------------------------- + */ +static int +test_noops_and_autofails(void) +{ +#ifndef H5_HAVE_LIBHDFS + TESTING("HDFS VFD always-fail and no-op routines"); + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; + +#else + + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + + hid_t fapl_id = -1; + H5FD_t *file = NULL; + const char data[36] = "The Force shall be with you, always"; + + TESTING("HDFS VFD always-fail and no-op routines"); + + /********* + * SETUP * + *********/ + + /* create HDFS fapl + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) ) + + /* open file + */ + file = H5FDopen( + filename_bard, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == file ) + + /********* + * TESTS * + *********/ + + /* auto-fail calls to write and truncate + */ + H5E_BEGIN_TRY { + JSVERIFY( FAIL, + H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, 1000, 35, data), + "write must fail" ) + } H5E_END_TRY; + + H5E_BEGIN_TRY { + JSVERIFY( FAIL, + H5FDtruncate(file, H5P_DEFAULT, FALSE), + "truncate must fail" ) + } H5E_END_TRY; + + H5E_BEGIN_TRY { + JSVERIFY( FAIL, + H5FDtruncate(file, H5P_DEFAULT, TRUE), + "truncate must fail (closing)" ) + } H5E_END_TRY; + + /* no-op calls to `lock()` and `unlock()` + */ + JSVERIFY( SUCCEED, + H5FDlock(file, TRUE), + "lock always succeeds; has no effect" ) + JSVERIFY( SUCCEED, + H5FDlock(file, FALSE), + NULL ) + JSVERIFY( SUCCEED, + H5FDunlock(file), + NULL ) + /* Lock/unlock with null file or similar error crashes tests. + * HDassert in calling heirarchy, `H5FD[un]lock()` and `H5FD_[un]lock()` + */ + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(file) ) + file = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; + +error: + + /*********** + * CLEANUP * + ***********/ + + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + if (file != NULL) { + (void)H5FDclose(file); + } + + return 1; + +#endif /* H5_HAVE_LIBHDFS */ + +} /* end test_noops_and_autofails() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_cmp() + * + * Purpose: + * + * Verify "file comparison" behavior. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-06 + * + *--------------------------------------------------------------------------- + */ +static int +test_cmp(void) +{ + TESTING("HDFS cmp (comparison)"); + SKIPPED(); + HDfprintf( + stderr, + " TODO: Distinct valid fapls to open the same file.\n"); + + return 0; + +} /* end test_cmp() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_H5F_integration() + * + * Purpose: + * + * Demonstrate H5F (File interface) behavior with files on HDFS. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-07 + * + * Changes: + * + modify from S3VFD codebase to HDFS; Minor changes, mostly. + * + Jacob Smith 2018-06-08 + * + *--------------------------------------------------------------------------- + */ +static int +test_H5F_integration(void) +{ +#ifndef H5_HAVE_LIBHDFS + TESTING("HDFS file access through HD5F library (H5F API)"); + SKIPPED(); + puts(" HDFS VFD is not enabled"); + fflush(stdout); + return 0; + +#else + + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + + hid_t file = -1; + hid_t fapl_id = -1; + + TESTING("HDFS file access through HD5F library (H5F API)"); + + /********* + * SETUP * + *********/ + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( 0 > fapl_id ) + FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) ) + + /********* + * TESTS * + *********/ + + /* Read-Write Open access is not allowed with this file driver. + */ + H5E_BEGIN_TRY { + FAIL_IF( 0 <= H5Fopen( + filename_example_h5, + H5F_ACC_RDWR, + fapl_id) ) + } H5E_END_TRY; + + /* H5Fcreate() is not allowed with this file driver. + */ + H5E_BEGIN_TRY { + FAIL_IF( 0 <= H5Fcreate( + filename_missing, + H5F_ACC_RDONLY, + H5P_DEFAULT, + fapl_id) ) + } H5E_END_TRY; + + /* Successful open. + */ + file = H5Fopen( + filename_example_h5, + H5F_ACC_RDONLY, + fapl_id); + FAIL_IF( file < 0 ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5Fclose(file) ) + file = -1; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + +#if HDFS_TEST_DEBUG + HDprintf("\nerror!"); fflush(stdout); +#endif /* HDFS_TEST_DEBUG */ + + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + if (file > 0) { + (void)H5Fclose(file); + } + + return 1; + +#endif /* H5_HAVE_LIBHDFS */ + +} /* test_H5F_integration */ + + +/*------------------------------------------------------------------------- + * + * Function: main + * + * Purpose: Tests the basic features of Virtual File Drivers + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + int nerrors = 0; + + /****************** + * commence tests * + ******************/ + + static char hdfs_namenode_name[HDFS_NAMENODE_NAME_MAX_SIZE] = ""; + const char *hdfs_namenode_name_env = NULL; + + hdfs_namenode_name_env = HDgetenv("HDFS_TEST_NAMENODE_NAME"); + if (hdfs_namenode_name_env == NULL || hdfs_namenode_name_env[0] == '\0') { + HDstrncpy(hdfs_namenode_name, "localhost", HDFS_NAMENODE_NAME_MAX_SIZE); + } + else { + HDstrncpy( /* TODO: error-check? */ + default_fa.namenode_name, + hdfs_namenode_name_env, + HDFS_NAMENODE_NAME_MAX_SIZE); + } + + h5_reset(); + + HDprintf("Testing hdfs VFD functionality.\n"); + + nerrors += test_fapl_config_validation(); + nerrors += test_hdfs_fapl(); + nerrors += test_vfd_open(); + nerrors += test_eof_eoa(); + nerrors += test_H5FDread_without_eoa_set_fails(); + nerrors += test_read(); + nerrors += test_noops_and_autofails(); + nerrors += test_cmp(); + nerrors += test_H5F_integration(); + + if (nerrors > 0) { + HDprintf("***** %d hdfs TEST%s FAILED! *****\n", + nerrors, + nerrors > 1 ? "S" : ""); + nerrors = 1; + } + else { + HDprintf("All hdfs tests passed.\n"); + } + return nerrors; /* 0 if no errors, 1 if any errors */ + +} /* end main() */ + + diff --git a/test/hyperslab.c b/test/hyperslab.c index 9e17a2f..d22a689 100644 --- a/test/hyperslab.c +++ b/test/hyperslab.c @@ -1180,13 +1180,11 @@ error: /*------------------------------------------------------------------------- * Function: main * - * Purpose: Test various hyperslab operations. Give the words - * `small' and/or `medium' on the command line or only `small' - * is assumed. + * Purpose: Test various hyperslab operations. Give the words + * 'small' and/or 'medium' on the command line or only 'small' + * is assumed. * - * Return: Success: exit(EXIT_SUCCESS) - * - * Failure: exit(EXIT_FAILURE) + * Return: EXIT_SUCCESS/EXIT_FAILURE * * Programmer: Robb Matzke * Friday, October 10, 1997 @@ -1442,6 +1440,6 @@ main(int argc, char *argv[]) H5close(); #endif /* H5_HAVE_THREADSAFE */ - return 0; + HDexit(EXIT_SUCCESS); } diff --git a/test/istore.c b/test/istore.c index 6656abd..ca60b03 100644 --- a/test/istore.c +++ b/test/istore.c @@ -579,17 +579,13 @@ error: /*------------------------------------------------------------------------- * Function: main * - * Purpose: Tests indexed storage stuff. + * Purpose: Tests indexed storage * - * Return: Success: exit(EXIT_SUCCESS) - * - * Failure: exit(EXIT_FAILURE) + * Return: EXIT_SUCCESS/EXIT_FAILURE * * Programmer: Robb Matzke * Wednesday, October 15, 1997 * - * Modifications: - * *------------------------------------------------------------------------- */ int @@ -618,9 +614,6 @@ main(int argc, char *argv[]) size_of_test |= TEST_LARGE; } else { HDprintf("unrecognized argument: %s\n", argv[i]); -#if 0 - exit(EXIT_FAILURE); -#endif } } } @@ -654,7 +647,7 @@ main(int argc, char *argv[]) h5_fixname(FILENAME[0], fapl, filename, sizeof filename); if ((file=H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) { HDprintf("Cannot create file %s; test aborted\n", filename); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } /* Initialize chunk dimensions */ @@ -722,13 +715,13 @@ main(int argc, char *argv[]) if (nerrors) { HDprintf("***** %d I-STORE TEST%s FAILED! *****\n", nerrors, 1 == nerrors ? "" : "S"); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } HDprintf("All i-store tests passed.\n"); h5_cleanup(FILENAME, fapl); - return 0; + HDexit(EXIT_SUCCESS); } diff --git a/test/links.c b/test/links.c index 89e62ab..6ba9992 100644 --- a/test/links.c +++ b/test/links.c @@ -14007,10 +14007,10 @@ main(void) HDrmdir(TMPDIR); HDrmdir(TMPDIR2); - return SUCCEED; + HDexit(EXIT_SUCCESS); error: HDputs("*** TESTS FAILED ***"); - return 1; + HDexit(EXIT_FAILURE); } diff --git a/test/links_env.c b/test/links_env.c index df14675..b07b1e0 100644 --- a/test/links_env.c +++ b/test/links_env.c @@ -134,10 +134,9 @@ external_link_env(hid_t fapl, hbool_t new_format) /*------------------------------------------------------------------------- * Function: main * - * Purpose: Test external link with environment variable HDF5_EXT_PREFIX + * Purpose: Test external link with environment variable HDF5_EXT_PREFIX * - * Return: Success: exit(EXIT_SUCCESS) - * Failure: exit(EXIT_FAILURE) + * Return: EXIT_SUCCESS/EXIT_FAILURE * * Programmer: Vailin Choi; Nov 2010 * @@ -175,9 +174,10 @@ main(void) /* clean up tmp_links_env directory created by external link tests */ HDrmdir(TMPDIR); - return 0; + HDexit(EXIT_SUCCESS); error: HDputs("*** TESTS FAILED ***"); - return 1; -} + HDexit(EXIT_FAILURE); +} /* end main() */ + diff --git a/test/mtime.c b/test/mtime.c index b50dfeb..2e4b2a4 100644 --- a/test/mtime.c +++ b/test/mtime.c @@ -15,10 +15,10 @@ * Programmer: Robb Matzke * Thursday, July 30, 1998 * - * Purpose: Determines if the modification time message is working - * properly. Specifically, the code in H5O_mtime_decode() is - * very OS-dependent and this test tries to figure out if it's - * working properly. + * Purpose: Determines if the modification time message is working + * properly. Specifically, the code in H5O_mtime_decode() is + * very OS-dependent and this test tries to figure out if it's + * working properly. */ #include "h5test.h" #include "H5srcdir.h" @@ -33,17 +33,17 @@ const char *FILENAME[] = { #define TESTFILE2 "tmtimen.h5" #define MTIME2 1041606478 - + /*------------------------------------------------------------------------- - * Function: main + * Function: main * - * Purpose: H5O_mtime_decode() test. + * Purpose: H5O_mtime_decode() test. * - * Return: Success: + * Return: Success: * - * Failure: + * Failure: * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Thursday, July 30, 1998 * * Modifications: @@ -58,13 +58,13 @@ const char *FILENAME[] = { int main(void) { - hid_t fapl, file, space, dset; - hsize_t size[1] = {2}; - time_t now; - struct tm *tm; - H5O_info_t oi1, oi2; - signed char buf1[32], buf2[32]; - char filename[1024]; + hid_t fapl, file, space, dset; + hsize_t size[1] = {2}; + time_t now; + struct tm *tm; + H5O_info_t oi1, oi2; + signed char buf1[32], buf2[32]; + char filename[1024]; h5_reset(); fapl = h5_fileaccess(); @@ -99,18 +99,18 @@ main(void) /* Compare addresses & times from the two ways of calling H5Oget_info() */ if(oi1.addr != oi2.addr || oi1.ctime != oi2.ctime) { H5_FAILED(); - puts(" Calling H5Oget_info() with the dataset ID returned"); - puts(" different values than calling it with a file and dataset"); - puts(" name."); + HDputs(" Calling H5Oget_info() with the dataset ID returned"); + HDputs(" different values than calling it with a file and dataset"); + HDputs(" name."); goto error; } /* Compare times -- they must be within 60 seconds of one another */ if(0 == oi1.ctime) { SKIPPED(); - puts(" The modification time could not be decoded on this OS."); - puts(" Modification times will be mantained in the file but"); - puts(" cannot be queried on this system. See H5O_mtime_decode()."); + HDputs(" The modification time could not be decoded on this OS."); + HDputs(" Modification times will be mantained in the file but"); + HDputs(" cannot be queried on this system. See H5O_mtime_decode()."); return 0; } else if(HDfabs(HDdifftime(now, oi1.ctime)) > (double)60.0F) { H5_FAILED(); @@ -130,14 +130,8 @@ main(void) TESTING("accessing old modification time messages"); { - char testfile[512]=""; - char *srcdir = HDgetenv("srcdir"); + const char *testfile = H5_get_srcdir_filename(TESTFILE1); /* Corrected test file name */ - if(srcdir && ((HDstrlen(srcdir) + strlen(TESTFILE1) + 1) < sizeof(testfile))){ - HDstrcpy(testfile, srcdir); - HDstrcat(testfile, "/"); - } - HDstrcat(testfile, TESTFILE1); file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); if(file >= 0){ if(H5Oget_info_by_name(file, "/Dataset1", &oi1, H5P_DEFAULT) < 0) @@ -166,21 +160,15 @@ main(void) TESTING("accessing new modification time messages"); { - char testfile[512]=""; - char *srcdir = HDgetenv("srcdir"); + const char *testfile = H5_get_srcdir_filename(TESTFILE2); /* Corrected test file name */ - if(srcdir && ((HDstrlen(srcdir) + strlen(TESTFILE2) + 1) < sizeof(testfile))){ - HDstrcpy(testfile, srcdir); - HDstrcat(testfile, "/"); - } - HDstrcat(testfile, TESTFILE2); file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); if(file >= 0){ if(H5Oget_info_by_name(file, "/Dataset1", &oi2, H5P_DEFAULT) < 0) TEST_ERROR; if(oi2.ctime != MTIME2) { H5_FAILED(); - puts(" Modification time incorrect."); + HDputs(" Modification time incorrect."); goto error; } if(H5Fclose(file) < 0) TEST_ERROR; @@ -198,7 +186,7 @@ main(void) if(h5_verify_cached_stabs(FILENAME, fapl) < 0) TEST_ERROR /* All looks good */ - puts("All modification time tests passed."); + HDputs("All modification time tests passed."); h5_cleanup(FILENAME, fapl); return 0; diff --git a/test/objcopy.c b/test/objcopy.c index db66285..b473e2b 100644 --- a/test/objcopy.c +++ b/test/objcopy.c @@ -12,10 +12,10 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Peter X. Cao - * May 01, 2005 + * Programmer: Peter X. Cao + * May 01, 2005 * - * Purpose: Test H5Ocopy(). + * Purpose: Test H5Ocopy(). */ #include "h5test.h" diff --git a/test/reserved.c b/test/reserved.c index e11ae19..7e53d9c 100644 --- a/test/reserved.c +++ b/test/reserved.c @@ -22,17 +22,17 @@ const char *FILENAME[] = { }; /*------------------------------------------------------------------------- - * Function: rsrv_heap + * Function: rsrv_heap * - * Purpose: Ensure that heaps reserve file address space. - * This function does this by creating datasets up to and past - * the limit of the file, then ensuring that an error (not an - * assert) was generated and that the file is readable. + * Purpose: Ensure that heaps reserve file address space. + * This function does this by creating datasets up to and past + * the limit of the file, then ensuring that an error (not an + * assert) was generated and that the file is readable. * - * Return: Success: 0 - * Failure: 1 + * Return: Success: 0 + * Failure: 1 * - * Programmer: James Laird + * Programmer: James Laird * Nat Furrer * Friday, May 28, 2004 * @@ -135,17 +135,17 @@ error: } /*------------------------------------------------------------------------- - * Function: rsrv_ohdr + * Function: rsrv_ohdr * - * Purpose: Ensure that object headers reserve file address space. - * This function does this by creating attributes of a dataset - * past the limit of the file, then ensuring that an error (not - * an assert) was generated and that the file is readable. + * Purpose: Ensure that object headers reserve file address space. + * This function does this by creating attributes of a dataset + * past the limit of the file, then ensuring that an error (not + * an assert) was generated and that the file is readable. * - * Return: Success: 0 - * Failure: 1 + * Return: Success: 0 + * Failure: 1 * - * Programmer: James Laird + * Programmer: James Laird * Nat Furrer * Friday, May 28, 2004 * @@ -257,20 +257,20 @@ error: } /*------------------------------------------------------------------------- - * Function: rsrv_vlen + * Function: rsrv_vlen * - * Purpose: Ensure that variable length datatypes properly ensure that + * Purpose: Ensure that variable length datatypes properly ensure that * enough file address space exists before writing. - * This function does this by creating a dataset containing + * This function does this by creating a dataset containing * variable length data past the limit of the file, then * ensuring that an error (not an assert) was generated and * that the file is readable. * - * Return: Success: 0 - * Failure: 1 + * Return: Success: 0 + * Failure: 1 * - * Programmer: James Laird - * Nat Furrer + * Programmer: James Laird + * Nat Furrer * Thursday, July 1, 2004 * * Modifications: @@ -401,15 +401,15 @@ error: #endif /* BROKEN */ /*------------------------------------------------------------------------- - * Function: main + * Function: main * * Purpose: * - * Return: Success: + * Return: Success: * - * Failure: + * Failure: * - * Programmer: Nat Furrer and James Laird + * Programmer: Nat Furrer and James Laird * Thursday, July 1, 2004 * * Modifications: @@ -434,22 +434,22 @@ main(void) envval = "nomatch"; /* QAK: should be able to use the core driver? */ if (HDstrcmp(envval, "core") && HDstrcmp(envval, "split") && HDstrcmp(envval, "multi") && HDstrcmp(envval, "family")) { - num_errs+=rsrv_ohdr(); - num_errs+=rsrv_heap(); - num_errs+=rsrv_vlen(); - - if(num_errs > 0) - HDprintf("**** %d FAILURE%s! ****\n", num_errs, num_errs==1?"":"S"); - else - puts("All address space reservation tests passed."); - - fapl = h5_fileaccess(); - h5_cleanup(FILENAME, fapl); - return num_errs; + num_errs+=rsrv_ohdr(); + num_errs+=rsrv_heap(); + num_errs+=rsrv_vlen(); + + if(num_errs > 0) + HDprintf("**** %d FAILURE%s! ****\n", num_errs, num_errs==1?"":"S"); + else + HDputs("All address space reservation tests passed."); + + fapl = h5_fileaccess(); + h5_cleanup(FILENAME, fapl); + return num_errs; } else { - puts("All address space reservation tests skippped - Incompatible with current Virtual File Driver"); + HDputs("All address space reservation tests skippped - Incompatible with current Virtual File Driver"); } #endif /* BROKEN */ diff --git a/test/ros3.c b/test/ros3.c new file mode 100644 index 0000000..73b6ac2 --- /dev/null +++ b/test/ros3.c @@ -0,0 +1,1937 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Read-Only S3 Virtual File Driver (VFD) + * + * Purpose: + * + * Verify behavior for Read-Only S3 VFD + * at the VFL (virtual file layer) level. + * + * Demonstrates basic use cases and fapl/dxpl interaction. + * + * Programmer: Jacob Smith + * 2017-10-11 + */ + +#include "h5test.h" + +#include "H5FDprivate.h" /* Virtual File Driver utilities */ +#include "H5FDros3.h" /* this file driver's utilities */ +#include "H5FDs3comms.h" /* for loading of credentials */ + +#ifdef H5_HAVE_ROS3_VFD + +/* only include the testing macros if needed */ + +/***************************************************************************** + * + * FILE-LOCAL TESTING MACROS + * + * Purpose: + * + * 1) Upon test failure, goto-jump to single-location teardown in test + * function. E.g., `error:` (consistency with HDF corpus) or + * `failed:` (reflects purpose). + * >>> using "error", in part because `H5E_BEGIN_TRY` expects it. + * 2) Increase clarity and reduce overhead found with `TEST_ERROR`. + * e.g., "if(somefunction(arg, arg2) < 0) TEST_ERROR:" + * requires reading of entire line to know whether this if/call is + * part of the test setup, test operation, or a test unto itself. + * 3) Provide testing macros with optional user-supplied failure message; + * if not supplied (NULL), generate comparison output in the spirit of + * test-driven development. E.g., "expected 5 but was -3" + * User messages clarify test's purpose in code, encouraging description + * without relying on comments. + * 4) Configurable expected-actual order in generated comparison strings. + * Some prefer `VERIFY(expected, actual)`, others + * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch + * to satifsy both parties, assuming one paradigm per test file. + * (One could #undef and redefine the flag through the file as desired, + * but _why_.) + * + * Provided as courtesy, per consideration for inclusion in the library + * proper. + * + * Macros: + * + * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order + * FAIL_IF() - check condition + * FAIL_UNLESS() - check _not_ condition + * JSVERIFY() - long-int equality check; prints reason/comparison + * JSVERIFY_NOT() - long-int inequality check; prints + * JSVERIFY_STR() - string equality check; prints + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *****************************************************************************/ + + +/*---------------------------------------------------------------------------- + * + * ifdef flag: JSVERIFY_EXP_ACT + * + * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason]) + * default, if this is undefined, is (ACTUAL, EXPECTED[, reason]) + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_EXP_ACT 1L + + +/*---------------------------------------------------------------------------- + * + * Macro: JSFAILED_AT() + * + * Purpose: + * + * Preface a test failure by printing "*FAILED*" and location to stdout + * Similar to `H5_FAILED(); AT();` from h5test.h + * + * *FAILED* at somefile.c:12 in function_name()... + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSFAILED_AT() { \ + HDprintf("*FAILED* at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_IF() + * + * Purpose: + * + * Make tests more accessible and less cluttered than + * `if (thing == otherthing()) TEST_ERROR` + * paradigm. + * + * The following lines are roughly equivalent: + * + * `if (myfunc() < 0) TEST_ERROR;` (as seen elsewhere in HDF tests) + * `FAIL_IF(myfunc() < 0)` + * + * Prints a generic "FAILED AT" line to stdout and jumps to `error`, + * similar to `TEST_ERROR` in h5test.h + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *---------------------------------------------------------------------------- + */ +#define FAIL_IF(condition) \ +if (condition) { \ + JSFAILED_AT() \ + goto error; \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_UNLESS() + * + * Purpose: + * + * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests", + * e.g., "a != b". + * + * Opposite of FAIL_IF; fails if the given condition is _not_ true. + * + * `FAIL_IF( 5 != my_op() )` + * is equivalent to + * `FAIL_UNLESS( 5 == my_op() )` + * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer. + * (see JSVERIFY) + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#if 0 /* UNUSED */ +#define FAIL_UNLESS(condition) \ +if (!(condition)) { \ + JSFAILED_AT() \ + goto error; \ +} +#endif + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_LONG() + * + * Purpose: + * + * Print an failure message for long-int arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:488 in somefunc()... + * forest must be made of trees. + * + * or + * + * *FAILED* at myfile.c:488 in somefunc()... + * ! Expected 425 + * ! Actual 3 + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_LONG(expected, actual, reason) { \ + JSFAILED_AT() \ + if (reason!= NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf(" ! Expected %ld\n ! Actual %ld\n", \ + (long)(expected), (long)(actual)); \ + } \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_STR() + * + * Purpose: + * + * Print an failure message for string arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:421 in myfunc()... + * Blue and Red strings don't match! + * + * or + * + * *FAILED* at myfile.c:421 in myfunc()... + * !!! Expected: + * this is my expected + * string + * !!! Actual: + * not what I expected at all + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_STR(expected, actual, reason) { \ + JSFAILED_AT() \ + if ((reason) != NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf("!!! Expected:\n%s\n!!!Actual:\n%s\n", \ + (expected), (actual)); \ + } \ +} + + + +#ifdef JSVERIFY_EXP_ACT + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY() + * + * Purpose: + * + * Verify that two long integers are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(expected, actual, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_NOT() + * + * Purpose: + * + * Verify that two long integers are _not_ equal. + * If equal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(expected, actual, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_STR() + * + * Purpose: + * + * Verify that two strings are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(expected, actual, reason) \ +if (strcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + + +#else +/* JSVERIFY_EXP_ACT not defined + * + * Repeats macros above, but with actual/expected parameters reversed. + */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY() + * See: JSVERIFY documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(actual, expected, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_NOT() + * See: JSVERIFY_NOT documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(actual, expected, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_STR() + * See: JSVERIFY_STR documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(actual, expected, reason) \ +if (strcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + +#endif /* ifdef/else JSVERIFY_EXP_ACT */ + +/******************************** + * OTHER MACROS AND DEFINITIONS * + ********************************/ + +#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1) + +#define S3_TEST_PROFILE_NAME "ros3_vfd_test" + +#define S3_TEST_MAX_URL_SIZE 256 + +#define S3_TEST_RESOURCE_TEXT_RESTRICTED "t8.shakespeare.txt" +#define S3_TEST_RESOURCE_TEXT_PUBLIC "Poe_Raven.txt" +#define S3_TEST_RESOURCE_H5_PUBLIC "GMODO-SVM01.h5" +#define S3_TEST_RESOURCE_MISSING "missing.csv" + +static char url_text_restricted[S3_TEST_MAX_URL_SIZE] = ""; +static char url_text_public[S3_TEST_MAX_URL_SIZE] = ""; +static char url_h5_public[S3_TEST_MAX_URL_SIZE] = ""; +static char url_missing[S3_TEST_MAX_URL_SIZE] = ""; +static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE] = ""; +static hbool_t s3_test_bucket_defined = FALSE; + +/* Global variables for aws test profile. + * An attempt is made to read ~/.aws/credentials and ~/.aws/config upon test + * startup -- if unable to open either file or cannot load region, id, and key, + * tests connecting with S3 will not be run + */ +static int s3_test_credentials_loaded = 0; +static char s3_test_aws_region[16]; +static char s3_test_aws_access_key_id[64]; +static char s3_test_aws_secret_access_key[128]; + +H5FD_ros3_fapl_t restricted_access_fa = { + H5FD_CURR_ROS3_FAPL_T_VERSION, /* fapl version */ + TRUE, /* authenticate */ + "", /* aws region */ + "", /* access key id */ + ""}; /* secret access key */ + +H5FD_ros3_fapl_t anonymous_fa = { + H5FD_CURR_ROS3_FAPL_T_VERSION, + FALSE, "", "", "" }; + + +/*--------------------------------------------------------------------------- + * + * Function: test_fapl_config_validation() + * + * Purpose: + * + * Test data consistency of fapl configuration. + * Tests `H5FD_ros3_validate_config` indirectly through `H5Pset_fapl_ros3`. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *--------------------------------------------------------------------------- + */ +static int +test_fapl_config_validation(void) +{ + + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char *msg; + herr_t expected; + H5FD_ros3_fapl_t config; + }; + + /************************ + * test-local variables * + ************************/ + + hid_t fapl_id = -1; /* file access property list ID */ + H5FD_ros3_fapl_t config; + H5FD_ros3_fapl_t fa_fetch; + herr_t success = SUCCEED; + unsigned int i = 0; + unsigned int ncases = 8; /* should equal number of cases */ + struct testcase *case_ptr = NULL; /* dumb work-around for possible */ + /* dynamic cases creation because */ + /* of compiler warnings Wlarger-than */ + struct testcase cases_arr[] = { + { "non-authenticating config allows empties.\n", + SUCCEED, + { H5FD_CURR_ROS3_FAPL_T_VERSION, /* version */ + FALSE, /* authenticate */ + "", /* aws_region */ + "", /* secret_id */ + "", /* secret_key */ + }, + }, + { "authenticating config asks for populated strings.\n", + FAIL, + { H5FD_CURR_ROS3_FAPL_T_VERSION, + TRUE, + "", + "", + "", + }, + }, + { "populated strings; key is the empty string?\n", + SUCCEED, + { H5FD_CURR_ROS3_FAPL_T_VERSION, + TRUE, + "region", + "me", + "", + }, + }, + { "id cannot be empty.\n", + FAIL, + { H5FD_CURR_ROS3_FAPL_T_VERSION, + TRUE, + "", + "me", + "", + }, + }, + { "region cannot be empty.\n", + FAIL, + { H5FD_CURR_ROS3_FAPL_T_VERSION, + TRUE, + "where", + "", + "", + }, + }, + { "all strings populated.\n", + SUCCEED, + { H5FD_CURR_ROS3_FAPL_T_VERSION, + TRUE, + "where", + "who", + "thisIsA GREAT seeeecrit", + }, + }, + { "incorrect version should fail\n", + FAIL, + { 12345, + FALSE, + "", + "", + "", + }, + }, + { "non-authenticating config cares not for (de)population" + "of strings.\n", + SUCCEED, + { H5FD_CURR_ROS3_FAPL_T_VERSION, + FALSE, + "someregion", + "someid", + "somekey", + }, + }, + }; + + TESTING("ROS3 fapl configuration validation"); + + /********* + * TESTS * + *********/ + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + for (i = 0; i < ncases; i++) { + + /*--------------- + * per-test setup + *--------------- + */ + case_ptr = &cases_arr[i]; + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) /* sanity-check */ + + /*----------------------------------- + * Actually test. + * Mute stack trace in failure cases. + *----------------------------------- + */ + H5E_BEGIN_TRY { + /* `H5FD_ros3_validate_config(...)` is static/private + * to src/ros3.c and cannot (and should not?) be tested directly? + * Instead, validate config through public api. + */ + success = H5Pset_fapl_ros3(fapl_id, &case_ptr->config); + } H5E_END_TRY; + + JSVERIFY( case_ptr->expected, success, case_ptr->msg ) + + /* Make sure we can get back what we put in. + * Only valid if the fapl configuration does not result in error. + */ + if (success == SUCCEED) { + config = case_ptr->config; + JSVERIFY( SUCCEED, + H5Pget_fapl_ros3(fapl_id, &fa_fetch), + "unable to get fapl" ) + + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, + fa_fetch.version, + "invalid version number" ) + JSVERIFY( config.version, + fa_fetch.version, + "version number mismatch" ) + JSVERIFY( config.authenticate, + fa_fetch.authenticate, + "authentication flag mismatch" ) + JSVERIFY_STR( config.aws_region, + fa_fetch.aws_region, + NULL ) + JSVERIFY_STR( config.secret_id, + fa_fetch.secret_id, + NULL ) + JSVERIFY_STR( config.secret_key, + fa_fetch.secret_key, + NULL ) + } + + /*----------------------------- + * per-test sanitation/teardown + *----------------------------- + */ + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + } /* for each test case */ + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fapl_id < 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + return 1; +} /* test_fapl_config_validation */ + + +/*------------------------------------------------------------------------- + * + * Function: test_ros3_fapl() + * + * Purpose: Tests the file handle interface for the ROS3 driver + * + * As the ROS3 driver is 1) read only, 2) requires access + * to an S3 server, this test is quite + * different from the other tests. + * + * For now, test only fapl & flags. Extend as the + * work on the VFD continues. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: John Mainzer + * 7/12/17 + * + *------------------------------------------------------------------------- + */ +static int +test_ros3_fapl(void) +{ + /************************ + * test-local variables * + ************************/ + + hid_t fapl_id = -1; /* file access property list ID */ + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ + H5FD_ros3_fapl_t ros3_fa_0 = { + H5FD_CURR_ROS3_FAPL_T_VERSION, /* version */ + FALSE, /* authenticate */ + "", /* aws_region */ + "", /* secret_id */ + "plugh", /* secret_key */ + }; + + TESTING("ROS3 fapl "); + + /* Set property list and file name for ROS3 driver. + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + + FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &ros3_fa_0) ) + + driver_id = H5Pget_driver(fapl_id); + FAIL_IF( driver_id < 0 ) + + /**************** + * Check that the VFD feature flags are correct + * SPEC MAY CHANGE + ******************/ + + FAIL_IF( H5FDdriver_query(driver_id, &driver_flags) < 0 ) + + JSVERIFY_NOT( 0, (driver_flags & H5FD_FEAT_DATA_SIEVE), + "bit(s) in `driver_flags` must align with " + "H5FD_FEAT_DATA_SIEVE" ) + + JSVERIFY( H5FD_FEAT_DATA_SIEVE, driver_flags, + "H5FD_FEAT_DATA_SIEVE should be the only supported flag") + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + + return 1; + +} /* test_ros3_fapl() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_vfd_open() + * + * Purpose: + * + * Demonstrate/specify VFD-level "Open" failure cases + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 1027-11-03 + * + *--------------------------------------------------------------------------- + */ +static int +test_vfd_open(void) +{ + + /********************* + * test-local macros * + *********************/ + + +#define FAPL_H5P_DEFAULT -2 +#define FAPL_FILE_ACCESS -3 +#define FAPL_ROS3_ANON -4 + + /************************* + * test-local structures * + *************************/ + + struct test_condition { + const char *message; + const char *url; + unsigned flags; + int which_fapl; + haddr_t maxaddr; + }; + + /************************ + * test-local variables * + ************************/ + + struct test_condition tests[] = { + { "default property list (H5P_DEFAULT) is invalid", + url_text_public, + H5F_ACC_RDONLY, + FAPL_H5P_DEFAULT, + MAXADDR, + }, + { "generic file access property list is invalid", + url_text_public, + H5F_ACC_RDONLY, + FAPL_FILE_ACCESS, + MAXADDR, + }, + { "filename cannot be null", + NULL, + H5F_ACC_RDONLY, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "filename cannot be empty", + "", + H5F_ACC_RDONLY, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "filename must exist", + url_missing, + H5F_ACC_RDONLY, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "read-write flag not supported", + url_text_public, + H5F_ACC_RDWR, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "truncate flag not supported", + url_text_public, + H5F_ACC_TRUNC, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "create flag not supported", + url_text_public, + H5F_ACC_CREAT, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "EXCL flag not supported", + url_text_public, + H5F_ACC_EXCL, + FAPL_ROS3_ANON, + MAXADDR, + }, + { "maxaddr cannot be 0 (caught in `H5FD_open()`)", + url_text_public, + H5F_ACC_RDONLY, + FAPL_ROS3_ANON, + 0, + }, + }; + H5FD_t *fd = NULL; + hbool_t curl_ready = FALSE; + hid_t fapl_id = -1; + hid_t fapl_file_access = -1; + unsigned i = 0; + unsigned tests_count = 10; + + TESTING("ROS3 VFD-level open"); + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) ) + curl_ready = TRUE; + + fapl_file_access = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_file_access < 0 ) + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &anonymous_fa) ) + + /********* + * TESTS * + *********/ + + /* all the test cases that will _not_ open + */ + for (i = 0; i < tests_count; i++) { + struct test_condition T = tests[i]; + hid_t _fapl_id = H5P_DEFAULT; + + fd = NULL; + + if (T.which_fapl == FAPL_FILE_ACCESS) + _fapl_id = fapl_file_access; + else if (T.which_fapl == FAPL_ROS3_ANON) + _fapl_id = fapl_id; + + H5E_BEGIN_TRY { + fd = H5FDopen(T.url, T.flags, _fapl_id, T.maxaddr); + } H5E_END_TRY; + if (NULL != fd) + JSVERIFY(1, 0, T.message); /* wrapper to print message and fail */ + } + + FAIL_IF( NULL != fd ) + + /* finally, show that a file can be opened + */ + fd = H5FDopen( + url_text_public, + H5F_ACC_RDONLY, + fapl_id, + MAXADDR); + FAIL_IF( NULL == fd ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(fd) ) + fd = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + FAIL_IF( FAIL == H5Pclose(fapl_file_access) ) + fapl_file_access = -1; + + curl_global_cleanup(); + curl_ready = FALSE; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fd) { + (void)H5FDclose(fd); + } + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + if (fapl_file_access >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_file_access); + } H5E_END_TRY; + } + if (curl_ready == TRUE) { + curl_global_cleanup(); + } + + return 1; + +#undef FAPL_FILE_ACCESS +#undef FAPL_H5P_DEFAULT +#undef FAPL_ROS3_ANON + +} /* test_vfd_open */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_eof_eoa() + * + * Purpose: + * + * Demonstrate behavior of get_eof, get_eoa, and set_eoa. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-08 + * + *--------------------------------------------------------------------------- + */ +static int +test_eof_eoa(void) +{ + + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + + H5FD_t *fd_shakespeare = NULL; + hbool_t curl_ready = FALSE; + hid_t fapl_id = -1; + + TESTING("ROS3 eof/eoa gets and sets"); + + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) ) + curl_ready = TRUE; + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( 0 > fapl_id ) + FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) ) + + fd_shakespeare = H5FDopen( + url_text_restricted, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_shakespeare ) + + /********* + * TESTS * + *********/ + + /* verify as found + */ + JSVERIFY( 5458199, H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), NULL ) + JSVERIFY( H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + H5FDget_eof(fd_shakespeare, H5FD_MEM_DRAW), + "mismatch between DEFAULT and RAW memory types" ) + JSVERIFY( 0, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoA should be unset by H5FDopen" ) + + /* set EoA below EoF + */ + JSVERIFY( SUCCEED, + H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 44442202), + "unable to set EoA (lower)" ) + JSVERIFY( 5458199, + H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoF changed" ) + JSVERIFY( 44442202, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoA unchanged" ) + + /* set EoA above EoF + */ + JSVERIFY( SUCCEED, + H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 6789012), + "unable to set EoA (higher)" ) + JSVERIFY( 5458199, + H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoF changed" ) + JSVERIFY( 6789012, + H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT), + "EoA unchanged" ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(fd_shakespeare) ) + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + curl_global_cleanup(); + curl_ready = FALSE; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fd_shakespeare) (void)H5FDclose(fd_shakespeare); + if (TRUE == curl_ready) curl_global_cleanup(); + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; + +} /* test_eof_eoa */ + + +/*----------------------------------------------------------------------------- + * + * Function: test_H5FDread_without_eoa_set_fails() + * + * Purpose: + * + * Demonstrate a not-obvious constraint by the library, preventing + * file read before EoA is set + * + * Programmer: Jacob Smith + * 2018-01-26 + * + *----------------------------------------------------------------------------- + */ +static int +test_H5FDread_without_eoa_set_fails(void) +{ + char buffer[256]; + unsigned int i = 0; + H5FD_t *file_shakespeare = NULL; + hid_t fapl_id = -1; + + TESTING("ROS3 VFD read-eoa temporal coupling library limitation "); + + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + /* create ROS3 fapl + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) ) + + file_shakespeare = H5FDopen( + url_text_restricted, + H5F_ACC_RDONLY, + fapl_id, + MAXADDR); + FAIL_IF( NULL == file_shakespeare ) + + JSVERIFY( 0, H5FDget_eoa(file_shakespeare, H5FD_MEM_DEFAULT), + "EoA should remain unset by H5FDopen" ) + + for (i = 0; i < 256; i++) + buffer[i] = 0; /* zero buffer contents */ + + /******** + * TEST * + ********/ + + H5E_BEGIN_TRY { /* mute stack trace on expected failure */ + JSVERIFY( FAIL, + H5FDread(file_shakespeare, + H5FD_MEM_DRAW, + H5P_DEFAULT, + 1200699, + 102, + buffer), + "cannot read before eoa is set" ) + } H5E_END_TRY; + JSVERIFY_STR( "", buffer, "buffer should remain untouched" ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(file_shakespeare) ) + file_shakespeare = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (file_shakespeare) { (void)H5FDclose(file_shakespeare); } + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; + +} /* test_H5FDread_without_eoa_set_fails */ + + + +/*--------------------------------------------------------------------------- + * + * Function: test_read() + * + * Purpose: + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-06 + * + *--------------------------------------------------------------------------- + */ +static int +test_read(void) +{ + + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + struct testcase { + const char *message; /* purpose of test case */ + haddr_t eoa_set; /* set file EOA to this prior to read */ + size_t addr; /* offset of read in file */ + size_t len; /* length of read in file */ + herr_t success; /* expected return value of read function */ + const char *expected; /* expected contents of buffer; failure ignores */ + }; + + /************************ + * test-local variables * + ************************/ + struct testcase cases[] = { + { "successful range-get", + 6464, + 5691, + 32, /* fancy quotes are three bytes each(?) */ + SUCCEED, + "Quoth the Raven “Nevermore.”", + }, + { "read past EOA fails (EOA < EOF < addr)", + 3000, + 4000, + 100, + FAIL, + NULL, + }, + { "read overlapping EOA fails (EOA < addr < EOF < (addr+len))", + 3000, + 8000, + 100, + FAIL, + NULL, + }, + { "read past EOA/EOF fails ((EOA==EOF) < addr)", + 6464, + 7000, + 100, + FAIL, + NULL, + }, + { "read overlapping EOA/EOF fails (addr < (EOA==EOF) < (addr+len))", + 6464, + 6400, + 100, + FAIL, + NULL, + }, + { "read between EOF and EOA fails (EOF < addr < (addr+len) < EOA)", + 8000, + 7000, + 100, + FAIL, + NULL, + }, + }; + unsigned testcase_count = 6; + unsigned test_i = 0; + struct testcase test; + herr_t open_return = FAIL; + char buffer[S3_TEST_MAX_URL_SIZE]; + unsigned int i = 0; + H5FD_t *file_raven = NULL; + hid_t fapl_id = -1; + + TESTING("ROS3 VFD read/range-gets"); + + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + /* create ROS3 fapl + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) ) + + /* open file + */ + file_raven = H5FDopen( /* will open with "authenticating" fapl */ + url_text_public, /* TODO: check return state: anon access of restricted says OK? (not NULL) */ + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); /* Demonstrate success with "automatic" value */ + FAIL_IF( NULL == file_raven ) + + JSVERIFY( 6464, H5FDget_eof(file_raven, H5FD_MEM_DEFAULT), NULL ) + + /********* + * TESTS * + *********/ + + for (test_i = 0; test_i < testcase_count; test_i++) { + + /* -------------- * + * per-test setup * + * -------------- */ + + test = cases[test_i]; + open_return = FAIL; + + FAIL_IF( S3_TEST_MAX_URL_SIZE < test.len ) /* buffer too small! */ + + FAIL_IF( FAIL == + H5FD_set_eoa( file_raven, H5FD_MEM_DEFAULT, test.eoa_set) ) + + for (i = 0; i < S3_TEST_MAX_URL_SIZE; i++) /* zero buffer contents */ + buffer[i] = 0; + + /* ------------ * + * conduct test * + * ------------ */ + + H5E_BEGIN_TRY { + open_return = H5FDread( + file_raven, + H5FD_MEM_DRAW, + H5P_DEFAULT, + test.addr, + test.len, + buffer); + } H5E_END_TRY; + + JSVERIFY( test.success, + open_return, + test.message ) + if (open_return == SUCCEED) + JSVERIFY_STR( test.expected, buffer, NULL ) + + } /* for each testcase */ + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(file_raven) ) + file_raven = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (file_raven) + (void)H5FDclose(file_raven); + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; + +} /* test_read */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_noops_and_autofails() + * + * Purpose: + * + * Demonstrate the unavailable and do-nothing routines unique to + * Read-Only VFD. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-06 + * + *--------------------------------------------------------------------------- + */ +static int +test_noops_and_autofails(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + + hbool_t curl_ready = FALSE; + hid_t fapl_id = -1; + H5FD_t *file = NULL; + const char data[36] = "The Force shall be with you, always"; + + TESTING("ROS3 VFD always-fail and no-op routines"); + + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) ) + curl_ready = TRUE; + + /* create ROS3 fapl + */ + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + JSVERIFY( SUCCEED, H5Pset_fapl_ros3(fapl_id, &anonymous_fa), NULL ) + + /* open file + */ + file = H5FDopen( + url_text_public, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == file ) + + /********* + * TESTS * + *********/ + + /* auto-fail calls to write and truncate + */ + H5E_BEGIN_TRY { + JSVERIFY( FAIL, + H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, 1000, 35, data), + "write must fail" ) + } H5E_END_TRY; + + H5E_BEGIN_TRY { + JSVERIFY( FAIL, + H5FDtruncate(file, H5P_DEFAULT, FALSE), + "truncate must fail" ) + } H5E_END_TRY; + + H5E_BEGIN_TRY { + JSVERIFY( FAIL, + H5FDtruncate(file, H5P_DEFAULT, TRUE), + "truncate must fail (closing)" ) + } H5E_END_TRY; + + /* no-op calls to `lock()` and `unlock()` + */ + JSVERIFY( SUCCEED, + H5FDlock(file, TRUE), + "lock always succeeds; has no effect" ) + JSVERIFY( SUCCEED, + H5FDlock(file, FALSE), + NULL ) + JSVERIFY( SUCCEED, + H5FDunlock(file), + NULL ) + /* Lock/unlock with null file or similar error crashes tests. + * HDassert in calling heirarchy, `H5FD[un]lock()` and `H5FD_[un]lock()` + */ + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(file) ) + file = NULL; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + curl_global_cleanup(); + curl_ready = FALSE; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + if (file) { (void)H5FDclose(file); } + if (curl_ready == TRUE) { curl_global_cleanup(); } + + return 1; + +} /* test_noops_and_autofails*/ + + +/*--------------------------------------------------------------------------- + * + * Function: test_cmp() + * + * Purpose: + * + * Verify "file comparison" behavior. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-06 + * + *--------------------------------------------------------------------------- + */ +static int +test_cmp(void) +{ + + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + + H5FD_t *fd_raven = NULL; + H5FD_t *fd_shakes = NULL; + H5FD_t *fd_raven_2 = NULL; + hbool_t curl_ready = FALSE; + hid_t fapl_id = -1; + + TESTING("ROS3 cmp (comparison)"); + + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) ) + curl_ready = TRUE; + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( 0 > fapl_id ) + JSVERIFY( SUCCEED, H5Pset_fapl_ros3(fapl_id, &restricted_access_fa), NULL ) + + fd_raven = H5FDopen( + url_text_public, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_raven ) + + fd_shakes = H5FDopen( + url_text_restricted, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_shakes ) + + fd_raven_2 = H5FDopen( + url_text_public, + H5F_ACC_RDONLY, + fapl_id, + HADDR_UNDEF); + FAIL_IF( NULL == fd_raven_2 ) + + /********* + * TESTS * + *********/ + + JSVERIFY( 0, H5FDcmp(fd_raven, fd_raven_2), NULL ) + JSVERIFY( -1, H5FDcmp(fd_raven, fd_shakes), NULL ) + JSVERIFY( -1, H5FDcmp(fd_shakes, fd_raven_2), NULL ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5FDclose(fd_raven) ) + fd_raven = NULL; + FAIL_IF( FAIL == H5FDclose(fd_shakes) ) + fd_shakes = NULL; + FAIL_IF( FAIL == H5FDclose(fd_raven_2) ) + fd_raven_2 = NULL; + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + curl_global_cleanup(); + curl_ready = FALSE; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (fd_raven != NULL) (void)H5FDclose(fd_raven); + if (fd_raven_2 != NULL) (void)H5FDclose(fd_raven_2); + if (fd_shakes != NULL) (void)H5FDclose(fd_shakes); + if (TRUE == curl_ready) curl_global_cleanup(); + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + + return 1; + +} /* test_cmp */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_H5F_integration() + * + * Purpose: + * + * Demonstrate S3 file-open through H5F API. + * + * Return: + * + * PASSED : 0 + * FAILED : 1 + * + * Programmer: Jacob Smith + * 2017-11-07 + * + *--------------------------------------------------------------------------- + */ +static int +test_H5F_integration(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + /************************ + * test-local variables * + ************************/ + + hid_t file = -1; + hid_t fapl_id = -1; + + TESTING("S3 file access through HD5F library (H5F API)"); + + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /********* + * SETUP * + *********/ + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( 0 > fapl_id ) + FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) ) + + /********* + * TESTS * + *********/ + + /* Read-Write Open access is not allowed with this file driver. + */ + H5E_BEGIN_TRY { + FAIL_IF( 0 <= H5Fopen( + url_h5_public, + H5F_ACC_RDWR, + fapl_id) ) + } H5E_END_TRY; + + /* H5Fcreate() is not allowed with this file driver. + */ + H5E_BEGIN_TRY { + FAIL_IF( 0 <= H5Fcreate( + url_missing, + H5F_ACC_RDONLY, + H5P_DEFAULT, + fapl_id) ) + } H5E_END_TRY; + + /* Successful open. + */ + file = H5Fopen( + url_h5_public, + H5F_ACC_RDONLY, + fapl_id); + FAIL_IF( file < 0 ) + + /************ + * TEARDOWN * + ************/ + + FAIL_IF( FAIL == H5Fclose(file) ) + file = -1; + + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + fapl_id = -1; + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ +HDprintf("\nerror!"); fflush(stdout); + + if (fapl_id >= 0) { + H5E_BEGIN_TRY { + (void)H5Pclose(fapl_id); + } H5E_END_TRY; + } + if (file > 0) + (void)H5Fclose(file); + + return 1; + +} /* test_H5F_integration */ + +#endif /* H5_HAVE_ROS3_VFD */ + + +/*------------------------------------------------------------------------- + * + * Function: main + * + * Purpose: Tests the basic features of Virtual File Drivers + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ +#ifdef H5_HAVE_ROS3_VFD + int nerrors = 0; + const char *bucket_url_env = NULL; + +#endif /* H5_HAVE_ROS3_VFD */ + + HDprintf("Testing ros3 VFD functionality.\n"); + +#ifdef H5_HAVE_ROS3_VFD + + /************************ + * initialize test urls * + ************************/ + + bucket_url_env = HDgetenv("HDF5_ROS3_TEST_BUCKET_URL"); + if (bucket_url_env == NULL || bucket_url_env[0] == '\0') { + HDprintf("WARNING: S3 bucket url is not defined in enviornment " \ + "variable 'HDF5_ROS3_TEST_BUCKET_URL'!\n"); + } else { + HDstrncpy(s3_test_bucket_url, bucket_url_env, S3_TEST_MAX_URL_SIZE); + s3_test_bucket_defined = TRUE; + } + + if (S3_TEST_MAX_URL_SIZE < HDsnprintf( + url_text_restricted, + (size_t)S3_TEST_MAX_URL_SIZE, + "%s/%s", + (const char *)s3_test_bucket_url, + (const char *)S3_TEST_RESOURCE_TEXT_RESTRICTED)) + { + HDprintf("* ros3 setup failed (text_restricted) ! *\n"); + return 1; + } + if (S3_TEST_MAX_URL_SIZE < HDsnprintf( + url_text_public, + (size_t)S3_TEST_MAX_URL_SIZE, + "%s/%s", + (const char *)s3_test_bucket_url, + (const char *)S3_TEST_RESOURCE_TEXT_PUBLIC)) + { + HDprintf("* ros3 setup failed (text_public) ! *\n"); + return 1; + } + if (S3_TEST_MAX_URL_SIZE < HDsnprintf( + url_h5_public, + (size_t)S3_TEST_MAX_URL_SIZE, + "%s/%s", + (const char *)s3_test_bucket_url, + (const char *)S3_TEST_RESOURCE_H5_PUBLIC)) + { + HDprintf("* ros3 setup failed (h5_public) ! *\n"); + return 1; + } + if (S3_TEST_MAX_URL_SIZE < HDsnprintf( + url_missing, + S3_TEST_MAX_URL_SIZE, + "%s/%s", + (const char *)s3_test_bucket_url, + (const char *)S3_TEST_RESOURCE_MISSING)) + { + HDprintf("* ros3 setup failed (missing) ! *\n"); + return 1; + } + + /************************************** + * load credentials and prepare fapls * + **************************************/ + + /* "clear" profile data strings */ + s3_test_aws_access_key_id[0] = '\0'; + s3_test_aws_secret_access_key[0] = '\0'; + s3_test_aws_region[0] = '\0'; + + /* attempt to load test credentials + * if unable, certain tests will be skipped + */ + if (SUCCEED == H5FD_s3comms_load_aws_profile( + S3_TEST_PROFILE_NAME, + s3_test_aws_access_key_id, + s3_test_aws_secret_access_key, + s3_test_aws_region)) + { + s3_test_credentials_loaded = 1; + HDstrncpy(restricted_access_fa.aws_region, + (const char *)s3_test_aws_region, + H5FD_ROS3_MAX_REGION_LEN); + HDstrncpy(restricted_access_fa.secret_id, + (const char *)s3_test_aws_access_key_id, + H5FD_ROS3_MAX_SECRET_ID_LEN); + HDstrncpy(restricted_access_fa.secret_key, + (const char *)s3_test_aws_secret_access_key, + H5FD_ROS3_MAX_SECRET_KEY_LEN); + } + + /****************** + * commence tests * + ******************/ + + h5_reset(); + + nerrors += test_fapl_config_validation(); + nerrors += test_ros3_fapl(); + nerrors += test_vfd_open(); + nerrors += test_eof_eoa(); + nerrors += test_H5FDread_without_eoa_set_fails(); + nerrors += test_read(); + nerrors += test_noops_and_autofails(); + nerrors += test_cmp(); + nerrors += test_H5F_integration(); + + if (nerrors > 0) { + HDprintf("***** %d ros3 TEST%s FAILED! *****\n", + nerrors, + nerrors > 1 ? "S" : ""); + nerrors = 1; + } else { + HDprintf("All ros3 tests passed.\n"); + } + return nerrors; /* 0 if no errors, 1 if any errors */ + +#else + + HDprintf("SKIPPED - read-only S3 VFD not built\n"); + return EXIT_SUCCESS; + +#endif /* H5_HAVE_ROS3_VFD */ + +} /* main() */ + diff --git a/test/s3comms.c b/test/s3comms.c new file mode 100644 index 0000000..9453b75 --- /dev/null +++ b/test/s3comms.c @@ -0,0 +1,2730 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Read-Only S3 Virtual File Driver (VFD) + * + * Purpose: Unit tests for the S3 Communications (s3comms) module. + * + * Programmer: Jacob Smith + * 2017-10-11 + */ + +#include "h5test.h" +#include "H5FDs3comms.h" +#include "H5MMprivate.h" /* memory management */ + +#ifdef H5_HAVE_ROS3_VFD + +/***************************************************************************** + * + * FILE-LOCAL TESTING MACROS + * + * Purpose: + * + * 1) Upon test failure, goto-jump to single-location teardown in test + * function. E.g., `error:` (consistency with HDF corpus) or + * `failed:` (reflects purpose). + * >>> using "error", in part because `H5E_BEGIN_TRY` expects it. + * 2) Increase clarity and reduce overhead found with `TEST_ERROR`. + * e.g., "if(somefunction(arg, arg2) < 0) TEST_ERROR:" + * requires reading of entire line to know whether this if/call is + * part of the test setup, test operation, or a test unto itself. + * 3) Provide testing macros with optional user-supplied failure message; + * if not supplied (NULL), generate comparison output in the spirit of + * test-driven development. E.g., "expected 5 but was -3" + * User messages clarify test's purpose in code, encouraging description + * without relying on comments. + * 4) Configurable expected-actual order in generated comparison strings. + * Some prefer `VERIFY(expected, actual)`, others + * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch + * to satifsy both parties, assuming one paradigm per test file. + * (One could #undef and redefine the flag through the file as desired, + * but _why_.) + * Provided as courtesy, per consideration for inclusion in the library + * proper. + * + * Macros: + * + * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order + * FAIL_IF() - check condition + * FAIL_UNLESS() - check _not_ condition + * JSVERIFY() - long-int equality check; prints reason/comparison + * JSVERIFY_NOT() - long-int inequality check; prints + * JSVERIFY_STR() - string equality check; prints + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *****************************************************************************/ + + +/*---------------------------------------------------------------------------- + * + * ifdef flag: JSVERIFY_EXP_ACT + * + * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason]) + * default, if this is undefined, is (ACTUAL, EXPECTED[, reason]) + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_EXP_ACT 1L + + +/*---------------------------------------------------------------------------- + * + * Macro: JSFAILED_AT() + * + * Purpose: + * + * Preface a test failure by printing "*FAILED*" and location to stdout + * Similar to `H5_FAILED(); AT();` from h5test.h + * + * *FAILED* at somefile.c:12 in function_name()... + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSFAILED_AT() { \ + HDprintf("*FAILED* at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_IF() + * + * Purpose: + * + * Make tests more accessible and less cluttered than + * `if (thing == otherthing()) TEST_ERROR` + * paradigm. + * + * The following lines are roughly equivalent: + * + * `if (myfunc() < 0) TEST_ERROR;` (as seen elsewhere in HDF tests) + * `FAIL_IF(myfunc() < 0)` + * + * Prints a generic "FAILED AT" line to stdout and jumps to `error`, + * similar to `TEST_ERROR` in h5test.h + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *---------------------------------------------------------------------------- + */ +#define FAIL_IF(condition) \ +if (condition) { \ + JSFAILED_AT() \ + goto error; \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_UNLESS() + * + * Purpose: + * + * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests", + * e.g., "a != b". + * + * Opposite of FAIL_IF; fails if the given condition is _not_ true. + * + * `FAIL_IF( 5 != my_op() )` + * is equivalent to + * `FAIL_UNLESS( 5 == my_op() )` + * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer. + * (see JSVERIFY) + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define FAIL_UNLESS(condition) \ +if (!(condition)) { \ + JSFAILED_AT() \ + goto error; \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_LONG() + * + * Purpose: + * + * Print an failure message for long-int arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:488 in somefunc()... + * forest must be made of trees. + * + * or + * + * *FAILED* at myfile.c:488 in somefunc()... + * ! Expected 425 + * ! Actual 3 + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_LONG(expected, actual, reason) { \ + JSFAILED_AT() \ + if (reason!= NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf(" ! Expected %ld\n ! Actual %ld\n", \ + (long)(expected), (long)(actual)); \ + } \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_STR() + * + * Purpose: + * + * Print an failure message for string arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:421 in myfunc()... + * Blue and Red strings don't match! + * + * or + * + * *FAILED* at myfile.c:421 in myfunc()... + * !!! Expected: + * this is my expected + * string + * !!! Actual: + * not what I expected at all + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_STR(expected, actual, reason) { \ + JSFAILED_AT() \ + if ((reason) != NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf("!!! Expected:\n%s\n!!!Actual:\n%s\n", \ + (expected), (actual)); \ + } \ +} + +#ifdef JSVERIFY_EXP_ACT +/* VERIFY rountines with paramter order (, [, ]) + */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY() + * + * Purpose: + * + * Verify that two long integers are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(expected, actual, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY */ + +#if 0 /* UNUSED */ + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_NOT() + * + * Purpose: + * + * Verify that two long integers are _not_ equal. + * If equal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(expected, actual, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ +#endif /* JSVERIFY_NOT unused */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_STR() + * + * Purpose: + * + * Verify that two strings are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(expected, actual, reason) \ +if (strcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + + +#else +/* JSVERIFY_EXP_ACT not defined + * + * Repeats macros above, but with actual/expected parameters reversed. + */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY() + * See: JSVERIFY documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(actual, expected, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY */ + +#if 0 /* UNUSED */ + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_NOT() + * See: JSVERIFY_NOT documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(actual, expected, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ +#endif /* JSVERIFY_NOT unused */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_STR() + * See: JSVERIFY_STR documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(actual, expected, reason) \ +if (strcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + +#endif /* ifdef/else JSVERIFY_EXP_ACT */ + + +#define S3_TEST_PROFILE_NAME "ros3_vfd_test" + +#define S3_TEST_RESOURCE_TEXT_RESTRICTED "t8.shakespeare.txt" +#define S3_TEST_RESOURCE_TEXT_PUBLIC "Poe_Raven.txt" +#define S3_TEST_RESOURCE_MISSING "missing.csv" + +#define S3_TEST_RUN_TIMEOUT 0 /* run tests that might hang */ +#define S3_TEST_MAX_URL_SIZE 256 /* char array size */ + +/* Global variables for aws test profile. + * An attempt is made to read ~/.aws/credentials and ~/.aws/config upon test + * startup -- if unable to open either file or cannot load region, id, and key, + * tests connecting with S3 will not be run + */ +static int s3_test_credentials_loaded = 0; +static char s3_test_aws_region[16] = ""; +static char s3_test_aws_access_key_id[64] = ""; +static char s3_test_aws_secret_access_key[128] = ""; +static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE] = ""; +static hbool_t s3_test_bucket_defined = FALSE; + + +/*--------------------------------------------------------------------------- + * + * Function: test_macro_format_credential() + * + * Purpose: + * + * Demonstrate that the macro `S3COMMS_FORMAT_CREDENTIAL` + * performs as expected. + * + * Programmer: Jacob Smith + * 2017-09-19 + * + *---------------------------------------------------------------------------- + */ +static herr_t +test_macro_format_credential(void) +{ + /************************ + * test-local variables * + ************************/ + + char dest[256]; + const char access[] = "AKIAIOSFODNN7EXAMPLE"; + const char date[] = "20130524"; + const char region[] = "us-east-1"; + const char service[] = "s3"; + const char expected[] = + "AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request"; + + TESTING("test_macro_format_credential"); + + FAIL_IF( S3COMMS_MAX_CREDENTIAL_SIZE < + S3COMMS_FORMAT_CREDENTIAL(dest, access, date, region, service) ) + + JSVERIFY_STR( expected, dest, NULL ) + + PASSED(); + return 0; + +error: + return -1; + +} /* end test_macro_format_credential() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_aws_canonical_request() + * + * Purpose: + * + * Demonstrate the construction of a Canoncial Request (and Signed Headers) + * + * Elided / not yet implemented: + * Query strings + * request "body" + * + * Programmer: Jacob Smith + * 2017-10-04 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_aws_canonical_request(void) +{ + /************************* + * test-local structures * + *************************/ + + struct header { + const char *name; + const char *value; + }; + + struct testcase { + const char *exp_request; + const char *exp_headers; + const char *verb; + const char *resource; + unsigned int listsize; + struct header list[5]; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { "GET\n/some/path.file\n\nhost:somebucket.someserver.somedomain\nrange:bytes=150-244\n\nhost;range\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "host;range", + "GET", + "/some/path.file", + 2, + { {"Range", "bytes=150-244"}, + {"Host", "somebucket.someserver.somedomain"}, + }, + }, + { "HEAD\n/bucketpath/myfile.dat\n\nhost:place.domain\nx-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nx-amz-date:19411207T150803Z\n\nhost;x-amz-content-sha256;x-amz-date\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "host;x-amz-content-sha256;x-amz-date", + "HEAD", + "/bucketpath/myfile.dat", + 3, + { {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"}, + {"host", "place.domain"}, + {"x-amz-date", "19411207T150803Z"}, + } + }, + { "PUT\n/\n\n\n\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "", + "PUT", + "/", + 0, + {{"",""},}, /* unused; satisfies compiler */ + }, + }; /* struct testcase cases[] */ + struct testcase *C = NULL; + char cr_dest[512]; /* canonical request */ + hrb_t *hrb = NULL; /* http request buffer object */ + unsigned int i = 0; /* looping/indexing */ + unsigned int j = 0; /* looping/indexing */ + hrb_node_t *node = NULL; /* http headers list pointer */ + unsigned int n_cases = 3; + char sh_dest[64]; /* signed headers */ + + TESTING("test_aws_canonical_request"); + + for (i = 0; i < n_cases; i++) { + /* pre-test bookkeeping + */ + C = &cases[i]; + for (j = 0; j < 256; j++) { cr_dest[j] = 0; } /* zero request buffer */ + for (j = 0; j < 64; j++) { sh_dest[j] = 0; } /* zero headers buffer */ + + /* create HTTP request object with given verb, resource/path + */ + hrb = H5FD_s3comms_hrb_init_request(C->verb, + C->resource, + "HTTP/1.1"); + HDassert(hrb->body == NULL); + + /* Create headers list from test case input + */ + for (j = 0; j < C->listsize; j++) { + FAIL_IF( FAIL == + H5FD_s3comms_hrb_node_set( + &node, + C->list[j].name, + C->list[j].value)); + } + + hrb->first_header = node; + + /* test + */ + JSVERIFY( SUCCEED, + H5FD_s3comms_aws_canonical_request( + cr_dest, + 512, + sh_dest, + 64, + hrb), + " unable to compose canonical request" ) + JSVERIFY_STR( C->exp_headers, sh_dest, NULL ) + JSVERIFY_STR( C->exp_request, cr_dest, NULL ) + + /* tear-down + */ + while (node != NULL) { + FAIL_IF( FAIL == + H5FD_s3comms_hrb_node_set(&node, node->name, NULL)); + } + HDassert(NULL == node); + FAIL_IF( FAIL == H5FD_s3comms_hrb_destroy(&hrb)); + HDassert(NULL == hrb); + + } /* for each test case */ + + /*************** + * ERROR CASES * + ***************/ + + /* malformed hrb and/or node-list + */ + JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request( + cr_dest, + 20, + sh_dest, + 20, + NULL), + "http request object cannot be null" ) + + hrb = H5FD_s3comms_hrb_init_request("GET", "/", "HTTP/1.1"); + JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request( + NULL, + 20, + sh_dest, + 20, + hrb), + "canonical request destination cannot be NULL" ) + + JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request( + cr_dest, + 20, + NULL, + 20, + hrb), + "signed headers destination cannot be null" ) + + FAIL_IF( FAIL == H5FD_s3comms_hrb_destroy(&hrb) ) + HDassert( NULL == hrb ); + + PASSED(); + return 0; + +error: + + if (node != NULL) { + while (node != NULL) + (void)H5FD_s3comms_hrb_node_set(&node, node->name, NULL); + HDassert( node == NULL ); + } + if (hrb != NULL) { + (void)H5FD_s3comms_hrb_destroy(&hrb); + } + + return -1; + +} /* end test_aws_canonical_request() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_bytes_to_hex + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_bytes_to_hex()`. + * + * Return: + * + * Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2017-09-14 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_bytes_to_hex(void) +{ + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char exp[17]; /* in size * 2 + 1 for null terminator */ + const unsigned char in[8]; + size_t size; + hbool_t lower; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { "52F3000C9A", + {82,243,0,12,154}, + 5, + FALSE, + }, + { "009a0cf3005200", /* lowercase alphas */ + {0,154,12,243,0,82,0}, + 7, + TRUE, + }, + { "", + {17,63,26,56}, + 0, + FALSE, /* irrelevant */ + }, + }; + int i = 0; + int n_cases = 3; + char out[17]; + int out_off = 0; + + + + TESTING("bytes-to-hex"); + + for (i = 0; i < n_cases; i++) { + for (out_off = 0; out_off < 17; out_off++) { + out[out_off] = 0; + } + + JSVERIFY( SUCCEED, + H5FD_s3comms_bytes_to_hex(out, + cases[i].in, + cases[i].size, + cases[i].lower), + NULL ) + + JSVERIFY_STR(cases[i].exp, out, NULL) + } + + /* dest cannot be null + */ + JSVERIFY( FAIL, + H5FD_s3comms_bytes_to_hex( + NULL, + (const unsigned char *)"nada", + 5, + FALSE), + "destination cannot be null" ) + + PASSED(); + return 0; + +error: + return -1; + +} /* end test_bytes_to_hex() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_hrb_init_request() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_hrb_init_request()` + * + * Programmer: Jacob Smith + * 2017-09-20 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_hrb_init_request(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char msg[64]; + const char *verb; + const char *resource; + const char *exp_res; + const char *version; + hbool_t ret_null; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { "get HTTP request just as we provided", + "GET", + "/path/to/some/file", + "/path/to/some/file", + "HTTP/1.1", + FALSE, + }, + { "null verb substitues to GET", + NULL, + "/MYPATH/MYFILE.tiff", + "/MYPATH/MYFILE.tiff", + "HTTP/1.1", + FALSE, + }, + { "demonstrate non-GET verb", + "HEAD", + "/MYPATH/MYFILE.tiff", + "/MYPATH/MYFILE.tiff", + "HTTP/1.1", + FALSE, + }, + { "slash prepended to resource path, if necessary", + NULL, + "MYPATH/MYFILE.tiff", + "/MYPATH/MYFILE.tiff", + NULL, + FALSE, + }, + { "null resource path causes problem", + "GET", + NULL, + NULL, + NULL, + TRUE, + }, + }; + struct testcase *C = NULL; + unsigned int i = 0; + unsigned int ncases = 5; + hrb_t *req = NULL; + + TESTING("hrb_init_request"); + + for (i = 0; i < ncases; i++) { + C = &cases[i]; + req = H5FD_s3comms_hrb_init_request( + C->verb, + C->resource, + C->version); + if (cases[i].ret_null == TRUE) { + FAIL_IF( req != NULL ); + } + else { + FAIL_IF( req == NULL ); + JSVERIFY( S3COMMS_HRB_MAGIC, req->magic, NULL ) + if (C->verb == NULL) { + JSVERIFY_STR( "GET", req->verb, NULL ) + } + else { + JSVERIFY_STR( req->verb, C->verb, NULL ) + } + JSVERIFY_STR( "HTTP/1.1", req->version, NULL ) + JSVERIFY_STR( C->exp_res, req->resource, NULL ) + FAIL_IF( req->first_header != NULL ); + FAIL_IF( req->body != NULL ); + JSVERIFY( 0, req->body_len, NULL ) + JSVERIFY( SUCCEED, H5FD_s3comms_hrb_destroy(&req), + "unable to destroy hrb_t" ) + FAIL_IF( NULL != req ); /* should annull pointer as well as free */ + } + + } /* end for each testcase */ + + PASSED(); + return 0; + +error: + (void)H5FD_s3comms_hrb_destroy(&req); + + return -1; + +} /* end test_hrb_init_request() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_hrb_node_set() + * + * Purpose: + * + * Test operations on hrb_node_t structure + * + * Programmer: Jacob Smith + * 2017-09-22 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_hrb_node_set(void) +{ + /************************* + * test-local structures * + *************************/ + + /* bundle of name/value representing an hrb_node_t + */ + typedef struct node_mock_t { + const char *name; + const char *value; + } node_mock_t; + + /* bundle for a testcase + * + * `message` + * purpose of the testcase + * + * `delta` + * container for name and value strings to pass into node-set function + * to to modify the list. + * + * `returned` + * expected return value of node-set function + * + * `given` + * `expected` + * string arrays representing the state of the list before and after + * modification. The number of strings must be even, with each name + * paired to a value. `NULL` terminates the list, with `{NULL}` + * representing the empty list. + */ + typedef struct testcase { + const char *message; + node_mock_t delta; + herr_t returned; + const char *given[11]; /* name/value pairs in array; NULL sentinel */ + const char *expected[11]; + } testcase; + + /************************ + * test-local variables * + ************************/ + + testcase cases[] = { + { "cannot remove node from null list", + { "Host", NULL }, + FAIL, + {NULL}, + {NULL}, + }, + { "cannot create list with NULL field name", + { NULL, "somevalue" }, + FAIL, + {NULL}, + {NULL}, + }, + { "create a new list", + { "Host", "somevalue" }, + SUCCEED, + {NULL}, + { "Host", "somevalue", + NULL, + }, + }, + { "insert new node at head list", + { "Host", "somevalue" }, + SUCCEED, + { "Range", "bytes=20-40", + NULL, + }, + { "Host", "somevalue", + "Range", "bytes=20-40", + NULL, + }, + }, + { "append new node at list end", + { "x-amz-date", "somevalue" }, + SUCCEED, + { "Range", "bytes=20-40", + NULL, + }, + { "Range", "bytes=20-40", + "x-amz-date", "somevalue", + NULL, + }, + }, + { "insert new node inside list", + { "Intermediary", "somevalue" }, + SUCCEED, + { "Host", "somehost" , + "Range", "bytes=20-40", + NULL, + }, + { "Host", "somehost", + "Intermediary", "somevalue", + "Range", "bytes=20-40", + NULL, + }, + }, + { "modify node", + { "Range", "bytes=40-80" }, + SUCCEED, + { "Host", "somehost", + "Range", "bytes=20-40", + NULL, + }, + { "Host", "somehost", + "Range", "bytes=40-80", + NULL, + }, + }, + { "modify node with new case", + { "RANGE", "bytes=40-80" }, + SUCCEED, + { "Host", "somehost", + "Range", "bytes=20-40", + NULL, + }, + { "Host", "somehost", + "RANGE", "bytes=40-80", + NULL, + }, + }, + { "cannot add node with no name", + { NULL, "bytes=40-80" }, + FAIL, + { "Host", "somehost", + NULL, + }, + { "Host", "somehost", + NULL, + }, + }, + { "add node with 'empty' name", + { "", "bytes=40-80" }, + SUCCEED, + { "Host", "somehost", + NULL, + }, + { "", "bytes=40-80", + "Host", "somehost", + NULL, + }, + }, + { "remove node from end of list", + { "Host", NULL }, + SUCCEED, + { "Date", "Thr, 25 Jan 2018", + "Host", "somehost", + NULL, + }, + { "Date", "Thr, 25 Jan 2018", + NULL, + }, + }, + { "remove node from middle of list", + { "Host", NULL }, + SUCCEED, + { "Date", "Thr, 25 Jan 2018", + "Host", "somehost", + "Range", "bytes=20-40", + NULL, + }, + { "Date", "Thr, 25 Jan 2018", + "Range", "bytes=20-40", + NULL, + }, + }, + { "remove node from start of list", + { "Date", NULL }, + SUCCEED, + { "Date", "Thr, 25 Jan 2018", + "Host", "somehost", + "Range", "bytes=20-40", + NULL, + }, + { "Host", "somehost", + "Range", "bytes=20-40", + NULL, + }, + }, + { "remove only node in list", + { "Date", NULL }, + SUCCEED, + { "Date", "Thr, 25 Jan 2018", + NULL, + }, + { NULL, + }, + }, + { "attempt to remove absent node fails", + { "Host", NULL }, + FAIL, + { "Date", "Thr, 25 Jan 2018", + "Range", "bytes=20-40", + NULL, + }, + { "Date", "Thr, 25 Jan 2018", + "Range", "bytes=20-40", + NULL, + }, + }, + { "removal is case-insensitive", + { "hOsT", NULL }, + SUCCEED, + { "Date", "Thr, 25 Jan 2018", + "Host", "somehost", + "Range", "bytes=20-40", + NULL, + }, + { "Date", "Thr, 25 Jan 2018", + "Range", "bytes=20-40", + NULL, + }, + }, + }; + unsigned testcases_count = 16; + unsigned test_i = 0; + hrb_node_t *list = NULL; + + TESTING("hrb_node_t (test_hrb_node_set)"); + + for (test_i = 0; test_i < testcases_count; test_i++) { + const hrb_node_t *node = NULL; + const testcase *test = &(cases[test_i]); + unsigned mock_i = 0; + + /********* + * SETUP * + *********/ + + for (mock_i = 0; test->given[mock_i] != NULL; mock_i += 2) { + const char *name = test->given[mock_i]; + const char *valu = test->given[mock_i+1]; + + FAIL_IF( SUCCEED != + H5FD_s3comms_hrb_node_set(&list, name, valu) ) + } + /******** + * TEST * + ********/ + + /* perform modification on list + */ + JSVERIFY( test->returned, + H5FD_s3comms_hrb_node_set(&list, + test->delta.name, + test->delta.value), + test->message ) + + + /* verify resulting list + */ + node = list; + mock_i = 0; + while (test->expected[mock_i] != NULL && node != NULL) { + const char *name = test->expected[mock_i]; + const char *valu = test->expected[mock_i+1]; + + JSVERIFY_STR( name, node->name, NULL ) + JSVERIFY_STR( valu, node->value, NULL ) + + mock_i += 2; + node = node->next; + } + FAIL_IF( test->expected[mock_i] != NULL ) + FAIL_IF( node != NULL ) + + /************ + * TEARDOWN * + ************/ + + while (list != NULL) { + FAIL_IF( SUCCEED != + H5FD_s3comms_hrb_node_set(&list, list->name, NULL) ) + } + } /* end for each testcase */ + + PASSED(); + return 0; + +error: + while (list != NULL) { + (void)H5FD_s3comms_hrb_node_set(&list, list->name, NULL); + } + + return -1; + +} /* end test_hrb_node_t() */ + + + +/*--------------------------------------------------------------------------- + * + * Function: test_HMAC_SHA256() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_HMAC_SHA256()` + * + * Programmer: Jacob Smith + * 2017-09-19 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_HMAC_SHA256(void) +{ + + /************************* + * test-local structures * + *************************/ + + struct testcase { + herr_t ret; /* SUCCEED/FAIL expected from call */ + const unsigned char key[SHA256_DIGEST_LENGTH]; + size_t key_len; + const char *msg; + size_t msg_len; + const char *exp; /* not used if ret == FAIL */ + size_t dest_size; /* if 0, `dest` is not malloc'd */ + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { SUCCEED, + { 0xdb, 0xb8, 0x93, 0xac, 0xc0, 0x10, 0x96, 0x49, + 0x18, 0xf1, 0xfd, 0x43, 0x3a, 0xdd, 0x87, 0xc7, + 0x0e, 0x8b, 0x0d, 0xb6, 0xbe, 0x30, 0xc1, 0xfb, + 0xea, 0xfe, 0xfa, 0x5e, 0xc6, 0xba, 0x83, 0x78, + }, + SHA256_DIGEST_LENGTH, + "AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/aws4_request\n7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972", + HDstrlen("AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/aws4_request\n7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972"), + "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41", + SHA256_DIGEST_LENGTH * 2 + 1, /* +1 for null terminator */ + }, + { SUCCEED, + {'J','e','f','e'}, + 4, + "what do ya want for nothing?", + 28, + "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843", + SHA256_DIGEST_LENGTH * 2 + 1, + }, + { FAIL, + "DOESN'T MATTER", + 14, + "ALSO IRRELEVANT", + 15, + NULL, + 0, /* dest -> null, resulting in immediate error */ + }, + }; + char *dest = NULL; + int i = 0; + int n_cases = 3; + + TESTING("HMAC_SHA256"); + + for (i = 0; i < n_cases; i++) { + if (cases[i].dest_size == 0) { + dest = NULL; + } else { + dest = (char *)HDmalloc(sizeof(char) * cases[i].dest_size); + HDassert(dest != NULL); + } + + JSVERIFY( cases[i].ret, + H5FD_s3comms_HMAC_SHA256( + cases[i].key, + cases[i].key_len, + cases[i].msg, + cases[i].msg_len, + dest), + cases[i].msg ); + if (cases[i].ret == SUCCEED) { +#ifdef VERBOSE + if (0 != + strncmp(cases[i].exp, + dest, + HDstrlen(cases[i].exp))) + { + /* print out how wrong things are, and then fail + */ + dest = (char *)realloc(dest, cases[i].dest_size + 1); + HDassert(dest != NULL); + dest[cases[i].dest_size] = 0; + HDfprintf(stdout, + "ERROR:\n!!! \"%s\"\n != \"%s\"\n", + cases[i].exp, + dest); + TEST_ERROR; + } +#else /* VERBOSE not defined */ + /* simple pass/fail test + */ + JSVERIFY( 0, + strncmp(cases[i].exp, dest, HDstrlen(cases[i].exp)), + NULL); +#endif /* VERBOSE */ + } + free(dest); + } + + PASSED(); + return 0; + +error: + free(dest); + return -1; + +} /* end test_HMAC_SHA256() */ + + +/*---------------------------------------------------------------------------- + * + * Function: test_nlowercase() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_nlowercase()` + * + * Programmer: Jacob Smith + * 2017-19-18 + * + *---------------------------------------------------------------------------- + */ +static herr_t +test_nlowercase(void) +{ + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char *in; + size_t len; + const char *exp; + }; + + /************************ + * test-local variables * + ************************/ + + /* any character after in exp on or after exp[len] is undefined. + * in this test, kept as the null character for simplicity. + */ + struct testcase cases[] = { + { "HALlEluJAh", + 6, + "hallel", + }, + { "all\0 lower", + 10, + "all\0 lower", + }, + { "to meeeeeee", + 0, + "", + }, + }; + char *dest = NULL; + int i = 0; + int n_cases = 3; + + TESTING("nlowercase"); + + for (i = 0; i < n_cases; i++) { + dest = (char *)HDmalloc(sizeof(char) * 16); + + JSVERIFY( SUCCEED, + H5FD_s3comms_nlowercase(dest, + cases[i].in, + cases[i].len), + cases[i].in ) + if (cases[i].len > 0) { + JSVERIFY( 0, strncmp(dest, cases[i].exp, cases[i].len), NULL ) + } + free(dest); + } /* end for each testcase */ + + JSVERIFY( FAIL, + H5FD_s3comms_nlowercase(NULL, + cases[0].in, + cases[0].len), + "null distination should fail" ) + + PASSED(); + return 0; + +error: + free(dest); + return -1; + +} /* end test_nlowercase() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_parse_url() + * + * Programmer: Jacob Smith + * 2017-11-?? + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_parse_url(void) +{ + /********************* + * test-local macros * + *********************/ + + /************************* + * test-local structures * + *************************/ + + typedef struct { + const char *scheme; + const char *host; + const char *port; + const char *path; + const char *query; + } const_purl_t; + + struct testcase { + const char *url; + herr_t exp_ret; /* expected return; */ + /* if FAIL, `expected` is unused */ + const_purl_t expected; + const char *msg; + }; + + /************************ + * test-local variables * + ************************/ + + parsed_url_t *purl = NULL; + unsigned int i = 0; + unsigned int ncases = 15; + struct testcase cases[] = { + { NULL, + FAIL, + { NULL, NULL, NULL, NULL, NULL }, + "null url", + }, + { "", + FAIL, + { NULL, NULL, NULL, NULL, NULL }, + "empty url", + }, + { "ftp://[1000:4000:0002:2010]", + SUCCEED, + { "ftp", + "[1000:4000:0002:2010]", + NULL, + NULL, + NULL, + }, + "IPv6 ftp and empty path (root)", + }, + { "ftp://[1000:4000:0002:2010]:2040", + SUCCEED, + { "ftp", + "[1000:4000:0002:2010]", + "2040", + NULL, + NULL, + }, + "root IPv6 ftp with port", + }, + { "http://some.domain.org:9000/path/to/resource.txt", + SUCCEED, + { "http", + "some.domain.org", + "9000", + "path/to/resource.txt", + NULL, + }, + "without query", + }, + { "https://domain.me:00/file.txt?some_params unchecked", + SUCCEED, + { "https", + "domain.me", + "00", + "file.txt", + "some_params unchecked", + }, + "with query", + }, + { "ftp://domain.com/", + SUCCEED, + { "ftp", + "domain.com", + NULL, + NULL, + NULL, + }, + "explicit root w/out port", + }, + { "ftp://domain.com:1234/", + SUCCEED, + { "ftp", + "domain.com", + "1234", + NULL, + NULL, + }, + "explicit root with port", + }, + { "ftp://domain.com:1234/file?", + FAIL, + { NULL, NULL, NULL, NULL, NULL, }, + "empty query is invalid", + }, + { "ftp://:1234/file", + FAIL, + { NULL, NULL, NULL, NULL, NULL, }, + "no host", + }, + { "h&r block", + FAIL, + { NULL, NULL, NULL, NULL, NULL, }, + "no scheme (bad URL)", + }, + { "http://domain.com?a=b&d=b", + SUCCEED, + { "http", + "domain.com", + NULL, + NULL, + "a=b&d=b", + }, + "QUERY with implict PATH", + }, + { "http://[5]/path?a=b&d=b", + SUCCEED, + { "http", + "[5]", + NULL, + "path", + "a=b&d=b", + }, + "IPv6 extraction is really dumb", + }, + { "http://[1234:5678:0910:1112]:port/path", + FAIL, + { NULL, NULL, NULL, NULL, NULL, }, + "non-decimal PORT (port)", + }, + { "http://mydomain.com:01a3/path", + FAIL, + { NULL, NULL, NULL, NULL, NULL, }, + "non-decimal PORT (01a3)", + }, + }; + + TESTING("url-parsing functionality"); + + /********* + * TESTS * + *********/ + + for (i = 0; i < ncases; i++) { + HDassert( purl == NULL ); + + JSVERIFY( cases[i].exp_ret, + H5FD_s3comms_parse_url(cases[i].url, &purl), + cases[i].msg ) + + if (cases[i].exp_ret == FAIL) { + /* on FAIL, `purl` should be untouched--remains NULL */ + FAIL_UNLESS( purl == NULL ) + } + else { + /* on SUCCEED, `purl` should be set */ + FAIL_IF( purl == NULL ) + + if (cases[i].expected.scheme != NULL) { + FAIL_IF( NULL == purl->scheme ) + JSVERIFY_STR( cases[i].expected.scheme, + purl->scheme, + cases[i].msg ) + } else { + FAIL_UNLESS( NULL == purl->scheme ) + } + + if (cases[i].expected.host != NULL) { + FAIL_IF( NULL == purl->host ) + JSVERIFY_STR( cases[i].expected.host, + purl->host, + cases[i].msg ) + } else { + FAIL_UNLESS( NULL == purl->host ) + } + + if (cases[i].expected.port != NULL) { + FAIL_IF( NULL == purl->port ) + JSVERIFY_STR( cases[i].expected.port, + purl->port, + cases[i].msg ) + } else { + FAIL_UNLESS( NULL == purl->port ) + } + + if (cases[i].expected.path != NULL) { + FAIL_IF( NULL == purl->path ) + JSVERIFY_STR( cases[i].expected.path, + purl->path, + cases[i].msg ) + } else { + FAIL_UNLESS( NULL == purl->path ) + } + + if (cases[i].expected.query != NULL) { + FAIL_IF( NULL == purl->query ) + JSVERIFY_STR( cases[i].expected.query, + purl->query, + cases[i].msg ) + } else { + FAIL_UNLESS( NULL == purl->query ) + } + } /* end if parse-url return SUCCEED/FAIL */ + + /* per-test cleanup + * well-behaved, even if `purl` is NULL + */ + FAIL_IF( FAIL == H5FD_s3comms_free_purl(purl) ) + purl = NULL; + + } /* end for each testcase */ + + PASSED(); + return 0; + +error: + /*********** + * cleanup * + ***********/ + (void)H5FD_s3comms_free_purl(purl); + + return -1; + +} /* end test_parse_url() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_percent_encode_char() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_percent_encode_char()` + * + * Return: + * + * Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2017-09-14 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_percent_encode_char(void) +{ + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char c; + const char *exp; + size_t exp_len; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + {'$', "%24", 3}, /* u+0024 dollar sign */ + {' ', "%20", 3}, /* u+0020 space */ + {'^', "%5E", 3}, /* u+0094 carat */ + {'/', "%2F", 3}, /* u+002f solidus (forward slash) */ + /* {??, "%C5%8C", 6},*/ /* u+014c Latin Capital Letter O with Macron */ + /* Not included because it is multibyte "wide" character that poses */ + /* issues both in the underlying function and in being written in */ + /* this file. */ + /* {'¢', "%C2%A2", 6}, */ /* u+00a2 cent sign */ + /* above works, but complains about wide character overflow */ + /* Elide for now, until it is determined (a) unnecessary or */ + /* (b) requiring signature change to accommodate wide characters */ + {'\0', "%00", 3}, /* u+0000 null */ + }; + char dest[13]; + size_t dest_len = 0; + int i = 0; + int n_cases = 5; + + TESTING("percent encode characters"); + + for (i = 0; i < n_cases; i++) { + JSVERIFY( SUCCEED, + H5FD_s3comms_percent_encode_char( + dest, + (const unsigned char)cases[i].c, + &dest_len), + NULL ) + JSVERIFY(cases[i].exp_len, dest_len, NULL ) + JSVERIFY(0, strncmp(dest, cases[i].exp, dest_len), NULL ) + JSVERIFY_STR( cases[i].exp, dest, NULL ) + } + + JSVERIFY( FAIL, + H5FD_s3comms_percent_encode_char( + NULL, + (const unsigned char)'^', + &dest_len), + NULL ) + + PASSED(); + return 0; + +error: + return -1; +} /* end test_percent_encode_char() */ + + +/*--------------------------------------------------------------------------- + * Function: test_s3r_open() + * + * Programmer: Jacob Smith 2018-01-24 + * + * Changes: None + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_s3r_get_filesize(void) +{ + + /************************ + * test-local variables * + ************************/ + + char url_raven[S3_TEST_MAX_URL_SIZE]; + s3r_t *handle = NULL; + + TESTING("s3r_get_filesize"); + + /* setup -- compose url to target resource + */ + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + FAIL_IF( S3_TEST_MAX_URL_SIZE < + HDsnprintf(url_raven, + S3_TEST_MAX_URL_SIZE, + "%s/%s", + s3_test_bucket_url, + S3_TEST_RESOURCE_TEXT_PUBLIC) ); + + JSVERIFY( 0, H5FD_s3comms_s3r_get_filesize(NULL), + "filesize of the null handle should be 0" ) + + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + FAIL_IF( handle == NULL ) + + JSVERIFY( 6464, H5FD_s3comms_s3r_get_filesize(handle), NULL ) + + + FAIL_IF( SUCCEED != H5FD_s3comms_s3r_close(handle) ) + + PASSED(); + return 0; + +error: + if (handle != NULL) + (void)H5FD_s3comms_s3r_close(handle); + + return -1; + +} /* end test_s3r_get_filesize() */ + + +/*--------------------------------------------------------------------------- + * Function: test_s3r_open() + * + * Programmer: Jacob Smith 2018-01-?? + * + * Changes: None + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_s3r_open(void) +{ + + /************************ + * test-local variables * + ************************/ + + char url_missing[S3_TEST_MAX_URL_SIZE]; + char url_raven[S3_TEST_MAX_URL_SIZE]; + char url_raven_badport[S3_TEST_MAX_URL_SIZE]; + char url_shakespeare[S3_TEST_MAX_URL_SIZE]; + unsigned char signing_key[SHA256_DIGEST_LENGTH]; + struct tm *now = NULL; + char iso8601now[ISO8601_SIZE]; + s3r_t *handle = NULL; + hbool_t curl_ready = FALSE; + parsed_url_t *purl = NULL; + + TESTING("s3r_open"); + + if (s3_test_credentials_loaded == 0) { + SKIPPED(); + puts(" s3 credentials are not loaded"); + fflush(stdout); + return 0; + } + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + /****************** + * PRE-TEST SETUP * + ******************/ + + FAIL_IF( S3_TEST_MAX_URL_SIZE < + HDsnprintf(url_shakespeare, + S3_TEST_MAX_URL_SIZE, + "%s/%s", + s3_test_bucket_url, + S3_TEST_RESOURCE_TEXT_RESTRICTED) ); + + FAIL_IF( S3_TEST_MAX_URL_SIZE < + HDsnprintf(url_missing, + S3_TEST_MAX_URL_SIZE, + "%s/%s", + s3_test_bucket_url, + S3_TEST_RESOURCE_MISSING) ); + + FAIL_IF( S3_TEST_MAX_URL_SIZE < + HDsnprintf(url_raven, + S3_TEST_MAX_URL_SIZE, + "%s/%s", + s3_test_bucket_url, + S3_TEST_RESOURCE_TEXT_PUBLIC) ); + + /* Set given bucket url with invalid/inactive port number for badport. + * Note, this sort of micro-management of parsed_url_t is not advised + */ + FAIL_IF( FAIL == H5FD_s3comms_parse_url(s3_test_bucket_url, &purl) ) + if (purl->port == NULL) { + purl->port = (char *)H5MM_malloc(sizeof(char) * 5); + FAIL_IF( purl->port == NULL ); + FAIL_IF( 5 < HDsnprintf(purl->port, 5, "9000") ) + } else if (strcmp(purl->port, "9000") != 0) { + FAIL_IF( 5 < HDsnprintf(purl->port, 5, "9000") ) + } else { + FAIL_IF( 5 < HDsnprintf(purl->port, 5, "1234") ) + } + FAIL_IF( S3_TEST_MAX_URL_SIZE < + HDsnprintf(url_raven_badport, + S3_TEST_MAX_URL_SIZE, + "%s://%s:%s/%s", + purl->scheme, + purl->host, + purl->port, + S3_TEST_RESOURCE_TEXT_PUBLIC) ); + + curl_global_init(CURL_GLOBAL_DEFAULT); + curl_ready = TRUE; + + now = gmnow(); + FAIL_IF( now == NULL ) + FAIL_IF( ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1) ); + + /* It is desired to have means available to verify that signing_key + * was set successfully and to an expected value. + */ + FAIL_IF( FAIL == + H5FD_s3comms_signing_key( + signing_key, + (const char *)s3_test_aws_secret_access_key, + (const char *)s3_test_aws_region, + (const char *)iso8601now) ); + + /************************* + * OPEN NONEXISTENT FILE * + *************************/ + + /* attempt anonymously + */ + handle = H5FD_s3comms_s3r_open(url_missing, NULL, NULL, NULL); + FAIL_IF( handle != NULL ); + + /* attempt with authentication + */ + handle = H5FD_s3comms_s3r_open( + url_missing, + (const char *)s3_test_aws_region, + (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key); + FAIL_IF( handle != NULL ); + + /************************* + * INACTIVE PORT ON HOST * + *************************/ + +#if S3_TEST_RUN_TIMEOUT + HDprintf("Opening on inactive port may hang for a minute; waiting for timeout\n"); + handle = H5FD_s3comms_s3r_open(url_raven_badport, NULL, NULL, NULL); + FAIL_IF( handle != NULL ); +#endif + + /******************************* + * INVALID AUTHENTICATION INFO * + *******************************/ + + /* anonymous access on restricted file + */ + handle = H5FD_s3comms_s3r_open(url_shakespeare, NULL, NULL, NULL); + FAIL_IF( handle != NULL ); + + /* passed in a bad ID + */ + handle = H5FD_s3comms_s3r_open( + url_shakespeare, + (const char *)s3_test_aws_region, + "I_MADE_UP_MY_ID", + (const unsigned char *)signing_key); + FAIL_IF( handle != NULL ); + + /* using an invalid signing key + */ + handle = H5FD_s3comms_s3r_open( + url_shakespeare, + (const char *)s3_test_aws_region, + (const char *)s3_test_aws_access_key_id, + (const unsigned char *)EMPTY_SHA256); + FAIL_IF( handle != NULL ); + + /******************************* + * SUCCESSFUL OPEN (AND CLOSE) * + *******************************/ + + /* anonymous + */ + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + FAIL_IF( handle == NULL ); + JSVERIFY( 6464, H5FD_s3comms_s3r_get_filesize(handle), + "did not get expected filesize" ) + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_close(handle), + "unable to close file" ) + handle = NULL; + + /* using authentication on anonymously-accessible file? + */ + handle = H5FD_s3comms_s3r_open( + url_raven, + (const char *)s3_test_aws_region, + (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key); + FAIL_IF( handle == NULL ); + JSVERIFY( 6464, H5FD_s3comms_s3r_get_filesize(handle), NULL ) + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_close(handle), + "unable to close file" ) + handle = NULL; + + /* authenticating + */ + handle = H5FD_s3comms_s3r_open( + url_shakespeare, + (const char *)s3_test_aws_region, + (const char *)s3_test_aws_access_key_id, + (const unsigned char *)signing_key); + FAIL_IF( handle == NULL ); + JSVERIFY( 5458199, H5FD_s3comms_s3r_get_filesize(handle), NULL ) + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_close(handle), + "unable to close file" ) + handle = NULL; + + + + curl_global_cleanup(); + curl_ready = FALSE; + + FAIL_IF( FAIL == H5FD_s3comms_free_purl(purl) ) + purl = NULL; + + PASSED(); + return 0; +error: + /*********** + * cleanup * + ***********/ + + if (handle != NULL) + H5FD_s3comms_s3r_close(handle); + if (purl != NULL) + H5FD_s3comms_free_purl(purl); + if (curl_ready == TRUE) + curl_global_cleanup(); + + return -1; + +} /* end test_s3r_open() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_s3r_read() + * + * Purpose: + * + * Specify and demonstrate the use and life cycle of an S3 Request handle + * `s3r_t`, through its related functions. + * + * H5FD_s3comms_s3r_open + * H5FD_s3comms_s3r_getsize << called by open() _only_ + * H5FD_s3comms_s3r_read << called by getsize(), multiple times working + * H5FD_s3comms_s3r_close + * + * Shows most basic curl interation. + * + * Programmer: Jacob Smith + * 2017-10-06 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_s3r_read(void) +{ + +#define S3COMMS_TEST_BUFFER_SIZE 256 + + /************************ + * test-local variables * + ************************/ + + char url_raven[S3_TEST_MAX_URL_SIZE]; + char buffer[S3COMMS_TEST_BUFFER_SIZE]; + s3r_t *handle = NULL; + hbool_t curl_ready = FALSE; + unsigned int i = 0; + + TESTING("test_s3r_read"); + + /* + * initial setup + */ + if (FALSE == s3_test_bucket_defined) { + SKIPPED(); + puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined"); + fflush(stdout); + return 0; + } + + curl_global_init(CURL_GLOBAL_DEFAULT); + curl_ready = TRUE; + FAIL_IF( S3_TEST_MAX_URL_SIZE < + HDsnprintf(url_raven, + S3_TEST_MAX_URL_SIZE, + "%s/%s", + s3_test_bucket_url, + S3_TEST_RESOURCE_TEXT_PUBLIC) ); + + for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++) + buffer[i] = '\0'; + + /* open file + */ + handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL); + FAIL_IF( handle == NULL ) + JSVERIFY( 6464, H5FD_s3comms_s3r_get_filesize(handle), NULL ) + + for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++) + buffer[i] = '\0'; + + /********************** + * read start of file * + **********************/ + + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)0, + (size_t)118, + buffer), + NULL ) + JSVERIFY_STR ( + "Once upon a midnight dreary, while I pondered, weak and weary,\n" \ + "Over many a quaint and curious volume of forgotten lore", + buffer, + NULL ) + + for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++) + buffer[i] = '\0'; + + /************************ + * read arbitrary range * + ************************/ + + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)2540, + (size_t)54, + buffer), + NULL ) + JSVERIFY_STR( "the grave and stern decorum of the countenance it wore", + buffer, + NULL ) + + for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++) + buffer[i] = '\0'; + + /********************** + * read one character * + **********************/ + + JSVERIFY(SUCCEED, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)2540, + (size_t)1, + buffer), + NULL ) + JSVERIFY_STR( "t", buffer, NULL ) + + + for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++) + buffer[i] = '\0'; + + /*************** + * read to EoF * + ***************/ + + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)6370, + (size_t)0, + buffer), + NULL ) + JSVERIFY( 0, + strncmp(buffer, + "And my soul from out that shadow that lies floating on the floor\nShall be lifted—nevermore!\n", + 94), + buffer ) + + for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++) + buffer[i] = '\0'; + + /***************** + * read past eof * + *****************/ + + JSVERIFY( FAIL, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)6400, + (size_t)100, /* 6400+100 > 6464 */ + buffer), + NULL ) + JSVERIFY( 0, strcmp("", buffer), NULL ) + + /************************ + * read starts past eof * + ************************/ + + JSVERIFY( FAIL, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)1200699, /* 1200699 > 6464 */ + (size_t)100, + buffer), + NULL ) + JSVERIFY( 0, strcmp("", buffer), NULL ) + + /********************** + * read starts on eof * + **********************/ + + JSVERIFY( FAIL, + H5FD_s3comms_s3r_read( + handle, + (haddr_t)6464, + (size_t)0, + buffer), + NULL ) + JSVERIFY( 0, strcmp("", buffer), NULL ) + + /************* + * TEAR DOWN * + *************/ + + JSVERIFY( SUCCEED, + H5FD_s3comms_s3r_close(handle), + "unable to close file" ) + handle = NULL; + + curl_global_cleanup(); + curl_ready = FALSE; + + PASSED(); + return 0; + +error: + /*********** + * cleanup * + ***********/ + + if (handle != NULL) + H5FD_s3comms_s3r_close(handle); + + if (curl_ready == TRUE) + curl_global_cleanup(); + + return -1; + +#undef S3COMMS_TEST_BUFFER_SIZE + +} /* end test_s3r_read() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_signing_key() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_signing_key()` + * + * More test cases would be a very good idea. + * + * Programmer: Jacob Smith + * 2017-09-18 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_signing_key(void) +{ + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char *region; + const char *secret_key; + const char *when; + unsigned char exp[SHA256_DIGEST_LENGTH]; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { "us-east-1", + "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "20130524T000000Z", + { 0xdb, 0xb8, 0x93, 0xac, 0xc0, 0x10, 0x96, 0x49, + 0x18, 0xf1, 0xfd, 0x43, 0x3a, 0xdd, 0x87, 0xc7, + 0x0e, 0x8b, 0x0d, 0xb6, 0xbe, 0x30, 0xc1, 0xfb, + 0xea, 0xfe, 0xfa, 0x5e, 0xc6, 0xba, 0x83, 0x78, + }, + }, + }; + int i = 0; + unsigned char *key = NULL; + int ncases = 1; + + TESTING("signing_key"); + + for (i = 0; i < ncases; i++) { + key = (unsigned char *)HDmalloc(sizeof(unsigned char) * \ + SHA256_DIGEST_LENGTH); + HDassert(key != NULL); + + JSVERIFY( SUCCEED, + H5FD_s3comms_signing_key( + key, + cases[i].secret_key, + cases[i].region, + cases[i].when), + NULL ) + + JSVERIFY( 0, + strncmp((const char *)cases[i].exp, + (const char *)key, + SHA256_DIGEST_LENGTH), + cases[i].exp ) + + free(key); + key = NULL; + } + + + /*************** + * ERROR CASES * + ***************/ + + key = (unsigned char *)HDmalloc(sizeof(unsigned char) * \ + SHA256_DIGEST_LENGTH); + HDassert(key != NULL); + + JSVERIFY( FAIL, + H5FD_s3comms_signing_key( + NULL, + cases[0].secret_key, + cases[0].region, + cases[0].when), + "destination cannot be NULL" ) + + JSVERIFY( FAIL, + H5FD_s3comms_signing_key( + key, + NULL, + cases[0].region, + cases[0].when), + "secret key cannot be NULL" ) + + JSVERIFY( FAIL, + H5FD_s3comms_signing_key( + key, + cases[0].secret_key, + NULL, + cases[0].when), + "aws region cannot be NULL" ) + + JSVERIFY( FAIL, + H5FD_s3comms_signing_key( + key, + cases[0].secret_key, + cases[0].region, + NULL), + "time string cannot be NULL" ) + + free(key); + key = NULL; + + PASSED(); + return 0; + +error: + if (key != NULL) { + free(key); + } + + return -1; + +} /* end test_signing_key() */ + + +/*--------------------------------------------------------------------------- + * + * Function: test_tostringtosign() + * + * Purpose: + * + * Verify that we can get the "string to sign" from a Canonical Request and + * related information. + * + * Demonstrate failure cases. + * + * Return: + * + * Success: 0 + * Failure: -1 + * + * Programmer: Jacob Smith + * 2017-09-13 + * + *--------------------------------------------------------------------------- + */ +static herr_t +test_tostringtosign(void) +{ + /************************ + * test-local variables * + ************************/ + + const char canonreq[] = "GET\n/test.txt\n\nhost:examplebucket.s3.amazonaws.com\nrange:bytes=0-9\nx-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nx-amz-date:20130524T000000Z\n\nhost;range;x-amz-content-sha256;x-amz-date\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + const char iso8601now[] = "20130524T000000Z"; + const char region[] = "us-east-1"; + char s2s[512]; + + TESTING("s3comms tostringtosign"); + + JSVERIFY( SUCCEED, + H5FD_s3comms_tostringtosign(s2s, canonreq, iso8601now, region), + "unable to create string to sign" ) + + JSVERIFY_STR( "AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/aws4_request\n7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972", + s2s, NULL ) + + JSVERIFY( FAIL, + H5FD_s3comms_tostringtosign(s2s, NULL, iso8601now, region), + "canonical request string cannot be NULL" ) + + JSVERIFY( FAIL, + H5FD_s3comms_tostringtosign(s2s, canonreq, NULL, region), + "time string cannot be NULL" ) + + JSVERIFY( FAIL, + H5FD_s3comms_tostringtosign(s2s, canonreq, iso8601now, NULL), + "aws region cannot be NULL" ) + + PASSED(); + return 0; + +error : + return -1; + +} /* end test_tostringtosign() */ + + +/*---------------------------------------------------------------------------- + * + * Function: test_trim() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_trim()`. + * + * Programmer: Jacob Smith + * 2017-09-14 + * + *---------------------------------------------------------------------------- + */ +static herr_t +test_trim(void) +{ + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char *in; + size_t in_len; + const char *exp; + size_t exp_len; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { "block string", + 12, + "block string", + 12, + }, + { " \n\r \t", + 6, + "", + 0, + }, + { " \twhite b4", + 10, + "white b4", + 8, + }, + { "white after\r\n ", + 15, + "white after", + 11, + }, + { " on\nends\t", + 9, + "on\nends", + 7, + }, + }; + char dest[32]; + size_t dest_len = 0; + int i = 0; + int n_cases = 5; + char *str = NULL; + + + + TESTING("s3comms trim"); + + for (i = 0; i < n_cases; i++) { + HDassert(str == NULL); + str = (char *)HDmalloc(sizeof(char) * cases[i].in_len); + HDassert(str != NULL); + HDstrncpy(str, cases[i].in, cases[i].in_len); + + JSVERIFY( SUCCEED, + H5FD_s3comms_trim(dest, str, cases[i].in_len, &dest_len), + NULL ) + JSVERIFY( cases[i].exp_len, dest_len, cases[i].in ) + if (dest_len > 0) { + JSVERIFY( 0, strncmp(cases[i].exp, dest, dest_len), + cases[i].exp ) + } + free(str); + str = NULL; + } /* end for each testcase */ + + JSVERIFY( SUCCEED, H5FD_s3comms_trim(dest, NULL, 3, &dest_len), + "should not fail when trimming a null string" ); + JSVERIFY( 0, dest_len, "trimming NULL string writes 0 characters" ) + + HDassert(str == NULL); + str = (char *)HDmalloc(sizeof(char *) * 11); + HDassert(str != NULL); + memcpy(str, "some text ", 11); /* string with null terminator */ + JSVERIFY( FAIL, H5FD_s3comms_trim(NULL, str, 10, &dest_len), + "destination for trim cannot be NULL" ); + free(str); + str = NULL; + + PASSED(); + return 0; + +error: + if (str != NULL) { + free(str); + } + return -1; + +} /* end test_trim() */ + + +/*---------------------------------------------------------------------------- + * + * Function: test_uriencode() + * + * Purpose: + * + * Define and verify behavior of `H5FD_s3comms_uriencode()`. + * + * Programmer: Jacob Smith + * 2017-09-14 + * + *---------------------------------------------------------------------------- + */ +static herr_t +test_uriencode(void) +{ + /************************* + * test-local structures * + *************************/ + + struct testcase { + const char *str; + size_t s_len; + hbool_t encode_slash; + const char *expected; + }; + + /************************ + * test-local variables * + ************************/ + + struct testcase cases[] = { + { "/path/to/resource.jpg", + 21, + FALSE, + "/path/to/resource.jpg", + }, + { "/path/to/resource.jpg", + 21, + TRUE, + "%2Fpath%2Fto%2Fresource.jpg", + }, + { "string got_spaa ces", + 20, + TRUE, + "string%20got_spaa%20%20ces", + }, + { "sp ac~es/and-sl ash.encoded", + 27, + TRUE, + "sp%20ac~es%2Fand-sl%20ash.encoded", + }, + { "sp ac~es/and-sl ash.unencoded", + 29, + FALSE, + "sp%20ac~es/and-sl%20ash.unencoded", + }, + { "/path/to/resource.txt", + 0, + FALSE, + "", + + } + }; + char *dest = NULL; + size_t dest_written = 0; + int i = 0; + int ncases = 6; + size_t str_len = 0; + + + + TESTING("s3comms uriencode") + + for (i = 0; i < ncases; i++) { + str_len = cases[i].s_len; + dest = (char *)HDmalloc(sizeof(char) * str_len * 3 + 1); + FAIL_IF( dest == NULL ) + + JSVERIFY( SUCCEED, + H5FD_s3comms_uriencode( + dest, + cases[i].str, + str_len, + cases[i].encode_slash, + &dest_written), + NULL ); + JSVERIFY( HDstrlen(cases[i].expected), + dest_written, + NULL ) + JSVERIFY( 0, + strncmp(dest, cases[i].expected, dest_written), + cases[i].expected ); + + free(dest); + dest = NULL; + } /* end for each testcase */ + + /*************** + * ERROR CASES * + ***************/ + + dest = (char *)HDmalloc(sizeof(char) * 15); + HDassert(dest != NULL); + + JSVERIFY( FAIL, + H5FD_s3comms_uriencode(NULL, "word$", 5, false, &dest_written), + "destination cannot be NULL" ); + JSVERIFY( FAIL, + H5FD_s3comms_uriencode(dest, NULL, 5, false, &dest_written), + "source string cannot be NULL" ); + + free(dest); + dest = NULL; + + PASSED(); + return 0; + +error: + if (dest != NULL) { + free(dest); + } + return -1; + +} /* end test_uriencode() */ + +#endif /* H5_HAVE_ROS3_VFD */ + + + +/*------------------------------------------------------------------------- + * Function: main() + * + * Purpose: + * + * Run unit tests for S3 Communications (s3comms). + * + * Return: + * + * Success: 0 + * Failure: 1 + * + * Programmer: Jacob Smith + * 2017-10-12 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ +#ifdef H5_HAVE_ROS3_VFD + int nerrors = 0; + const char *bucket_url_env = NULL; + + h5_reset(); + +#endif /* H5_HAVE_ROS3_VFD */ + + HDprintf("Testing S3Communications functionality.\n"); + +#ifdef H5_HAVE_ROS3_VFD + + /* "clear" profile data strings */ + s3_test_aws_access_key_id[0] = '\0'; + s3_test_aws_secret_access_key[0] = '\0'; + s3_test_aws_region[0] = '\0'; + s3_test_bucket_url[0] = '\0'; + +/* TODO: unit/regression test for H5FD_s3comms_load_aws_profile() + * requires a few test files and/or manipulation of default path + */ + /* attempt to load test credentials + * if unable, certain tests will be skipped + */ + if (SUCCEED == H5FD_s3comms_load_aws_profile( + S3_TEST_PROFILE_NAME, + s3_test_aws_access_key_id, + s3_test_aws_secret_access_key, + s3_test_aws_region)) + { + s3_test_credentials_loaded = 1; + } + + bucket_url_env = HDgetenv("HDF5_ROS3_TEST_BUCKET_URL"); + if (bucket_url_env == NULL || bucket_url_env[0] == '\0') { + HDprintf("WARNING: S3 bucket url is not defined in enviornment " \ + "variable 'HDF5_ROS3_TEST_BUCKET_URL'!\n"); + } + else { + HDstrncpy(s3_test_bucket_url, bucket_url_env, S3_TEST_MAX_URL_SIZE); + s3_test_bucket_defined = TRUE; + } + + /* tests ordered rougly by dependence */ + nerrors += test_macro_format_credential() < 0 ? 1 : 0; + nerrors += test_trim() < 0 ? 1 : 0; + nerrors += test_nlowercase() < 0 ? 1 : 0; + nerrors += test_uriencode() < 0 ? 1 : 0; + nerrors += test_percent_encode_char() < 0 ? 1 : 0; + nerrors += test_bytes_to_hex() < 0 ? 1 : 0; + nerrors += test_HMAC_SHA256() < 0 ? 1 : 0; + nerrors += test_signing_key() < 0 ? 1 : 0; + nerrors += test_hrb_node_set() < 0 ? 1 : 0; + nerrors += test_hrb_init_request() < 0 ? 1 : 0; + nerrors += test_parse_url() < 0 ? 1 : 0; + nerrors += test_aws_canonical_request() < 0 ? 1 : 0; + nerrors += test_tostringtosign() < 0 ? 1 : 0; + nerrors += test_s3r_open() < 0 ? 1 : 0; + nerrors += test_s3r_get_filesize() < 0 ? 1 : 0; + nerrors += test_s3r_read() < 0 ? 1 : 0; + + if (nerrors) { + HDprintf("***** %d S3comms TEST%s FAILED! *****\n", + nerrors, + nerrors > 1 ? "S" : ""); + return 1; + } + + HDprintf("All S3comms tests passed.\n"); + + return 0; + +#else + + HDprintf("SKIPPED - read-only S3 VFD not built\n"); + return EXIT_SUCCESS; + +#endif /* H5_HAVE_ROS3_VFD */ + +} /* end main() */ + diff --git a/test/testframe.c b/test/testframe.c index 231bcc9..d16d1bc 100644 --- a/test/testframe.c +++ b/test/testframe.c @@ -83,8 +83,7 @@ AddTest(const char *TheName, void (*TheCall) (void), void (*Cleanup) (void), con HDexit(EXIT_FAILURE); } /* end if */ if (HDstrlen(TheName) >= MAXTESTNAME) { - HDprintf("Test name too long, increase MAXTESTNAME(%d).\n", - MAXTESTNAME); + HDprintf("Test name too long, increase MAXTESTNAME(%d).\n", MAXTESTNAME); HDexit(EXIT_FAILURE); } /* end if */ @@ -96,7 +95,7 @@ AddTest(const char *TheName, void (*TheCall) (void), void (*Cleanup) (void), con /* Reallocate array */ if(NULL == (newTest = (TestStruct *)HDrealloc(Test, newAlloc * sizeof(TestStruct)))) { HDprintf("Out of memory for tests, Index = %u, TestAlloc = %u, newAlloc = %u\n", Index, TestAlloc, newAlloc); - exit(EXIT_FAILURE); + HDexit(EXIT_FAILURE); } /* end if */ /* Update info */ diff --git a/test/vfd.c b/test/vfd.c index ee177c4..1c2e656 100644 --- a/test/vfd.c +++ b/test/vfd.c @@ -25,8 +25,13 @@ #define FAMILY_SIZE (1*KB) #define FAMILY_SIZE2 (5*KB) #define MULTI_SIZE 128 + #define CORE_INCREMENT (4*KB) -#define CORE_PAGE_SIZE (1024 * 1024) +#define CORE_PAGE_SIZE (1024*KB) +#define CORE_DSET_NAME "core dset" +#define CORE_DSET_DIM1 1024 +#define CORE_DSET_DIM2 32 + #define DSET1_NAME "dset1" #define DSET1_DIM1 1024 #define DSET1_DIM2 32 @@ -53,6 +58,7 @@ const char *FILENAME[] = { "stdio_file", /*7*/ "windows_file", /*8*/ "new_multi_file_v16",/*9*/ + "ro_s3_file", /*10*/ NULL }; @@ -61,7 +67,7 @@ const char *FILENAME[] = { #define COMPAT_BASENAME "family_v16_" #define MULTI_COMPAT_BASENAME "multi_file_v16" - + /*------------------------------------------------------------------------- * Function: test_sec2 * @@ -78,56 +84,83 @@ const char *FILENAME[] = { static herr_t test_sec2(void) { - hid_t file = -1; /* file ID */ - hid_t fapl = -1; /* file access property list ID */ - hid_t access_fapl = -1; - char filename[1024]; - int *fhandle = NULL; - hsize_t file_size = 0; + hid_t fid = -1; /* file ID */ + hid_t fapl_id = -1; /* file access property list ID */ + hid_t fapl_id_out = -1; /* from H5Fget_access_plist */ + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ + char filename[1024]; /* filename */ + void *os_file_handle = NULL; /* OS file handle */ + hsize_t file_size; /* file size */ TESTING("SEC2 file driver"); /* Set property list and file name for SEC2 driver. */ - if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR; - if(H5Pset_fapl_sec2(fapl) < 0) + if(H5Pset_fapl_sec2(fapl_id) < 0) TEST_ERROR; - h5_fixname(FILENAME[0], fapl, filename, sizeof filename); + h5_fixname(FILENAME[0], fapl_id, filename, sizeof(filename)); - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + /* Check that the VFD feature flags are correct */ + if ((driver_id = H5Pget_driver(fapl_id)) < 0) + TEST_ERROR + if (H5FDdriver_query(driver_id, &driver_flags) < 0) + TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_METADATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_ACCUMULATE_METADATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_POSIX_COMPAT_HANDLE)) TEST_ERROR + /* Check for extra flags not accounted for above */ + if(driver_flags != (H5FD_FEAT_AGGREGATE_METADATA + | H5FD_FEAT_ACCUMULATE_METADATA + | H5FD_FEAT_DATA_SIEVE + | H5FD_FEAT_AGGREGATE_SMALLDATA + | H5FD_FEAT_POSIX_COMPAT_HANDLE)) + TEST_ERROR + + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) TEST_ERROR; /* Retrieve the access property list... */ - if((access_fapl = H5Fget_access_plist(file)) < 0) + if((fapl_id_out = H5Fget_access_plist(fid)) < 0) TEST_ERROR; /* Check that the driver is correct */ - if(H5FD_SEC2 != H5Pget_driver(access_fapl)) + if(H5FD_SEC2 != H5Pget_driver(fapl_id_out)) TEST_ERROR; /* ...and close the property list */ - if(H5Pclose(access_fapl) < 0) + if(H5Pclose(fapl_id_out) < 0) TEST_ERROR; - /* Check file handle API */ - if(H5Fget_vfd_handle(file, H5P_DEFAULT, (void **)&fhandle) < 0) - TEST_ERROR; - if(*fhandle < 0) + /* Check that we can get an operating-system-specific handle from + * the library. + */ + if(H5Fget_vfd_handle(fid, H5P_DEFAULT, &os_file_handle) < 0) TEST_ERROR; + if(os_file_handle == NULL) + FAIL_PUTS_ERROR("NULL os-specific vfd/file handle was returned from H5Fget_vfd_handle"); - /* Check file size API */ - if(H5Fget_filesize(file, &file_size) < 0) - TEST_ERROR; - /* There is no guarantee the size of metadata in file is constant. - * Just try to check if it's reasonable. It's 2KB right now. + /* There is no garantee the size of metadata in file is constant. + * Just try to check if it's reasonable. + * + * Currently it should be around 2 KB. */ + if(H5Fget_filesize(fid, &file_size) < 0) + TEST_ERROR; if(file_size < 1 * KB || file_size > 4 * KB) + FAIL_PUTS_ERROR("suspicious file size obtained from H5Fget_filesize"); + + /* Close and delete the file */ + if(H5Fclose(fid) < 0) TEST_ERROR; - h5_delete_test_file(FILENAME[0], fapl); + h5_delete_test_file(FILENAME[0], fapl_id); /* Close the fapl */ - if(H5Pclose(fapl) < 0) + if(H5Pclose(fapl_id) < 0) TEST_ERROR; PASSED(); @@ -135,209 +168,347 @@ test_sec2(void) error: H5E_BEGIN_TRY { - H5Pclose(fapl); - H5Fclose(file); + H5Pclose(fapl_id); + H5Pclose(fapl_id_out); + H5Fclose(fid); } H5E_END_TRY; return -1; -} +} /* end test_sec2() */ - /*------------------------------------------------------------------------- - * Function: test_direct + * Function: test_core * - * Purpose: Tests the file handle interface for DIRECT I/O driver + * Purpose: Tests the file handle interface for CORE driver * * Return: Success: 0 * Failure: -1 * * Programmer: Raymond Lu - * Wednesday, 20 September 2006 + * Tuesday, Sept 24, 2002 * *------------------------------------------------------------------------- */ static herr_t -test_direct(void) +test_core(void) { -#ifdef H5_HAVE_DIRECT - hid_t file=(-1), fapl, access_fapl = -1; - hid_t dset1=-1, dset2=-1, space1=-1, space2=-1; - char filename[1024]; - int *fhandle=NULL; - hsize_t file_size; - hsize_t dims1[2], dims2[1]; - size_t mbound; - size_t fbsize; - size_t cbsize; - int *points = NULL, *check = NULL, *p1, *p2; - int wdata2[DSET2_DIM] = {11,12,13,14}; - int rdata2[DSET2_DIM]; - int i, j, n; -#endif /*H5_HAVE_DIRECT*/ + hid_t fid = -1; /* file ID */ + hid_t fapl_id = -1; /* file access property list ID */ + hid_t fapl_id_out = -1; /* from H5Fget_access_plist */ + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ + hid_t did = -1; /* dataset ID */ + hid_t sid = -1; /* dataspace ID */ + char filename[1024]; /* filename */ + void *os_file_handle = NULL; /* OS file handle */ + hsize_t file_size; /* file size */ + size_t increment; /* core VFD increment */ + hbool_t backing_store; /* use backing store? */ + hbool_t use_write_tracking; /* write tracking flag */ + size_t write_tracking_page_size; /* write tracking page size */ + int *data_w = NULL; /* data written to the dataset */ + int *data_r = NULL; /* data read from the dataset */ + int val; /* data value */ + int *pw = NULL, *pr = NULL; /* pointers for iterating over + data arrays (write & read) */ + hsize_t dims[2]; /* dataspace dimensions */ + int i, j; /* iterators */ + htri_t status; /* return value from H5Lexists */ - TESTING("DIRECT I/O file driver"); + TESTING("CORE file driver"); -#ifndef H5_HAVE_DIRECT - SKIPPED(); - return 0; -#else /*H5_HAVE_DIRECT*/ + /* Get a file access property list and fix up the file name */ + if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + h5_fixname(FILENAME[1], fapl_id, filename, sizeof(filename)); - /* Set property list and file name for Direct driver. Set memory alignment boundary - * and file block size to 512 which is the minimum for Linux 2.6. */ - if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + /************************************************************************ + * Check that the backing store flag works by creating a file, close + * it, and ensure that the file does not exist. + ************************************************************************/ + + /* Make sure it's not present at the start of the test */ + if(HDaccess(filename, F_OK) != -1) + if(HDremove(filename) < 0) + FAIL_PUTS_ERROR("unable to remove backing store file"); + + /* Create and close file w/ backing store off */ + if(H5Pset_fapl_core(fapl_id, (size_t)CORE_INCREMENT, FALSE) < 0) TEST_ERROR; - if(H5Pset_fapl_direct(fapl, MBOUNDARY, FBSIZE, CBSIZE) < 0) + + /* Check that the VFD feature flags are correct. + * Note that the H5FDdriver_query() API call does not require a file + * so backing-store related flags will not be returned here. + */ + if ((driver_id = H5Pget_driver(fapl_id)) < 0) + TEST_ERROR + if (H5FDdriver_query(driver_id, &driver_flags) < 0) + TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_METADATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_ACCUMULATE_METADATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_ALLOW_FILE_IMAGE)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_CAN_USE_FILE_IMAGE_CALLBACKS)) TEST_ERROR + /* Check for extra flags not accounted for above */ + if(driver_flags != (H5FD_FEAT_AGGREGATE_METADATA + | H5FD_FEAT_ACCUMULATE_METADATA + | H5FD_FEAT_DATA_SIEVE + | H5FD_FEAT_AGGREGATE_SMALLDATA + | H5FD_FEAT_ALLOW_FILE_IMAGE + | H5FD_FEAT_CAN_USE_FILE_IMAGE_CALLBACKS)) + TEST_ERROR + + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) TEST_ERROR; - h5_fixname(FILENAME[5], fapl, filename, sizeof filename); + if(H5Fclose(fid) < 0) + TEST_ERROR; + /* Check for the backing store file */ + if(HDaccess(filename, F_OK) != -1) + FAIL_PUTS_ERROR("file created when backing store set to FALSE"); - /* Verify the file access properties */ - if(H5Pget_fapl_direct(fapl, &mbound, &fbsize, &cbsize) < 0) + /************************************************************************ + * Check basic core VFD operation and properties. This is done with the + * backing store on so a file will be created for later use. + ************************************************************************/ + + /* Turn the backing store on */ + if(H5Pset_fapl_core(fapl_id, (size_t)CORE_INCREMENT, TRUE) < 0) TEST_ERROR; - if(mbound != MBOUNDARY || fbsize != FBSIZE || cbsize != CBSIZE) + + /* Check that write tracking is off by default and that the default + * page size is non-zero. + */ + if(H5Pget_core_write_tracking(fapl_id, &use_write_tracking, &write_tracking_page_size) < 0) TEST_ERROR; + if(FALSE != use_write_tracking) + FAIL_PUTS_ERROR("write tracking should be off by default"); + if(0 == write_tracking_page_size) + FAIL_PUTS_ERROR("write tracking page size should never be zero"); - if(H5Pset_alignment(fapl, (hsize_t)THRESHOLD, (hsize_t)FBSIZE) < 0) + /* Set core VFD properties */ + if(H5Pset_core_write_tracking(fapl_id, TRUE, CORE_PAGE_SIZE) < 0) TEST_ERROR; - H5E_BEGIN_TRY { - file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); - } H5E_END_TRY; - if(file<0) { - H5Pclose (fapl); - SKIPPED(); - HDprintf(" Probably the file system doesn't support Direct I/O\n"); - return 0; - } + /* Create the file */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) + TEST_ERROR; /* Retrieve the access property list... */ - if ((access_fapl = H5Fget_access_plist(file)) < 0) + if((fapl_id_out = H5Fget_access_plist(fid)) < 0) TEST_ERROR; /* Check that the driver is correct */ - if(H5FD_DIRECT != H5Pget_driver(access_fapl)) + if(H5FD_CORE != H5Pget_driver(fapl_id_out)) TEST_ERROR; - /* ...and close the property list */ - if (H5Pclose(access_fapl) < 0) - TEST_ERROR; + /* Get the basic VFD properties from the fapl and ensure that + * they are correct. + */ + if(H5Pget_fapl_core(fapl_id_out, &increment, &backing_store) < 0) + TEST_ERROR + if(increment != (size_t)CORE_INCREMENT) + FAIL_PUTS_ERROR("incorrect increment from file fapl"); + if(backing_store != TRUE) + FAIL_PUTS_ERROR("incorrect backing store flag from file fapl"); - /* Check file handle API */ - if(H5Fget_vfd_handle(file, H5P_DEFAULT, (void **)&fhandle) < 0) - TEST_ERROR; - if(*fhandle<0) + /* Check that the backing store write tracking info was saved */ + /* TODO: There is a bug where H5Fget_access_plist() does not return + * the write tracking properties. Until this bug is fixed, just + * test the main fapl_id. + */ + if(H5Pget_core_write_tracking(fapl_id, &use_write_tracking, &write_tracking_page_size) < 0) TEST_ERROR; + if(TRUE != use_write_tracking) + FAIL_PUTS_ERROR("write tracking flag incorrect in fapl obtained from H5Fget_access_plist"); + if(CORE_PAGE_SIZE != write_tracking_page_size) + FAIL_PUTS_ERROR("write tracking page size incorrect in fapl obtained from H5Fget_access_plist"); - /* Check file size API */ - if(H5Fget_filesize(file, &file_size) < 0) + /* Close the property list */ + if(H5Pclose(fapl_id_out) < 0) TEST_ERROR; - /* There is no guarantee of the number of metadata allocations, but it's - * 4 currently and the size of the file should be between 3 & 4 file buffer - * sizes.. + /* Check that we can get an operating-system-specific handle from + * the library. */ - if(file_size < (FBSIZE * 3) || file_size >= (FBSIZE * 4)) + if(H5Fget_vfd_handle(fid, H5P_DEFAULT, &os_file_handle) < 0) TEST_ERROR; + if(os_file_handle == NULL) + FAIL_PUTS_ERROR("NULL os-specific vfd/file handle was returned from H5Fget_vfd_handle"); - /* Allocate aligned memory for data set 1. For data set 1, everything is aligned including - * memory address, size of data, and file address. */ - if(0 != HDposix_memalign(&points, (size_t)FBSIZE, (size_t)(DSET1_DIM1 * DSET1_DIM2 * sizeof(int)))) + /* There is no garantee the size of metadata in file is constant. + * Just try to check if it's reasonable. + * + * TODO: Needs justification of why is this is a reasonable size. + */ + if(H5Fget_filesize(fid, &file_size) < 0) TEST_ERROR; - if(0 != HDposix_memalign(&check, (size_t)FBSIZE, (size_t)(DSET1_DIM1 * DSET1_DIM2 * sizeof(int)))) + if(file_size < 2 * KB || file_size > 6 * KB) + FAIL_PUTS_ERROR("suspicious file size obtained from H5Fget_filesize"); + + /* Close the file */ + if(H5Fclose(fid) < 0) TEST_ERROR; - /* Initialize the dset1 */ - p1 = points; - for(i = n = 0; i < DSET1_DIM1; i++) - for(j = 0; j < DSET1_DIM2; j++) - *p1++ = n++; - /* Create the data space1 */ - dims1[0] = DSET1_DIM1; - dims1[1] = DSET1_DIM2; - if((space1 = H5Screate_simple(2, dims1, NULL)) < 0) + /************************************************************************ + * Make changes to the file with the backing store flag OFF to ensure + * that they ARE NOT propagated. + ************************************************************************/ + + /* Open the file with backing store off for read and write. + * Changes won't be saved in file. + */ + if(H5Pset_fapl_core(fapl_id, (size_t)CORE_INCREMENT, FALSE) < 0) + TEST_ERROR; + if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) TEST_ERROR; - /* Create the dset1 */ - if((dset1 = H5Dcreate2(file, DSET1_NAME, H5T_NATIVE_INT, space1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + /* Allocate memory for data set. */ + if(NULL == (data_w = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int)))) + FAIL_PUTS_ERROR("unable to allocate memory for input array"); + if(NULL == (data_r = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int)))) + FAIL_PUTS_ERROR("unable to allocate memory for output array"); + + /* Initialize the buffers */ + val = 0; + pw = data_w; + for(i = 0; i < CORE_DSET_DIM1; i++) + for(j = 0; j < CORE_DSET_DIM2; j++) + *pw++ = val++; + HDmemset(data_r, 0, DSET1_DIM1 * DSET1_DIM2 * sizeof(int)); + + /* Create the dataspace */ + dims[0] = CORE_DSET_DIM1; + dims[1] = CORE_DSET_DIM2; + if((sid = H5Screate_simple(2, dims, NULL)) < 0) TEST_ERROR; - /* Write the data to the dset1 */ - if(H5Dwrite(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, points) < 0) + /* Create the dataset */ + if((did = H5Dcreate2(fid, CORE_DSET_NAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; - if(H5Dclose(dset1) < 0) + /* Write the data to the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_w) < 0) TEST_ERROR; - if((dset1 = H5Dopen2(file, DSET1_NAME, H5P_DEFAULT)) < 0) + /* Close and reopen the dataset */ + if(H5Dclose(did) < 0) + TEST_ERROR; + if((did = H5Dopen2(fid, CORE_DSET_NAME, H5P_DEFAULT)) < 0) TEST_ERROR; /* Read the data back from dset1 */ - if(H5Dread(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, check) < 0) + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_r) < 0) TEST_ERROR; /* Check that the values read are the same as the values written */ - p1 = points; - p2 = check; - for(i = 0; i < DSET1_DIM1; i++) - for(j = 0; j < DSET1_DIM2; j++) - if(*p1++ != *p2++) { + pw = data_w; + pr = data_r; + for(i = 0; i < CORE_DSET_DIM1; i++) + for(j = 0; j < CORE_DSET_DIM2; j++) + if(*pr++ != *pw++) { H5_FAILED(); - HDprintf(" Read different values than written in data set 1.\n"); + HDprintf(" Read different values than written in data set.\n"); HDprintf(" At index %d,%d\n", i, j); TEST_ERROR; - } /* end if */ + } /* end if */ - /* Create the data space2. For data set 2, memory address and data size are not aligned. */ - dims2[0] = DSET2_DIM; - if((space2 = H5Screate_simple(1, dims2, NULL)) < 0) + /* Close everything except the dataspace ID (needed below)*/ + if(H5Dclose(did) < 0) TEST_ERROR; - - /* Create the dset2 */ - if((dset2 = H5Dcreate2(file, DSET2_NAME, H5T_NATIVE_INT, space2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + if(H5Fclose(fid) < 0) TEST_ERROR; - /* Write the data to the dset1 */ - if(H5Dwrite(dset2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata2) < 0) + /* Reopen the file and ensure that the dataset does not exist */ + if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) + TEST_ERROR; + status = H5Lexists(fid, CORE_DSET_NAME, H5P_DEFAULT); + if(status < 0) TEST_ERROR; + if(status > 0) + FAIL_PUTS_ERROR("core VFD dataset created in file when backing store disabled"); - if(H5Dclose(dset2) < 0) + /* Close the file */ + if(H5Fclose(fid) < 0) TEST_ERROR; - if((dset2 = H5Dopen2(file, DSET2_NAME, H5P_DEFAULT)) < 0) + + /************************************************************************ + * Make changes to the file with the backing store flag ON to ensure + * that they ARE propagated. + ************************************************************************/ + + /* Open the file with backing store on for read and write. + * Changes will be saved in file. + */ + if(H5Pset_fapl_core(fapl_id, (size_t)CORE_INCREMENT, TRUE) < 0) + TEST_ERROR; + if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) TEST_ERROR; - /* Read the data back from dset1 */ - if(H5Dread(dset2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata2) < 0) + /* Create the dataset */ + if((did = H5Dcreate2(fid, CORE_DSET_NAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; - /* Check that the values read are the same as the values written */ - for(i = 0; i < DSET2_DIM; i++) - if(wdata2[i] != rdata2[i]) { - H5_FAILED(); - HDprintf(" Read different values than written in data set 2.\n"); - HDprintf(" At index %d\n", i); - TEST_ERROR; - } /* end if */ + /* Write the data to the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_w) < 0) + TEST_ERROR; - if(H5Sclose(space1) < 0) + /* Close everything and reopen */ + if(H5Dclose(did) < 0) TEST_ERROR; - if(H5Dclose(dset1) < 0) + if(H5Fclose(fid) < 0) TEST_ERROR; - if(H5Sclose(space2) < 0) + if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0) TEST_ERROR; - if(H5Dclose(dset2) < 0) + if((did = H5Dopen2(fid, CORE_DSET_NAME, H5P_DEFAULT)) < 0) TEST_ERROR; - if(H5Fclose(file) < 0) + + /* Read the data back from the dataset */ + HDmemset(data_r, 0, DSET1_DIM1 * DSET1_DIM2 * sizeof(int)); + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_r) < 0) TEST_ERROR; - HDassert(points); - HDfree(points); - HDassert(check); - HDfree(check); - h5_delete_test_file(FILENAME[1], fapl); + /* Check that the values read are the same as the values written */ + pw = data_w; + pr = data_r; + for(i = 0; i < CORE_DSET_DIM1; i++) + for(j = 0; j < CORE_DSET_DIM2; j++) + if(*pw++ != *pr++) { + H5_FAILED(); + HDprintf(" Read different values than written in data set.\n"); + HDprintf(" At index %d,%d\n", i, j); + TEST_ERROR; + } /* end if */ + + /* Check file size API. + * There is no garantee the size of metadata in file is constant. + * Just try to check if it's reasonable. + * + * TODO: Needs justification of why is this is a reasonable size. + */ + if(H5Fget_filesize(fid, &file_size) < 0) + TEST_ERROR; + if(file_size < 64 * KB || file_size > 256 * KB) + FAIL_PUTS_ERROR("suspicious file size obtained from H5Fget_filesize"); + + /* Close everything */ + if(H5Sclose(sid) < 0) + TEST_ERROR; + if(H5Dclose(did) < 0) + TEST_ERROR; + + HDfree(data_w); + HDfree(data_r); + + /* Close and delete the file */ + if(H5Fclose(fid) < 0) + TEST_ERROR; + h5_delete_test_file(FILENAME[1], fapl_id); /* Close the fapl */ - if(H5Pclose(fapl) < 0) + if(H5Pclose(fapl_id) < 0) TEST_ERROR; PASSED(); @@ -345,119 +516,122 @@ test_direct(void) error: H5E_BEGIN_TRY { - H5Pclose(fapl); - H5Sclose(space1); - H5Dclose(dset1); - H5Sclose(space2); - H5Dclose(dset2); - H5Fclose(file); + H5Sclose(sid); + H5Dclose(did); + H5Pclose(fapl_id_out); + H5Pclose(fapl_id); + H5Fclose(fid); } H5E_END_TRY; - if(points) - HDfree(points); - if(check) - HDfree(check); + if(data_w) + HDfree(data_w); + if(data_r) + HDfree(data_r); return -1; -#endif /*H5_HAVE_DIRECT*/ -} +} /* end test_core() */ + - /*------------------------------------------------------------------------- - * Function: test_core + * Function: test_direct * - * Purpose: Tests the file handle interface for CORE driver + * Purpose: Tests the file handle interface for DIRECT I/O driver * * Return: Success: 0 * Failure: -1 * * Programmer: Raymond Lu - * Tuesday, Sept 24, 2002 + * Wednesday, 20 September 2006 * *------------------------------------------------------------------------- */ static herr_t -test_core(void) +test_direct(void) { - hid_t file=(-1), fapl, access_fapl = -1; +#ifdef H5_HAVE_DIRECT + hid_t file=-1, fapl=-1, access_fapl = -1; + hid_t dset1=-1, dset2=-1, space1=-1, space2=-1; char filename[1024]; - void *fhandle=NULL; + int *fhandle=NULL; hsize_t file_size; - hbool_t use_write_tracking; - size_t write_tracking_page_size; - int *points = NULL, *check = NULL, *p1, *p2; - hid_t dset1=-1, space1=-1; - hsize_t dims1[2]; + hsize_t dims1[2], dims2[1]; + size_t mbound; + size_t fbsize; + size_t cbsize; + int *points = NULL, *check = NULL, *p1 = NULL, *p2 = NULL; + int wdata2[DSET2_DIM] = {11,12,13,14}; + int rdata2[DSET2_DIM]; int i, j, n; +#endif /*H5_HAVE_DIRECT*/ - TESTING("CORE file driver"); + TESTING("DIRECT I/O file driver"); - /* Set property list and file name for CORE driver */ +#ifndef H5_HAVE_DIRECT + SKIPPED(); + return 0; +#else /*H5_HAVE_DIRECT*/ + + /* Set property list and file name for Direct driver. Set memory alignment boundary + * and file block size to 512 which is the minimum for Linux 2.6. */ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR; - if(H5Pset_fapl_core(fapl, (size_t)CORE_INCREMENT, TRUE) < 0) + if(H5Pset_fapl_direct(fapl, MBOUNDARY, FBSIZE, CBSIZE) < 0) TEST_ERROR; - if(H5Pset_core_write_tracking(fapl, TRUE, CORE_PAGE_SIZE) < 0) + h5_fixname(FILENAME[5], fapl, filename, sizeof filename); + + /* Verify the file access properties */ + if(H5Pget_fapl_direct(fapl, &mbound, &fbsize, &cbsize) < 0) + TEST_ERROR; + if(mbound != MBOUNDARY || fbsize != FBSIZE || cbsize != CBSIZE) TEST_ERROR; - h5_fixname(FILENAME[1], fapl, filename, sizeof filename); - if((file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + if(H5Pset_alignment(fapl, (hsize_t)THRESHOLD, (hsize_t)FBSIZE) < 0) TEST_ERROR; + H5E_BEGIN_TRY { + file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + } H5E_END_TRY; + if(file<0) { + H5Pclose (fapl); + SKIPPED(); + HDprintf(" Probably the file system doesn't support Direct I/O\n"); + return 0; + } + /* Retrieve the access property list... */ if ((access_fapl = H5Fget_access_plist(file)) < 0) TEST_ERROR; /* Check that the driver is correct */ - if(H5FD_CORE != H5Pget_driver(access_fapl)) - TEST_ERROR; - - /* Check that the backing store write tracking info was saved */ - if(H5Pget_core_write_tracking(fapl, &use_write_tracking, &write_tracking_page_size) < 0) - TEST_ERROR; - if(TRUE != use_write_tracking) - TEST_ERROR; - if(CORE_PAGE_SIZE != write_tracking_page_size) + if(H5FD_DIRECT != H5Pget_driver(access_fapl)) TEST_ERROR; /* ...and close the property list */ if (H5Pclose(access_fapl) < 0) TEST_ERROR; - if(H5Fget_vfd_handle(file, H5P_DEFAULT, &fhandle) < 0) + /* Check file handle API */ + if(H5Fget_vfd_handle(file, H5P_DEFAULT, (void **)&fhandle) < 0) + TEST_ERROR; + if(*fhandle<0) TEST_ERROR; - if(fhandle==NULL) - { - HDprintf("fhandle==NULL\n"); - TEST_ERROR; - } /* Check file size API */ if(H5Fget_filesize(file, &file_size) < 0) TEST_ERROR; - /* There is no garantee the size of metadata in file is constant. - * Just try to check if it's reasonable. Why is this 4KB? + /* There is no guarantee of the number of metadata allocations, but it's + * 4 currently and the size of the file should be between 3 & 4 file buffer + * sizes.. */ - if(file_size<2*KB || file_size>6*KB) - TEST_ERROR; - - if(H5Fclose(file) < 0) - TEST_ERROR; - - - /* Open the file with backing store off for read and write. - * Changes won't be saved in file. */ - if(H5Pset_fapl_core(fapl, (size_t)CORE_INCREMENT, FALSE) < 0) - TEST_ERROR; - - if((file=H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + if(file_size < (FBSIZE * 3) || file_size >= (FBSIZE * 4)) TEST_ERROR; - /* Allocate memory for data set. */ - if(NULL == (points = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int)))) + /* Allocate aligned memory for data set 1. For data set 1, everything is aligned including + * memory address, size of data, and file address. */ + if(0 != HDposix_memalign(&points, (size_t)FBSIZE, (size_t)(DSET1_DIM1 * DSET1_DIM2 * sizeof(int)))) TEST_ERROR; - if(NULL == (check = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int)))) + if(0 != HDposix_memalign(&check, (size_t)FBSIZE, (size_t)(DSET1_DIM1 * DSET1_DIM2 * sizeof(int)))) TEST_ERROR; /* Initialize the dset1 */ @@ -500,74 +674,50 @@ test_core(void) HDprintf(" Read different values than written in data set 1.\n"); HDprintf(" At index %d,%d\n", i, j); TEST_ERROR; - } /* end if */ - - if(H5Dclose(dset1) < 0) - TEST_ERROR; - - if(H5Fclose(file) < 0) - TEST_ERROR; - - /* Open the file with backing store on for read and write. - * Changes will be saved in file. */ - if(H5Pset_fapl_core(fapl, (size_t)CORE_INCREMENT, TRUE) < 0) - TEST_ERROR; + } /* end if */ - if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + /* Create the data space2. For data set 2, memory address and data size are not aligned. */ + dims2[0] = DSET2_DIM; + if((space2 = H5Screate_simple(1, dims2, NULL)) < 0) TEST_ERROR; - /* Create the dset1 */ - if((dset1 = H5Dcreate2(file, DSET1_NAME, H5T_NATIVE_INT, space1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + /* Create the dset2 */ + if((dset2 = H5Dcreate2(file, DSET2_NAME, H5T_NATIVE_INT, space2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; /* Write the data to the dset1 */ - if(H5Dwrite(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, points) < 0) - TEST_ERROR; - - if(H5Dclose(dset1) < 0) + if(H5Dwrite(dset2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata2) < 0) TEST_ERROR; - if((dset1 = H5Dopen2(file, DSET1_NAME, H5P_DEFAULT)) < 0) + if(H5Dclose(dset2) < 0) TEST_ERROR; - /* Reallocate memory for reading buffer. */ - HDassert(check); - HDfree(check); - if(NULL == (check = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int)))) + if((dset2 = H5Dopen2(file, DSET2_NAME, H5P_DEFAULT)) < 0) TEST_ERROR; /* Read the data back from dset1 */ - if(H5Dread(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, check) < 0) + if(H5Dread(dset2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata2) < 0) TEST_ERROR; /* Check that the values read are the same as the values written */ - p1 = points; - p2 = check; - for(i = 0; i < DSET1_DIM1; i++) - for(j = 0; j < DSET1_DIM2; j++) - if(*p1++ != *p2++) { - H5_FAILED(); - HDprintf(" Read different values than written in data set 1.\n"); - HDprintf(" At index %d,%d\n", i, j); - TEST_ERROR; - } /* end if */ - - /* Check file size API */ - if(H5Fget_filesize(file, &file_size) < 0) - TEST_ERROR; - - /* There is no garantee the size of metadata in file is constant. - * Just try to check if it's reasonable. */ - if(file_size<64*KB || file_size>256*KB) - TEST_ERROR; + for(i = 0; i < DSET2_DIM; i++) + if(wdata2[i] != rdata2[i]) { + H5_FAILED(); + HDprintf(" Read different values than written in data set 2.\n"); + HDprintf(" At index %d\n", i); + TEST_ERROR; + } /* end if */ if(H5Sclose(space1) < 0) TEST_ERROR; if(H5Dclose(dset1) < 0) TEST_ERROR; - HDassert(points); + if(H5Sclose(space2) < 0) + TEST_ERROR; + if(H5Dclose(dset2) < 0) + TEST_ERROR; + HDfree(points); - HDassert(check); HDfree(check); /* Close and delete the file */ @@ -585,6 +735,10 @@ test_core(void) error: H5E_BEGIN_TRY { H5Pclose(fapl); + H5Sclose(space1); + H5Dclose(dset1); + H5Sclose(space2); + H5Dclose(dset2); H5Fclose(file); } H5E_END_TRY; @@ -594,9 +748,10 @@ error: HDfree(check); return -1; +#endif /*H5_HAVE_DIRECT*/ } - + /*------------------------------------------------------------------------- * Function: test_family_opens * @@ -611,6 +766,14 @@ error: * *------------------------------------------------------------------------- */ +/* Disable warning for "format not a string literal" here -QAK */ +/* + * This pragma only needs to surround the snprintf() calls with + * 'first_name' in the code below, but early (4.4.7, at least) gcc only + * allows diagnostic pragmas to be toggled outside of functions. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" static herr_t test_family_opens(char *fname, hid_t fa_pl) { @@ -667,8 +830,9 @@ test_family_opens(char *fname, hid_t fa_pl) error: return -1; } /* end test_family_opens() */ +#pragma GCC diagnostic pop + - /*------------------------------------------------------------------------- * Function: test_family * @@ -687,6 +851,8 @@ test_family(void) { hid_t file=-1, fapl=-1, fapl2=-1, space=-1, dset=-1; hid_t access_fapl = -1; + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ char filename[1024]; char dname[]="dataset"; unsigned int i, j; @@ -704,6 +870,22 @@ test_family(void) TEST_ERROR; h5_fixname(FILENAME[2], fapl, filename, sizeof(filename)); + /* Check that the VFD feature flags are correct */ + if ((driver_id = H5Pget_driver(fapl)) < 0) + TEST_ERROR + if (H5FDdriver_query(driver_id, &driver_flags) < 0) + TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_METADATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_ACCUMULATE_METADATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR + /* Check for extra flags not accounted for above */ + if(driver_flags != (H5FD_FEAT_AGGREGATE_METADATA + | H5FD_FEAT_ACCUMULATE_METADATA + | H5FD_FEAT_DATA_SIEVE + | H5FD_FEAT_AGGREGATE_SMALLDATA)) + TEST_ERROR + if((file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR; @@ -832,7 +1014,7 @@ error: return -1; } - + /*------------------------------------------------------------------------- * Function: test_family_compat * @@ -851,6 +1033,14 @@ error: * *------------------------------------------------------------------------- */ +/* Disable warning for "format not a string literal" here -QAK */ +/* + * This pragma only needs to surround the snprintf() calls with + * 'newname_individual', etc. in the code below, but early (4.4.7, at least) gcc only + * allows diagnostic pragmas to be toggled outside of functions. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" static herr_t test_family_compat(void) { @@ -934,26 +1124,34 @@ error: return -1; } /* end test_family_compat() */ +#pragma GCC diagnostic pop + - /*------------------------------------------------------------------------- * Function: test_multi_opens * * Purpose: Private function for test_multi() to tests wrong ways of * reopening multi file. * - * Return: Success: 0 - * Failure: -1 + * Return: SUCCEED/FAIL * * Programmer: Raymond Lu * Thursday, May 19, 2005 * *------------------------------------------------------------------------- */ +/* Disable warning for "format not a string literal" here -QAK */ +/* + * This pragma only needs to surround the snprintf() calls with + * 'sf_name' in the code below, but early (4.4.7, at least) gcc only + * allows diagnostic pragmas to be toggled outside of functions. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wformat-nonliteral" static herr_t test_multi_opens(char *fname) { - hid_t file=-1; + hid_t fid = H5I_INVALID_HID; char super_name[1024]; /*name string "%%s-s.h5"*/ char sf_name[1024]; /*name string "multi_file-s.h5"*/ @@ -962,20 +1160,20 @@ test_multi_opens(char *fname) HDsnprintf(sf_name, sizeof(sf_name), super_name, fname); H5E_BEGIN_TRY { - file = H5Fopen(sf_name, H5F_ACC_RDWR, H5P_DEFAULT); + fid = H5Fopen(sf_name, H5F_ACC_RDWR, H5P_DEFAULT); } H5E_END_TRY; - return(file >= 0 ? -1 : 0); -} + return(fid >= 0 ? FAIL : SUCCEED); +} /* end test_multi_opens() */ +#pragma GCC diagnostic pop + - /*------------------------------------------------------------------------- * Function: test_multi * * Purpose: Tests the file handle interface for MUTLI driver * - * Return: Success: 0 - * Failure: -1 + * Return: SUCCEED/FAIL * * Programmer: Raymond Lu * Tuesday, Sept 24, 2002 @@ -988,6 +1186,8 @@ test_multi(void) hid_t file=-1, fapl=-1, fapl2=-1, dset=-1, space=-1; hid_t root=-1, attr=-1, aspace=-1, atype=-1; hid_t access_fapl = -1; + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ char filename[1024]; int *fhandle2=NULL, *fhandle=NULL; hsize_t file_size; @@ -1044,6 +1244,18 @@ test_multi(void) TEST_ERROR; h5_fixname(FILENAME[4], fapl, filename, sizeof filename); + /* Check that the VFD feature flags are correct */ + if ((driver_id = H5Pget_driver(fapl)) < 0) + TEST_ERROR + if (H5FDdriver_query(driver_id, &driver_flags) < 0) + TEST_ERROR + if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR + /* Check for extra flags not accounted for above */ + if(driver_flags != (H5FD_FEAT_DATA_SIEVE + | H5FD_FEAT_AGGREGATE_SMALLDATA)) + TEST_ERROR + if((file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR; @@ -1136,7 +1348,7 @@ test_multi(void) if((atype = H5Tcopy(H5T_C_S1)) < 0) TEST_ERROR; - if(H5Tset_size(atype, strlen(meta) + 1) < 0) + if(H5Tset_size(atype, HDstrlen(meta) + 1) < 0) TEST_ERROR; if(H5Tset_strpad(atype, H5T_STR_NULLTERM) < 0) @@ -1171,7 +1383,7 @@ test_multi(void) PASSED(); - return 0; + return SUCCEED; error: H5E_BEGIN_TRY { @@ -1180,18 +1392,19 @@ error: H5Pclose(fapl); H5Pclose(fapl2); H5Fclose(file); + H5Aclose(attr); } H5E_END_TRY; - return -1; -} + return FAIL; +} /* end test_multi() */ + - /*------------------------------------------------------------------------- * Function: test_multi_compat * * Purpose: Tests the backward compatibility for MULTI driver. * See if we can open files created with v1.6 library. * The source file was created by the test/file_handle.c - * of the v1.6 library. This test verifies the fix for + * of the v1.6 library. This test verifies the fix for * Issue 2598. In v1.6 library, there was EOA for the whole * MULTI file saved in the super block. We took it out in * v1.8 library because it's meaningless for the MULTI file. @@ -1252,8 +1465,9 @@ test_multi_compat(void) h5_fixname(FILENAME[9], fapl, newname, sizeof newname); - /* Make copy for the data file in the build directory, to protect the - * original file in the source directory */ + /* Make copy for the data file in the build directory, to protect the + * original file in the source directory + */ HDsprintf(filename_s, "%s-%c.h5", MULTI_COMPAT_BASENAME, 's'); HDsprintf(newname_s, "%s-%c.h5", FILENAME[9], 's'); h5_make_local_copy(filename_s, newname_s); @@ -1263,7 +1477,8 @@ test_multi_compat(void) h5_make_local_copy(filename_r, newname_r); /* Reopen the file for read only. Verify 1.8 library can open file - * created with 1.6 library. */ + * created with 1.6 library. + */ if((file=H5Fopen(newname, H5F_ACC_RDONLY, fapl)) < 0) TEST_ERROR; @@ -1289,7 +1504,7 @@ test_multi_compat(void) if(H5Fclose(file) < 0) TEST_ERROR; - /* Reopen the file for adding another dataset. The new EOA for metadata file + /* Reopen the file for adding another dataset. The new EOA for metadata file * should be written to the file */ if((file=H5Fopen(newname, H5F_ACC_RDWR, fapl)) < 0) TEST_ERROR; @@ -1316,7 +1531,7 @@ test_multi_compat(void) if(H5Fclose(file) < 0) TEST_ERROR; - /* Reopen the file for read only again. Verify the library can handle + /* Reopen the file for read only again. Verify the library can handle * the EOA correctly */ if((file=H5Fopen(newname, H5F_ACC_RDONLY, fapl)) < 0) TEST_ERROR; @@ -1356,7 +1571,7 @@ error: return -1; } - + /*------------------------------------------------------------------------- * Function: test_log * @@ -1376,6 +1591,8 @@ test_log(void) hid_t file = -1; hid_t fapl = -1; hid_t access_fapl = -1; + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ char filename[1024]; int *fhandle = NULL; hsize_t file_size = 0; @@ -1391,6 +1608,24 @@ test_log(void) TEST_ERROR; h5_fixname(FILENAME[6], fapl, filename, sizeof filename); + /* Check that the VFD feature flags are correct */ + if ((driver_id = H5Pget_driver(fapl)) < 0) + TEST_ERROR + if (H5FDdriver_query(driver_id, &driver_flags) < 0) + TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_METADATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_ACCUMULATE_METADATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_POSIX_COMPAT_HANDLE)) TEST_ERROR + /* Check for extra flags not accounted for above */ + if(driver_flags != (H5FD_FEAT_AGGREGATE_METADATA + | H5FD_FEAT_ACCUMULATE_METADATA + | H5FD_FEAT_DATA_SIEVE + | H5FD_FEAT_AGGREGATE_SMALLDATA + | H5FD_FEAT_POSIX_COMPAT_HANDLE)) + TEST_ERROR + /* Create the test file */ if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR; @@ -1443,7 +1678,7 @@ error: return -1; } - + /*------------------------------------------------------------------------- * Function: test_stdio * @@ -1463,6 +1698,8 @@ test_stdio(void) hid_t file = -1; hid_t fapl = -1; hid_t access_fapl = -1; + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ char filename[1024]; FILE *fhandle = NULL; hsize_t file_size = 0; @@ -1476,6 +1713,22 @@ test_stdio(void) TEST_ERROR; h5_fixname(FILENAME[7], fapl, filename, sizeof filename); + /* Check that the VFD feature flags are correct */ + if ((driver_id = H5Pget_driver(fapl)) < 0) + TEST_ERROR + if (H5FDdriver_query(driver_id, &driver_flags) < 0) + TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_METADATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_ACCUMULATE_METADATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR + /* Check for extra flags not accounted for above */ + if(driver_flags != (H5FD_FEAT_AGGREGATE_METADATA + | H5FD_FEAT_ACCUMULATE_METADATA + | H5FD_FEAT_DATA_SIEVE + | H5FD_FEAT_AGGREGATE_SMALLDATA)) + TEST_ERROR + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR; @@ -1528,7 +1781,7 @@ error: } - + /*------------------------------------------------------------------------- * Function: test_windows * @@ -1550,6 +1803,8 @@ test_windows(void) hid_t file = -1; hid_t fapl = -1; hid_t access_fapl = -1; + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ char filename[1024]; int *fhandle = NULL; hsize_t file_size = 0; @@ -1572,6 +1827,24 @@ test_windows(void) TEST_ERROR; h5_fixname(FILENAME[8], fapl, filename, sizeof filename); + /* Check that the VFD feature flags are correct */ + if ((driver_id = H5Pget_driver(fapl)) < 0) + TEST_ERROR + if (H5FDdriver_query(driver_id, &driver_flags) < 0) + TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_METADATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_ACCUMULATE_METADATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR + if(!(driver_flags & H5FD_FEAT_POSIX_COMPAT_HANDLE)) TEST_ERROR + /* Check for extra flags not accounted for above */ + if(driver_flags != (H5FD_FEAT_AGGREGATE_METADATA + | H5FD_FEAT_ACCUMULATE_METADATA + | H5FD_FEAT_DATA_SIEVE + | H5FD_FEAT_AGGREGATE_SMALLDATA + | H5FD_FEAT_POSIX_COMPAT_HANDLE)) + TEST_ERROR + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR; @@ -1623,10 +1896,107 @@ error: return -1; #endif /* H5_HAVE_WINDOWS */ -} +} /* end test_windows() */ - +/*------------------------------------------------------------------------- + * Function: test_ros3 + * + * Purpose: Tests the file handle interface for the ROS3 driver + * + * As the ROS3 driver is 1) read only, 2) requires access + * to an S3 server (minio for now), this test is quite + * different from the other tests. + * + * For now, test only fapl & flags. Extend as the + * work on the VFD continues. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: John Mainzer + * 7/12/17 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_ros3(void) +{ +#ifdef H5_HAVE_ROS3_VFD + hid_t fid = -1; /* file ID */ + hid_t fapl_id = -1; /* file access property list ID */ + hid_t fapl_id_out = -1; /* from H5Fget_access_plist */ + hid_t driver_id = -1; /* ID for this VFD */ + unsigned long driver_flags = 0; /* VFD feature flags */ + char filename[1024]; /* filename */ + void *os_file_handle = NULL; /* OS file handle */ + hsize_t file_size; /* file size */ + H5FD_ros3_fapl_t test_ros3_fa; + H5FD_ros3_fapl_t ros3_fa_0 = + { + /* version = */ H5FD_CURR_ROS3_FAPL_T_VERSION, + /* authenticate = */ FALSE, + /* aws_region = */ "", + /* secret_id = */ "", + /* secret_key = */ "plugh", + }; +#endif /*H5_HAVE_ROS3_VFD */ + + TESTING("Read-only S3 file driver"); + +#ifndef H5_HAVE_ROS3_VFD + SKIPPED(); + return 0; +#else /* H5_HAVE_ROS3_VFD */ + + /* Set property list and file name for ROS3 driver. */ + if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) + TEST_ERROR; + + if(H5Pset_fapl_ros3(fapl_id, &ros3_fa_0) < 0) + TEST_ERROR; + + /* verify that the ROS3 FAPL entry is set as expected */ + if(H5Pget_fapl_ros3(fapl_id, &test_ros3_fa) < 0) + TEST_ERROR; + + /* need a macro to compare instances of H5FD_ros3_fapl_t */ + if((test_ros3_fa.version != ros3_fa_0.version) || + (test_ros3_fa.authenticate != ros3_fa_0.authenticate) || + (strcmp(test_ros3_fa.aws_region, ros3_fa_0.aws_region) != 0) || + (strcmp(test_ros3_fa.secret_id, ros3_fa_0.secret_id) != 0) || + (strcmp(test_ros3_fa.secret_key, ros3_fa_0.secret_key) != 0)) + TEST_ERROR; + + h5_fixname(FILENAME[10], fapl_id, filename, sizeof(filename)); + + /* Check that the VFD feature flags are correct */ + if ((driver_id = H5Pget_driver(fapl_id)) < 0) + TEST_ERROR; + + if (H5FDdriver_query(driver_id, &driver_flags) < 0) + TEST_ERROR; + + if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) + TEST_ERROR + + /* Check for extra flags not accounted for above */ + if(driver_flags != (H5FD_FEAT_DATA_SIEVE)) + TEST_ERROR + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(fapl_id); + H5Pclose(fapl_id_out); + H5Fclose(fid); + } H5E_END_TRY; + return -1; +#endif /* H5_HAVE_ROS3_VFD */ +} /* end test_ros3() */ + /*------------------------------------------------------------------------- * Function: main * @@ -1651,22 +2021,24 @@ main(void) nerrors += test_sec2() < 0 ? 1 : 0; nerrors += test_core() < 0 ? 1 : 0; + nerrors += test_direct() < 0 ? 1 : 0; nerrors += test_family() < 0 ? 1 : 0; nerrors += test_family_compat() < 0 ? 1 : 0; nerrors += test_multi() < 0 ? 1 : 0; nerrors += test_multi_compat() < 0 ? 1 : 0; - nerrors += test_direct() < 0 ? 1 : 0; nerrors += test_log() < 0 ? 1 : 0; nerrors += test_stdio() < 0 ? 1 : 0; nerrors += test_windows() < 0 ? 1 : 0; + nerrors += test_ros3() < 0 ? 1 : 0; if(nerrors) { - HDprintf("***** %d Virtual File Driver TEST%s FAILED! *****\n", - nerrors, nerrors > 1 ? "S" : ""); - return 1; - } + HDprintf("***** %d Virtual File Driver TEST%s FAILED! *****\n", + nerrors, nerrors > 1 ? "S" : ""); + return 1; + } /* end if */ HDprintf("All Virtual File Driver tests passed.\n"); + return 0; -} +} /* end main() */ diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index f088d3b..f19186e 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -6,6 +6,15 @@ add_subdirectory (lib) #-- Add the h5diff and test executables add_subdirectory (h5diff) +if (BUILD_TESTING) +# -------------------------------------------------------------------- +# If S3 or HDFS enabled, then we need to test the tools library +# -------------------------------------------------------------------- + if (HDF5_ENABLE_ROS3_VFD OR HDF5_ENABLE_HDFS) + add_subdirectory (libtest) + endif () +endif () + #-- Add the h5ls executable add_subdirectory (h5ls) diff --git a/tools/h5copy/h5copy.c b/tools/h5copy/h5copy.c index 1800810..9490cd9 100644 --- a/tools/h5copy/h5copy.c +++ b/tools/h5copy/h5copy.c @@ -381,10 +381,10 @@ main (int argc, const char *argv[]) *-------------------------------------------------------------------------*/ if (verbose) { - printf("Copying file <%s> and object <%s> to file <%s> and object <%s>\n", + HDprintf("Copying file <%s> and object <%s> to file <%s> and object <%s>\n", fname_src, oname_src, fname_dst, oname_dst); if (flag) { - printf("Using %s flag\n", str_flag); + HDprintf("Using %s flag\n", str_flag); } } @@ -419,7 +419,7 @@ main (int argc, const char *argv[]) /* Display some output if requested */ if(verbose) - printf("%s: Creating parent groups\n", h5tools_getprogname()); + HDprintf("%s: Creating parent groups\n", h5tools_getprogname()); } /* end if */ else { /* error, if parent groups doesn't already exist in destination file */ @@ -490,7 +490,7 @@ main (int argc, const char *argv[]) leave(EXIT_SUCCESS); done: - printf("Error in copy...Exiting\n"); + HDprintf("Error in copy...Exiting\n"); /* free link info path */ if (linkinfo.trg_path) diff --git a/tools/h5copy/h5copygentest.c b/tools/h5copy/h5copygentest.c index fdc165e..049f2d5 100644 --- a/tools/h5copy/h5copygentest.c +++ b/tools/h5copy/h5copygentest.c @@ -325,7 +325,7 @@ static void gent_nested_vl(hid_t loc_id) * Function: gent_att_compound_vlstr * * Purpose: Generate a dataset and a group. - * Both has an attribute with a compound datatype consisting + * Both has an attribute with a compound datatype consisting * of a variable length string * *------------------------------------------------------------------------- @@ -336,15 +336,15 @@ static void gent_att_compound_vlstr(hid_t loc_id) int i; const char *v; } s1; - hsize_t dim[1] = {1}; /* Dimension size */ - hid_t sid = -1; /* Dataspace ID */ - hid_t tid = -1; /* Datatype ID */ - hid_t aid = -1; /* Attribute ID */ - hid_t did = -1; /* Dataset ID */ - hid_t gid = -1; /* Group ID */ - hid_t vl_str_tid = -1; /* Variable length datatype ID */ - hid_t cmpd_tid = -1; /* Compound datatype ID */ - hid_t null_sid = -1; /* Null dataspace ID */ + hsize_t dim[1] = {1}; /* Dimension size */ + hid_t sid = -1; /* Dataspace ID */ + hid_t tid = -1; /* Datatype ID */ + hid_t aid = -1; /* Attribute ID */ + hid_t did = -1; /* Dataset ID */ + hid_t gid = -1; /* Group ID */ + hid_t vl_str_tid = -1; /* Variable length datatype ID */ + hid_t cmpd_tid = -1; /* Compound datatype ID */ + hid_t null_sid = -1; /* Null dataspace ID */ s1 buf; /* Buffer */ buf.i = 9; diff --git a/tools/h5diff/h5diffgentest.c b/tools/h5diff/h5diffgentest.c index f59cb47..e6b5103 100644 --- a/tools/h5diff/h5diffgentest.c +++ b/tools/h5diff/h5diffgentest.c @@ -106,10 +106,16 @@ size_t H5TOOLS_MALLOCSIZE = (128 * 1024 * 1024); #define SPACE1_DIM1 0 #define SPACE1_DIM2 0 +/* Error macros */ +#define AT() HDprintf("ERROR at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); +#define PROGRAM_ERROR {AT(); goto error;} + /* A UD link traversal function. Shouldn't actually be called. */ -static hid_t UD_traverse(H5_ATTR_UNUSED const char * link_name, +static hid_t +UD_traverse(H5_ATTR_UNUSED const char * link_name, H5_ATTR_UNUSED hid_t cur_group, H5_ATTR_UNUSED const void * udata, - H5_ATTR_UNUSED size_t udata_size, H5_ATTR_UNUSED hid_t lapl_id) { + H5_ATTR_UNUSED size_t udata_size, H5_ATTR_UNUSED hid_t lapl_id) +{ return -1; } @@ -270,7 +276,7 @@ int main(void) /* string dataset and attribute. HDFFV-10028 */ test_objs_strings(DIFF_STRINGS1, DIFF_STRINGS2); - return 0; + return EXIT_SUCCESS; } /*------------------------------------------------------------------------- @@ -4802,9 +4808,9 @@ out: * types. * h5diff should show non-comparable output from these common objects. *-------------------------------------------------------------------------*/ -static void test_objs_nocomparables(const char *fname1, const char *fname2) +static void +test_objs_nocomparables(const char *fname1, const char *fname2) { - herr_t status = SUCCEED; hid_t fid1 = -1; hid_t fid2 = -1; hid_t topgid1 = -1; @@ -4821,110 +4827,90 @@ static void test_objs_nocomparables(const char *fname1, const char *fname2) * Open file(s) to add objects *------------------------------------------------------------------------*/ /* file1 */ - fid1 = H5Fopen(fname1, H5F_ACC_RDWR, H5P_DEFAULT); - if (fid1 < 0) { - HDfprintf(stderr, "Error: %s> H5Fopen failed.\n", fname1); - status = FAIL; - goto out; - } + if((fid1 = H5Fopen(fname1, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) + PROGRAM_ERROR /* file2 */ - fid2 = H5Fopen(fname2, H5F_ACC_RDWR, H5P_DEFAULT); - if (fid2 < 0) { - HDfprintf(stderr, "Error: %s> H5Fopen failed.\n", fname2); - status = FAIL; - goto out; - } + if((fid2 = H5Fopen(fname2, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) + PROGRAM_ERROR /*----------------------------------------------------------------------- * in file1 : add member objects *------------------------------------------------------------------------*/ /* parent group */ - topgid1 = H5Gcreate2(fid1, "diffobjtypes", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (topgid1 < 0) { - HDfprintf(stderr, "Error: %s> H5Gcreate2 failed.\n", fname1); - status = FAIL; - goto out; - } + if((topgid1 = H5Gcreate2(fid1, "diffobjtypes", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + PROGRAM_ERROR /* dataset */ - status = write_dset(topgid1, 1, dims, "obj1", H5T_NATIVE_INT, data1); - if (status == FAIL) { - HDfprintf(stderr, "Error: %s> write_dset failed\n", fname1); - goto out; - } + if(write_dset(topgid1, 1, dims, "obj1", H5T_NATIVE_INT, data1) < 0) + PROGRAM_ERROR /* group */ - gid1 = H5Gcreate2(topgid1, "obj2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (gid1 < 0) { - HDfprintf(stderr, "Error: %s> H5Gcreate2 failed.\n", fname1); - status = FAIL; - goto out; - } + if((gid1 = H5Gcreate2(topgid1, "obj2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + PROGRAM_ERROR /* committed type */ - tid1 = H5Tcopy(H5T_NATIVE_INT); - status = H5Tcommit2(topgid1, "obj3", tid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (status < 0) { - HDfprintf(stderr, "Error: %s> H5Tcommit2 failed.\n", fname1); - goto out; - } + if((tid1 = H5Tcopy(H5T_NATIVE_INT)) < 0) + PROGRAM_ERROR + if(H5Tcommit2(topgid1, "obj3", tid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) + PROGRAM_ERROR /*----------------------------------------------------------------------- * in file2 : add member objects *------------------------------------------------------------------------*/ /* parent group */ - topgid2 = H5Gcreate2(fid2, "diffobjtypes", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (topgid2 < 0) { - HDfprintf(stderr, "Error: %s> H5Gcreate2 failed.\n", fname2); - status = FAIL; - goto out; - } + if((topgid2 = H5Gcreate2(fid2, "diffobjtypes", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + PROGRAM_ERROR /* group */ - gid2 = H5Gcreate2(topgid2, "obj1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (gid2 < 0) { - HDfprintf(stderr, "Error: %s> H5Gcreate2 failed.\n", fname2); - status = FAIL; - goto out; - } + if((gid2 = H5Gcreate2(topgid2, "obj1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + PROGRAM_ERROR /* committed type */ - tid2 = H5Tcopy(H5T_NATIVE_INT); - status = H5Tcommit2(topgid2, "obj2", tid2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - if (status < 0) { - HDfprintf(stderr, "Error: %s> H5Tcommit2 failed.\n", fname2); - goto out; - } + if((tid2 = H5Tcopy(H5T_NATIVE_INT)) < 0) + PROGRAM_ERROR + if(H5Tcommit2(topgid2, "obj2", tid2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) + PROGRAM_ERROR /* dataset */ - status = write_dset(topgid2, 1, dims, "obj3", H5T_NATIVE_INT, data2); - if (status == FAIL) { - HDfprintf(stderr, "Error: %s> write_dset failed\n", fname2); - goto out; - } + if(write_dset(topgid2, 1, dims, "obj3", H5T_NATIVE_INT, data2) < 0) + PROGRAM_ERROR -out: /*----------------------------------------------------------------------- * Close IDs *-----------------------------------------------------------------------*/ - if (fid1) + if(H5Fclose(fid1) < 0) + PROGRAM_ERROR + if(H5Fclose(fid2) < 0) + PROGRAM_ERROR + if(H5Gclose(topgid1) < 0) + PROGRAM_ERROR + if(H5Gclose(topgid2) < 0) + PROGRAM_ERROR + if(H5Gclose(gid1) < 0) + PROGRAM_ERROR + if(H5Gclose(gid2) < 0) + PROGRAM_ERROR + if(H5Tclose(tid1) < 0) + PROGRAM_ERROR + if(H5Tclose(tid2) < 0) + PROGRAM_ERROR + + return; + +error: + H5E_BEGIN_TRY { H5Fclose(fid1); - if (fid2) H5Fclose(fid2); - if (topgid1) H5Gclose(topgid1); - if (topgid2) H5Gclose(topgid2); - if (gid1) H5Gclose(gid1); - if (gid2) H5Gclose(gid2); - if (tid1) H5Tclose(tid1); - if (tid2) H5Tclose(tid2); + } H5E_END_TRY; + return; } static hid_t mkstr(int size, H5T_str_t pad) @@ -7986,29 +7972,33 @@ int write_dset(hid_t loc_id, int rank, hsize_t *dims, const char *name, hid_t ti hid_t sid = -1; /* create a space */ - if ((sid = H5Screate_simple(rank, dims, NULL)) < 0) - goto out; + if((sid = H5Screate_simple(rank, dims, NULL)) < 0) + PROGRAM_ERROR /* create the dataset */ - if ((did = H5Dcreate2(loc_id, name, tid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) - goto out; + if((did = H5Dcreate2(loc_id, name, tid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + PROGRAM_ERROR /* write */ - if (buf) { - if (H5Dwrite(did, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) - goto out; - } + if(buf) + if(H5Dwrite(did, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) + PROGRAM_ERROR /* close */ - H5Dclose(did); - H5Sclose(sid); + if(H5Dclose(did) < 0) + PROGRAM_ERROR + if(H5Sclose(sid) < 0) + PROGRAM_ERROR return SUCCEED; -out: +error: + + H5E_BEGIN_TRY { + H5Dclose(did); + H5Sclose(sid); + } H5E_END_TRY; - H5Dclose(did); - H5Sclose(sid); return FAIL; -} +} /* end write_dset() */ diff --git a/tools/h5diff/ph5diff_main.c b/tools/h5diff/ph5diff_main.c index 83240cb..380ab3b 100644 --- a/tools/h5diff/ph5diff_main.c +++ b/tools/h5diff/ph5diff_main.c @@ -68,7 +68,7 @@ int main(int argc, const char *argv[]) if(g_nTasks == 1) { - printf("Only 1 task available...doing serial diff\n"); + HDprintf("Only 1 task available...doing serial diff\n"); g_Parallel = 0; @@ -148,12 +148,12 @@ ph5diff_worker(int nID) /* Open the files */ if ((file1_id = H5Fopen (filenames[0], H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) { - printf ("h5diff Task [%d]: <%s>: unable to open file\n", nID, filenames[0]); + HDprintf("h5diff Task [%d]: <%s>: unable to open file\n", nID, filenames[0]); MPI_Abort(MPI_COMM_WORLD, 0); } if ((file2_id = H5Fopen (filenames[1], H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) { - printf ("h5diff Task [%d]: <%s>: unable to open file\n", nID, filenames[1]); + HDprintf("h5diff Task [%d]: <%s>: unable to open file\n", nID, filenames[1]); MPI_Abort(MPI_COMM_WORLD, 0); } /* enable error reporting */ @@ -170,7 +170,7 @@ ph5diff_worker(int nID) /* Make certain we've received the filenames and opened the files already */ if(file1_id < 0 || file2_id < 0) { - printf("ph5diff_worker: ERROR: work received before/without filenames\n"); + HDprintf("ph5diff_worker: ERROR: work received before/without filenames\n"); break; } @@ -199,7 +199,7 @@ ph5diff_worker(int nID) char out_data[PRINT_DATA_MAX_SIZE]; int tmp; - memset(out_data, 0, PRINT_DATA_MAX_SIZE); + HDmemset(out_data, 0, PRINT_DATA_MAX_SIZE); i=0; rewind(overflow_file); @@ -210,7 +210,7 @@ ph5diff_worker(int nID) { MPI_Send(out_data, PRINT_DATA_MAX_SIZE, MPI_BYTE, 0, MPI_TAG_PRINT_DATA, MPI_COMM_WORLD); i=0; - memset(out_data, 0, PRINT_DATA_MAX_SIZE); + HDmemset(out_data, 0, PRINT_DATA_MAX_SIZE); } } @@ -221,8 +221,8 @@ ph5diff_worker(int nID) overflow_file = NULL; } - fflush(stdout); - memset(outBuff, 0, OUTBUFF_SIZE); + HDfflush(stdout); + HDmemset(outBuff, 0, OUTBUFF_SIZE); outBuffOffset = 0; MPI_Send(&diffs, sizeof(diffs), MPI_BYTE, 0, MPI_TAG_TOK_RETURN, MPI_COMM_WORLD); @@ -238,7 +238,7 @@ ph5diff_worker(int nID) } else { - printf("ph5diff_worker: ERROR: invalid tag (%d) received\n", Status.MPI_TAG); + HDprintf("ph5diff_worker: ERROR: invalid tag (%d) received\n", Status.MPI_TAG); break; } @@ -266,14 +266,14 @@ void print_manager_output(void) /* If there was something we buffered, let's print it now */ if( (outBuffOffset>0) && g_Parallel) { - printf("%s", outBuff); + HDprintf("%s", outBuff); if(overflow_file) { int tmp; rewind(overflow_file); - while((tmp = getc(overflow_file)) >= 0) - putchar(tmp); + while((tmp = HDgetc(overflow_file)) >= 0) + HDputchar(tmp); fclose(overflow_file); overflow_file = NULL; } @@ -321,6 +321,6 @@ void h5diff_exit(int status) /* Always exit(0), since MPI implementations do weird stuff when they * receive a non-zero exit value. - QAK */ - exit(0); + HDexit(0); } diff --git a/tools/h5dump/h5dump.c b/tools/h5dump/h5dump.c index 9c1dfa3..7c5b3fa 100644 --- a/tools/h5dump/h5dump.c +++ b/tools/h5dump/h5dump.c @@ -24,6 +24,27 @@ static int doxml = 0; static int useschema = 1; static const char *xml_dtd_uri = NULL; +#ifdef H5_HAVE_ROS3_VFD +static H5FD_ros3_fapl_t ros3_fa = { + 1, /* version */ + false, /* authenticate */ + "", /* aws region */ + "", /* access key id */ + "", /* secret access key */ +}; +#endif /* H5_HAVE_ROS3_VFD */ + +#ifdef H5_HAVE_LIBHDFS +static H5FD_hdfs_fapl_t hdfs_fa = { + 1, /* fapl version */ + "localhost", /* namenode name */ + 0, /* namenode port */ + "", /* kerberos ticket cache */ + "", /* user name */ + 2048, /* stream buffer size */ +}; +#endif /* H5_HAVE_LIBHDFS */ + /* module-scoped variables for XML option */ #define DEFAULT_XSD "http://www.hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd" #define DEFAULT_DTD "http://www.hdfgroup.org/HDF5/XML/DTD/HDF5-File.dtd" @@ -186,6 +207,8 @@ static struct long_options l_opts[] = { { "no-compact-subset", no_arg, 'C' }, { "ddl", optional_arg, 'O' }, { "any_path", require_arg, 'N' }, + { "s3-cred", require_arg, '$' }, + { "hdfs-attrs", require_arg, '#' }, { NULL, 0, '\0' } }; @@ -239,6 +262,16 @@ usage(const char *prog) PRINTVALSTREAM(rawoutstream, " -b B, --binary=B Binary file output, of form B\n"); PRINTVALSTREAM(rawoutstream, " -O F, --ddl=F Output ddl text into file F\n"); PRINTVALSTREAM(rawoutstream, " Use blank(empty) filename F to suppress ddl display\n"); + PRINTVALSTREAM(rawoutstream, " --s3-cred= Supply S3 authentication information to \"ros3\" vfd.\n"); + PRINTVALSTREAM(rawoutstream, " :: \"(,,)\"\n"); + PRINTVALSTREAM(rawoutstream, " If absent or -> \"(,,)\", no authentication.\n"); + PRINTVALSTREAM(rawoutstream, " Has no effect is filedriver is not `ros3'.\n"); + PRINTVALSTREAM(rawoutstream, " --hdfs-attrs= Supply configuration information for HDFS file access.\n"); + PRINTVALSTREAM(rawoutstream, " For use with \"--filedriver=hdfs\"\n"); + PRINTVALSTREAM(rawoutstream, " :: (,,\n"); + PRINTVALSTREAM(rawoutstream, " ,,\n"); + PRINTVALSTREAM(rawoutstream, " )\n"); + PRINTVALSTREAM(rawoutstream, " Any absent attribute will use a default value.\n"); PRINTVALSTREAM(rawoutstream, "--------------- Object Options ---------------\n"); PRINTVALSTREAM(rawoutstream, " -a P, --attribute=P Print the specified attribute\n"); PRINTVALSTREAM(rawoutstream, " If an attribute name contains a slash (/), escape the\n"); @@ -1268,6 +1301,126 @@ end_collect: hand = NULL; h5tools_setstatus(EXIT_SUCCESS); goto done; + + case '$': +#ifndef H5_HAVE_ROS3_VFD + error_msg("Read-Only S3 VFD not enabled.\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; +#else + /* s3 credential */ + { + char **s3_cred = NULL; + char *s3_cred_string = NULL; + const char *ccred[3]; + unsigned nelems = 0; + if ( FAIL == + parse_tuple(opt_arg, ',', + &s3_cred_string, &nelems, &s3_cred)) + { + error_msg("unable to parse malformed s3 credentials\n"); + usage(h5tools_getprogname()); + free_handler(hand, argc); + hand= NULL; + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + if (nelems != 3) { + error_msg("s3 credentials expects 3 elements\n"); + usage(h5tools_getprogname()); + free_handler(hand, argc); + hand= NULL; + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + ccred[0] = (const char *)s3_cred[0]; + ccred[1] = (const char *)s3_cred[1]; + ccred[2] = (const char *)s3_cred[2]; + if (0 == h5tools_populate_ros3_fapl(&ros3_fa, ccred)) { + error_msg("Invalid S3 credentials\n"); + usage(h5tools_getprogname()); + free_handler(hand, argc); + hand= NULL; + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + HDfree(s3_cred); + HDfree(s3_cred_string); + } /* s3 credential block */ + break; +#endif /* H5_HAVE_ROS3_VFD */ + + case '#': +#ifndef H5_HAVE_LIBHDFS + error_msg("HDFS VFD is not enabled.\n"); + goto error; +#else + { + /* read hdfs properties tuple and store values in `hdfs_fa` + */ + unsigned nelems = 0; + char *props_src = NULL; + char **props = NULL; + unsigned long k = 0; + if (FAIL == parse_tuple( + (const char *)opt_arg, + ',', + &props_src, + &nelems, + &props)) + { + error_msg("unable to parse hdfs properties tuple\n"); + goto error; + } + /* sanity-check tuple count + */ + if (nelems != 5) { + h5tools_setstatus(EXIT_FAILURE); + goto error; + } + /* Populate fapl configuration structure with given + * properties. + * WARNING: No error-checking is done on length of input + * strings... Silent overflow is possible, albeit + * unlikely. + */ + if (strncmp(props[0], "", 1)) { + HDstrncpy(hdfs_fa.namenode_name, + (const char *)props[0], + HDstrlen(props[0])); + } + if (strncmp(props[1], "", 1)) { + k = strtoul((const char *)props[1], NULL, 0); + if (errno == ERANGE) { + h5tools_setstatus(EXIT_FAILURE); + goto error; + } + hdfs_fa.namenode_port = (int32_t)k; + } + if (strncmp(props[2], "", 1)) { + HDstrncpy(hdfs_fa.kerberos_ticket_cache, + (const char *)props[2], + HDstrlen(props[2])); + } + if (strncmp(props[3], "", 1)) { + HDstrncpy(hdfs_fa.user_name, + (const char *)props[3], + HDstrlen(props[3])); + } + if (strncmp(props[4], "", 1)) { + k = strtoul((const char *)props[4], NULL, 0); + if (errno == ERANGE) { + h5tools_setstatus(EXIT_FAILURE); + goto error; + } + hdfs_fa.stream_buffer_size = (int32_t)k; + } + HDfree(props); + HDfree(props_src); + } +#endif /* H5_HAVE_LIBHDFS */ + break; + case '?': default: usage(h5tools_getprogname()); @@ -1340,6 +1493,7 @@ main(int argc, const char *argv[]) { hid_t fid = -1; hid_t gid = -1; + hid_t fapl_id = H5P_DEFAULT; H5E_auto2_t func; H5E_auto2_t tools_func; H5O_info_t oi; @@ -1426,10 +1580,55 @@ main(int argc, const char *argv[]) /* Initialize indexing options */ h5trav_set_index(sort_by, sort_order); + if (driver != NULL) { + void *conf_fa = NULL; + + if (!strcmp(driver, "ros3")) { +#ifndef H5_HAVE_ROS3_VFD + error_msg("Read-Only S3 VFD not enabled.\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; +#else + conf_fa = (void *)&ros3_fa; +#endif /* H5_HAVE_ROS3_VFD */ + } else if (!HDstrcmp(driver, "hdfs")) { +#ifndef H5_HAVE_LIBHDFS + error_msg("HDFS VFD is not enabled.\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; +#else + conf_fa = (void *)&hdfs_fa; +#endif /* H5_HAVE_LIBHDFS */ + } + + if (conf_fa != NULL) { + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (fapl_id < 0) { + error_msg("unable to create fapl entry\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + if (0 == h5tools_set_configured_fapl( + fapl_id, + driver, /* guaranteed "ros3" or "hdfs" */ + conf_fa)) /* appropriate to driver */ + { + error_msg("unable to set fapl\n"); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + } + } /* driver defined */ + while(opt_ind < argc) { fname = HDstrdup(argv[opt_ind++]); - fid = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, driver, NULL, 0); + if (fapl_id != H5P_DEFAULT) { + fid = H5Fopen(fname, H5F_ACC_RDONLY, fapl_id); + } + else { + fid = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, driver, NULL, 0); + } if (fid < 0) { error_msg("unable to open file \"%s\"\n", fname); @@ -1610,6 +1809,11 @@ done: /* Free tables for objects */ table_list_free(); + if (fapl_id != H5P_DEFAULT && 0 < H5Pclose(fapl_id)) { + error_msg("Can't close fapl entry\n"); + h5tools_setstatus(EXIT_FAILURE); + } + if(fid >=0) if (H5Fclose(fid) < 0) h5tools_setstatus(EXIT_FAILURE); @@ -1631,127 +1835,7 @@ done: H5Eset_auto2(H5E_DEFAULT, func, edata); leave(h5tools_getstatus()); -} - -/*------------------------------------------------------------------------- - * Function: h5_fileaccess - * - * Purpose: Returns a file access template which is the default template - * but with a file driver set according to the constant or - * environment variable HDF5_DRIVER - * - * Return: Success: A file access property list - * - * Failure: -1 - * - * Programmer: Robb Matzke - * Thursday, November 19, 1998 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -hid_t -h5_fileaccess(void) -{ - static const char *multi_letters = "msbrglo"; - const char *val = NULL; - const char *name; - char s[1024]; - hid_t fapl = -1; - - /* First use the environment variable, then the constant */ - val = HDgetenv("HDF5_DRIVER"); -#ifdef HDF5_DRIVER - if (!val) val = HDF5_DRIVER; -#endif - - if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) return -1; - if (!val || !*val) return fapl; /*use default*/ - - HDstrncpy(s, val, sizeof s); - s[sizeof(s)-1] = '\0'; - if (NULL==(name=HDstrtok(s, " \t\n\r"))) return fapl; - - if (!HDstrcmp(name, "sec2")) { - /* Unix read() and write() system calls */ - if (H5Pset_fapl_sec2(fapl)<0) return -1; - } - else if (!HDstrcmp(name, "stdio")) { - /* Standard C fread() and fwrite() system calls */ - if (H5Pset_fapl_stdio(fapl)<0) return -1; - } - else if (!HDstrcmp(name, "core")) { - /* In-core temporary file with 1MB increment */ - if (H5Pset_fapl_core(fapl, 1024*1024, FALSE)<0) return -1; - } - else if (!HDstrcmp(name, "split")) { - /* Split meta data and raw data each using default driver */ - if (H5Pset_fapl_split(fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0) - return -1; - } - else if (!HDstrcmp(name, "multi")) { - /* Multi-file driver, general case of the split driver */ - H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; - hid_t memb_fapl[H5FD_MEM_NTYPES]; - const char *memb_name[H5FD_MEM_NTYPES]; - char sv[H5FD_MEM_NTYPES][1024]; - haddr_t memb_addr[H5FD_MEM_NTYPES]; - H5FD_mem_t mt; - - HDmemset(memb_map, 0, sizeof memb_map); - HDmemset(memb_fapl, 0, sizeof memb_fapl); - HDmemset(memb_name, 0, sizeof memb_name); - HDmemset(memb_addr, 0, sizeof memb_addr); - - if(HDstrlen(multi_letters)==H5FD_MEM_NTYPES) { - for (mt=H5FD_MEM_DEFAULT; mt= 0) { + if((obj = H5Dopen2(group, name, dapl_id)) >= 0) { if(oinfo.rc > 1 || hit_elink) { obj_t *found_obj; /* Found object */ @@ -304,6 +305,8 @@ dump_all_cb(hid_t group, const char *name, const H5L_info_t *linfo, void H5_ATTR h5tools_setstatus(EXIT_FAILURE); ret = FAIL; + if (dapl_id != H5P_DEFAULT) + H5Pclose(dapl_id); H5Dclose(obj); goto done; } @@ -341,6 +344,8 @@ dump_all_cb(hid_t group, const char *name, const H5L_info_t *linfo, void H5_ATTR h5tools_str_append(&buffer, "%s", h5tools_dump_header_format->datasetend); h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, (size_t)outputformat->line_ncols, (hsize_t)0, (hsize_t)0); + if (dapl_id != H5P_DEFAULT) + H5Pclose(dapl_id); H5Dclose(obj); goto done; } @@ -350,9 +355,13 @@ dump_all_cb(hid_t group, const char *name, const H5L_info_t *linfo, void H5_ATTR } /* end if */ dump_function_table->dump_dataset_function(obj, name, NULL); + if (dapl_id != H5P_DEFAULT) + H5Pclose(dapl_id); H5Dclose(obj); } else { + if (dapl_id != H5P_DEFAULT) + H5Pclose(dapl_id); error_msg("unable to dump dataset \"%s\"\n", name); h5tools_setstatus(EXIT_FAILURE); ret = FAIL; @@ -1133,7 +1142,7 @@ dump_data(hid_t obj_id, int obj_data, struct subset_t *sset, int display_index) h5tools_dump_data(rawoutstream, outputformat, &ctx, obj_id, print_dataset, sset, display_index, display_char); } - + /*------------------------------------------------------------------------- * Function: dump_fcpl * @@ -1648,10 +1657,11 @@ handle_datasets(hid_t fid, const char *dset, void *data, int pe, const char *dis { H5O_info_t oinfo; hid_t dsetid; + hid_t dapl_id = H5P_DEFAULT; /* dataset access property list ID */ struct subset_t *sset = (struct subset_t *)data; const char *real_name = display_name ? display_name : dset; - if((dsetid = H5Dopen2(fid, dset, H5P_DEFAULT)) < 0) { + if((dsetid = H5Dopen2(fid, dset, dapl_id)) < 0) { if (pe) handle_links(fid, dset, data, pe, display_name); return; @@ -1775,7 +1785,8 @@ handle_datasets(hid_t fid, const char *dset, void *data, int pe, const char *dis dump_dataset(dsetid, real_name, sset); dump_indent -= COL; } - + if (dapl_id != H5P_DEFAULT) + H5Pclose(dapl_id); if(H5Dclose(dsetid) < 0) h5tools_setstatus(EXIT_FAILURE); } @@ -1883,9 +1894,6 @@ handle_links(hid_t fid, const char *links, void H5_ATTR_UNUSED * data, int H5_AT break; case H5L_TYPE_EXTERNAL: - begin_obj(h5tools_dump_header_format->udlinkbegin, links, h5tools_dump_header_format->udlinkblockbegin); - PRINTVALSTREAM(rawoutstream, "\n"); - indentation(COL); begin_obj(h5tools_dump_header_format->extlinkbegin, links, h5tools_dump_header_format->extlinkblockbegin); PRINTVALSTREAM(rawoutstream, "\n"); if(H5Lget_val(fid, links, buf, linfo.u.val_size, H5P_DEFAULT) >= 0) { @@ -1894,8 +1902,6 @@ handle_links(hid_t fid, const char *links, void H5_ATTR_UNUSED * data, int H5_AT if(H5Lunpack_elink_val(buf, linfo.u.val_size, NULL, &elink_file, &elink_path)>=0) { indentation(COL); - PRINTSTREAM(rawoutstream, "LINKCLASS %d\n", linfo.type); - indentation(COL); PRINTSTREAM(rawoutstream, "TARGETFILE \"%s\"\n", elink_file); indentation(COL); PRINTSTREAM(rawoutstream, "TARGETPATH \"%s\"\n", elink_path); @@ -1919,9 +1925,6 @@ handle_links(hid_t fid, const char *links, void H5_ATTR_UNUSED * data, int H5_AT begin_obj(h5tools_dump_header_format->udlinkbegin, links, h5tools_dump_header_format->udlinkblockbegin); PRINTVALSTREAM(rawoutstream, "\n"); indentation(COL); - begin_obj(h5tools_dump_header_format->udlinkbegin, links, h5tools_dump_header_format->udlinkblockbegin); - PRINTVALSTREAM(rawoutstream, "\n"); - indentation(COL); PRINTSTREAM(rawoutstream, "LINKCLASS %d\n", linfo.type); end_obj(h5tools_dump_header_format->udlinkend, h5tools_dump_header_format->udlinkblockend); break; @@ -2007,7 +2010,7 @@ handle_datatypes(hid_t fid, const char *type, void H5_ATTR_UNUSED * data, int pe } } - + /*------------------------------------------------------------------------- * Function: dump_extlink * diff --git a/tools/h5dump/h5dump_defines.h b/tools/h5dump/h5dump_defines.h index b08fbb1..7a9d4c0 100644 --- a/tools/h5dump/h5dump_defines.h +++ b/tools/h5dump/h5dump_defines.h @@ -21,25 +21,25 @@ #define COL 3 /* Macros for displaying objects */ -#define begin_obj(obj,name,begin) \ - do { \ - if ((name)) { \ +#define begin_obj(obj,name,begin) \ + do { \ + if ((name)) { \ PRINTSTREAM(rawoutstream, "%s \"%s\" %s", (obj), (name), (begin)); \ } \ else { \ - PRINTSTREAM(rawoutstream, "%s %s", (obj), (begin)); \ - } \ + PRINTSTREAM(rawoutstream, "%s %s", (obj), (begin)); \ + } \ } while(0); -#define end_obj(obj,end) \ - do { \ - if(HDstrlen(end)) { \ - PRINTSTREAM(rawoutstream, "%s", end); \ +#define end_obj(obj,end) \ + do { \ + if(HDstrlen(end)) { \ + PRINTSTREAM(rawoutstream, "%s", end); \ if(HDstrlen(obj)) \ - PRINTVALSTREAM(rawoutstream, " "); \ - } \ + PRINTVALSTREAM(rawoutstream, " "); \ + } \ if(HDstrlen(obj)) \ - PRINTSTREAM(rawoutstream, "%s", obj); \ + PRINTSTREAM(rawoutstream, "%s", obj); \ } while(0); diff --git a/tools/h5dump/h5dump_xml.c b/tools/h5dump/h5dump_xml.c index 95b81b5..c19685c 100644 --- a/tools/h5dump/h5dump_xml.c +++ b/tools/h5dump/h5dump_xml.c @@ -1408,7 +1408,6 @@ xml_print_datatype(hid_t type, unsigned in_group) /* Print lead-in */ ctx.need_prefix = TRUE; - h5tools_simple_prefix(rawoutstream, outputformat, &ctx, (hsize_t)0, 0); /* Render the element */ h5tools_str_reset(&buffer); @@ -2881,7 +2880,7 @@ xml_print_refs(hid_t did, int source) goto error; } - refbuf = (hobj_ref_t *) buf; + refbuf = (hobj_ref_t *)((void *)buf); /* setup */ HDmemset(&buffer, 0, sizeof(h5tools_str_t)); @@ -3057,7 +3056,7 @@ xml_print_strs(hid_t did, int source) for (i = 0; i < (hsize_t)ssiz; i++) { if (is_vlstr) { - onestring = *(char **) bp; + onestring = *(char **)((void *)bp); if (onestring) str_size = HDstrlen(onestring); } diff --git a/tools/h5dump/h5dumpgentest.c b/tools/h5dump/h5dumpgentest.c index e6ddbd9..8ba752d 100644 --- a/tools/h5dump/h5dumpgentest.c +++ b/tools/h5dump/h5dumpgentest.c @@ -112,6 +112,7 @@ #define FILE83 "tvlenstr_array.h5" #define FILE84 "tudfilter.h5" #define FILE85 "tgrpnullspace.h5" +#define FILE87 "tintsnodata.h5" /*------------------------------------------------------------------------- * prototypes @@ -2683,14 +2684,14 @@ gent_vldatatypes2(void) for(i = 0; i < SPACE1_DIM1; i++) { wdata[i].p = (hvl_t *)HDmalloc((i + 1) * sizeof(hvl_t)); if(wdata[i].p == NULL) { - printf("Cannot allocate memory for VL data! i=%u\n", i); + HDprintf("Cannot allocate memory for VL data! i=%u\n", i); return; } /* end if */ wdata[i].len = i + 1; for(t1 = (hvl_t *)wdata[i].p, j = 0; j < (i + 1); j++, t1++) { t1->p = (unsigned *)HDmalloc((j + 1) * sizeof(unsigned)); if(t1->p == NULL) { - printf("Cannot allocate memory for VL data! i=%u, j=%u\n",i,j); + HDprintf("Cannot allocate memory for VL data! i=%u, j=%u\n",i,j); return; } /* end if */ t1->len=j+1; @@ -3739,7 +3740,7 @@ void gent_multi(void) memb_map[mt] = mt; HDsprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]); memb_name[mt] = sv[mt]; - /*printf("memb_name[%d]=%s, memb_map[%d]=%d; ", mt, memb_name[mt], mt, memb_map[mt]);*/ + /*HDprintf("memb_name[%d]=%s, memb_map[%d]=%d; ", mt, memb_name[mt], mt, memb_map[mt]);*/ memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10); } memb_map[H5FD_MEM_DEFAULT] = H5FD_MEM_SUPER; @@ -6248,8 +6249,8 @@ static int gent_ldouble(void) return 0; - error: - printf("error !\n"); +error: + HDprintf("error !\n"); return -1; } @@ -6412,7 +6413,7 @@ gent_bigdims(void) return; out: - printf("Error.....\n"); + HDprintf("Error.....\n"); H5E_BEGIN_TRY { H5Pclose(dcpl); H5Sclose(f_sid); @@ -6604,7 +6605,7 @@ gent_group_creation_order(void) return; out: - printf("Error.....\n"); + HDprintf("Error.....\n"); H5E_BEGIN_TRY { H5Gclose(gid); H5Pclose(gcpl_id); @@ -6873,7 +6874,7 @@ gent_attr_creation_order(void) return; out: - printf("Error.....\n"); + HDprintf("Error.....\n"); H5E_BEGIN_TRY { H5Gclose(gid); H5Dclose(did); @@ -7389,6 +7390,80 @@ gent_attr_intsize(void) H5Fclose(fid); } +static void +gent_nodata(void) +{ + hid_t fid, dataset, space; + hsize_t dims[2]; + + fid = H5Fcreate(FILE87, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + + /* Dataset of 8 bits unsigned int */ + dims[0] = F66_XDIM; dims[1] = F66_YDIM8; + space = H5Screate_simple(2, dims, NULL); + dataset = H5Dcreate2(fid, F66_DATASETU08, H5T_STD_U8LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Sclose(space); + H5Dclose(dataset); + + /* Dataset of 16 bits unsigned int */ + dims[0] = F66_XDIM; dims[1] = F66_YDIM16; + space = H5Screate_simple(2, dims, NULL); + dataset = H5Dcreate2(fid, F66_DATASETU16, H5T_STD_U16LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Sclose(space); + H5Dclose(dataset); + + /* Dataset of 32 bits unsigned int */ + dims[0] = F66_XDIM; dims[1] = F66_YDIM32; + space = H5Screate_simple(2, dims, NULL); + dataset = H5Dcreate2(fid, F66_DATASETU32, H5T_STD_U32LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Sclose(space); + H5Dclose(dataset); + + /* Dataset of 64 bits unsigned int */ + dims[0] = F66_XDIM; dims[1] = F66_YDIM64; + space = H5Screate_simple(2, dims, NULL); + dataset = H5Dcreate2(fid, F66_DATASETU64, H5T_STD_U64LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Sclose(space); + H5Dclose(dataset); + + /* Dataset of 8 bits signed int */ + dims[0] = F66_XDIM; dims[1] = F66_YDIM8; + space = H5Screate_simple(2, dims, NULL); + dataset = H5Dcreate2(fid, F66_DATASETS08, H5T_STD_I8LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Sclose(space); + H5Dclose(dataset); + + /* Dataset of 16 bits signed int */ + dims[0] = F66_XDIM; dims[1] = F66_YDIM16; + space = H5Screate_simple(2, dims, NULL); + dataset = H5Dcreate2(fid, F66_DATASETS16, H5T_STD_I16LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Sclose(space); + H5Dclose(dataset); + + /* Dataset of 32 bits signed int */ + dims[0] = F66_XDIM; dims[1] = F66_YDIM32; + space = H5Screate_simple(2, dims, NULL); + dataset = H5Dcreate2(fid, F66_DATASETS32, H5T_STD_I32LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Sclose(space); + H5Dclose(dataset); + + /* Dataset of 64 bits signed int */ + dims[0] = F66_XDIM; dims[1] = F66_YDIM64; + space = H5Screate_simple(2, dims, NULL); + dataset = H5Dcreate2(fid, F66_DATASETS64, H5T_STD_I64LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Sclose(space); + H5Dclose(dataset); + + /* Double Dummy set for failure tests */ + dims[0] = F66_XDIM; dims[1] = F66_YDIM8; + space = H5Screate_simple(2, dims, NULL); + dataset = H5Dcreate2(fid, F66_DUMMYDBL, H5T_IEEE_F64BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + H5Sclose(space); + H5Dclose(dataset); + H5Fclose(fid); +} + + /*------------------------------------------------------------------------- * Function: gent_charsets * @@ -9294,15 +9369,13 @@ gent_intattrscalars(void) } /*------------------------------------------------------------------------- - * Function: gent_packedbits + * Function: gent_intsattrs * - * Purpose: Generate a file to be used in the h5dump packed bits tests. + * Purpose: Generate a file to be used in the h5dump tests. * Four datasets of 1, 2, 4 and 8 bytes of unsigned int types are created. * Four more datasets of 1, 2, 4 and 8 bytes of signed int types are created. * Fill them with raw data such that no bit will be all zero in a dataset. * A dummy dataset of double type is created for failure test. - * Created: Albert Cheng, 2010/5/10. - * Modified: Allen Byrne, 2011/1/5 Use file to test Signed/Unsigned datatypes *------------------------------------------------------------------------- */ static void @@ -10051,7 +10124,6 @@ static void gent_compound_complex2(void) } H5Tclose(type); } - */ /* CompoundComplex4D */ /* @@ -10417,6 +10489,7 @@ int main(void) gent_intattrscalars(); gent_intsattrs(); gent_bitnopaquefields(); + gent_nodata(); gent_intsfourdims(); gent_null_space_group(); diff --git a/tools/h5import/h5importtest.c b/tools/h5import/h5importtest.c index 27995d2..560f1b3 100644 --- a/tools/h5import/h5importtest.c +++ b/tools/h5import/h5importtest.c @@ -436,7 +436,7 @@ main(void) { char c = bin8w[i]; if ( HDfwrite( &c, sizeof(char), 1, sp) != 1 ) - printf("error writing file\n"); + HDprintf("error writing file\n"); } HDfclose(sp); diff --git a/tools/h5jam/h5jam.c b/tools/h5jam/h5jam.c index 28a62fd..fae7b9e 100644 --- a/tools/h5jam/h5jam.c +++ b/tools/h5jam/h5jam.c @@ -19,10 +19,7 @@ /* Name of tool */ #define PROGRAMNAME "h5jam" -#define TRUE 1 -#define FALSE 0 - -hsize_t write_pad (int, hsize_t); +herr_t write_pad(int ofile, hsize_t old_where, hsize_t *new_where); hsize_t compute_user_block_size (hsize_t); hsize_t copy_some_to_file (int, int, hsize_t, hsize_t, ssize_t); void parse_command_line (int, const char *[]); @@ -313,7 +310,7 @@ main (int argc, const char *argv[]) h5fsize = (hsize_t)sbuf2.st_size; if (output_file == NULL) { - ofid = HDopen (input_file, O_WRONLY, 0); + ofid = HDopen(input_file, O_WRONLY, 0); if (ofid < 0) { error_msg("unable to open output file \"%s\"\n", output_file); @@ -323,7 +320,7 @@ main (int argc, const char *argv[]) } } else { - ofid = HDopen (output_file, O_WRONLY | O_CREAT | O_TRUNC, 0644); + ofid = HDopen(output_file, O_WRONLY | O_CREAT | O_TRUNC, 0644); if (ofid < 0) { error_msg("unable to create output file \"%s\"\n", output_file); @@ -366,7 +363,13 @@ main (int argc, const char *argv[]) where = copy_some_to_file(ufid, ofid, (hsize_t) 0, startub, (ssize_t) - 1); /* pad the ub */ - where = write_pad (ofid, where); + if(write_pad(ofid, where, &where) < 0) { + error_msg("Can't pad file \"%s\"\n", output_file); + HDclose (h5fid); + HDclose (ufid); + HDclose (ofid); + leave (EXIT_FAILURE); + } /* end if */ if(ub_file) HDfree(ub_file); @@ -503,7 +506,7 @@ copy_some_to_file(int infid, int outfid, hsize_t startin, hsize_t startout, *------------------------------------------------------------------------- */ hsize_t -compute_user_block_size (hsize_t ublock_size) +compute_user_block_size(hsize_t ublock_size) { hsize_t where = 512; @@ -521,23 +524,30 @@ compute_user_block_size (hsize_t ublock_size) * * Returns the size of the padded file. */ -hsize_t -write_pad(int ofile, hsize_t where) +herr_t +write_pad(int ofile, hsize_t old_where, hsize_t *new_where) { unsigned int i; char buf[1]; hsize_t psize; + if(new_where == NULL) + return FAIL; + buf[0] = '\0'; - HDlseek(ofile, (off_t) where, SEEK_SET); + HDlseek(ofile, (off_t)old_where, SEEK_SET); - psize = compute_user_block_size (where); - psize -= where; + psize = compute_user_block_size(old_where); + psize -= old_where; for(i = 0; i < psize; i++) - HDwrite (ofile, buf, 1); + if(HDwrite(ofile, buf, 1) < 0) + return FAIL; - return(where + psize); /* the new size of the file. */ -} + /* Set the new size of the file. */ + *new_where = old_where + psize; + + return SUCCEED; +} /* end write_pad() */ diff --git a/tools/h5jam/h5jamgentest.c b/tools/h5jam/h5jamgentest.c index e7a1878..c8b0801 100644 --- a/tools/h5jam/h5jamgentest.c +++ b/tools/h5jam/h5jamgentest.c @@ -275,10 +275,7 @@ gent_ub(const char * filename, size_t ub_size, size_t ub_fill) H5Fclose(fid); /* If a user block is being used, write to it here */ - if(ub_size > 0) - { - ssize_t nbytes; - + if(ub_size > 0) { HDassert(ub_size <= BUF_SIZE); fd = HDopen(filename, O_RDWR, 0); @@ -290,8 +287,7 @@ gent_ub(const char * filename, size_t ub_size, size_t ub_fill) for (u = 0; u < ub_fill; u++) *bp++ = pattern[u % 10]; - nbytes = HDwrite(fd, buf, ub_size); - HDassert(nbytes >= 0); + HDwrite(fd, buf, ub_size); HDclose(fd); } @@ -304,7 +300,6 @@ create_textfile(const char *name, size_t size) int fd; size_t i; char *bp; - ssize_t nbytes; fd = HDcreat(name,0777); HDassert(fd >= 0); @@ -316,8 +311,7 @@ create_textfile(const char *name, size_t size) for(i = 0; i < size; i++) *bp++ = pattern[i % 10]; - nbytes = HDwrite(fd, buf, size); - HDassert(nbytes >= 0); + HDwrite(fd, buf, size); HDfree(buf); diff --git a/tools/h5jam/h5unjam.c b/tools/h5jam/h5unjam.c index 59e5dae..ffe2aca 100644 --- a/tools/h5jam/h5unjam.c +++ b/tools/h5jam/h5unjam.c @@ -19,8 +19,6 @@ /* Name of tool */ #define PROGRAMNAME "h5unjam" -#define TRUE 1 -#define FALSE 0 #define COPY_BUF_SIZE 1024 hsize_t write_pad( int , hsize_t ); @@ -334,17 +332,19 @@ done: *------------------------------------------------------------------------- */ herr_t -copy_to_file( FILE *infid, FILE *ofid, ssize_t _where, ssize_t how_much ) +copy_to_file( FILE *infid, FILE *ofid, ssize_t _where, ssize_t show_much ) { static char buf[COPY_BUF_SIZE]; + size_t how_much; off_t where = (off_t)_where; off_t to; off_t from; herr_t ret_value = 0; /* nothing to copy */ - if(how_much <= 0) + if(show_much <= 0) goto done; + how_much = (size_t)show_much; /* rewind */ HDfseek(infid, 0L, 0); @@ -379,8 +379,8 @@ copy_to_file( FILE *infid, FILE *ofid, ssize_t _where, ssize_t how_much ) /* Update positions/size */ how_much -= bytes_read; - from += bytes_read; - to += bytes_read; + from += (off_t)bytes_read; + to += (off_t)bytes_read; /* Write nchars bytes to output file */ bytes_wrote = HDfwrite(buf, (size_t)1, bytes_read, ofid); diff --git a/tools/h5ls/h5ls.c b/tools/h5ls/h5ls.c index 78a2cce..fe58610 100644 --- a/tools/h5ls/h5ls.c +++ b/tools/h5ls/h5ls.c @@ -157,19 +157,13 @@ static hbool_t print_int_type(h5tools_str_t *buffer, hid_t type, int ind); static hbool_t print_float_type(h5tools_str_t *buffer, hid_t type, int ind); static herr_t visit_obj(hid_t file, const char *oname, iter_t *iter); - + /*------------------------------------------------------------------------- * Function: usage * * Purpose: Prints a usage message on stderr and then returns. * * Return: void - * - * Programmer: Robb Matzke - * Thursday, July 16, 1998 - * - * Modifications: - * *------------------------------------------------------------------------- */ static void @@ -215,6 +209,15 @@ usage (void) PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n"); PRINTVALSTREAM(rawoutstream, " --vfd=DRIVER Use the specified virtual file driver\n"); PRINTVALSTREAM(rawoutstream, " -x, --hexdump Show raw data in hexadecimal format\n"); + PRINTVALSTREAM(rawoutstream, " --s3-cred=C Supply S3 authentication information to \"ros3\" vfd.\n"); + PRINTVALSTREAM(rawoutstream, " Accepts tuple of \"(,,)\".\n"); + PRINTVALSTREAM(rawoutstream, " If absent or C->\"(,,)\", defaults to no-authentication.\n"); + PRINTVALSTREAM(rawoutstream, " Has no effect if vfd flag not set to \"ros3\".\n"); + PRINTVALSTREAM(rawoutstream, " --hdfs-attrs=A Supply configuration information to Hadoop VFD.\n"); + PRINTVALSTREAM(rawoutstream, " Accepts tuple of (,,\n"); + PRINTVALSTREAM(rawoutstream, " ...,,)\n"); + PRINTVALSTREAM(rawoutstream, " If absent or A == '(,,,,)', all default values are used.\n"); + PRINTVALSTREAM(rawoutstream, " Has no effect if vfd flag is not 'hdfs'.\n"); PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, " file/OBJECT\n"); PRINTVALSTREAM(rawoutstream, " Each object consists of an HDF5 file name optionally followed by a\n"); @@ -236,22 +239,14 @@ usage (void) PRINTVALSTREAM(rawoutstream, " Replaced by --enable-error-stack.\n"); } - /*------------------------------------------------------------------------- * Function: print_string * * Purpose: Print a string value by escaping unusual characters. If - * STREAM is null then we only count how large the output would - * be. + * STREAM is null then we only count how large the output would be. * * Return: Number of characters printed. - * - * Programmer: Robb Matzke - * Thursday, November 5, 1998 - * - * Modifications: - * *------------------------------------------------------------------------- */ static int @@ -262,27 +257,33 @@ print_string(h5tools_str_t *buffer, const char *s, hbool_t escape_spaces) for (/*void*/; s && *s; s++) { switch (*s) { case '"': - if (buffer) h5tools_str_append(buffer, "\\\""); + if (buffer) + h5tools_str_append(buffer, "\\\""); nprint += 2; break; case '\\': - if (buffer) h5tools_str_append(buffer, "\\\\"); + if (buffer) + h5tools_str_append(buffer, "\\\\"); nprint += 2; break; case '\b': - if (buffer) h5tools_str_append(buffer, "\\b"); + if (buffer) + h5tools_str_append(buffer, "\\b"); nprint += 2; break; case '\f': - if (buffer) h5tools_str_append(buffer, "\\f"); + if (buffer) + h5tools_str_append(buffer, "\\f"); nprint += 2; break; case '\n': - if (buffer) h5tools_str_append(buffer, "\\n"); + if (buffer) + h5tools_str_append(buffer, "\\n"); nprint += 2; break; case '\r': - if (buffer) h5tools_str_append(buffer, "\\r"); + if (buffer) + h5tools_str_append(buffer, "\\r"); nprint += 2; break; case '\t': @@ -291,21 +292,25 @@ print_string(h5tools_str_t *buffer, const char *s, hbool_t escape_spaces) break; case ' ': if (escape_spaces) { - if (buffer) h5tools_str_append(buffer, "\\ "); + if (buffer) + h5tools_str_append(buffer, "\\ "); nprint += 2; } else { - if (buffer) h5tools_str_append(buffer, " "); + if (buffer) + h5tools_str_append(buffer, " "); nprint++; } break; default: if (isprint((int)*s)) { - if (buffer) h5tools_str_append(buffer, "%c", *s); + if (buffer) + h5tools_str_append(buffer, "%c", *s); nprint++; } else { - if (buffer) h5tools_str_append(buffer, "\\%03o", *((const unsigned char*)s)); + if (buffer) + h5tools_str_append(buffer, "\\%03o", *((const unsigned char*)s)); nprint += 4; } break; @@ -314,21 +319,14 @@ print_string(h5tools_str_t *buffer, const char *s, hbool_t escape_spaces) return nprint; } - + /*------------------------------------------------------------------------- * Function: print_obj_name * * Purpose: Print an object name and another string. * * Return: Success: TRUE - * - * Failure: FALSE, nothing printed - * - * Programmer: Quincey Koziol - * Tuesday, November 6, 2007 - * - * Modifications: - * + * Failure: FALSE, nothing printed *------------------------------------------------------------------------- */ static int @@ -339,19 +337,19 @@ print_obj_name(h5tools_str_t *buffer, const iter_t *iter, const char *oname, const char *name = fullname; /* Pointer to buffer for printing */ int n; - if(show_file_name_g) + if (show_file_name_g) HDsnprintf(fullname, sizeof(fullname), "%s/%s", iter->fname, oname + iter->name_start); else name = oname + iter->name_start; /* Print the object name, either full name or base name */ - if(fullname_g) + if (fullname_g) n = print_string(buffer, name, TRUE); else { const char *last_sep; /* The location of the last group separator */ /* Find the last component of the path name */ - if(NULL == (last_sep = HDstrrchr(name, '/'))) + if (NULL == (last_sep = HDstrrchr(name, '/'))) last_sep = name; else { last_sep++; @@ -363,123 +361,158 @@ print_obj_name(h5tools_str_t *buffer, const iter_t *iter, const char *oname, return TRUE; } - + /*------------------------------------------------------------------------- * Function: print_native_type * * Purpose: Prints the name of a native C data type. * * Return: Success: TRUE - * - * Failure: FALSE, nothing printed. - * - * Programmer: Robb Matzke - * Thursday, November 5, 1998 - * - * Modifications: - * Robb Matzke, 1999-06-11 - * Added the C9x types, but we still prefer to display the types - * from the C language itself (like `int' vs. `int32_t'). - * + * Failure: FALSE, nothing printed. *------------------------------------------------------------------------- */ static hbool_t -print_native_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind) +print_native_type(h5tools_str_t *buffer, hid_t type, int ind) { - if(!simple_output_g) { - if (H5Tequal(type, H5T_NATIVE_SCHAR)==TRUE) { + if (!simple_output_g) { + if (H5Tequal(type, H5T_NATIVE_SCHAR) == TRUE) { h5tools_str_append(buffer, "native signed char"); - } else if (H5Tequal(type, H5T_NATIVE_UCHAR)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UCHAR) == TRUE) { h5tools_str_append(buffer, "native unsigned char"); - } else if (H5Tequal(type, H5T_NATIVE_INT)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_INT) == TRUE) { h5tools_str_append(buffer, "native int"); - } else if (H5Tequal(type, H5T_NATIVE_UINT)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT) == TRUE) { h5tools_str_append(buffer, "native unsigned int"); - } else if (H5Tequal(type, H5T_NATIVE_SHORT)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_SHORT) == TRUE) { h5tools_str_append(buffer, "native short"); - } else if (H5Tequal(type, H5T_NATIVE_USHORT)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_USHORT) == TRUE) { h5tools_str_append(buffer, "native unsigned short"); - } else if (H5Tequal(type, H5T_NATIVE_LONG)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_LONG) == TRUE) { h5tools_str_append(buffer, "native long"); - } else if (H5Tequal(type, H5T_NATIVE_ULONG)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_ULONG) == TRUE) { h5tools_str_append(buffer, "native unsigned long"); - } else if (H5Tequal(type, H5T_NATIVE_LLONG)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_LLONG) == TRUE) { h5tools_str_append(buffer, "native long long"); - } else if (H5Tequal(type, H5T_NATIVE_ULLONG)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_ULLONG) == TRUE) { h5tools_str_append(buffer, "native unsigned long long"); - } else if (H5Tequal(type, H5T_NATIVE_FLOAT)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_FLOAT) == TRUE) { h5tools_str_append(buffer, "native float"); - } else if (H5Tequal(type, H5T_NATIVE_DOUBLE)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_DOUBLE) == TRUE) { h5tools_str_append(buffer, "native double"); + } #if H5_SIZEOF_LONG_DOUBLE !=0 - } else if (H5Tequal(type, H5T_NATIVE_LDOUBLE)==TRUE) { + else if (H5Tequal(type, H5T_NATIVE_LDOUBLE) == TRUE) { h5tools_str_append(buffer, "native long double"); + } #endif - } else if (H5Tequal(type, H5T_NATIVE_INT8)==TRUE) { + else if (H5Tequal(type, H5T_NATIVE_INT8) == TRUE) { h5tools_str_append(buffer, "native int8_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT8)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT8) == TRUE) { h5tools_str_append(buffer, "native uint8_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT16)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_INT16) == TRUE) { h5tools_str_append(buffer, "native int16_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT16)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT16) == TRUE) { h5tools_str_append(buffer, "native uint16_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT32)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_INT32) == TRUE) { h5tools_str_append(buffer, "native int32_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT32)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT32) == TRUE) { h5tools_str_append(buffer, "native uint32_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT64)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_INT64) == TRUE) { h5tools_str_append(buffer, "native int64_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT64)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT64) == TRUE) { h5tools_str_append(buffer, "native uint64_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST8)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_INT_LEAST8) == TRUE) { h5tools_str_append(buffer, "native int_least8_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST8)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST8) == TRUE) { h5tools_str_append(buffer, "native uint_least8_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST16)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_INT_LEAST16) == TRUE) { h5tools_str_append(buffer, "native int_least16_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST16)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST16) == TRUE) { h5tools_str_append(buffer, "native uint_least16_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST32)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_INT_LEAST32) == TRUE) { h5tools_str_append(buffer, "native int_least32_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST32)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST32) == TRUE) { h5tools_str_append(buffer, "native uint_least32_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST64)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_INT_LEAST64) == TRUE) { h5tools_str_append(buffer, "native int_least64_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST64)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST64) == TRUE) { h5tools_str_append(buffer, "native uint_least64_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_FAST8)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_INT_FAST8) == TRUE) { h5tools_str_append(buffer, "native int_fast8_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST8)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT_FAST8) == TRUE) { h5tools_str_append(buffer, "native uint_fast8_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_FAST16)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_INT_FAST16) == TRUE) { h5tools_str_append(buffer, "native int_fast16_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST16)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT_FAST16) == TRUE) { h5tools_str_append(buffer, "native uint_fast16_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_FAST32)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_INT_FAST32) == TRUE) { h5tools_str_append(buffer, "native int_fast32_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST32)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT_FAST32) == TRUE) { h5tools_str_append(buffer, "native uint_fast32_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_FAST64)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_INT_FAST64) == TRUE) { h5tools_str_append(buffer, "native int_fast64_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST64)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_UINT_FAST64) == TRUE) { h5tools_str_append(buffer, "native uint_fast64_t"); - } else if (H5Tequal(type, H5T_NATIVE_B8)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_B8) == TRUE) { h5tools_str_append(buffer, "native 8-bit field"); - } else if (H5Tequal(type, H5T_NATIVE_B16)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_B16) == TRUE) { h5tools_str_append(buffer, "native 16-bit field"); - } else if (H5Tequal(type, H5T_NATIVE_B32)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_B32) == TRUE) { h5tools_str_append(buffer, "native 32-bit field"); - } else if (H5Tequal(type, H5T_NATIVE_B64)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_B64) == TRUE) { h5tools_str_append(buffer, "native 64-bit field"); - } else if (H5Tequal(type, H5T_NATIVE_HSIZE)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_HSIZE) == TRUE) { h5tools_str_append(buffer, "native hsize_t"); - } else if (H5Tequal(type, H5T_NATIVE_HSSIZE)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_HSSIZE) == TRUE) { h5tools_str_append(buffer, "native hssize_t"); - } else if (H5Tequal(type, H5T_NATIVE_HERR)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_HERR) == TRUE) { h5tools_str_append(buffer, "native herr_t"); - } else if (H5Tequal(type, H5T_NATIVE_HBOOL)==TRUE) { + } + else if (H5Tequal(type, H5T_NATIVE_HBOOL) == TRUE) { h5tools_str_append(buffer, "native hbool_t"); - } else { + } + else { return print_int_type(buffer, type, ind); } } else { @@ -488,36 +521,29 @@ print_native_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind) return TRUE; } - + /*------------------------------------------------------------------------- * Function: print_ieee_type * * Purpose: Print the name of an IEEE floating-point data type. * * Return: Success: TRUE - * - * Failure: FALSE, nothing printed - * - * Programmer: Robb Matzke - * Thursday, November 5, 1998 - * - * Modifications: - * + * Failure: FALSE, nothing printed *------------------------------------------------------------------------- */ static hbool_t -print_ieee_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind) +print_ieee_type(h5tools_str_t *buffer, hid_t type, int ind) { - if (H5Tequal(type, H5T_IEEE_F32BE)==TRUE) { + if (H5Tequal(type, H5T_IEEE_F32BE) == TRUE) { h5tools_str_append(buffer, "IEEE 32-bit big-endian float"); } - else if (H5Tequal(type, H5T_IEEE_F32LE)==TRUE) { + else if (H5Tequal(type, H5T_IEEE_F32LE) == TRUE) { h5tools_str_append(buffer, "IEEE 32-bit little-endian float"); } - else if (H5Tequal(type, H5T_IEEE_F64BE)==TRUE) { + else if (H5Tequal(type, H5T_IEEE_F64BE) == TRUE) { h5tools_str_append(buffer, "IEEE 64-bit big-endian float"); } - else if (H5Tequal(type, H5T_IEEE_F64LE)==TRUE) { + else if (H5Tequal(type, H5T_IEEE_F64LE) == TRUE) { h5tools_str_append(buffer, "IEEE 64-bit little-endian float"); } else { @@ -526,21 +552,15 @@ print_ieee_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind) return TRUE; } - + /*------------------------------------------------------------------------- * Function: print_precision * * Purpose: Prints information on the next line about precision and - * padding if the precision is less than the total data type - * size. + * padding if the precision is less than the total data type + * size. * * Return: void - * - * Programmer: Robb Matzke - * Thursday, November 5, 1998 - * - * Modifications: - * *------------------------------------------------------------------------- */ static void @@ -555,13 +575,12 @@ print_precision(h5tools_str_t *buffer, hid_t type, int ind) /* If the precision is less than the total size then show the precision * and offset on the following line. Also display the padding * information. */ - if(8 * H5Tget_size(type) != (prec = H5Tget_precision(type))) { + if (8 * H5Tget_size(type) != (prec = H5Tget_precision(type))) { h5tools_str_append(buffer, "\n%*s(%lu bit%s of precision beginning at bit %lu)", - ind, "", (unsigned long)prec, 1 == prec ? "" : "s", - (unsigned long)H5Tget_offset(type)); + ind, "", (unsigned long)prec, 1 == prec ? "" : "s", (unsigned long)H5Tget_offset(type)); H5Tget_pad(type, &plsb, &pmsb); - if(H5Tget_offset(type) > 0) { + if (H5Tget_offset(type) > 0) { switch(plsb) { case H5T_PAD_ZERO: plsb_s = "zero"; @@ -580,7 +599,7 @@ print_precision(h5tools_str_t *buffer, hid_t type, int ind) break; } } - if((unsigned)H5Tget_offset(type) + prec < 8 * H5Tget_size(type)) { + if ((unsigned)H5Tget_offset(type) + prec < 8 * H5Tget_size(type)) { switch(pmsb) { case H5T_PAD_ZERO: pmsb_s = "zero"; @@ -603,61 +622,53 @@ print_precision(h5tools_str_t *buffer, hid_t type, int ind) h5tools_str_append(buffer, "\n%*s(", ind, ""); if (plsb_s) { nbits = (unsigned)H5Tget_offset(type); - h5tools_str_append(buffer, "%lu %s bit%s at bit 0", - (unsigned long)nbits, plsb_s, 1 == nbits ? "" : "s"); + h5tools_str_append(buffer, "%lu %s bit%s at bit 0", (unsigned long)nbits, plsb_s, 1 == nbits ? "" : "s"); } - if (plsb_s && pmsb_s) h5tools_str_append(buffer, ", "); + if (plsb_s && pmsb_s) + h5tools_str_append(buffer, ", "); if (pmsb_s) { nbits = (8 * H5Tget_size(type)) - ((unsigned)H5Tget_offset(type) + prec); - h5tools_str_append(buffer, "%lu %s bit%s at bit %lu", - (unsigned long)nbits, pmsb_s, 1 == nbits ? "" : "s", - (unsigned long)(8 * H5Tget_size(type) - nbits)); + h5tools_str_append(buffer, "%lu %s bit%s at bit %lu", (unsigned long)nbits, pmsb_s, 1 == nbits ? "" : "s", (unsigned long)(8 * H5Tget_size(type) - nbits)); } h5tools_str_append(buffer, ")"); } } } - + /*------------------------------------------------------------------------- * Function: print_int_type * * Purpose: Print the name of an integer data type. Common information - * like number of bits, byte order, and sign scheme appear on - * the first line. Additional information might appear in - * parentheses on the following lines. + * like number of bits, byte order, and sign scheme appear on + * the first line. Additional information might appear in + * parentheses on the following lines. * * Return: Success: TRUE - * - * Failure: FALSE, nothing printed - * - * Programmer: Robb Matzke - * Thursday, November 5, 1998 - * - * Modifications: - * + * Failure: FALSE, nothing printed *------------------------------------------------------------------------- */ static hbool_t print_int_type(h5tools_str_t *buffer, hid_t type, int ind) { - H5T_order_t order; /* byte order value */ - const char *order_s=NULL; /* byte order string */ - H5T_sign_t sign; /* sign scheme value */ - const char *sign_s=NULL; /* sign scheme string */ + H5T_order_t order; /* byte order value */ + const char *order_s = NULL; /* byte order string */ + H5T_sign_t sign; /* sign scheme value */ + const char *sign_s = NULL; /* sign scheme string */ - if (H5T_INTEGER!=H5Tget_class(type)) return FALSE; + if (H5T_INTEGER != H5Tget_class(type)) + return FALSE; /* Byte order */ - if (H5Tget_size(type)>1) { + if (H5Tget_size(type) > 1) { order = H5Tget_order(type); - if (H5T_ORDER_LE==order) { + if (H5T_ORDER_LE == order) { order_s = " little-endian"; } - else if (H5T_ORDER_BE==order) { + else if (H5T_ORDER_BE == order) { order_s = " big-endian"; } - else if (H5T_ORDER_VAX==order) { + else if (H5T_ORDER_VAX == order) { order_s = " mixed-endian"; } else { @@ -669,11 +680,11 @@ print_int_type(h5tools_str_t *buffer, hid_t type, int ind) } /* Sign */ - if ((sign=H5Tget_sign(type))>=0) { - if (H5T_SGN_NONE==sign) { + if ((sign = H5Tget_sign(type)) >= 0) { + if (H5T_SGN_NONE == sign) { sign_s = " unsigned"; } - else if (H5T_SGN_2==sign) { + else if (H5T_SGN_2 == sign) { sign_s = ""; } else { @@ -686,27 +697,19 @@ print_int_type(h5tools_str_t *buffer, hid_t type, int ind) /* Print size, order, and sign on first line, precision and padding * information on the subsequent lines */ - h5tools_str_append(buffer, "%lu-bit%s%s integer", - (unsigned long)(8*H5Tget_size(type)), order_s, sign_s); + h5tools_str_append(buffer, "%lu-bit%s%s integer", (unsigned long)(8*H5Tget_size(type)), order_s, sign_s); print_precision(buffer, type, ind); return TRUE; } - + /*------------------------------------------------------------------------- * Function: print_float_type * * Purpose: Print info about a floating point data type. * * Return: Success: TRUE - * - * Failure: FALSE, nothing printed - * - * Programmer: Robb Matzke - * Thursday, November 5, 1998 - * - * Modifications: - * + * Failure: FALSE, nothing printed *------------------------------------------------------------------------- */ static hbool_t @@ -723,18 +726,19 @@ print_float_type(h5tools_str_t *buffer, hid_t type, int ind) H5T_pad_t pad; /* internal padding value */ const char *pad_s=NULL; /* internal padding string */ - if (H5T_FLOAT!=H5Tget_class(type)) return FALSE; + if (H5T_FLOAT != H5Tget_class(type)) + return FALSE; /* Byte order */ - if (H5Tget_size(type)>1) { + if (H5Tget_size(type) > 1) { order = H5Tget_order(type); - if (H5T_ORDER_LE==order) { + if (H5T_ORDER_LE == order) { order_s = " little-endian"; } - else if (H5T_ORDER_BE==order) { + else if (H5T_ORDER_BE == order) { order_s = " big-endian"; } - else if (H5T_ORDER_VAX==order) { + else if (H5T_ORDER_VAX == order) { order_s = " mixed-endian"; } else { @@ -747,8 +751,7 @@ print_float_type(h5tools_str_t *buffer, hid_t type, int ind) /* Print size and byte order on first line, precision and padding on * subsequent lines. */ - h5tools_str_append(buffer, "%lu-bit%s floating-point", - (unsigned long)(8*H5Tget_size(type)), order_s); + h5tools_str_append(buffer, "%lu-bit%s floating-point", (unsigned long)(8*H5Tget_size(type)), order_s); print_precision(buffer, type, ind); /* Print sizes, locations, and other information about each field */ @@ -773,15 +776,13 @@ print_float_type(h5tools_str_t *buffer, hid_t type, int ind) break; } h5tools_str_append(buffer, "\n%*s(significant for %lu bit%s at bit %lu%s)", ind, "", - (unsigned long)msize, 1==msize?"":"s", (unsigned long)mpos, - norm_s); + (unsigned long)msize, 1==msize?"":"s", (unsigned long)mpos, norm_s); h5tools_str_append(buffer, "\n%*s(exponent for %lu bit%s at bit %lu, bias is 0x%lx)", - ind, "", (unsigned long)esize, 1==esize?"":"s", - (unsigned long)epos, (unsigned long)ebias); + ind, "", (unsigned long)esize, 1==esize?"":"s", (unsigned long)epos, (unsigned long)ebias); h5tools_str_append(buffer, "\n%*s(sign bit at %lu)", ind, "", (unsigned long)spos); /* Display internal padding */ - if (1+esize+msize 0) { + if (nmembs > 0) { char **name; /* member names */ unsigned char *value; /* value array */ hid_t native = -1; /* native integer data type */ @@ -904,9 +890,9 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind) * 1. long long -- the largest native signed integer * 2. unsigned long long -- the largest native unsigned integer * 3. raw format */ - if(H5Tget_size(type) <= sizeof(long long)) { + if (H5Tget_size(type) <= sizeof(long long)) { dst_size = sizeof(long long); - if(H5T_SGN_NONE == H5Tget_sign(type)) + if (H5T_SGN_NONE == H5Tget_sign(type)) native = H5T_NATIVE_ULLONG; else native = H5T_NATIVE_LLONG; @@ -917,16 +903,16 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind) /* Get the names and raw values of all members */ name = (char **)HDcalloc((size_t)nmembs, sizeof(char *)); value = (unsigned char *)HDcalloc((size_t)nmembs, MAX(H5Tget_size(type), dst_size)); - for(i = 0; i < (unsigned)nmembs; i++) { + for (i = 0; i < (unsigned)nmembs; i++) { name[i] = H5Tget_member_name(type, i); H5Tget_member_value(type, i, value + i * H5Tget_size(type)); } /* Convert values to native data type */ - if(native > 0) - if(H5Tconvert(super, native, (size_t)nmembs, value, NULL, H5P_DEFAULT) < 0) { + if (native > 0) + if (H5Tconvert(super, native, (size_t)nmembs, value, NULL, H5P_DEFAULT) < 0) { /* Release resources */ - for(i = 0; i < (unsigned)nmembs; i++) + for (i = 0; i < (unsigned)nmembs; i++) H5free_memory(name[i]); HDfree(name); HDfree(value); @@ -938,7 +924,7 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind) /*not implemented yet*/ /* Print members */ - for(i = 0; i < (unsigned)nmembs; i++) { + for (i = 0; i < (unsigned)nmembs; i++) { unsigned char *copy; /* a pointer to value array */ int nchars; /* number of output characters */ @@ -946,14 +932,14 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind) nchars = print_string(buffer, name[i], TRUE); h5tools_str_append(buffer, "%*s = ", MAX(0, 16 - nchars), ""); - if(native < 0) { + if (native < 0) { size_t j; h5tools_str_append(buffer, "0x"); - for(j = 0; j < dst_size; j++) + for (j = 0; j < dst_size; j++) h5tools_str_append(buffer, "%02x", value[i*dst_size+j]); } - else if(H5T_SGN_NONE == H5Tget_sign(native)) { + else if (H5T_SGN_NONE == H5Tget_sign(native)) { /*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size" *strangely, unless use another pointer "copy".*/ copy = value + i * dst_size; @@ -963,8 +949,7 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind) /*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size" *strangely, unless use another pointer "copy".*/ copy = value + i * dst_size; - h5tools_str_append(buffer, "%"H5_PRINTF_LL_WIDTH"d", - *((long long*)((void*)copy))); + h5tools_str_append(buffer, "%"H5_PRINTF_LL_WIDTH"d", *((long long*)((void*)copy))); } } @@ -984,21 +969,14 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind) return TRUE; } - + /*------------------------------------------------------------------------- * Function: print_string_type * * Purpose: Print information about a string data type. * * Return: Success: TRUE - * - * Failure: FALSE, nothing printed - * - * Programmer: Robb Matzke - * Thursday, November 5, 1998 - * - * Modifications: - * + * Failure: FALSE, nothing printed *------------------------------------------------------------------------- */ static hbool_t @@ -1009,7 +987,8 @@ print_string_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind) H5T_cset_t cset; const char *cset_s=NULL; - if (H5T_STRING!=H5Tget_class(type)) return FALSE; + if (H5T_STRING != H5Tget_class(type)) + return FALSE; /* Padding */ pad = H5Tget_strpad(type); @@ -1085,59 +1064,43 @@ print_string_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind) return TRUE; } - + /*------------------------------------------------------------------------- * Function: print_reference_type * * Purpose: Prints information about a reference data type. * * Return: Success: TRUE - * - * Failure: FALSE, nothing printed - * - * Programmer: Robb Matzke - * Thursday, November 5, 1998 - * - * Modifications: - * Robb Matzke, 1999-06-04 - * Knows about object and dataset region references. - * + * Failure: FALSE, nothing printed *------------------------------------------------------------------------- */ static hbool_t print_reference_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind) { - if (H5T_REFERENCE!=H5Tget_class(type)) return FALSE; + if (H5T_REFERENCE != H5Tget_class(type)) + return FALSE; - if (H5Tequal(type, H5T_STD_REF_OBJ)==TRUE) { + if (H5Tequal(type, H5T_STD_REF_OBJ) == TRUE) { h5tools_str_append(buffer, "object reference"); } - else if (H5Tequal(type, H5T_STD_REF_DSETREG)==TRUE) { + else if (H5Tequal(type, H5T_STD_REF_DSETREG) == TRUE) { h5tools_str_append(buffer, "dataset region reference"); } else { - h5tools_str_append(buffer, "%lu-byte unknown reference", - (unsigned long)H5Tget_size(type)); + h5tools_str_append(buffer, "%lu-byte unknown reference", (unsigned long)H5Tget_size(type)); } return TRUE; } - + /*------------------------------------------------------------------------- * Function: print_opaque_type * * Purpose: Prints information about an opaque data type. * * Return: Success: TRUE - * - * Failure: FALSE, nothing printed - * - * Programmer: Robb Matzke - * Monday, June 7, 1999 - * - * Modifications: - * + * Failure: FALSE, nothing printed *------------------------------------------------------------------------- */ static hbool_t @@ -1146,11 +1109,12 @@ print_opaque_type(h5tools_str_t *buffer, hid_t type, int ind) char *tag; size_t size; - if (H5T_OPAQUE!=H5Tget_class(type)) return FALSE; + if (H5T_OPAQUE != H5Tget_class(type)) + return FALSE; size = H5Tget_size(type); h5tools_str_append(buffer, "%lu-byte opaque type", (unsigned long)size); - if ((tag=H5Tget_tag(type))) { + if ((tag = H5Tget_tag(type))) { h5tools_str_append(buffer, "\n%*s(tag = \"", ind, ""); print_string(buffer, tag, FALSE); h5tools_str_append(buffer, "\")"); @@ -1159,20 +1123,14 @@ print_opaque_type(h5tools_str_t *buffer, hid_t type, int ind) return TRUE; } - + /*------------------------------------------------------------------------- * Function: print_vlen_type * * Purpose: Print information about a variable-length type * * Return: Success: TRUE - * * Failure: FALSE - * - * Programmer: Robb Matzke - * Friday, December 1, 2000 - * - * Modifications: *------------------------------------------------------------------------- */ static hbool_t @@ -1180,27 +1138,22 @@ print_vlen_type(h5tools_str_t *buffer, hid_t type, int ind) { hid_t super; - if (H5T_VLEN!=H5Tget_class(type)) return FALSE; + if (H5T_VLEN != H5Tget_class(type)) + return FALSE; - h5tools_str_append(buffer, "variable length of\n%*s", ind+4, ""); + h5tools_str_append(buffer, "variable length of\n%*s", ind + 4, ""); super = H5Tget_super(type); - print_type(buffer, super, ind+4); + print_type(buffer, super, ind + 4); H5Tclose(super); return TRUE; } - + /*--------------------------------------------------------------------------- * Purpose: Print information about an array type * * Return: Success: TRUE - * * Failure: FALSE - * - * Programmer: Robb Matzke - * Thursday, January 31, 2002 - * - * Modifications: *--------------------------------------------------------------------------- */ static hbool_t @@ -1210,7 +1163,7 @@ print_array_type(h5tools_str_t *buffer, hid_t type, int ind) int ndims, i; hsize_t *dims=NULL; - if (H5T_ARRAY!=H5Tget_class(type)) + if (H5T_ARRAY != H5Tget_class(type)) return FALSE; ndims = H5Tget_array_ndims(type); if (ndims) { @@ -1218,7 +1171,7 @@ print_array_type(h5tools_str_t *buffer, hid_t type, int ind) H5Tget_array_dims2(type, dims); /* Print dimensions */ - for (i=0; i1) { order = H5Tget_order(type); - if (H5T_ORDER_LE==order) { + if (H5T_ORDER_LE == order) { order_s = " little-endian"; - } else if (H5T_ORDER_BE==order) { + } + else if (H5T_ORDER_BE == order) { order_s = " big-endian"; - } else if (H5T_ORDER_VAX==order) { + } + else if (H5T_ORDER_VAX == order) { order_s = " mixed-endian"; - } else { + } + else { order_s = "unknown-byte-order"; } - } else { + } + else { order_s = ""; } - h5tools_str_append(buffer, "%lu-bit%s bitfield", - (unsigned long)(8*H5Tget_size(type)), order_s); + h5tools_str_append(buffer, "%lu-bit%s bitfield", (unsigned long)(8*H5Tget_size(type)), order_s); print_precision(buffer, type, ind); return TRUE; } @@ -1292,16 +1240,9 @@ print_bitfield_type(h5tools_str_t *buffer, hid_t type, int ind) * there might be line-feeds inside the type definition). The * first line is assumed to have IND characters before it on * the same line (printed by the caller). - * - * Return: void - * - * Programmer: Robb Matzke - * Thursday, November 5, 1998 - * - * Modifications: - * Robb Matzke, 1999-06-11 * Prints the OID of shared data types. * + * Return: void *------------------------------------------------------------------------- */ static void @@ -1310,24 +1251,23 @@ print_type(h5tools_str_t *buffer, hid_t type, int ind) H5T_class_t data_class = H5Tget_class(type); /* Bad data type */ - if (type<0) { + if (type < 0) { h5tools_str_append(buffer,""); return; } /* Shared? If so then print the type's OID */ - if(H5Tcommitted(type)) { + if (H5Tcommitted(type)) { H5O_info_t oi; - if(H5Oget_info(type, &oi) >= 0) - h5tools_str_append(buffer,"shared-%lu:"H5_PRINTF_HADDR_FMT" ", - oi.fileno, oi.addr); + if (H5Oget_info(type, &oi) >= 0) + h5tools_str_append(buffer,"shared-%lu:"H5_PRINTF_HADDR_FMT" ", oi.fileno, oi.addr); else h5tools_str_append(buffer,"shared "); } /* end if */ /* Print the type */ - if(print_native_type(buffer, type, ind) || + if (print_native_type(buffer, type, ind) || print_ieee_type(buffer, type, ind) || print_cmpd_type(buffer, type, ind) || print_enum_type(buffer, type, ind) || @@ -1340,25 +1280,16 @@ print_type(h5tools_str_t *buffer, hid_t type, int ind) return; /* Unknown type */ - h5tools_str_append(buffer,"%lu-byte class-%u unknown", - (unsigned long)H5Tget_size(type), (unsigned)data_class); + h5tools_str_append(buffer,"%lu-byte class-%u unknown", (unsigned long)H5Tget_size(type), (unsigned)data_class); } - + /*------------------------------------------------------------------------- * Function: dump_dataset_values * * Purpose: Prints all values of a dataset. * * Return: void - * - * Programmer: Robb Matzke - * Tuesday, July 21, 1998 - * - * Modifications: - * Robb Matzke, 1999-09-27 - * Understands the simple_output_g switch which causes data to - * be displayed in a more machine-readable format. *------------------------------------------------------------------------- */ static void @@ -1398,7 +1329,8 @@ dump_dataset_values(hid_t dset) } outputformat.cmpd_sep = " "; - if (label_g) outputformat.cmpd_name = "%s="; + if (label_g) + outputformat.cmpd_name = "%s="; outputformat.elmt_suf1 = " "; outputformat.str_locale = ESCAPE_HTML; @@ -1441,7 +1373,7 @@ dump_dataset_values(hid_t dset) * command line switch was given. */ outputformat.raw = TRUE; } - else if (string_g && 1==size && H5T_INTEGER==H5Tget_class(f_type)) { + else if (string_g && 1 == size && H5T_INTEGER == H5Tget_class(f_type)) { /* Print 1-byte integer data as an ASCI character string instead of * integers if the `-s' or `--string' command-line option was given. */ outputformat.ascii = TRUE; @@ -1474,21 +1406,14 @@ dump_dataset_values(hid_t dset) PRINTVALSTREAM(rawoutstream, "\n"); } - + /*------------------------------------------------------------------------- * Function: list_attr * * Purpose: Prints information about attributes. * * Return: Success: 0 - * - * Failure: -1 - * - * Programmer: Robb Matzke - * Friday, June 5, 1998 - * - * Modifications: - * + * Failure: -1 *------------------------------------------------------------------------- */ static herr_t @@ -1521,7 +1446,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain print_string(&buffer, attr_name, TRUE); - if((attr = H5Aopen(obj, attr_name, H5P_DEFAULT))) { + if ((attr = H5Aopen(obj, attr_name, H5P_DEFAULT))) { space = H5Aget_space(attr); type = H5Aget_type(attr); @@ -1538,7 +1463,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain case H5S_SIMPLE: /* simple dataspace */ h5tools_str_append(&buffer, " {"); - for (i=0; i= 0) { + if (p_type >= 0) { /* VL data special information */ unsigned int vl_data = 0; /* contains VL datatypes */ @@ -1661,7 +1585,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain return 0; } - + /*------------------------------------------------------------------------- * Function: dataset_list1 * @@ -1671,14 +1595,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain * printed by the caller. * * Return: Success: 0 - * - * Failure: -1 - * - * Programmer: Robb Matzke - * Thursday, August 27, 1998 - * - * Modifications: - * + * Failure: -1 *------------------------------------------------------------------------- */ static herr_t @@ -1706,17 +1623,19 @@ dataset_list1(hid_t dset) space_type = H5Sget_simple_extent_type(space); ndims = H5Sget_simple_extent_dims(space, cur_size, max_size); h5tools_str_append(&buffer, " {"); - for (i=0; i0) { + else if (max_size[i] != cur_size[i] || verbose_g > 0) { h5tools_str_append(&buffer, "/"HSIZE_T_FORMAT, max_size[i]); } } - if (space_type==H5S_SCALAR) h5tools_str_append(&buffer, "SCALAR"); - else if (space_type==H5S_NULL) h5tools_str_append(&buffer, "NULL"); + if (space_type == H5S_SCALAR) + h5tools_str_append(&buffer, "SCALAR"); + else if (space_type == H5S_NULL) + h5tools_str_append(&buffer, "NULL"); h5tools_str_append(&buffer, "}"); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); H5Sclose (space); @@ -1726,7 +1645,7 @@ dataset_list1(hid_t dset) return 0; } - + /*------------------------------------------------------------------------- * Function: dataset_list2 * @@ -1734,14 +1653,7 @@ dataset_list1(hid_t dset) * information which is general to all objects. * * Return: Success: 0 - * - * Failure: -1 - * - * Programmer: Robb Matzke - * Thursday, August 27, 1998 - * - * Modifications: - * + * Failure: -1 *------------------------------------------------------------------------- */ static herr_t @@ -1777,7 +1689,7 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) h5tools_str_reset(&buffer); - if(verbose_g > 0) { + if (verbose_g > 0) { dcpl = H5Dget_create_plist(dset); space = H5Dget_space(dset); type = H5Dget_type(dset); @@ -1791,7 +1703,7 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) ndims = H5Pget_chunk(dcpl, (int)NELMTS(chsize), chsize/*out*/); h5tools_str_append(&buffer, " %-10s {", "Chunks:"); total = H5Tget_size(type); - for (i=0; i 0) { - for(i = 0, max_len = 0; i < nf; i++) { - if(H5Pget_external(dcpl, (unsigned)i, sizeof(f_name), f_name, NULL, NULL) < 0) + if ((nf = H5Pget_external_count(dcpl)) > 0) { + for (i = 0, max_len = 0; i < nf; i++) { + if (H5Pget_external(dcpl, (unsigned)i, sizeof(f_name), f_name, NULL, NULL) < 0) continue; n = print_string(NULL, f_name, TRUE); max_len = MAX(max_len, n); } /* end for */ - h5tools_str_append(&buffer, " %-10s %d external file%s\n", - "Extern:", nf, 1==nf?"":"s"); - h5tools_str_append(&buffer, " %4s %10s %10s %10s %s\n", - "ID", "DSet-Addr", "File-Addr", "Bytes", "File"); - h5tools_str_append(&buffer, " %4s %10s %10s %10s ", - "----", "----------", "----------", "----------"); - for (i=0; i0) - { + h5tools_str_append(&buffer, HSIZE_T_FORMAT" logical byte%s, "HSIZE_T_FORMAT" allocated byte%s", total, 1==total?"":"s", used, 1==used?"":"s"); + if (used>0) { utilization = ((double)total * (double)100.0f) / (double)used; h5tools_str_append(&buffer, ", %1.2f%% utilization", utilization); } @@ -1892,18 +1793,14 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) h5tools_str_append(&buffer, "\n"); /* Print information about raw data filters */ - if((nf = H5Pget_nfilters(dcpl)) > 0) { - for(i = 0; i < nf; i++) { + if ((nf = H5Pget_nfilters(dcpl)) > 0) { + for (i = 0; i < nf; i++) { cd_nelmts = NELMTS(cd_values); - filt_id = H5Pget_filter2(dcpl, (unsigned)i, &filt_flags, &cd_nelmts, - cd_values, sizeof(f_name), f_name, NULL); + filt_id = H5Pget_filter2(dcpl, (unsigned)i, &filt_flags, &cd_nelmts, cd_values, sizeof(f_name), f_name, NULL); f_name[sizeof(f_name) - 1] = '\0'; HDsnprintf(s, sizeof(s), "Filter-%d:", i); - h5tools_str_append(&buffer, " %-10s %s-%u %s {", s, - (f_name[0] ? f_name : "method"), - (unsigned)filt_id, - ((filt_flags & H5Z_FLAG_OPTIONAL) ? "OPT" : "")); - for(cd_num = 0; cd_num < cd_nelmts; cd_num++) + h5tools_str_append(&buffer, " %-10s %s-%u %s {", s, (f_name[0] ? f_name : "method"), (unsigned)filt_id, ((filt_flags & H5Z_FLAG_OPTIONAL) ? "OPT" : "")); + for (cd_num = 0; cd_num < cd_nelmts; cd_num++) h5tools_str_append(&buffer, "%s%u", (cd_num ? ", " : ""), cd_values[cd_num]); h5tools_str_append(&buffer, "}\n"); } /* end for */ @@ -1916,7 +1813,7 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); /* Print address information */ - if(address_g) + if (address_g) H5Ddebug(dset); /* Close stuff */ @@ -1927,13 +1824,13 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) h5tools_str_close(&buffer); - if(data_g) + if (data_g) dump_dataset_values(dset); return 0; } /* end dataset_list2() */ - + /*------------------------------------------------------------------------- * Function: datatype_list2 * @@ -1941,20 +1838,13 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) * information which is general to all objects. * * Return: Success: 0 - * - * Failure: -1 - * - * Programmer: Robb Matzke - * Thursday, November 5, 1998 - * - * Modifications: - * + * Failure: -1 *------------------------------------------------------------------------- */ static herr_t datatype_list2(hid_t type, const char H5_ATTR_UNUSED *name) { - if (verbose_g>0) { + if (verbose_g > 0) { hsize_t curr_pos = 0; /* total data element position */ h5tools_str_t buffer; /* string into which to render */ h5tools_context_t ctx; /* print context */ @@ -1975,19 +1865,14 @@ datatype_list2(hid_t type, const char H5_ATTR_UNUSED *name) return 0; } - + /*------------------------------------------------------------------------- * Function: list_obj * * Purpose: Prints information about an object * * Return: Success: 0 - * - * Failure: -1 - * - * Programmer: Quincey Koziol - * Tuesday, November 6, 2007 - * + * Failure: -1 *------------------------------------------------------------------------- */ static herr_t @@ -2006,26 +1891,26 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void h5tools_str_reset(&buffer); /* Print the link's name, either full name or base name */ - if(!iter->symlink_target) + if (!iter->symlink_target) print_obj_name(&buffer, iter, name, ""); /* Check object information */ - if(oinfo->type < 0 || oinfo->type >= H5O_TYPE_NTYPES) { + if (oinfo->type < 0 || oinfo->type >= H5O_TYPE_NTYPES) { h5tools_str_append(&buffer, "Unknown type(%d)", (int)oinfo->type); obj_type = H5O_TYPE_UNKNOWN; } - if(iter->symlink_target) + if (iter->symlink_target) h5tools_str_append(&buffer, "{"); - if(obj_type >= 0 && dispatch_g[obj_type].name) + if (obj_type >= 0 && dispatch_g[obj_type].name) h5tools_str_append(&buffer, "%s", dispatch_g[obj_type].name); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); /* Check if we've seen this object before */ - if(first_seen) { + if (first_seen) { h5tools_str_reset(&buffer); h5tools_str_append(&buffer, ", same as "); print_string(&buffer, first_seen, TRUE); - if(!iter->symlink_target) { + if (!iter->symlink_target) { h5tools_str_append(&buffer, "\n"); } h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); @@ -2036,7 +1921,7 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void /* Open the object. Not all objects can be opened. If this is the case * then return right away. */ - if(obj_type >= 0 && (obj = H5Oopen(iter->fid, name, H5P_DEFAULT)) < 0) { + if (obj_type >= 0 && (obj = H5Oopen(iter->fid, name, H5P_DEFAULT)) < 0) { h5tools_str_reset(&buffer); h5tools_str_append(&buffer, " *ERROR*\n"); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); @@ -2044,9 +1929,9 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void } /* end if */ /* List the first line of information for the object. */ - if(obj_type >= 0 && dispatch_g[obj_type].list1) + if (obj_type >= 0 && dispatch_g[obj_type].list1) (dispatch_g[obj_type].list1)(obj); - if(!iter->symlink_target || (verbose_g > 0)) { + if (!iter->symlink_target || (verbose_g > 0)) { h5tools_str_reset(&buffer); h5tools_str_append(&buffer, "\n"); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); @@ -2054,13 +1939,13 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void /* Show detailed information about the object, beginning with information * which is common to all objects. */ - if(verbose_g > 0) { + if (verbose_g > 0) { size_t buf_size = 0; char* comment = NULL; ssize_t cmt_bufsize = -1; /* Display attributes */ - if(obj_type >= 0) + if (obj_type >= 0) H5Aiterate2(obj, H5_INDEX_NAME, H5_ITER_INC, NULL, list_attr, NULL); /* Object location & reference count */ @@ -2070,7 +1955,7 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); /* Modification time */ - if(oinfo->mtime > 0) { + if (oinfo->mtime > 0) { char buf[256]; struct tm *tm; @@ -2078,7 +1963,7 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void tm = HDgmtime(&(oinfo->mtime)); else tm = HDlocaltime(&(oinfo->mtime)); - if(tm) { + if (tm) { HDstrftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S %Z", tm); h5tools_str_reset(&buffer); h5tools_str_append(&buffer, " %-10s %s\n", "Modified:", buf); @@ -2094,9 +1979,9 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void * If the call to H5Oget_comment returned an error, skip this block */ if (cmt_bufsize > 0) { comment = (char *)HDmalloc((size_t)cmt_bufsize + 1); /* new_size including null terminator */ - if(comment) { + if (comment) { cmt_bufsize = H5Oget_comment(obj, comment, (size_t)cmt_bufsize); - if(cmt_bufsize > 0) { + if (cmt_bufsize > 0) { comment[cmt_bufsize] = 0; h5tools_str_reset(&buffer); h5tools_str_append(&buffer, " %-10s \"", "Comment:"); @@ -2110,16 +1995,16 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void } /* end if */ /* Detailed list for object */ - if(obj_type >= 0 && dispatch_g[obj_type].list2) + if (obj_type >= 0 && dispatch_g[obj_type].list2) (dispatch_g[obj_type].list2)(obj, name); /* Close the object. */ - if(obj_type >= 0) + if (obj_type >= 0) H5Oclose(obj); } /* end else */ done: - if(iter->symlink_target) { + if (iter->symlink_target) { h5tools_str_reset(&buffer); h5tools_str_append(&buffer, "}\n"); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); @@ -2131,19 +2016,13 @@ done: } /* end list_obj() */ - /*------------------------------------------------------------------------- * Function: list_lnk * * Purpose: Prints information about a link * * Return: Success: 0 - * - * Failure: -1 - * - * Programmer: Quincey Koziol - * Thursday, November 8, 2007 - * + * Failure: -1 *------------------------------------------------------------------------- */ static herr_t @@ -2190,15 +2069,13 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) h5tools_str_append(&buffer, buf); h5tools_str_append(&buffer, "}"); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); - if(follow_symlink_g) - { + if (follow_symlink_g) { hbool_t orig_grp_literal = grp_literal_g; h5tools_str_reset(&buffer); h5tools_str_append(&buffer, " "); /* Check if we have already seen this softlink */ - if(symlink_is_visited(iter->symlink_list, linfo->type, NULL, buf)) - { + if (symlink_is_visited(iter->symlink_list, linfo->type, NULL, buf)) { h5tools_str_append(&buffer, "{Already Visited}\n"); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); goto done; @@ -2206,7 +2083,7 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); /* Add this link to the list of seen softlinks */ - if(symlink_visit_add(iter->symlink_list, linfo->type, NULL, buf) < 0) + if (symlink_visit_add(iter->symlink_list, linfo->type, NULL, buf) < 0) goto done; /* Adjust user data to specify that we are operating on the @@ -2215,11 +2092,10 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) /* Prevent recursive listing of soft link target if * recursive_g is off */ - if(!recursive_g) + if (!recursive_g) grp_literal_g = TRUE; /* Recurse through the soft link */ - if(visit_obj(iter->fid, name, iter) < 0) - { + if (visit_obj(iter->fid, name, iter) < 0) { grp_literal_g = orig_grp_literal; goto done; } @@ -2251,7 +2127,7 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) else if (no_dangling_link_g && ret == 0) iter->symlink_list->dangle_link = TRUE; - if(H5Lunpack_elink_val(buf, linfo->u.val_size, NULL, &filename, &path) < 0) + if (H5Lunpack_elink_val(buf, linfo->u.val_size, NULL, &filename, &path) < 0) goto done; h5tools_str_append(&buffer, "External Link {"); @@ -2259,21 +2135,19 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) h5tools_str_append(&buffer, "/"); if(*path != '/') h5tools_str_append(&buffer, "/"); - h5tools_str_append(&buffer, path); + h5tools_str_append(&buffer, path); h5tools_str_append(&buffer, "}"); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); /* Recurse through the external link */ /* keep the follow_elink_g for backward compatibility with -E */ - if(follow_link) - { + if (follow_link) { hbool_t orig_grp_literal = grp_literal_g; h5tools_str_reset(&buffer); h5tools_str_append(&buffer, " "); /* Check if we have already seen this elink */ - if(symlink_is_visited(iter->symlink_list, linfo->type, filename, path)) - { + if (symlink_is_visited(iter->symlink_list, linfo->type, filename, path)) { h5tools_str_append(&buffer, "{Already Visited}\n"); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); goto done; @@ -2281,8 +2155,7 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); /* Add this link to the list of seen elinks */ - if(symlink_visit_add(iter->symlink_list, linfo->type, filename, path) < 0) - { + if (symlink_visit_add(iter->symlink_list, linfo->type, filename, path) < 0) { goto done; } @@ -2292,11 +2165,11 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) /* Prevent recursive listing of external link target if * recursive_g is off */ - if(!recursive_g) + if (!recursive_g) grp_literal_g = TRUE; /* Recurse through the external link */ - if(visit_obj(iter->fid, name, iter) < 0) { + if (visit_obj(iter->fid, name, iter) < 0) { grp_literal_g = orig_grp_literal; goto done; } @@ -2325,7 +2198,7 @@ done: return 0; } /* end list_lnk() */ - + /*------------------------------------------------------------------------- * Function: visit_obj * @@ -2334,11 +2207,6 @@ done: * Return: * Success: 0 * Failure: -1 - * - * Programmer: Neil Fortner - * Wednesday, August 21, 2008 - * Mostly copied from main() - * *------------------------------------------------------------------------- */ static herr_t @@ -2357,8 +2225,8 @@ visit_obj(hid_t file, const char *oname, iter_t *iter) h5tools_str_reset(&buffer); /* Retrieve info for object to list */ - if(H5Oget_info_by_name(file, oname, &oi, H5P_DEFAULT) < 0) { - if(iter->symlink_target) { + if (H5Oget_info_by_name(file, oname, &oi, H5P_DEFAULT) < 0) { + if (iter->symlink_target) { h5tools_str_append(&buffer, "{**NOT FOUND**}\n"); iter->symlink_target = FALSE; } @@ -2370,9 +2238,9 @@ visit_obj(hid_t file, const char *oname, iter_t *iter) } /* end if */ /* Check for group iteration */ - if(H5O_TYPE_GROUP == oi.type && !grp_literal_g) { + if (H5O_TYPE_GROUP == oi.type && !grp_literal_g) { /* Get ID for group */ - if(!iter->symlink_target && (iter->gid = H5Gopen2(file, oname, H5P_DEFAULT)) < 0) { + if (!iter->symlink_target && (iter->gid = H5Gopen2(file, oname, H5P_DEFAULT)) < 0) { h5tools_str_append(&buffer, "%s: unable to open '%s' as group\n", iter->fname, oname); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); goto done; /* Previously "continue", when this code was in main(). @@ -2388,7 +2256,7 @@ visit_obj(hid_t file, const char *oname, iter_t *iter) h5trav_visit(file, oname, (hbool_t) (display_root_g || iter->symlink_target), recursive_g, list_obj, list_lnk, iter); /* Close group */ - if(!iter->symlink_target) + if (!iter->symlink_target) H5Gclose(iter->gid); } /* end if */ else { @@ -2405,7 +2273,7 @@ done: return retval; } - + /*------------------------------------------------------------------------- * Function: get_width * @@ -2415,14 +2283,7 @@ done: * were borrowed from the GNU less(1). * * Return: Success: Number of columns. - * - * Failure: Some default number of columms. - * - * Programmer: Robb Matzke - * Friday, November 6, 1998 - * - * Modifications: - * + * Failure: Some default number of columms. *------------------------------------------------------------------------- */ static int @@ -2433,7 +2294,7 @@ get_width(void) /* Try to get it from the COLUMNS environment variable first since it's * value is sometimes wrong. */ - if ((s=HDgetenv("COLUMNS")) && *s && isdigit((int)*s)) + if ((s = HDgetenv("COLUMNS")) && *s && isdigit((int)*s)) width = (int)HDstrtol(s, NULL, 0); #if defined(H5_HAVE_STRUCT_VIDEOCONFIG) && defined(H5_HAVE__GETVIDEOCONFIG) @@ -2481,7 +2342,8 @@ get_width(void) #endif /* Set to at least 1 */ - if (width<1) width = 1; + if (width < 1) + width = 1; return width; } @@ -2493,25 +2355,19 @@ get_width(void) * Return: * Success: TRUE (1) * Failure: FALSE (0) - * - * Programmer: - * Jonathan Kim (06/15/2010) - * *-------------------------------------------------------------------------*/ static hbool_t is_valid_args(void) { hbool_t ret = TRUE; - if(recursive_g && grp_literal_g) - { + if (recursive_g && grp_literal_g) { HDfprintf(rawerrorstream, "Error: 'recursive' option not compatible with 'group info' option!\n\n"); ret = FALSE; goto out; } - if(no_dangling_link_g && !follow_symlink_g) - { + if (no_dangling_link_g && !follow_symlink_g) { HDfprintf(rawerrorstream, "Error: --no-dangling-links must be used along with --follow-symlinks option!\n\n"); ret = FALSE; goto out; @@ -2521,19 +2377,13 @@ out: return ret; } - + /*------------------------------------------------------------------------- * Function: leave * * Purpose: Close HDF5 and MPI and call exit() * * Return: Does not return - * - * Programmer: Quincey Koziol - * Saturday, January 31, 2004 - * - * Modifications: - * *------------------------------------------------------------------------- */ static void @@ -2544,35 +2394,52 @@ leave(int ret) HDexit(ret); } - + /*------------------------------------------------------------------------- * Function: main * * Purpose: Opens a file and lists the specified group * * Return: Success: 0 - * - * Failure: 1 - * - * Programmer: Robb Matzke - * Monday, March 23, 1998 - * - * Modifications: - * + * Failure: 1 *------------------------------------------------------------------------- */ int main(int argc, const char *argv[]) { - hid_t file = -1; - char *fname = NULL, *oname = NULL, *x; + hid_t file = -1; + char *fname = NULL, *oname = NULL, *x; const char *s = NULL; - char *rest; - int argno; + char *rest; + int argno; static char root_name[] = "/"; char drivername[50]; const char *preferred_driver = NULL; - int err_exit = 0; + int err_exit = 0; + hid_t fapl_id = H5P_DEFAULT; + +#ifdef H5_HAVE_ROS3_VFD + /* default "anonymous" s3 configuration */ + H5FD_ros3_fapl_t ros3_fa = { + 1, /* fapl version */ + false, /* authenticate */ + "", /* aws region */ + "", /* access key id */ + "", /* secret access key */ + }; +#endif /* H5_HVAE_ROS3_VFD */ + +#ifdef H5_HAVE_LIBHDFS + /* "default" HDFS configuration */ + H5FD_hdfs_fapl_t hdfs_fa = { + 1, /* fapl version */ + "localhost", /* namenode name */ + 0, /* namenode port */ + "", /* kerberos ticket cache */ + "", /* user name */ + 2048, /* stream buffer size */ + }; +#endif /* H5_HAVE_LIBHDFS */ h5tools_setprogname(PROGRAMNAME); h5tools_setstatus(EXIT_SUCCESS); @@ -2589,90 +2456,258 @@ main(int argc, const char *argv[]) width_g = get_width(); /* Switches come before non-switch arguments */ - for(argno = 1; argno < argc && '-' == argv[argno][0]; argno++) { - if(!HDstrcmp(argv[argno], "--")) { + for (argno = 1; argno < argc && '-' == argv[argno][0]; argno++) { + if (!HDstrcmp(argv[argno], "--")) { /* Last switch */ argno++; break; - } else if(!HDstrcmp(argv[argno], "--help")) { + } + else if (!HDstrcmp(argv[argno], "--help")) { usage(); leave(EXIT_SUCCESS); - } else if(!HDstrcmp(argv[argno], "--address")) { + } + else if (!HDstrcmp(argv[argno], "--address")) { address_g = TRUE; - } else if(!HDstrcmp(argv[argno], "--data")) { + } + else if(!HDstrcmp(argv[argno], "--data")) { data_g = TRUE; - } else if(!HDstrcmp(argv[argno], "--enable-error-stack")) { + } + else if (!HDstrcmp(argv[argno], "--enable-error-stack")) { show_errors_g = TRUE; /* deprecated --errors */ - } else if(!HDstrcmp(argv[argno], "--errors")) { + } + else if (!HDstrcmp(argv[argno], "--errors")) { show_errors_g = TRUE; - } else if(!HDstrcmp(argv[argno], "--follow-symlinks")) { + } + else if (!HDstrcmp(argv[argno], "--follow-symlinks")) { follow_symlink_g = TRUE; - } else if(!HDstrcmp(argv[argno], "--no-dangling-links")) { + } + else if (!HDstrcmp(argv[argno], "--no-dangling-links")) { no_dangling_link_g = TRUE; - } else if(!HDstrcmp(argv[argno], "--external")) { + } + else if (!HDstrcmp(argv[argno], "--external")) { follow_elink_g = TRUE; - } else if(!HDstrcmp(argv[argno], "--full")) { + } + else if (!HDstrcmp(argv[argno], "--full")) { fullname_g = TRUE; - } else if(!HDstrcmp(argv[argno], "--group")) { + } + else if (!HDstrcmp(argv[argno], "--group")) { grp_literal_g = TRUE; - } else if(!HDstrcmp(argv[argno], "--label")) { + } + else if (!HDstrcmp(argv[argno], "--label")) { label_g = TRUE; - } else if(!HDstrcmp(argv[argno], "--recursive")) { + } + else if (!HDstrcmp(argv[argno], "--recursive")) { recursive_g = TRUE; fullname_g = TRUE; - } else if(!HDstrcmp(argv[argno], "--simple")) { + } + else if (!HDstrcmp(argv[argno], "--simple")) { simple_output_g = TRUE; - } else if(!HDstrcmp(argv[argno], "--string")) { + } + else if (!HDstrcmp(argv[argno], "--string")) { string_g = TRUE; - } else if(!HDstrncmp(argv[argno], "--vfd=", (size_t)6)) { + } + else if (!HDstrncmp(argv[argno], "--vfd=", (size_t)6)) { preferred_driver = argv[argno]+6; - } else if(!HDstrncmp(argv[argno], "--width=", (size_t)8)) { + } + else if (!HDstrncmp(argv[argno], "--width=", (size_t)8)) { width_g = (int)HDstrtol(argv[argno]+8, &rest, 0); - if(0 == width_g) + if (0 == width_g) no_line_wrap_g = TRUE; - else if(width_g < 0 || *rest) { + else if (width_g < 0 || *rest) { usage(); leave(EXIT_FAILURE); } - } else if(!HDstrcmp(argv[argno], "--width")) { - if((argno + 1) >= argc) { + } + else if (!HDstrcmp(argv[argno], "--width")) { + if ((argno + 1) >= argc) { usage(); leave(EXIT_FAILURE); - } else { + } + else { s = argv[++argno]; } width_g = (int)HDstrtol(s, &rest, 0); - if(width_g <= 0 || *rest) { + if (width_g <= 0 || *rest) { usage(); leave(EXIT_FAILURE); } - } else if(!HDstrcmp(argv[argno], "--verbose")) { + } + else if (!HDstrcmp(argv[argno], "--verbose")) { verbose_g++; - } else if(!HDstrcmp(argv[argno], "--version")) { + } + else if (!HDstrcmp(argv[argno], "--version")) { print_version(h5tools_getprogname()); leave(EXIT_SUCCESS); - } else if(!HDstrcmp(argv[argno], "--hexdump")) { + } + else if (!HDstrcmp(argv[argno], "--hexdump")) { hexdump_g = TRUE; - } else if(!HDstrncmp(argv[argno], "-w", (size_t)2)) { - if(argv[argno][2]) { + } + else if (!HDstrncmp(argv[argno], "-w", (size_t)2)) { + if (argv[argno][2]) { s = argv[argno] + 2; - } else if((argno + 1) >= argc) { + } + else if ((argno + 1) >= argc) { usage(); leave(EXIT_FAILURE); - } else { + } + else { s = argv[++argno]; } width_g = (int)HDstrtol(s, &rest, 0); - if(0 == width_g) + if(0 == width_g) { no_line_wrap_g = TRUE; + } else if(width_g < 0 || *rest) { usage(); leave(EXIT_FAILURE); } - } else if('-'!=argv[argno][1]) { + + } else if (!HDstrncmp(argv[argno], "--s3-cred=", (size_t)10)) { +#ifndef H5_HAVE_ROS3_VFD + HDfprintf(rawerrorstream, "Error: Read-Only S3 VFD is not enabled\n\n"); + usage(); + leave(EXIT_FAILURE); +#else + unsigned nelems = 0; + char *start = NULL; + char *s3cred_src = NULL; + char **s3cred = NULL; + char const *ccred[3]; + /* try to parse s3 credentials tuple + */ + start = strchr(argv[argno], '='); + if (start == NULL) { + HDfprintf(rawerrorstream, + "Error: Unable to parse null credentials tuple\n" + " For anonymous access, omit \"--s3-cred\" and use only \"--vfd=ros3\"\n\n"); + usage(); + leave(EXIT_FAILURE); + } + start++; + if (FAIL == parse_tuple((const char *)start, ',', &s3cred_src, &nelems, &s3cred)) { + HDfprintf(rawerrorstream, "Error: Unable to parse S3 credentials\n\n"); + usage(); + leave(EXIT_FAILURE); + } + /* sanity-check tuple count + */ + if (nelems != 3) { + HDfprintf(rawerrorstream, "Error: Invalid S3 credentials\n\n"); + usage(); + leave(EXIT_FAILURE); + } + ccred[0] = (const char *)s3cred[0]; + ccred[1] = (const char *)s3cred[1]; + ccred[2] = (const char *)s3cred[2]; + if (0 == h5tools_populate_ros3_fapl(&ros3_fa, ccred)) { + HDfprintf(rawerrorstream, "Error: Invalid S3 credentials\n\n"); + usage(); + leave(EXIT_FAILURE); + } + HDfree(s3cred); + HDfree(s3cred_src); +#endif /* H5_HAVE_ROS3_VFD */ + + } + else if (!HDstrncmp(argv[argno], "--hdfs-attrs=", (size_t)13)) { +#ifndef H5_HAVE_LIBHDFS + PRINTVALSTREAM(rawoutstream, "The HDFS VFD is not enabled.\n"); + leave(EXIT_FAILURE); +#else + /* Parse received configuration data and set fapl config struct */ + + hbool_t _debug = FALSE; + unsigned nelems = 0; + char const *start = NULL; + char *props_src = NULL; + char **props = NULL; + unsigned long k = 0; + + /* try to parse tuple + */ + if (_debug) { + HDfprintf(stderr, "configuring hdfs...\n"); + } + start = argv[argno]+13; /* should never segfault: worst case of */ + if (*start != '(') { /* null-termintor after '='. */ + + if (_debug) { + HDfprintf(stderr, " no tuple.\n"); + } + usage(); + leave(EXIT_FAILURE); + } + if (FAIL == parse_tuple((const char *)start, ',', &props_src, &nelems, &props)) { + HDfprintf(stderr, " unable to parse tuple.\n"); + usage(); + leave(EXIT_FAILURE); + } + + /* sanity-check tuple count + */ + if (nelems != 5) { + HDfprintf(stderr, " expected 5-ple, got `%d`\n", nelems); + usage(); + leave(EXIT_FAILURE); + } + if (_debug) { + HDfprintf(stderr, " got hdfs-attrs tuple: `(%s,%s,%s,%s,%s)`\n", + props[0], props[1], props[2], props[3], props[4]); + } + + /* Populate fapl configuration structure with given properties. + * WARNING: No error-checking is done on length of input strings... + * Silent overflow is possible, albeit unlikely. + */ + if (HDstrncmp(props[0], "", 1)) { + if (_debug) { + HDfprintf(stderr, " setting namenode name: %s\n", props[0]); + } + HDstrncpy(hdfs_fa.namenode_name, (const char *)props[0], HDstrlen(props[0])); + } + if (HDstrncmp(props[1], "", 1)) { + k = strtoul((const char *)props[1], NULL, 0); + if (errno == ERANGE) { + HDfprintf(stderr, " supposed port number wasn't.\n"); + leave(EXIT_FAILURE); + } + if (_debug) { + HDfprintf(stderr, " setting namenode port: %lu\n", k); + } + hdfs_fa.namenode_port = (int32_t)k; + } + if (HDstrncmp(props[2], "", 1)) { + if (_debug) { + HDfprintf(stderr, " setting kerb cache path: %s\n", props[2]); + } + HDstrncpy(hdfs_fa.kerberos_ticket_cache, (const char *)props[2], HDstrlen(props[2])); + } + if (HDstrncmp(props[3], "", 1)) { + if (_debug) { + HDfprintf(stderr, " setting username: %s\n", props[3]); + } + HDstrncpy(hdfs_fa.user_name, (const char *)props[3], HDstrlen(props[3])); + } + if (HDstrncmp(props[4], "", 1)) { + k = HDstrtoul((const char *)props[4], NULL, 0); + if (errno == ERANGE) { + HDfprintf(stderr, " supposed buffersize number wasn't.\n"); + leave(EXIT_FAILURE); + } + if (_debug) { + HDfprintf(stderr, " setting stream buffer size: %lu\n", k); + } + hdfs_fa.stream_buffer_size = (int32_t)k; + } + HDfree(props); + HDfree(props_src); +#endif /* H5_HAVE_LIBHDFS */ + + } + else if('-'!=argv[argno][1]) { /* Single-letter switches */ for(s = argv[argno] + 1; *s; s++) { switch(*s) { @@ -2742,7 +2777,9 @@ main(int argc, const char *argv[]) leave(EXIT_FAILURE); } /* end switch */ } /* end for */ - } else { + } + else { + HDfprintf(stderr, "Unknown argument: %s\n", argv[argno]); usage(); leave(EXIT_FAILURE); } @@ -2750,18 +2787,54 @@ main(int argc, const char *argv[]) /* If no arguments remain then print a usage message (instead of doing * absolutely nothing ;-) */ - if(argno >= argc) { + if (argno >= argc) { usage(); leave(EXIT_FAILURE); } /* end if */ /* Check for conflicting arguments */ - if (!is_valid_args()) - { + if (!is_valid_args()) { usage(); leave(EXIT_FAILURE); } + if (preferred_driver) { + void *conf_fa = NULL; + + if (!HDstrcmp(preferred_driver, "ros3")) { +#ifndef H5_HAVE_ROS3_VFD + HDfprintf(rawerrorstream, "Error: Read-Only S3 VFD not enabled.\n\n"); + usage(); + leave(EXIT_FAILURE); +#else + conf_fa = (void *)&ros3_fa; +#endif /* H5_HAVE_ROS3_VFD */ + + } + else if (!HDstrcmp(preferred_driver, "hdfs")) { +#ifndef H5_HAVE_LIBHDFS + PRINTVALSTREAM(rawoutstream, "The HDFS VFD is not enabled.\n"); + leave(EXIT_FAILURE); +#else + conf_fa = (void *)&hdfs_fa; +#endif /* H5_HAVE_LIBHDFS */ + } + + if (conf_fa != NULL) { + HDassert(fapl_id == H5P_DEFAULT); + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (fapl_id < 0) { + HDfprintf(rawerrorstream, "Error: Unable to create fapl entry\n\n"); + leave(EXIT_FAILURE); + } + if (0 == h5tools_set_configured_fapl(fapl_id, preferred_driver, conf_fa)) { + HDfprintf(rawerrorstream, "Error: Unable to set fapl\n\n"); + usage(); + leave(EXIT_FAILURE); + } + } + } /* preferred_driver defined */ + /* Turn off HDF5's automatic error printing unless you're debugging h5ls */ if(!show_errors_g) H5Eset_auto2(H5E_DEFAULT, NULL, NULL); @@ -2790,11 +2863,16 @@ main(int argc, const char *argv[]) oname = NULL; file = -1; - while(fname && *fname) { - file = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, preferred_driver, drivername, sizeof drivername); + while (fname && *fname) { + if (fapl_id != H5P_DEFAULT) { + file = H5Fopen(fname, H5F_ACC_RDONLY, fapl_id); + } + else { + file = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, preferred_driver, drivername, sizeof drivername); + } - if(file >= 0) { - if(verbose_g) + if (file >= 0) { + if (verbose_g) PRINTSTREAM(rawoutstream, "Opened \"%s\" with %s driver.\n", fname, drivername); break; /*success*/ } /* end if */ @@ -2802,27 +2880,27 @@ main(int argc, const char *argv[]) /* Shorten the file name; lengthen the object name */ x = oname; oname = HDstrrchr(fname, '/'); - if(x) + if (x) *x = '/'; - if(!oname) + if (!oname) break; *oname = '\0'; } /* end while */ - if(file < 0) { + if (file < 0) { HDfprintf(rawerrorstream, "%s: unable to open file\n", argv[argno-1]); HDfree(fname); err_exit = 1; continue; } /* end if */ - if(oname) { + if (oname) { /* Always use absolute paths to avoid confusion, keep track of where * to begin path name output */ *oname = '/'; iter.base_len = HDstrlen(oname); iter.base_len -= oname[iter.base_len-1] == '/'; x = oname; - if(NULL == (oname = HDstrdup(oname))) { + if (NULL == (oname = HDstrdup(oname))) { HDfprintf(rawerrorstream, "memory allocation failed\n"); leave(EXIT_FAILURE); } @@ -2831,9 +2909,9 @@ main(int argc, const char *argv[]) * is displayed if it is a link or non-group object */ iter.name_start = 1; } - if(!oname || !*oname) { + if (!oname || !*oname) { oname = root_name; - if(recursive_g) + if (recursive_g) display_root_g = TRUE; iter.base_len = 0; iter.name_start = 0; @@ -2854,9 +2932,9 @@ main(int argc, const char *argv[]) symlink_list.objs = NULL; /* Check for root group as object name */ - if(HDstrcmp(oname, root_name)) { + if (HDstrcmp(oname, root_name)) { /* Check the type of link given */ - if(H5Lget_info(file, oname, &li, H5P_DEFAULT) < 0) { + if (H5Lget_info(file, oname, &li, H5P_DEFAULT) < 0) { hsize_t curr_pos = 0; /* total data element position */ h5tools_str_t buffer; /* string into which to render */ h5tools_context_t ctx; /* print context */ @@ -2875,8 +2953,8 @@ main(int argc, const char *argv[]) li.type = H5L_TYPE_HARD; /* Open the object and display it's information */ - if(li.type == H5L_TYPE_HARD) { - if(visit_obj(file, oname, &iter) < 0) + if (li.type == H5L_TYPE_HARD) { + if (visit_obj(file, oname, &iter) < 0) leave(EXIT_FAILURE); } /* end if(li.type == H5L_TYPE_HARD) */ else { @@ -2887,11 +2965,10 @@ main(int argc, const char *argv[]) } H5Fclose(file); HDfree(fname); - if(x) + if (x) HDfree(oname); - for(u=0; u < symlink_list.nused; u++) - { + for (u = 0; u < symlink_list.nused; u++) { if (symlink_list.objs[u].type == H5L_TYPE_EXTERNAL) HDfree(symlink_list.objs[u].file); @@ -2904,6 +2981,13 @@ main(int argc, const char *argv[]) err_exit = 1; } /* end while */ + if (fapl_id != H5P_DEFAULT) { + if (0 < H5Pclose(fapl_id)) { + HDfprintf(rawerrorstream, "Error: Unable to set close fapl entry\n\n"); + leave(EXIT_FAILURE); + } + } + if (err_exit) leave(EXIT_FAILURE); else diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index e8c3409..b6f332a 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -481,7 +481,7 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, } /*H5T_REFERENCE*/ if (options->verbose) - printf(FORMAT_OBJ_ATTR, "attr", name); + HDprintf(FORMAT_OBJ_ATTR, "attr", name); /*------------------------------------------------------------------------- * close @@ -543,7 +543,7 @@ static int check_options(pack_opt_t *options) { */ if (options->verbose && have_request(options) /* only print if requested */) { if (options->all_layout == 1) { - printf("All objects to modify layout are...\n"); + HDprintf("All objects to modify layout are...\n"); switch (options->layout_g) { case H5D_COMPACT: strcpy(slayout, "compact"); @@ -561,17 +561,17 @@ static int check_options(pack_opt_t *options) { strcpy(slayout, "invalid layout\n"); HGOTO_DONE(FAIL); } - printf(" Apply %s layout to all", slayout); + HDprintf(" Apply %s layout to all", slayout); if (H5D_CHUNKED == options->layout_g) { - printf("with dimension [ "); + HDprintf("with dimension [ "); for (j = 0; j < options->chunk_g.rank; j++) - printf("%d ", (int) options->chunk_g.chunk_lengths[j]); - printf("]"); + HDprintf("%d ", (int) options->chunk_g.chunk_lengths[j]); + HDprintf("]"); } - printf("\n"); + HDprintf("\n"); } else - printf("No all objects to modify layout\n"); + HDprintf("No all objects to modify layout\n"); }/* verbose */ for (i = 0; i < options->op_tbl->nelems; i++) { @@ -579,16 +579,16 @@ static int check_options(pack_opt_t *options) { if (options->op_tbl->objs[i].chunk.rank > 0) { if (options->verbose) { - printf(" <%s> with chunk size ", name); + HDprintf(" <%s> with chunk size ", name); for (k = 0; k < options->op_tbl->objs[i].chunk.rank; k++) - printf("%d ", (int) options->op_tbl->objs[i].chunk.chunk_lengths[k]); - printf("\n"); + HDprintf("%d ", (int) options->op_tbl->objs[i].chunk.chunk_lengths[k]); + HDprintf("\n"); } has_ck = 1; } else if (options->op_tbl->objs[i].chunk.rank == -2) { if (options->verbose) - printf(" <%s> %s\n", name, "NONE (contiguous)"); + HDprintf(" <%s> %s\n", name, "NONE (contiguous)"); has_ck = 1; } } @@ -603,33 +603,33 @@ static int check_options(pack_opt_t *options) { if (options->verbose && have_request(options) /* only print if requested */) { if (options->all_filter == 1) { - printf("All objects to apply filter are...\n"); + HDprintf("All objects to apply filter are...\n"); for (k = 0; k < options->n_filter_g; k++) { H5Z_filter_t filtn = options->filter_g[k].filtn; if (filtn < 0) { - printf(" Unknown\n"); + HDprintf(" Unknown\n"); continue; } switch (filtn) { case H5Z_FILTER_NONE: - printf(" Uncompress all\n"); + HDprintf(" Uncompress all\n"); break; case H5Z_FILTER_SHUFFLE: case H5Z_FILTER_FLETCHER32: - printf(" All with %s\n", get_sfilter(filtn)); + HDprintf(" All with %s\n", get_sfilter(filtn)); break; case H5Z_FILTER_SZIP: case H5Z_FILTER_DEFLATE: - printf(" All with %s, parameter %d\n", get_sfilter(filtn), options->filter_g[k].cd_values[0]); + HDprintf(" All with %s, parameter %d\n", get_sfilter(filtn), options->filter_g[k].cd_values[0]); break; default: - printf(" User Defined %d\n", filtn); + HDprintf(" User Defined %d\n", filtn); break; } /* k */ }; } else - printf("No all objects to apply filter\n"); + HDprintf("No all objects to apply filter\n"); } /* verbose */ for (i = 0; i < options->op_tbl->nelems; i++) { @@ -640,9 +640,9 @@ static int check_options(pack_opt_t *options) { if (options->verbose) { if(pack.filter[j].filtn >= 0) { if(pack.filter[j].filtn > H5Z_FILTER_SCALEOFFSET) - printf(" <%s> with %s filter %d\n", name, get_sfilter(pack.filter[j].filtn), pack.filter[j].filtn); + HDprintf(" <%s> with %s filter %d\n", name, get_sfilter(pack.filter[j].filtn), pack.filter[j].filtn); else - printf(" <%s> with %s filter\n", name, get_sfilter(pack.filter[j].filtn)); + HDprintf(" <%s> with %s filter\n", name, get_sfilter(pack.filter[j].filtn)); } } has_cp = 1; @@ -673,7 +673,7 @@ static int check_options(pack_opt_t *options) { */ if (options->ublock_filename != NULL && options->ublock_size == 0) { if (options->verbose) { - printf("Warning: user block size missing for file %s. Assigning a default size of 1024...\n", options->ublock_filename); + HDprintf("Warning: user block size missing for file %s. Assigning a default size of 1024...\n", options->ublock_filename); options->ublock_size = 1024; } } @@ -708,7 +708,7 @@ static int check_objects(const char* fname, pack_opt_t *options) { hid_t did = -1; hid_t sid = -1; unsigned int i; - unsigned int uf; + int ifil; trav_table_t *travt = NULL; /* nothing to do */ @@ -742,31 +742,31 @@ static int check_objects(const char* fname, pack_opt_t *options) { */ if (options->verbose) - printf("Opening file. Searching %d objects to modify ...\n", travt->nobjs); + HDprintf("Opening file. Searching %zu objects to modify ...\n", travt->nobjs); for (i = 0; i < options->op_tbl->nelems; i++) { char* name = options->op_tbl->objs[i].path; if (options->verbose) - printf(" <%s>", name); + HDprintf(" <%s>", name); /* the input object names are present in the file and are valid */ if (h5trav_getindext(name, travt) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "%s Could not find <%s> in file <%s>. Exiting...\n", (options->verbose ? "\n" : ""), name, fname); if (options->verbose) - printf("...Found\n"); + HDprintf("...Found\n"); - for (uf = 0; uf < options->op_tbl->objs[i].nfilters; uf++) { - if (options->op_tbl->objs[i].filter[uf].filtn < 0) + for (ifil = 0; ifil < options->op_tbl->objs[i].nfilters; ifil++) { + if (options->op_tbl->objs[i].filter[ifil].filtn < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "invalid filter"); /* check for extra filter conditions */ - switch (options->op_tbl->objs[i].filter[uf].filtn) { + switch (options->op_tbl->objs[i].filter[ifil].filtn) { /* chunk size must be smaller than pixels per block */ case H5Z_FILTER_SZIP: { int j; hsize_t csize = 1; - unsigned ppb = options->op_tbl->objs[i].filter[uf].cd_values[0]; + unsigned ppb = options->op_tbl->objs[i].filter[ifil].cd_values[0]; hsize_t dims[H5S_MAX_RANK]; int rank; @@ -794,7 +794,7 @@ static int check_objects(const char* fname, pack_opt_t *options) { } if (csize < ppb) { - printf(" \n"); + HDprintf(" \n"); HGOTO_DONE(0); } } @@ -802,7 +802,7 @@ static int check_objects(const char* fname, pack_opt_t *options) { default: break; } - } /* for uf */ + } /* for ifil */ } /* for i */ done: diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index 5b7bb0b..211d254 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -94,21 +94,21 @@ typedef struct { /* all the above, ready to go to the hrepack call */ typedef struct { - pack_opttbl_t *op_tbl; /*table with all -c and -f options */ - int all_layout; /*apply the layout to all objects */ - int all_filter; /*apply the filter to all objects */ + pack_opttbl_t *op_tbl; /*table with all -c and -f options */ + int all_layout; /*apply the layout to all objects */ + int all_filter; /*apply the filter to all objects */ filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */ - int n_filter_g; /*number of global filters */ - chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ - H5D_layout_t layout_g; /*global layout information for the ALL case */ - int verbose; /*verbose mode */ - hsize_t min_comp; /*minimum size to compress, in bytes */ - int use_native; /*use a native type in write */ - int latest; /*pack file with the latest file format */ - int grp_compact; /* Set the maximum number of links to store as header messages in the group */ - int grp_indexed; /* Set the minimum number of links to store in the indexed format */ - int msg_size[8]; /* Minimum size of shared messages: dataspace, - datatype, fill value, filter pipleline, attribute */ + int n_filter_g; /*number of global filters */ + chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ + H5D_layout_t layout_g; /*global layout information for the ALL case */ + int verbose; /*verbose mode */ + hsize_t min_comp; /*minimum size to compress, in bytes */ + int use_native; /*use a native type in write */ + int latest; /*pack file with the latest file format */ + int grp_compact; /* Set the maximum number of links to store as header messages in the group */ + int grp_indexed; /* Set the minimum number of links to store in the indexed format */ + int msg_size[8]; /* Minimum size of shared messages: dataspace, + datatype, fill value, filter pipleline, attribute */ const char *ublock_filename; /* user block file name */ hsize_t ublock_size; /* user block size */ hsize_t meta_block_size; /* metadata aggregation block size (for H5Pset_meta_block_size) */ diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 6b9b079..02a3c00 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -264,7 +264,7 @@ print_user_block(fnamein, fidin); *------------------------------------------------------------------------- */ if (options->verbose) - printf("Making new file ...\n"); + HDprintf("Making new file ...\n"); if ((fidout = H5Fcreate(fnameout, H5F_ACC_TRUNC, fcpl, fapl)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fcreate could not create file <%s>:", fnameout); @@ -592,9 +592,9 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, */ if (options->verbose) { - printf("-----------------------------------------\n"); - printf(" Type Filter (Compression) Name\n"); - printf("-----------------------------------------\n"); + HDprintf("-----------------------------------------\n"); + HDprintf(" Type Filter (Compression) Name\n"); + HDprintf("-----------------------------------------\n"); } if (travt->objs) { @@ -613,7 +613,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, */ case H5TRAV_TYPE_GROUP: if (options->verbose) - printf(FORMAT_OBJ, "group", travt->objs[i].name); + HDprintf(FORMAT_OBJ, "group", travt->objs[i].name); /* open input group */ if ((grp_in = H5Gopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) @@ -848,7 +848,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, if (dset_out == FAIL) { H5Epush2(H5tools_ERR_STACK_g, __FILE__, FUNC, __LINE__, H5tools_ERR_CLS_g, H5E_tools_g, H5E_tools_min_id_g, "H5Dcreate2 failed"); if (options->verbose) - printf(" warning: could not create dataset <%s>. Applying original settings\n", travt->objs[i].name); + HDprintf(" warning: could not create dataset <%s>. Applying original settings\n", travt->objs[i].name); if ((dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_in, H5P_DEFAULT)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dcreate2 failed"); @@ -1011,10 +1011,10 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, (in case there was a filter) */ if (has_filter && apply_s == 0) - printf(" \n", travt->objs[i].name, (int) options->min_comp); + HDprintf(" \n", travt->objs[i].name, (int) options->min_comp); if (has_filter && apply_f == 0) - printf(" \n", travt->objs[i].name); + HDprintf(" \n", travt->objs[i].name); } /* verbose */ /*------------------------------------------------------------------------- @@ -1095,7 +1095,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed"); if (options->verbose) - printf(FORMAT_OBJ, "dset", travt->objs[i].name); + HDprintf(FORMAT_OBJ, "dset", travt->objs[i].name); } /* end do we have request for filter/chunking */ break; @@ -1106,7 +1106,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, */ case H5TRAV_TYPE_NAMED_DATATYPE: if (options->verbose) - printf(FORMAT_OBJ, "type", travt->objs[i].name); + HDprintf(FORMAT_OBJ, "type", travt->objs[i].name); if ((type_in = H5Topen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Topen2 failed"); @@ -1145,13 +1145,13 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, case H5TRAV_TYPE_LINK: case H5TRAV_TYPE_UDLINK: if (options->verbose) - printf(FORMAT_OBJ, "link", travt->objs[i].name); + HDprintf(FORMAT_OBJ, "link", travt->objs[i].name); if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Lcopy failed"); if (options->verbose) - printf(FORMAT_OBJ, "link", travt->objs[i].name); + HDprintf(FORMAT_OBJ, "link", travt->objs[i].name); break; default: @@ -1294,15 +1294,15 @@ print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr) }/*i*/ if (!pr) - printf(FORMAT_OBJ, "dset", objname); + HDprintf(FORMAT_OBJ, "dset", objname); else { - char str[255], temp[28]; + char str[512], temp[512]; HDstrcpy(str, "dset "); HDstrcat(str, strfilter); HDsprintf(temp, " (%.3f:1)", ratio); HDstrcat(str, temp); - printf(FORMAT_OBJ, str, objname); + HDprintf(FORMAT_OBJ, str, objname); } } @@ -1429,10 +1429,10 @@ print_user_block(const char *filename, hid_t fid) for (i = 0; i < nread; i++) { - printf("%c ", rbuf[i]); + HDprintf("%c ", rbuf[i]); } - printf("\n"); + HDprintf("\n"); if (nread < 0) { HGOTO_ERROR(H5E_tools_g, H5E_tools_min_id_g, "nread < 0"); diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c index 8cc7b92..c42d284 100644 --- a/tools/h5repack/h5repack_filters.c +++ b/tools/h5repack/h5repack_filters.c @@ -42,7 +42,8 @@ static int aux_copy_obj(hid_t dcpl_id, /* dataset creation property list */ H5D_layout_t layout; int rank; /* rank of dataset */ hsize_t chsize[64]; /* chunk size in elements */ - unsigned int i; + int i; + unsigned u; /* get information about input filters */ if ((nfilters = H5Pget_nfilters(dcpl_id)) < 0) @@ -65,8 +66,8 @@ static int aux_copy_obj(hid_t dcpl_id, /* dataset creation property list */ if ((rank = H5Pget_chunk(dcpl_id, NELMTS(chsize), chsize/*out*/)) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_chunk failed"); objout->chunk.rank = rank; - for (i = 0; i < rank; i++) - objout->chunk.chunk_lengths[i] = chsize[i]; + for (u = 0; u < (unsigned)rank; u++) + objout->chunk.chunk_lengths[u] = chsize[u]; } done: @@ -85,8 +86,7 @@ static int aux_find_obj(const char* name, /* object name from traverse list */ pack_info_t *obj /*OUT*/) /* info about object to filter */ { char *pdest = NULL; - char *pname = NULL; - int result; + const char *pname = NULL; unsigned int i; for (i = 0; i < options->op_tbl->nelems; i++) { @@ -335,12 +335,14 @@ int apply_filters(const char* name, /* object name from traverse list */ sm_nbytes = msize; for (i = rank; i > 0; --i) { - hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes; + hsize_t size = 0; + if(sm_nbytes == 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "number of bytes per stripmine must be > 0"); + size = H5TOOLS_BUFSIZE / sm_nbytes; if (size == 0) /* datum size > H5TOOLS_BUFSIZE */ size = 1; sm_size[i - 1] = MIN(dims[i - 1], size); sm_nbytes *= sm_size[i - 1]; - HDassert(sm_nbytes > 0); } for (i = 0; i < rank; i++) { diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index 2bd0fbb..9c96e56 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -77,6 +77,7 @@ static void usage(const char *prog) { PRINTVALSTREAM(rawoutstream, " -v, --verbose Verbose mode, print object information\n"); PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n"); PRINTVALSTREAM(rawoutstream, " -n, --native Use a native HDF5 type when repacking\n"); + PRINTVALSTREAM(rawoutstream, " --enable-error-stack Prints messages from the HDF5 error stack as they occur\n"); PRINTVALSTREAM(rawoutstream, " -L, --latest Use latest version of file format\n"); PRINTVALSTREAM(rawoutstream, " -c L1, --compact=L1 Maximum number of links in header messages\n"); PRINTVALSTREAM(rawoutstream, " -d L2, --indexed=L2 Minimum number of links in the indexed format\n"); @@ -106,9 +107,6 @@ static void usage(const char *prog) { PRINTVALSTREAM(rawoutstream, " F - is the shared object header message type, any of . If F is not specified, S applies to all messages\n"); PRINTVALSTREAM(rawoutstream, "\n"); - PRINTVALSTREAM(rawoutstream, " --enable-error-stack Prints messages from the HDF5 error stack as they\n"); - PRINTVALSTREAM(rawoutstream, " occur.\n"); - PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, " FILT - is a string with the format:\n"); PRINTVALSTREAM(rawoutstream, "\n"); PRINTVALSTREAM(rawoutstream, " :=\n"); diff --git a/tools/h5repack/h5repack_refs.c b/tools/h5repack/h5repack_refs.c index 59b652c..fc9e897 100644 --- a/tools/h5repack/h5repack_refs.c +++ b/tools/h5repack/h5repack_refs.c @@ -158,7 +158,7 @@ int do_copy_refobjs(hid_t fidin, if(nelmts) { buf = (hobj_ref_t *)HDmalloc((unsigned)(nelmts * msize)); if(buf==NULL) { - printf("cannot read into memory\n" ); + HDprintf("cannot read into memory\n" ); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); } /* end if */ if(H5Dread(dset_in, mtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) @@ -166,7 +166,7 @@ int do_copy_refobjs(hid_t fidin, refbuf = (hobj_ref_t*) HDcalloc((unsigned)nelmts, msize); if(refbuf == NULL){ - printf("cannot allocate memory\n" ); + HDprintf("cannot allocate memory\n" ); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDcalloc failed"); } /* end if */ for(u = 0; u < nelmts; u++) { @@ -183,8 +183,8 @@ int do_copy_refobjs(hid_t fidin, if(H5Rcreate(&refbuf[u], fidout, refname, H5R_OBJECT, (hid_t)-1) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Rcreate failed"); if(options->verbose) { - printf(FORMAT_OBJ,"dset",travt->objs[i].name ); - printf("object <%s> object reference created to <%s>\n", + HDprintf(FORMAT_OBJ,"dset",travt->objs[i].name ); + HDprintf("object <%s> object reference created to <%s>\n", travt->objs[i].name, refname); } @@ -234,7 +234,7 @@ int do_copy_refobjs(hid_t fidin, if(nelmts) { buf = (hdset_reg_ref_t *)HDmalloc((unsigned)(nelmts * msize)); if(buf == NULL) { - printf("cannot read into memory\n"); + HDprintf("cannot read into memory\n"); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); } /* end if */ if(H5Dread(dset_in, mtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) @@ -246,7 +246,7 @@ int do_copy_refobjs(hid_t fidin, */ refbuf = (hdset_reg_ref_t *)HDcalloc(sizeof(hdset_reg_ref_t), (size_t)nelmts); /*init to zero */ if(refbuf == NULL) { - printf("cannot allocate memory\n"); + HDprintf("cannot allocate memory\n"); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDcalloc failed"); } /* end if */ @@ -271,8 +271,8 @@ int do_copy_refobjs(hid_t fidin, if(H5Sclose(region_id) < 0) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed"); if(options->verbose) { - printf(FORMAT_OBJ,"dset",travt->objs[i].name ); - printf("object <%s> region reference created to <%s>\n", + HDprintf(FORMAT_OBJ,"dset",travt->objs[i].name ); + HDprintf("object <%s> region reference created to <%s>\n", travt->objs[i].name, refname); } @@ -592,7 +592,7 @@ static int copy_refs_attr(hid_t loc_in, if((is_ref || is_ref_array) && (H5R_OBJ_REF_BUF_SIZE==msize)) { buf = (hobj_ref_t *)HDmalloc((unsigned)(nelmts * msize)); if(buf == NULL) { - printf("cannot read into memory\n"); + HDprintf("cannot read into memory\n"); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); } /* end if */ if(H5Aread(attr_id, mtype_id, buf) < 0) @@ -600,12 +600,12 @@ static int copy_refs_attr(hid_t loc_in, refbuf = (hobj_ref_t *)HDcalloc((unsigned)nelmts, msize); if(refbuf == NULL) { - printf("cannot allocate memory\n"); + HDprintf("cannot allocate memory\n"); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDcalloc failed"); } /* end if */ for(i = 0; i < (unsigned)nelmts; i++) { - if (update_ref_value(attr_id, H5R_OBJECT, &((hobj_ref_t *)buf)[i], fidout, &((hobj_ref_t *)refbuf)[i], travt)<0) + if(update_ref_value(attr_id, H5R_OBJECT, &((hobj_ref_t *)buf)[i], fidout, &((hobj_ref_t *)refbuf)[i], travt) < 0) continue; if(options->verbose) printf("object <%s> reference created to <%s>\n", name, refname); @@ -616,7 +616,7 @@ static int copy_refs_attr(hid_t loc_in, buf = (hdset_reg_ref_t *)HDmalloc((unsigned)(nelmts * msize)); if(buf == NULL) { - printf( "cannot read into memory\n" ); + HDprintf( "cannot read into memory\n" ); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); } /* end if */ if(H5Aread(attr_id, mtype_id, buf) < 0) @@ -628,12 +628,12 @@ static int copy_refs_attr(hid_t loc_in, */ refbuf = (hdset_reg_ref_t *)HDcalloc(sizeof(hdset_reg_ref_t), (size_t)nelmts); /*init to zero */ if(refbuf == NULL) { - printf( "cannot allocate memory\n" ); + HDprintf( "cannot allocate memory\n" ); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDcalloc failed"); } /* end if */ for(i = 0; i < (unsigned)nelmts; i++) { - if (update_ref_value(attr_id, H5R_DATASET_REGION, &((hdset_reg_ref_t *)buf)[i], fidout, &((hdset_reg_ref_t *)refbuf)[i], travt)<0) + if(update_ref_value(attr_id, H5R_DATASET_REGION, &((hdset_reg_ref_t *)buf)[i], fidout, &((hdset_reg_ref_t *)refbuf)[i], travt) < 0) continue; if(options->verbose) printf("object <%s> region reference created to <%s>\n", name, refname); @@ -646,7 +646,7 @@ static int copy_refs_attr(hid_t loc_in, refbuf = buf; /* reuse the read buffer for write */ if(buf == NULL) { - printf( "cannot read into memory\n" ); + HDprintf( "cannot read into memory\n" ); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); } /* end if */ @@ -687,7 +687,7 @@ static int copy_refs_attr(hid_t loc_in, refbuf = buf; /* reuse the read buffer for write */ if(buf == NULL) { - printf( "cannot read into memory\n" ); + HDprintf( "cannot read into memory\n" ); HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed"); } /* end if */ @@ -700,7 +700,7 @@ static int copy_refs_attr(hid_t loc_in, size_t idx = (i * msize) + H5Tget_member_offset(mtype_id, ref_comp_index[j]); hobj_ref_t ref_out; - if (update_ref_value(attr_id, H5R_OBJECT, (hobj_ref_t *)(((char *)buf)+idx), fidout, &ref_out, travt)<0) + if (update_ref_value(attr_id, H5R_OBJECT, (hobj_ref_t *)((void *)(((char *)buf)+idx)), fidout, &ref_out, travt) < 0) /* Extra (void *) cast to quiet "cast to create alignment" warning - 2019/07/05, QAK */ continue; HDmemcpy(((char *)buf)+idx, &ref_out, ref_comp_size[j]); } /* if */ diff --git a/tools/h5repack/testfiles/h5repack-help.txt b/tools/h5repack/testfiles/h5repack-help.txt index 337e004..8161157 100644 --- a/tools/h5repack/testfiles/h5repack-help.txt +++ b/tools/h5repack/testfiles/h5repack-help.txt @@ -6,6 +6,7 @@ usage: h5repack [OPTIONS] file1 file2 -v, --verbose Verbose mode, print object information -V, --version Print version number and exit -n, --native Use a native HDF5 type when repacking + --enable-error-stack Prints messages from the HDF5 error stack as they occur -L, --latest Use latest version of file format -c L1, --compact=L1 Maximum number of links in header messages -d L2, --indexed=L2 Minimum number of links in the indexed format @@ -35,9 +36,6 @@ usage: h5repack [OPTIONS] file1 file2 F - is the shared object header message type, any of . If F is not specified, S applies to all messages - --enable-error-stack Prints messages from the HDF5 error stack as they - occur. - FILT - is a string with the format: := diff --git a/tools/h5stat/h5stat.c b/tools/h5stat/h5stat.c index 54f252b..e3e19d5 100644 --- a/tools/h5stat/h5stat.c +++ b/tools/h5stat/h5stat.c @@ -63,14 +63,14 @@ typedef struct iter_t { ohdr_info_t group_ohdr_info; /* Object header information for groups */ hsize_t max_attrs; /* Maximum attributes from a group */ - unsigned long *num_small_attrs; /* Size of small attributes tracked */ + unsigned long *num_small_attrs; /* Size of small attributes tracked */ unsigned attr_nbins; /* Number of bins for attribute counts */ unsigned long *attr_bins; /* Pointer to array of bins for attribute counts */ unsigned max_dset_rank; /* Maximum rank of dataset */ unsigned long dset_rank_count[H5S_MAX_RANK]; /* Number of datasets of each rank */ hsize_t max_dset_dims; /* Maximum dimension size of dataset */ - unsigned long *small_dset_dims; /* Size of dimensions of small datasets tracked */ + unsigned long *small_dset_dims; /* Size of dimensions of small datasets tracked */ unsigned long dset_layouts[H5D_NLAYOUTS]; /* Type of storage for each dataset */ unsigned long dset_comptype[H5_NFILTERS_IMPL]; /* Number of currently implemented filters */ unsigned long dset_ntypes; /* Number of diff. dataset datatypes found */ @@ -97,6 +97,33 @@ typedef struct iter_t { } iter_t; +static const char *drivername = ""; + +#ifdef H5_HAVE_ROS3_VFD +/* default "anonymous" s3 configuration + */ +static H5FD_ros3_fapl_t ros3_fa = { + 1, /* fapl version */ + false, /* authenticate */ + "", /* aws region */ + "", /* access key id */ + "", /* secret access key */ +}; +#endif /* H5_HAVE_ROS3_VFD */ + +#ifdef H5_HAVE_LIBHDFS +/* default HDFS access configuration + */ +static H5FD_hdfs_fapl_t hdfs_fa = { + 1, /* fapl version */ + "localhost", /* namenode name */ + 0, /* namenode port */ + "", /* kerberos ticket cache */ + "", /* user name */ + 2048, /* stream buffer size */ +}; +#endif /* H5_HAVE_LIBHDFS */ + static int display_all = TRUE; /* Enable the printing of selected statistics */ @@ -124,7 +151,7 @@ struct handler_t { char **obj; }; -static const char *s_opts ="Aa:Ddm:EFfhGgl:STO:V"; +static const char *s_opts ="Aa:Ddm:EFfhGgl:STO:Vw:"; /* e.g. "filemetadata" has to precede "file"; "groupmetadata" has to precede "group" etc. */ static struct long_options l_opts[] = { {"help", no_arg, 'h'}, @@ -216,6 +243,8 @@ static struct long_options l_opts[] = { { "summ", no_arg, 'S' }, { "sum", no_arg, 'S' }, { "su", no_arg, 'S' }, + { "s3-cred", require_arg, 'w' }, + { "hdfs-attrs", require_arg, 'H' }, { NULL, 0, '\0' } }; @@ -227,7 +256,7 @@ leave(int ret) } - + /*------------------------------------------------------------------------- * Function: usage * @@ -264,9 +293,19 @@ static void usage(const char *prog) HDfprintf(stdout, " than 0. The default threshold is 10.\n"); HDfprintf(stdout, " -S, --summary Print summary of file space information\n"); HDfprintf(stdout, " --enable-error-stack Prints messages from the HDF5 error stack as they occur\n"); + HDfprintf(stdout, " --s3-cred= Access file on S3, using provided credential\n"); + HDfprintf(stdout, " :: (region,id,key)\n"); + HDfprintf(stdout, " If == \"(,,)\", no authentication is used.\n"); + HDfprintf(stdout, " --hdfs-attrs= Access a file on HDFS with given configuration\n"); + HDfprintf(stdout, " attributes.\n"); + HDfprintf(stdout, " :: (,,\n"); + HDfprintf(stdout, " ,,\n"); + HDfprintf(stdout, " )\n"); + HDfprintf(stdout, " If an attribute is empty, a default value will be\n"); + HDfprintf(stdout, " used.\n"); } - + /*------------------------------------------------------------------------- * Function: ceil_log10 * @@ -293,7 +332,7 @@ ceil_log10(unsigned long x) return ret; } /* ceil_log10() */ - + /*------------------------------------------------------------------------- * Function: attribute_stats * @@ -326,12 +365,12 @@ attribute_stats(iter_t *iter, const H5O_info_t *oi) /* Add attribute count to proper bin */ bin = ceil_log10((unsigned long)oi->num_attrs); if((bin + 1) > iter->attr_nbins) { - iter->attr_bins = (unsigned long *)HDrealloc(iter->attr_bins, (bin + 1) * sizeof(unsigned long)); + iter->attr_bins = (unsigned long *)HDrealloc(iter->attr_bins, (bin + 1) * sizeof(unsigned long)); HDassert(iter->attr_bins); /* Initialize counts for intermediate bins */ while(iter->attr_nbins < bin) - iter->attr_bins[iter->attr_nbins++] = 0; + iter->attr_bins[iter->attr_nbins++] = 0; iter->attr_nbins++; /* Initialize count for new bin */ @@ -343,7 +382,7 @@ attribute_stats(iter_t *iter, const H5O_info_t *oi) return 0; } /* end attribute_stats() */ - + /*------------------------------------------------------------------------- * Function: group_stats * @@ -425,7 +464,7 @@ done: return ret_value; } /* end group_stats() */ - + /*------------------------------------------------------------------------- * Function: dataset_stats * @@ -478,6 +517,7 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "attribute_stats() failed"); /* Get storage info */ + /* Failure 0 indistinguishable from no-data-stored 0 */ storage = H5Dget_storage_size(did); /* Gather layout statistics */ @@ -503,7 +543,8 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) if(num_ext) { iter->nexternal += (unsigned long)num_ext; iter->dset_external_storage_size += (unsigned long)storage; - } else + } + else iter->dset_storage_size += storage; /* Gather dataspace statistics */ @@ -616,7 +657,7 @@ done: return ret_value; } /* end dataset_stats() */ - + /*------------------------------------------------------------------------- * Function: datatype_stats * @@ -648,7 +689,7 @@ done: return ret_value; } /* end datatype_stats() */ - + /*------------------------------------------------------------------------- * Function: obj_stats * @@ -704,7 +745,7 @@ done: return ret_value; } /* end obj_stats() */ - + /*------------------------------------------------------------------------- * Function: lnk_stats * @@ -742,7 +783,6 @@ lnk_stats(const char H5_ATTR_UNUSED *path, const H5L_info_t *li, void *_iter) return 0; } /* end lnk_stats() */ - /*------------------------------------------------------------------------- * Function: hand_free * @@ -771,7 +811,7 @@ hand_free(struct handler_t *hand) } /* end if */ } /* end hand_free() */ - + /*------------------------------------------------------------------------- * Function: parse_command_line * @@ -839,7 +879,8 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) error_msg("Invalid threshold for small groups\n"); goto error; } - } else + } + else error_msg("Missing threshold for small groups\n"); break; @@ -861,7 +902,8 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) error_msg("Invalid threshold for small datasets\n"); goto error; } - } else + } + else error_msg("Missing threshold for small datasets\n"); break; @@ -883,7 +925,8 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) error_msg("Invalid threshold for small # of attributes\n"); goto error; } - } else + } + else error_msg("Missing threshold for small # of attributes\n"); break; @@ -918,6 +961,105 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) } /* end if */ break; + case 'w': +#ifndef H5_HAVE_ROS3_VFD + error_msg("Read-Only S3 VFD not enabled.\n"); + goto error; +#else + { + char *cred_str = NULL; + unsigned nelems = 0; + char **cred = NULL; + char const *ccred[3]; + + if (FAIL == parse_tuple((const char *)opt_arg, ',', &cred_str, &nelems, &cred)) { + error_msg("Unable to parse s3 credential\n"); + goto error; + } + if (nelems != 3) { + error_msg("s3 credential must have three elements\n"); + goto error; + } + ccred[0] = (const char *)cred[0]; + ccred[1] = (const char *)cred[1]; + ccred[2] = (const char *)cred[2]; + if (0 == h5tools_populate_ros3_fapl(&ros3_fa, ccred)) { + error_msg("Unable to set ros3 fapl config\n"); + goto error; + } + HDfree(cred); + HDfree(cred_str); + } /* parse s3-cred block */ + drivername = "ros3"; + break; +#endif /* H5_HAVE_ROS3_VFD */ + + case 'H': +#ifndef H5_HAVE_LIBHDFS + error_msg("HDFS VFD is not enabled.\n"); + goto error; +#else + { + unsigned nelems = 0; + char *props_src = NULL; + char **props = NULL; + unsigned long k = 0; + if (FAIL == parse_tuple((const char *)opt_arg, + ',', &props_src, &nelems, &props)) { + error_msg("unable to parse hdfs properties tuple\n"); + goto error; + } + /* sanity-check tuple count + */ + if (nelems != 5) { + char str[64] = ""; + HDsprintf(str, + "expected 5 elements in hdfs properties tuple " + "but found %u\n", + nelems); + HDfree(props); + HDfree(props_src); + error_msg(str); + goto error; + } + /* Populate fapl configuration structure with given + * properties. + * TODO/WARNING: No error-checking is done on length of + * input strings... Silent overflow is possible, + * albeit unlikely. + */ + if (HDstrncmp(props[0], "", 1)) { + HDstrncpy(hdfs_fa.namenode_name,(const char *)props[0], HDstrlen(props[0])); + } + if (HDstrncmp(props[1], "", 1)) { + k = strtoul((const char *)props[1], NULL, 0); + if (errno == ERANGE) { + error_msg("supposed port number wasn't.\n"); + goto error; + } + hdfs_fa.namenode_port = (int32_t)k; + } + if (HDstrncmp(props[2], "", 1)) { + HDstrncpy(hdfs_fa.kerberos_ticket_cache, (const char *)props[2], HDstrlen(props[2])); + } + if (HDstrncmp(props[3], "", 1)) { + HDstrncpy(hdfs_fa.user_name, (const char *)props[3], HDstrlen(props[3])); + } + if (strncmp(props[4], "", 1)) { + k = HDstrtoul((const char *)props[4], NULL, 0); + if (errno == ERANGE) { + error_msg("supposed buffersize number wasn't.\n"); + goto error; + } + hdfs_fa.stream_buffer_size = (int32_t)k; + } + HDfree(props); + HDfree(props_src); + drivername = "hdfs"; + } + break; +#endif /* H5_HAVE_LIBHDFS */ + default: usage(h5tools_getprogname()); goto error; @@ -944,7 +1086,7 @@ error: return -1; } - + /*------------------------------------------------------------------------- * Function: iter_free * @@ -1004,7 +1146,7 @@ iter_free(iter_t *iter) } /* end iter_free() */ - + /*------------------------------------------------------------------------- * Function: print_file_info * @@ -1024,19 +1166,19 @@ iter_free(iter_t *iter) static herr_t print_file_info(const iter_t *iter) { - printf("File information\n"); - printf("\t# of unique groups: %lu\n", iter->uniq_groups); - printf("\t# of unique datasets: %lu\n", iter->uniq_dsets); - printf("\t# of unique named datatypes: %lu\n", iter->uniq_dtypes); - printf("\t# of unique links: %lu\n", iter->uniq_links); - printf("\t# of unique other: %lu\n", iter->uniq_others); - printf("\tMax. # of links to object: %lu\n", iter->max_links); + HDprintf("File information\n"); + HDprintf("\t# of unique groups: %lu\n", iter->uniq_groups); + HDprintf("\t# of unique datasets: %lu\n", iter->uniq_dsets); + HDprintf("\t# of unique named datatypes: %lu\n", iter->uniq_dtypes); + HDprintf("\t# of unique links: %lu\n", iter->uniq_links); + HDprintf("\t# of unique other: %lu\n", iter->uniq_others); + HDprintf("\tMax. # of links to object: %lu\n", iter->max_links); HDfprintf(stdout, "\tMax. # of objects in group: %Hu\n", iter->max_fanout); return 0; } /* print_file_info() */ - + /*------------------------------------------------------------------------- * Function: print_file_metadata * @@ -1059,15 +1201,12 @@ print_file_metadata(const iter_t *iter) HDfprintf(stdout, "\tUser block: %Hu\n", iter->ublk_size); HDfprintf(stdout, "\tObject headers: (total/unused)\n"); - HDfprintf(stdout, "\t\tGroups: %Hu/%Hu\n", - iter->group_ohdr_info.total_size, - iter->group_ohdr_info.free_size); + HDfprintf(stdout, "\t\tGroups: %Hu/%Hu\n", iter->group_ohdr_info.total_size, + iter->group_ohdr_info.free_size); HDfprintf(stdout, "\t\tDatasets(exclude compact data): %Hu/%Hu\n", - iter->dset_ohdr_info.total_size, - iter->dset_ohdr_info.free_size); - HDfprintf(stdout, "\t\tDatatypes: %Hu/%Hu\n", - iter->dtype_ohdr_info.total_size, - iter->dtype_ohdr_info.free_size); + iter->dset_ohdr_info.total_size, iter->dset_ohdr_info.free_size); + HDfprintf(stdout, "\t\tDatatypes: %Hu/%Hu\n", iter->dtype_ohdr_info.total_size, + iter->dtype_ohdr_info.free_size); HDfprintf(stdout, "\tGroups:\n"); HDfprintf(stdout, "\t\tB-tree/List: %Hu\n", iter->groups_btree_storage_size); @@ -1091,7 +1230,7 @@ print_file_metadata(const iter_t *iter) return 0; } /* print_file_metadata() */ - + /*------------------------------------------------------------------------- * Function: print_group_info * @@ -1118,37 +1257,37 @@ print_group_info(const iter_t *iter) unsigned long total; /* Total count for various statistics */ unsigned u; /* Local index variable */ - printf("Small groups (with 0 to %u links):\n", sgroups_threshold-1); + HDprintf("Small groups (with 0 to %u links):\n", sgroups_threshold-1); total = 0; for(u = 0; u < (unsigned)sgroups_threshold; u++) { if(iter->num_small_groups[u] > 0) { - printf("\t# of groups with %u link(s): %lu\n", u, iter->num_small_groups[u]); + HDprintf("\t# of groups with %u link(s): %lu\n", u, iter->num_small_groups[u]); total += iter->num_small_groups[u]; } /* end if */ } /* end for */ - printf("\tTotal # of small groups: %lu\n", total); + HDprintf("\tTotal # of small groups: %lu\n", total); - printf("Group bins:\n"); + HDprintf("Group bins:\n"); total = 0; if((iter->group_nbins > 0) && (iter->group_bins[0] > 0)) { - printf("\t# of groups with 0 link: %lu\n", iter->group_bins[0]); + HDprintf("\t# of groups with 0 link: %lu\n", iter->group_bins[0]); total = iter->group_bins[0]; } /* end if */ power = 1; for(u = 1; u < iter->group_nbins; u++) { if(iter->group_bins[u] > 0) { - printf("\t# of groups with %lu - %lu links: %lu\n", power, (power * 10) - 1, + HDprintf("\t# of groups with %lu - %lu links: %lu\n", power, (power * 10) - 1, iter->group_bins[u]); total += iter->group_bins[u]; } /* end if */ power *= 10; } /* end for */ - printf("\tTotal # of groups: %lu\n", total); + HDprintf("\tTotal # of groups: %lu\n", total); return 0; } /* print_group_info() */ - + /*------------------------------------------------------------------------- * Function: print_group_metadata * @@ -1164,7 +1303,7 @@ print_group_info(const iter_t *iter) static herr_t print_group_metadata(const iter_t *iter) { - printf("File space information for groups' metadata (in bytes):\n"); + HDprintf("File space information for groups' metadata (in bytes):\n"); HDfprintf(stdout, "\tObject headers (total/unused): %Hu/%Hu\n", iter->group_ohdr_info.total_size, iter->group_ohdr_info.free_size); @@ -1175,7 +1314,7 @@ print_group_metadata(const iter_t *iter) return 0; } /* print_group_metadata() */ - + /*------------------------------------------------------------------------- * Function: print_dataset_info * @@ -1197,72 +1336,72 @@ print_dataset_info(const iter_t *iter) unsigned u; /* Local index variable */ if(iter->uniq_dsets > 0) { - printf("Dataset dimension information:\n"); - printf("\tMax. rank of datasets: %u\n", iter->max_dset_rank); - printf("\tDataset ranks:\n"); + HDprintf("Dataset dimension information:\n"); + HDprintf("\tMax. rank of datasets: %u\n", iter->max_dset_rank); + HDprintf("\tDataset ranks:\n"); for(u = 0; u < H5S_MAX_RANK; u++) if(iter->dset_rank_count[u] > 0) - printf("\t\t# of dataset with rank %u: %lu\n", u, iter->dset_rank_count[u]); + HDprintf("\t\t# of dataset with rank %u: %lu\n", u, iter->dset_rank_count[u]); - printf("1-D Dataset information:\n"); + HDprintf("1-D Dataset information:\n"); HDfprintf(stdout, "\tMax. dimension size of 1-D datasets: %Hu\n", iter->max_dset_dims); - printf("\tSmall 1-D datasets (with dimension sizes 0 to %u):\n", sdsets_threshold - 1); + HDprintf("\tSmall 1-D datasets (with dimension sizes 0 to %u):\n", sdsets_threshold - 1); total = 0; for(u = 0; u < (unsigned)sdsets_threshold; u++) { if(iter->small_dset_dims[u] > 0) { - printf("\t\t# of datasets with dimension sizes %u: %lu\n", u, + HDprintf("\t\t# of datasets with dimension sizes %u: %lu\n", u, iter->small_dset_dims[u]); total += iter->small_dset_dims[u]; } /* end if */ } /* end for */ - printf("\t\tTotal # of small datasets: %lu\n", total); + HDprintf("\t\tTotal # of small datasets: %lu\n", total); /* Protect against no datasets in file */ if(iter->dset_dim_nbins > 0) { - printf("\t1-D Dataset dimension bins:\n"); + HDprintf("\t1-D Dataset dimension bins:\n"); total = 0; if(iter->dset_dim_bins[0] > 0) { - printf("\t\t# of datasets with dimension size 0: %lu\n", iter->dset_dim_bins[0]); + HDprintf("\t\t# of datasets with dimension size 0: %lu\n", iter->dset_dim_bins[0]); total = iter->dset_dim_bins[0]; } /* end if */ power = 1; for(u = 1; u < iter->dset_dim_nbins; u++) { if(iter->dset_dim_bins[u] > 0) { - printf("\t\t# of datasets with dimension size %lu - %lu: %lu\n", power, (power * 10) - 1, + HDprintf("\t\t# of datasets with dimension size %lu - %lu: %lu\n", power, (power * 10) - 1, iter->dset_dim_bins[u]); total += iter->dset_dim_bins[u]; } /* end if */ power *= 10; } /* end for */ - printf("\t\tTotal # of datasets: %lu\n", total); + HDprintf("\t\tTotal # of datasets: %lu\n", total); } /* end if */ - printf("Dataset storage information:\n"); + HDprintf("Dataset storage information:\n"); HDfprintf(stdout, "\tTotal raw data size: %Hu\n", iter->dset_storage_size); HDfprintf(stdout, "\tTotal external raw data size: %Hu\n", iter->dset_external_storage_size); - printf("Dataset layout information:\n"); + HDprintf("Dataset layout information:\n"); for(u = 0; u < H5D_NLAYOUTS; u++) - printf("\tDataset layout counts[%s]: %lu\n", (u == H5D_COMPACT ? "COMPACT" : + HDprintf("\tDataset layout counts[%s]: %lu\n", (u == H5D_COMPACT ? "COMPACT" : (u == H5D_CONTIGUOUS ? "CONTIG" : (u == H5D_CHUNKED ? "CHUNKED" : "ERROR"))), iter->dset_layouts[u]); - printf("\tNumber of external files : %lu\n", iter->nexternal); - - printf("Dataset filters information:\n"); - printf("\tNumber of datasets with:\n"); - printf("\t\tNO filter: %lu\n", iter->dset_comptype[H5Z_FILTER_ERROR+1]); - printf("\t\tGZIP filter: %lu\n", iter->dset_comptype[H5Z_FILTER_DEFLATE]); - printf("\t\tSHUFFLE filter: %lu\n", iter->dset_comptype[H5Z_FILTER_SHUFFLE]); - printf("\t\tFLETCHER32 filter: %lu\n", iter->dset_comptype[H5Z_FILTER_FLETCHER32]); - printf("\t\tSZIP filter: %lu\n", iter->dset_comptype[H5Z_FILTER_SZIP]); - printf("\t\tNBIT filter: %lu\n", iter->dset_comptype[H5Z_FILTER_NBIT]); - printf("\t\tSCALEOFFSET filter: %lu\n", iter->dset_comptype[H5Z_FILTER_SCALEOFFSET]); - printf("\t\tUSER-DEFINED filter: %lu\n", iter->dset_comptype[H5_NFILTERS_IMPL-1]); + HDprintf("\tNumber of external files : %lu\n", iter->nexternal); + + HDprintf("Dataset filters information:\n"); + HDprintf("\tNumber of datasets with:\n"); + HDprintf("\t\tNO filter: %lu\n", iter->dset_comptype[H5Z_FILTER_ERROR+1]); + HDprintf("\t\tGZIP filter: %lu\n", iter->dset_comptype[H5Z_FILTER_DEFLATE]); + HDprintf("\t\tSHUFFLE filter: %lu\n", iter->dset_comptype[H5Z_FILTER_SHUFFLE]); + HDprintf("\t\tFLETCHER32 filter: %lu\n", iter->dset_comptype[H5Z_FILTER_FLETCHER32]); + HDprintf("\t\tSZIP filter: %lu\n", iter->dset_comptype[H5Z_FILTER_SZIP]); + HDprintf("\t\tNBIT filter: %lu\n", iter->dset_comptype[H5Z_FILTER_NBIT]); + HDprintf("\t\tSCALEOFFSET filter: %lu\n", iter->dset_comptype[H5Z_FILTER_SCALEOFFSET]); + HDprintf("\t\tUSER-DEFINED filter: %lu\n", iter->dset_comptype[H5_NFILTERS_IMPL-1]); } /* end if */ return 0; } /* print_dataset_info() */ - + /*------------------------------------------------------------------------- * Function: print_dataset_metadata * @@ -1279,7 +1418,7 @@ print_dataset_info(const iter_t *iter) static herr_t print_dset_metadata(const iter_t *iter) { - printf("File space information for datasets' metadata (in bytes):\n"); + HDprintf("File space information for datasets' metadata (in bytes):\n"); HDfprintf(stdout, "\tObject headers (total/unused): %Hu/%Hu\n", iter->dset_ohdr_info.total_size, iter->dset_ohdr_info.free_size); @@ -1291,7 +1430,7 @@ print_dset_metadata(const iter_t *iter) return 0; } /* print_dset_metadata() */ - + /*------------------------------------------------------------------------- * Function: print_dset_dtype_meta * @@ -1313,26 +1452,26 @@ print_dset_dtype_meta(const iter_t *iter) unsigned u; /* Local index variable */ if(iter->dset_ntypes) { - printf("Dataset datatype information:\n"); - printf("\t# of unique datatypes used by datasets: %lu\n", iter->dset_ntypes); + HDprintf("Dataset datatype information:\n"); + HDprintf("\t# of unique datatypes used by datasets: %lu\n", iter->dset_ntypes); total = 0; for(u = 0; u < iter->dset_ntypes; u++) { H5Tencode(iter->dset_type_info[u].tid, NULL, &dtype_size); - printf("\tDataset datatype #%u:\n", u); - printf("\t\tCount (total/named) = (%lu/%lu)\n", + HDprintf("\tDataset datatype #%u:\n", u); + HDprintf("\t\tCount (total/named) = (%lu/%lu)\n", iter->dset_type_info[u].count, iter->dset_type_info[u].named); - printf("\t\tSize (desc./elmt) = (%lu/%lu)\n", (unsigned long)dtype_size, + HDprintf("\t\tSize (desc./elmt) = (%lu/%lu)\n", (unsigned long)dtype_size, (unsigned long)H5Tget_size(iter->dset_type_info[u].tid)); H5Tclose(iter->dset_type_info[u].tid); total += iter->dset_type_info[u].count; } /* end for */ - printf("\tTotal dataset datatype count: %lu\n", total); + HDprintf("\tTotal dataset datatype count: %lu\n", total); } /* end if */ return 0; } /* print_dset_dtype_meta() */ - + /*------------------------------------------------------------------------- * Function: print_attr_info * @@ -1354,34 +1493,34 @@ print_attr_info(const iter_t *iter) unsigned long total; /* Total count for various statistics */ unsigned u; /* Local index variable */ - printf("Small # of attributes (objects with 1 to %u attributes):\n", sattrs_threshold); + HDprintf("Small # of attributes (objects with 1 to %u attributes):\n", sattrs_threshold); total = 0; for(u = 1; u <= (unsigned)sattrs_threshold; u++) { if(iter->num_small_attrs[u] > 0) { - printf("\t# of objects with %u attributes: %lu\n", u, iter->num_small_attrs[u]); + HDprintf("\t# of objects with %u attributes: %lu\n", u, iter->num_small_attrs[u]); total += iter->num_small_attrs[u]; } /* end if */ } /* end for */ - printf("\tTotal # of objects with small # of attributes: %lu\n", total); + HDprintf("\tTotal # of objects with small # of attributes: %lu\n", total); - printf("Attribute bins:\n"); + HDprintf("Attribute bins:\n"); total = 0; power = 1; for(u = 1; u < iter->attr_nbins; u++) { if(iter->attr_bins[u] > 0) { - printf("\t# of objects with %lu - %lu attributes: %lu\n", power, (power * 10) - 1, + HDprintf("\t# of objects with %lu - %lu attributes: %lu\n", power, (power * 10) - 1, iter->attr_bins[u]); total += iter->attr_bins[u]; } /* end if */ power *= 10; } /* end for */ - printf("\tTotal # of objects with attributes: %lu\n", total); - printf("\tMax. # of attributes to objects: %lu\n", (unsigned long)iter->max_attrs); + HDprintf("\tTotal # of objects with attributes: %lu\n", total); + HDprintf("\tMax. # of attributes to objects: %lu\n", (unsigned long)iter->max_attrs); return 0; } /* print_attr_info() */ - + /*------------------------------------------------------------------------- * Function: print_storage_summary * @@ -1437,7 +1576,7 @@ print_storage_summary(const iter_t *iter) return 0; } /* print_storage_summary() */ - + /*------------------------------------------------------------------------- * Function: print_file_statistics * @@ -1482,7 +1621,7 @@ print_file_statistics(const iter_t *iter) if(display_summary) print_storage_summary(iter); } /* print_file_statistics() */ - + /*------------------------------------------------------------------------- * Function: print_object_statistics * @@ -1502,10 +1641,10 @@ print_file_statistics(const iter_t *iter) static void print_object_statistics(const char *name) { - printf("Object name %s\n", name); + HDprintf("Object name %s\n", name); } /* print_object_statistics() */ - + /*------------------------------------------------------------------------- * Function: print_statistics * @@ -1531,7 +1670,7 @@ print_statistics(const char *name, const iter_t *iter) print_file_statistics(iter); } /* print_statistics() */ - + /*------------------------------------------------------------------------- * Function: main * @@ -1552,6 +1691,7 @@ main(int argc, const char *argv[]) void *edata; void *tools_edata; struct handler_t *hand = NULL; + hid_t fapl_id = H5P_DEFAULT; h5tools_setprogname(PROGRAMNAME); h5tools_setstatus(EXIT_SUCCESS); @@ -1572,6 +1712,42 @@ main(int argc, const char *argv[]) if(parse_command_line(argc, argv, &hand) < 0) goto done; + /* if drivername is not null, probably need to set the fapl */ + if (HDstrcmp(drivername, "")) { + void *conf_fa = NULL; + + if (!HDstrcmp(drivername, "ros3")) { +#ifndef H5_HAVE_ROS3_VFD + error_msg("Read-Only S3 VFD not enabled.\n\n"); + goto done; +#else + conf_fa = (void *)&ros3_fa; +#endif /* H5_HAVE_ROS3_VFD */ + + } + else if (!HDstrcmp(drivername, "hdfs")) { +#ifndef H5_HAVE_LIBHDFS + error_msg("HDFS VFD not enabled.\n\n"); + goto done; +#else + conf_fa = (void *)&hdfs_fa; +#endif /* H5_HAVE_LIBHDFS */ + } + + if (conf_fa != NULL) { + HDassert(fapl_id == H5P_DEFAULT); + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + if (fapl_id < 0) { + error_msg("Unable to create fapl entry\n"); + goto done; + } + if (1 > h5tools_set_configured_fapl(fapl_id, drivername, conf_fa)) { + error_msg("Unable to set fapl\n"); + goto done; + } + } + } /* drivername set */ + fname = argv[opt_ind]; if(enable_error_stack > 0) { @@ -1584,7 +1760,7 @@ main(int argc, const char *argv[]) hid_t fcpl; H5F_info_t finfo; - printf("Filename: %s\n", fname); + HDprintf("Filename: %s\n", fname); fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT); if(fid < 0) { @@ -1616,7 +1792,7 @@ main(int argc, const char *argv[]) if(iter.num_small_groups == NULL || iter.num_small_attrs == NULL || iter.small_dset_dims == NULL) { error_msg("Unable to allocate memory for tracking small groups/datasets/attributes\n"); - h5tools_setstatus(EXIT_FAILURE); + h5tools_setstatus(EXIT_FAILURE); goto done; } @@ -1642,7 +1818,8 @@ main(int argc, const char *argv[]) if(h5trav_visit(fid, "/", TRUE, TRUE, obj_stats, lnk_stats, &iter) < 0) { error_msg("unable to traverse objects/links in file \"%s\"\n", fname); h5tools_setstatus(EXIT_FAILURE); - } else + } + else print_statistics("/", &iter); } /* end else */ } /* end if */ @@ -1653,6 +1830,13 @@ done: /* Free iter structure */ iter_free(&iter); + if (fapl_id != H5P_DEFAULT) { + if (0 < H5Pclose(fapl_id)) { + error_msg("unable to close fapl entry\n"); + h5tools_setstatus(EXIT_FAILURE); + } + } + if(fid >= 0 && H5Fclose(fid) < 0) { error_msg("unable to close file \"%s\"\n", fname); h5tools_setstatus(EXIT_FAILURE); diff --git a/tools/h5stat/testfiles/h5stat_help1.ddl b/tools/h5stat/testfiles/h5stat_help1.ddl index ce8bb73..e722f74 100644 --- a/tools/h5stat/testfiles/h5stat_help1.ddl +++ b/tools/h5stat/testfiles/h5stat_help1.ddl @@ -22,3 +22,13 @@ Usage: h5stat [OPTIONS] file than 0. The default threshold is 10. -S, --summary Print summary of file space information --enable-error-stack Prints messages from the HDF5 error stack as they occur + --s3-cred= Access file on S3, using provided credential + :: (region,id,key) + If == "(,,)", no authentication is used. + --hdfs-attrs= Access a file on HDFS with given configuration + attributes. + :: (,, + ,, + ) + If an attribute is empty, a default value will be + used. diff --git a/tools/h5stat/testfiles/h5stat_help2.ddl b/tools/h5stat/testfiles/h5stat_help2.ddl index ce8bb73..e722f74 100644 --- a/tools/h5stat/testfiles/h5stat_help2.ddl +++ b/tools/h5stat/testfiles/h5stat_help2.ddl @@ -22,3 +22,13 @@ Usage: h5stat [OPTIONS] file than 0. The default threshold is 10. -S, --summary Print summary of file space information --enable-error-stack Prints messages from the HDF5 error stack as they occur + --s3-cred= Access file on S3, using provided credential + :: (region,id,key) + If == "(,,)", no authentication is used. + --hdfs-attrs= Access a file on HDFS with given configuration + attributes. + :: (,, + ,, + ) + If an attribute is empty, a default value will be + used. diff --git a/tools/h5stat/testfiles/h5stat_nofile.ddl b/tools/h5stat/testfiles/h5stat_nofile.ddl index ce8bb73..e722f74 100644 --- a/tools/h5stat/testfiles/h5stat_nofile.ddl +++ b/tools/h5stat/testfiles/h5stat_nofile.ddl @@ -22,3 +22,13 @@ Usage: h5stat [OPTIONS] file than 0. The default threshold is 10. -S, --summary Print summary of file space information --enable-error-stack Prints messages from the HDF5 error stack as they occur + --s3-cred= Access file on S3, using provided credential + :: (region,id,key) + If == "(,,)", no authentication is used. + --hdfs-attrs= Access a file on HDFS with given configuration + attributes. + :: (,, + ,, + ) + If an attribute is empty, a default value will be + used. diff --git a/tools/lib/h5diff_array.c b/tools/lib/h5diff_array.c index cbfabad..d178866 100644 --- a/tools/lib/h5diff_array.c +++ b/tools/lib/h5diff_array.c @@ -782,12 +782,11 @@ static hsize_t diff_datum( } /* check object type */ - if (ret_value >= 0) - if (obj1_type != obj2_type) { - parallel_print("Different object types referenced: <%s> and <%s>", obj1, obj2); - opts->not_cmp = 1; - HGOTO_DONE (opts->err_stat); - } + if (obj1_type != obj2_type) { + parallel_print("Different object types referenced: <%s> and <%s>", obj1, obj2); + opts->not_cmp = 1; + HGOTO_DONE (opts->err_stat); + } if ((obj1_id = H5Rdereference(container1_id, H5R_OBJECT, _mem1)) < 0) { opts->err_stat = 1; diff --git a/tools/lib/h5tools_dump.c b/tools/lib/h5tools_dump.c index 7ec5da8..e76328c 100644 --- a/tools/lib/h5tools_dump.c +++ b/tools/lib/h5tools_dump.c @@ -93,7 +93,7 @@ h5tool_format_t h5tools_dataformat = { const h5tools_dump_header_t h5tools_standardformat = { "standardformat", /*name */ -"HDF5", /*fileebgin */ +"HDF5", /*filebegin */ "", /*fileend */ SUPER_BLOCK, /*bootblockbegin */ "", /*bootblockend */ @@ -2257,18 +2257,26 @@ h5tools_print_datatype(FILE *stream, h5tools_str_t *buffer, const h5tool_format_ h5tools_render_element(stream, info, ctx, buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); ctx->indent_level++; { - char *ttag; + char *ttag; - if(NULL == (ttag = H5Tget_tag(type))) - H5E_THROW(FAIL, H5E_tools_min_id_g, "H5Tget_tag failed"); + if(NULL == (ttag = H5Tget_tag(type))) + H5E_THROW(FAIL, H5E_tools_min_id_g, "H5Tget_tag failed"); - ctx->need_prefix = TRUE; + ctx->need_prefix = TRUE; + + h5tools_str_reset(buffer); + h5tools_str_append(buffer, "OPAQUE_TAG \"%s\";", ttag); + h5tools_render_element(stream, info, ctx, buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); + + H5free_memory(ttag); - h5tools_str_reset(buffer); - h5tools_str_append(buffer, "OPAQUE_TAG \"%s\";", ttag); - h5tools_render_element(stream, info, ctx, buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); + if((size = H5Tget_size(type)) <= 0) { + ctx->need_prefix = TRUE; - H5free_memory(ttag); + h5tools_str_reset(buffer); + h5tools_str_append(buffer, "OPAQUE_SIZE \"%s\";", size); + h5tools_render_element(stream, info, ctx, buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0); + } } ctx->indent_level--; diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index 37d137c..9b564c0 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -21,6 +21,10 @@ #include "H5private.h" #include "h5trav.h" +#ifdef H5_HAVE_ROS3_VFD +#include "H5FDros3.h" +#endif + /* global variables */ unsigned h5tools_nCols = 80; /* ``get_option'' variables */ @@ -322,7 +326,229 @@ get_option(int argc, const char **argv, const char *opts, const struct long_opti return opt_opt; } - + +/***************************************************************************** + * + * Function: parse_tuple() + * + * Purpose: + * + * Create array of pointers to strings, identified as elements in a tuple + * of arbitrary length separated by provided character. + * ("tuple" because "nple" looks strange) + * + * * Receives pointer to start of tuple sequence string, '('. + * * Attempts to separate elements by token-character `sep`. + * * If the separator character is preceded by a backslash '\', + * the backslash is deleted and the separator is included in the + * element string as any other character. + * * To end an element with a backslash, escape the backslash, e.g. + * "(myelem\\,otherelem) -> {"myelem\", "otherelem"} + * * In all other cases, a backslash appearing not as part of "\\" or + * "\" digraph will be included berbatim. + * * Last two characters in the string MUST be ")\0". + * + * * Generates a copy of the input string `start`, (src..")\0"), replacing + * separators and close-paren with null charaters. + * * This string is allocated at runtime and should be freed when done. + * * Generates array of char pointers, and directs start of each element + * (each pointer) into this copy. + * * Each tuple element points to the start of its string (substring) + * and ends with a null terminator. + * * This array is allocated at runtime and should be freed when done. + * * Reallocates and expands elements array during parsing. + * * Initially allocated for 2 (plus one null entry), and grows by + * powers of 2. + * * The final 'slot' in the element array (elements[nelements], e.g.) + * always points to NULL. + * * The number of elements found and stored are passed out through pointer + * to unsigned, `nelems`. + * + * Return: + * + * FAIL If malformed--does not look like a tuple "(...)" + * or major error was encountered while parsing. + * or + * SUCCEED String looks properly formed "(...)" and no major errors. + * + * Stores number of elements through pointer `nelems`. + * Stores list of pointers to char (first char in each element + * string) through pointer `ptrs_out`. + * NOTE: `ptrs_out[nelems] == NULL` should be true. + * NOTE: list is malloc'd by function, and should be freed + * when done. + * Stores "source string" for element pointers through `cpy_out`. + * NOTE: Each element substring is null-terminated. + * NOTE: There may be extra characters after the last element + * (past its null terminator), but is guaranteed to + * be null-terminated. + * NOTE: `cpy_out` string is malloc'd by function, + * and should be freed when done. + * + * Programmer: Jacob Smith + * 2017-11-10 + * + * Changes: None. + * + ***************************************************************************** + */ +herr_t +parse_tuple(const char *start, + int sep, + char **cpy_out, + unsigned *nelems, + char ***ptrs_out) +{ + char *elem_ptr = NULL; + char *dest_ptr = NULL; + unsigned elems_count = 0; + char **elems = NULL; /* more like *elems[], but complier... */ + char **elems_re = NULL; /* temporary pointer, for realloc */ + char *cpy = NULL; + herr_t ret_value = SUCCEED; + unsigned init_slots = 2; + + + + /***************** + * SANITY-CHECKS * + *****************/ + + /* must start with "(" + */ + if (start[0] != '(') { + ret_value = FAIL; + goto done; + } + + /* must end with ")" + */ + while (start[elems_count] != '\0') { + elems_count++; + } + if (start[elems_count - 1] != ')') { + ret_value = FAIL; + goto done; + } + + elems_count = 0; + + + + /*********** + * PREPARE * + ***********/ + + /* create list + */ + elems = (char **)HDmalloc(sizeof(char *) * (init_slots + 1)); + if (elems == NULL) { ret_value = FAIL; goto done; } /* CANTALLOC */ + + /* create destination string + */ + start++; /* advance past opening paren '(' */ + cpy = (char *)HDmalloc(sizeof(char) * (HDstrlen(start))); /* no +1; less '(' */ + if (cpy == NULL) { ret_value = FAIL; goto done; } /* CANTALLOC */ + + /* set pointers + */ + dest_ptr = cpy; /* start writing copy here */ + elem_ptr = cpy; /* first element starts here */ + elems[elems_count++] = elem_ptr; /* set first element pointer into list */ + + + + /********* + * PARSE * + *********/ + + while (*start != '\0') { + /* For each character in the source string... + */ + if (*start == '\\') { + /* Possibly an escape digraph. + */ + if ((*(start + 1) == '\\') || + (*(start + 1) == sep) ) + { + /* Valid escape digraph of "\\" or "\". + */ + start++; /* advance past escape char '\' */ + *(dest_ptr++) = *(start++); /* Copy subsequent char */ + /* and advance pointers. */ + } else { + /* Not an accepted escape digraph. + * Copy backslash character. + */ + *(dest_ptr++) = *(start++); + } + } else if (*start == sep) { + /* Non-escaped separator. + * Terminate elements substring in copy, record element, advance. + * Expand elements list if appropriate. + */ + *(dest_ptr++) = 0; /* Null-terminate elem substring in copy */ + /* and advance pointer. */ + start++; /* Advance src pointer past separator. */ + elem_ptr = dest_ptr; /* Element pointer points to start of first */ + /* character after null sep in copy. */ + elems[elems_count++] = elem_ptr; /* Set elem pointer in list */ + /* and increment count. */ + + /* Expand elements list, if necessary. + */ + if (elems_count == init_slots) { + init_slots *= 2; + elems_re = (char **)realloc(elems, sizeof(char *) * \ + (init_slots + 1)); + if (elems_re == NULL) { + /* CANTREALLOC */ + ret_value = FAIL; + goto done; + } + elems = elems_re; + } + } else if (*start == ')' && *(start + 1) == '\0') { + /* Found terminal, non-escaped close-paren. Last element. + * Write null terminator to copy. + * Advance source pointer to gently break from loop. + * Requred to prevent ")" from always being added to last element. + */ + start++; + } else { + /* Copy character into destination. Advance pointers. + */ + *(dest_ptr++) = *(start++); + } + } + *dest_ptr = '\0'; /* Null-terminate destination string. */ + elems[elems_count] = NULL; /* Null-terminate elements list. */ + + + + /******************** + * PASS BACK VALUES * + ********************/ + + *ptrs_out = elems; + *nelems = elems_count; + *cpy_out = cpy; + +done: + if (ret_value == FAIL) { + /* CLEANUP */ + if (cpy) free(cpy); + if (elems) free(elems); + } + + return ret_value; + +} /* parse_tuple */ + + + + + /*------------------------------------------------------------------------- * Function: indentation * @@ -841,3 +1067,260 @@ done: return ret_value; } + +/*---------------------------------------------------------------------------- + * + * Function: h5tools_populate_ros3_fapl() + * + * Purpose: + * + * Set the values of a ROS3 fapl configuration object. + * + * If the values pointer is NULL, sets fapl target `fa` to a default + * (valid, current-version, non-authenticating) fapl config. + * + * If `values` pointer is _not_ NULL, expects `values` to contain at least + * three non-null pointers to null-terminated strings, corresponding to: + * { aws_region, + * secret_id, + * secret_key, + * } + * If all three strings are empty (""), the default fapl will be default. + * Both aws_region and secret_id values must be both empty or both + * populated. If + * Only secret_key is allowed to be empty (the empty string, ""). + * All values are checked against overflow as defined in the ros3 vfd + * header file; if a value overruns the permitted space, FAIL is returned + * and the function aborts without resetting the fapl to values initially + * present. + * + * Return: + * + * 0 (failure) if... + * * Read-Only S3 VFD is not enabled. + * * NULL fapl pointer: (NULL, {...} ) + * * Warning: In all cases below, fapl will be set as "default" + * before error occurs. + * * NULL value strings: (&fa, {NULL?, NULL? NULL?, ...}) + * * Incomplete fapl info: + * * empty region, non-empty id, key either way + * * (&fa, {"", "...", "?"}) + * * empty id, non-empty region, key either way + * * (&fa, {"...", "", "?"}) + * * "non-empty key and either id or region empty + * * (&fa, {"", "", "...") + * * (&fa, {"", "...", "...") + * * (&fa, {"...", "", "...") + * * Any string would overflow allowed space in fapl definition. + * or + * 1 (success) + * * Sets components in fapl_t pointer, copying strings as appropriate. + * * "Default" fapl (valid version, authenticate->False, empty strings) + * * `values` pointer is NULL + * * (&fa, NULL) + * * first three strings in `values` are empty ("") + * * (&fa, {"", "", "", ...} + * * Authenticating fapl + * * region, id, and optional key provided + * * (&fa, {"...", "...", ""}) + * * (&fa, {"...", "...", "..."}) + * + * Programmer: Jacob Smith + * 2017-11-13 + * + *---------------------------------------------------------------------------- + */ +#ifdef H5_HAVE_ROS3_VFD +int +h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, + const char **values) +{ + int show_progress = 0; /* set to 1 for debugging */ + int ret_value = 1; /* 1 for success, 0 for failure */ + /* e.g.? if (!populate()) { then failed } */ + + if (show_progress) { + HDprintf("called h5tools_populate_ros3_fapl\n"); + } + + if (fa == NULL) { + if (show_progress) { + HDprintf(" ERROR: null pointer to fapl_t\n"); + } + ret_value = 0; + goto done; + } + + if (show_progress) { + HDprintf(" preset fapl with default values\n"); + } + fa->version = H5FD_CURR_ROS3_FAPL_T_VERSION; + fa->authenticate = FALSE; + *(fa->aws_region) = '\0'; + *(fa->secret_id) = '\0'; + *(fa->secret_key) = '\0'; + + /* sanity-check supplied values + */ + if (values != NULL) { + if (values[0] == NULL) { + if (show_progress) { + HDprintf(" ERROR: aws_region value cannot be NULL\n"); + } + ret_value = 0; + goto done; + } + if (values[1] == NULL) { + if (show_progress) { + HDprintf(" ERROR: secret_id value cannot be NULL\n"); + } + ret_value = 0; + goto done; + } + if (values[2] == NULL) { + if (show_progress) { + HDprintf(" ERROR: secret_key value cannot be NULL\n"); + } + ret_value = 0; + goto done; + } + + /* if region and ID are supplied (key optional), write to fapl... + * fail if value would overflow + */ + if (*values[0] != '\0' && + *values[1] != '\0') + { + if (HDstrlen(values[0]) > H5FD_ROS3_MAX_REGION_LEN) { + if (show_progress) { + HDprintf(" ERROR: aws_region value too long\n"); + } + ret_value = 0; + goto done; + } + HDmemcpy(fa->aws_region, values[0], + (HDstrlen(values[0]) + 1)); + if (show_progress) { + HDprintf(" aws_region set\n"); + } + + + if (HDstrlen(values[1]) > H5FD_ROS3_MAX_SECRET_ID_LEN) { + if (show_progress) { + HDprintf(" ERROR: secret_id value too long\n"); + } + ret_value = 0; + goto done; + } + HDmemcpy(fa->secret_id, + values[1], + (HDstrlen(values[1]) + 1)); + if (show_progress) { + HDprintf(" secret_id set\n"); + } + + if (HDstrlen(values[2]) > H5FD_ROS3_MAX_SECRET_KEY_LEN) { + if (show_progress) { + HDprintf(" ERROR: secret_key value too long\n"); + } + ret_value = 0; + goto done; + } + HDmemcpy(fa->secret_key, + values[2], + (HDstrlen(values[2]) + 1)); + if (show_progress) { + HDprintf(" secret_key set\n"); + } + + fa->authenticate = TRUE; + if (show_progress) { + HDprintf(" set to authenticate\n"); + } + + } else if (*values[0] != '\0' || + *values[1] != '\0' || + *values[2] != '\0') + { + if (show_progress) { + HDprintf( + " ERROR: invalid assortment of empty/non-empty values\n" + ); + } + ret_value = 0; + goto done; + } + } /* values != NULL */ + +done: + return ret_value; + +} /* h5tools_populate_ros3_fapl */ +#endif /* H5_HAVE_ROS3_VFD */ + + +/*----------------------------------------------------------------------------- + * + * Function: h5tools_set_configured_fapl + * + * Purpose: prepare fapl_id with the given property list, according to + * VFD prototype. + * + * Return: 0 on failure, 1 on success + * + * Programmer: Jacob Smith + * 2018-05-21 + * + * Changes: None. + * + *----------------------------------------------------------------------------- + */ +int +h5tools_set_configured_fapl(hid_t fapl_id, + const char vfd_name[], + void *fapl_t_ptr) +{ + int ret_value = 1; + + if (fapl_id < 0) { + return 0; + } + + if (!strcmp("", vfd_name)) { + goto done; + +#ifdef H5_HAVE_ROS3_VFD + } else if (!strcmp("ros3", vfd_name)) { + if ((fapl_id == H5P_DEFAULT) || + (fapl_t_ptr == NULL) || + (FAIL == H5Pset_fapl_ros3( + fapl_id, + (H5FD_ros3_fapl_t *)fapl_t_ptr))) + { + ret_value = 0; + goto done; + } +#endif /* H5_HAVE_ROS3_VFD */ + +#ifdef H5_HAVE_LIBHDFS + } else if (!strcmp("hdfs", vfd_name)) { + if ((fapl_id == H5P_DEFAULT) || + (fapl_t_ptr == NULL) || + (FAIL == H5Pset_fapl_hdfs( + fapl_id, + (H5FD_hdfs_fapl_t *)fapl_t_ptr))) + { + ret_value = 0; + goto done; + } +#endif /* H5_HAVE_LIBHDFS */ + + } else { + ret_value = 0; /* unrecognized fapl type "name" */ + } + +done: + return ret_value; + +} /* h5tools_set_configured_fapl() */ + diff --git a/tools/lib/h5tools_utils.h b/tools/lib/h5tools_utils.h index 4c2bf1e..0fa5250 100644 --- a/tools/lib/h5tools_utils.h +++ b/tools/lib/h5tools_utils.h @@ -123,6 +123,11 @@ H5TOOLS_DLLVAR unsigned h5tools_nCols; /*max number of columns for H5TOOLS_DLL void indentation(unsigned); H5TOOLS_DLL void print_version(const char *progname); H5TOOLS_DLL void parallel_print(const char* format, ... ); +H5TOOLS_DLL herr_t parse_tuple(const char *start, + int sep, + char **cpy_out, + unsigned *nelems, + char ***ptrs_out); H5TOOLS_DLL void error_msg(const char *fmt, ...); H5TOOLS_DLL void warn_msg(const char *fmt, ...); H5TOOLS_DLL void help_ref_msg(FILE *output); @@ -174,6 +179,14 @@ H5TOOLS_DLL void h5tools_setprogname(const char*progname); H5TOOLS_DLL int h5tools_getstatus(void); H5TOOLS_DLL void h5tools_setstatus(int d_status); H5TOOLS_DLL int h5tools_getenv_update_hyperslab_bufsize(void); +H5TOOLS_DLL int h5tools_set_configured_fapl(hid_t fapl_id, + const char vfd_name[], + void *fapl_t_ptr); +#ifdef H5_HAVE_ROS3_VFD +H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa, + const char **values); +#endif /* H5_HAVE_ROS3_VFD */ + #ifdef __cplusplus } #endif diff --git a/tools/libtest/CMakeLists.txt b/tools/libtest/CMakeLists.txt new file mode 100644 index 0000000..f3d28da --- /dev/null +++ b/tools/libtest/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required (VERSION 3.10) +project (HDF5_TOOLS_LIBTEST C) + +#----------------------------------------------------------------------------- +# Add the h5tools_utils test executables +#----------------------------------------------------------------------------- +add_executable (h5tools_test_utils ${HDF5_TOOLS_LIBTEST_SOURCE_DIR}/h5tools_test_utils.c) +target_include_directories(h5tools_test_utils PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") +if (NOT ONLY_SHARED_LIBS) + TARGET_C_PROPERTIES (h5tools_test_utils STATIC) + target_link_libraries (h5tools_test_utils PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} ${HDF5_TEST_LIB_TARGET}) +else () + TARGET_C_PROPERTIES (h5tools_test_utils SHARED) + target_link_libraries (h5tools_test_utils PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} ${HDF5_TEST_LIBSH_TARGET}) +endif () +set_target_properties (h5tools_test_utils PROPERTIES FOLDER tools) + +include (CMakeTests.cmake) diff --git a/tools/libtest/CMakeTests.cmake b/tools/libtest/CMakeTests.cmake new file mode 100644 index 0000000..4feee9b --- /dev/null +++ b/tools/libtest/CMakeTests.cmake @@ -0,0 +1,49 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +############################################################################## +############################################################################## +### T E S T I N G ### +############################################################################## +############################################################################## + + +############################################################################## +############################################################################## +### T H E T E S T S M A C R O S ### +############################################################################## +############################################################################## + + macro (ADD_H5_TEST resultfile resultcode) + add_test ( + NAME H5LIBTEST-${resultfile}-clear-objects + COMMAND ${CMAKE_COMMAND} + -E remove + ${resultfile}.out + ${resultfile}.out.err + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5LIBTEST-${resultfile}-clear-objects PROPERTIES DEPENDS ${last_test}) + endif () + add_test (NAME H5LIBTEST-${resultfile} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ ${ARGN}) + if (NOT "${resultcode}" STREQUAL "0") + set_tests_properties (H5LIBTEST-${resultfile} PROPERTIES WILL_FAIL "true") + endif () + set_tests_properties (H5LIBTEST-${resultfile} PROPERTIES DEPENDS H5LIBTEST-${resultfile}-clear-objects) + endmacro () + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + ADD_H5_TEST (h5tools_utils-default 0) diff --git a/tools/libtest/Makefile.am b/tools/libtest/Makefile.am new file mode 100644 index 0000000..a93e25d --- /dev/null +++ b/tools/libtest/Makefile.am @@ -0,0 +1,34 @@ +# +# Read-Only S3 Virtual File Driver (VFD) +# Copyright (c) 2017-2018, The HDF Group. +# +# All rights reserved. +# +# NOTICE: +# All information contained herein is, and remains, the property of The HDF +# Group. The intellectual and technical concepts contained herein are +# proprietary to The HDF Group. Dissemination of this information or +# reproduction of this material is strictly forbidden unless prior written +# permission is obtained from The HDF Group. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +# Include src and tools/lib directories +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib + +# All programs depend on the hdf5 and h5tools libraries +LDADD=$(LIBH5TOOLS) $(LIBHDF5) + + +# main target +bin_PROGRAMS=h5tools_test_utils +# check_PROGRAMS=$(TEST_PROG) + + +include $(top_srcdir)/config/conclude.am diff --git a/tools/libtest/h5tools_test_utils.c b/tools/libtest/h5tools_test_utils.c new file mode 100644 index 0000000..7908519 --- /dev/null +++ b/tools/libtest/h5tools_test_utils.c @@ -0,0 +1,1265 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: unit-test functionality of the routines in `tools/lib/h5tools_utils` + * + * Jacob Smith 2017-11-10 + */ + +#include "h5tools_utils.h" +#include "h5test.h" + +#define UTIL_TEST_DEBUG 0 + +#ifndef __js_test__ + +#define __js_test__ 1L + +/***************************************************************************** + * + * FILE-LOCAL TESTING MACROS + * + * Purpose: + * + * 1. Upon test failure, goto-jump to single-location teardown in test + * function. E.g., `error:` (consistency with HDF corpus) or + * `failed:` (reflects purpose). + * >>> using "error", in part because `H5E_BEGIN_TRY` expects it. + * 2. Increase clarity and reduce overhead found with `TEST_ERROR`. + * e.g., "if(somefunction(arg, arg2) < 0) TEST_ERROR:" + * requires reading of entire line to know whether this if/call is + * part of the test setup, test operation, or a test unto itself. + * 3. Provide testing macros with optional user-supplied failure message; + * if not supplied (NULL), generate comparison output in the spirit of + * test-driven development. E.g., "expected 5 but was -3" + * User messages clarify test's purpose in code, encouraging description + * without relying on comments. + * 4. Configurable expected-actual order in generated comparison strings. + * Some prefer `VERIFY(expected, actual)`, others + * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch + * to satifsy both parties, assuming one paradigm per test file. + * (One could #undef and redefine the flag through the file as desired, + * but _why_.) + * + * Provided as courtesy, per consideration for inclusion in the library + * proper. + * + * Macros: + * + * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order + * FAIL_IF() - check condition + * FAIL_UNLESS() - check _not_ condition + * JSVERIFY() - long-int equality check; prints reason/comparison + * JSVERIFY_NOT() - long-int inequality check; prints + * JSVERIFY_STR() - string equality check; prints + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *****************************************************************************/ + + +/*---------------------------------------------------------------------------- + * + * ifdef flag: JSVERIFY_EXP_ACT + * + * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason]) + * default, if this is undefined, is (ACTUAL, EXPECTED[, reason]) + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_EXP_ACT 1L + + +/*---------------------------------------------------------------------------- + * + * Macro: JSFAILED_AT() + * + * Purpose: + * + * Preface a test failure by printing "*FAILED*" and location to stdout + * Similar to `H5_FAILED(); AT();` from h5test.h + * + * *FAILED* at somefile.c:12 in function_name()... + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSFAILED_AT() { \ + HDprintf("*FAILED* at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_IF() + * + * Purpose: + * + * Make tests more accessible and less cluttered than + * `if (thing == otherthing()) TEST_ERROR` + * paradigm. + * + * The following lines are roughly equivalent: + * + * `if (myfunc() < 0) TEST_ERROR;` (as seen elsewhere in HDF tests) + * `FAIL_IF(myfunc() < 0)` + * + * Prints a generic "FAILED AT" line to stdout and jumps to `error`, + * similar to `TEST_ERROR` in h5test.h + * + * Programmer: Jacob Smith + * 2017-10-23 + * + *---------------------------------------------------------------------------- + */ +#define FAIL_IF(condition) \ +if (condition) { \ + JSFAILED_AT() \ + goto error; \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: FAIL_UNLESS() + * + * Purpose: + * + * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests", + * e.g., "a != b". + * + * Opposite of FAIL_IF; fails if the given condition is _not_ true. + * + * `FAIL_IF( 5 != my_op() )` + * is equivalent to + * `FAIL_UNLESS( 5 == my_op() )` + * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer. + * (see JSVERIFY) + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define FAIL_UNLESS(condition) \ +if (!(condition)) { \ + JSFAILED_AT() \ + goto error; \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_LONG() + * + * Purpose: + * + * Print an failure message for long-int arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:488 in somefunc()... + * forest must be made of trees. + * + * or + * + * *FAILED* at myfile.c:488 in somefunc()... + * ! Expected 425 + * ! Actual 3 + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_LONG(expected, actual, reason) { \ + JSFAILED_AT() \ + if (reason!= NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf(" ! Expected %ld\n ! Actual %ld\n", \ + (long)(expected), (long)(actual)); \ + } \ +} + + +/*---------------------------------------------------------------------------- + * + * Macro: JSERR_STR() + * + * Purpose: + * + * Print an failure message for string arguments. + * ERROR-AT printed first. + * If `reason` is given, it is printed on own line and newlined after + * else, prints "expected/actual" aligned on own lines. + * + * *FAILED* at myfile.c:421 in myfunc()... + * Blue and Red strings don't match! + * + * or + * + * *FAILED* at myfile.c:421 in myfunc()... + * !!! Expected: + * this is my expected + * string + * !!! Actual: + * not what I expected at all + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSERR_STR(expected, actual, reason) { \ + JSFAILED_AT() \ + if ((reason) != NULL) { \ + HDprintf("%s\n", (reason)); \ + } else { \ + HDprintf("!!! Expected:\n%s\n!!!Actual:\n%s\n", \ + (expected), (actual)); \ + } \ +} + +#ifdef JSVERIFY_EXP_ACT + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY() + * + * Purpose: + * + * Verify that two long integers are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(expected, actual, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_NOT() + * + * Purpose: + * + * Verify that two long integers are _not_ equal. + * If equal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(expected, actual, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ + + +/*---------------------------------------------------------------------------- + * + * Macro: JSVERIFY_STR() + * + * Purpose: + * + * Verify that two strings are equal. + * If unequal, print failure message + * (with `reason`, if not NULL; expected/actual if NULL) + * and jump to `error` at end of function + * + * Programmer: Jacob Smith + * 2017-10-24 + * + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(expected, actual, reason) \ +if (HDstrcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + + +#else /* JSVERIFY_EXP_ACT not defined */ + /* Repeats macros above, but with actual/expected parameters reversed. */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY() + * See: JSVERIFY documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY(actual, expected, reason) \ +if ((long)(actual) != (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_NOT() + * See: JSVERIFY_NOT documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_NOT(actual, expected, reason) \ +if ((long)(actual) == (long)(expected)) { \ + JSERR_LONG((expected), (actual), (reason)) \ + goto error; \ +} /* JSVERIFY_NOT */ + + +/*---------------------------------------------------------------------------- + * Macro: JSVERIFY_STR() + * See: JSVERIFY_STR documentation above. + * Programmer: Jacob Smith + * 2017-10-14 + *---------------------------------------------------------------------------- + */ +#define JSVERIFY_STR(actual, expected, reason) \ +if (HDstrcmp((actual), (expected)) != 0) { \ + JSERR_STR((expected), (actual), (reason)); \ + goto error; \ +} /* JSVERIFY_STR */ + +#endif /* ifdef/else JSVERIFY_EXP_ACT */ + +#endif /* __js_test__ */ + +/* if > 0, be very verbose when performing tests */ +#define H5TOOLS_UTILS_TEST_DEBUG 0 + +/******************/ +/* TEST FUNCTIONS */ +/******************/ + + +/*---------------------------------------------------------------------------- + * + * Function: test_parse_tuple() + * + * Purpose: + * + * Provide unit tests and specification for the `parse_tuple()` function. + * + * Return: + * + * 0 Tests passed. + * 1 Tests failed. + * + * Programmer: Jacob Smith + * 2017-11-11 + * + * Changes: None. + * + *---------------------------------------------------------------------------- + */ +static unsigned +test_parse_tuple(void) +{ + /************************* + * TEST-LOCAL STRUCTURES * + *************************/ + + struct testcase { + const char *test_msg; /* info about test case */ + const char *in_str; /* input string */ + int sep; /* separator "character" */ + herr_t exp_ret; /* expected SUCCEED / FAIL */ + unsigned exp_nelems; /* expected number of elements */ + /* (no more than 7!) */ + const char *exp_elems[7]; /* list of elements (no more than 7!) */ + }; + + /****************** + * TEST VARIABLES * + ******************/ + + struct testcase cases[] = { + { "bad start", + "words(before)", + ';', + FAIL, + 0, + {NULL}, + }, + { "tuple not closed", + "(not ok", + ',', + FAIL, + 0, + {NULL}, + }, + { "empty tuple", + "()", + '-', + SUCCEED, + 1, + {""}, + }, + { "no separator", + "(stuff keeps on going)", + ',', + SUCCEED, + 1, + {"stuff keeps on going"}, + }, + { "4-ple, escaped seperator", + "(elem0,elem1,el\\,em2,elem3)", /* "el\,em" */ + ',', + SUCCEED, + 4, + {"elem0", "elem1", "el,em2", "elem3"}, + }, + { "5-ple, escaped escaped separator", + "(elem0,elem1,el\\\\,em2,elem3)", + ',', + SUCCEED, + 5, + {"elem0", "elem1", "el\\", "em2", "elem3"}, + }, + { "escaped non-comma separator", + "(5-2-7-2\\-6-2)", + '-', + SUCCEED, + 5, + {"5","2","7","2-6","2"}, + }, + { "embedded close-paren", + "(be;fo)re)", + ';', + SUCCEED, + 2, + {"be", "fo)re"}, + }, + { "embedded non-escaping backslash", + "(be;fo\\re)", + ';', + SUCCEED, + 2, + {"be", "fo\\re"}, + }, + { "double close-paren at end", + "(be;fore))", + ';', + SUCCEED, + 2, + {"be", "fore)"}, + }, + { "empty elements", + "(;a1;;a4;)", + ';', + SUCCEED, + 5, + {"", "a1", "", "a4", ""}, + }, + { "nested tuples with different separators", + "((4,e,a);(6,2,a))", + ';', + SUCCEED, + 2, + {"(4,e,a)","(6,2,a)"}, + }, + { "nested tuples with same separators", + "((4,e,a),(6,2,a))", + ',', + SUCCEED, + 6, + {"(4","e","a)","(6","2","a)"}, + }, + { "real-world use case", + "(us-east-2,AKIAIMC3D3XLYXLN5COA,ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+)", + ',', + SUCCEED, + 3, + {"us-east-2", + "AKIAIMC3D3XLYXLN5COA", + "ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+"}, + } + }; + struct testcase tc; + unsigned n_tests = 14; + unsigned i = 0; + unsigned count = 0; + unsigned elem_i = 0; + char **parsed = NULL; + char *cpy = NULL; + herr_t success = TRUE; + hbool_t show_progress = FALSE; + + + + TESTING("arbitrary-count tuple parsing"); + +#if H5TOOLS_UTILS_TEST_DEBUG > 0 + show_progress = TRUE; +#endif /* H5TOOLS_UTILS_TEST_DEBUG */ + + /********* + * TESTS * + *********/ + + for (i = 0; i < n_tests; i++) { + + /* SETUP + */ + HDassert(parsed == NULL); + HDassert(cpy == NULL); + tc = cases[i]; + if (show_progress == TRUE) { + HDprintf("testing %d: %s...\n", i, tc.test_msg); + } + + /* VERIFY + */ + success = parse_tuple(tc.in_str, tc.sep, &cpy, &count, &parsed); + + JSVERIFY( tc.exp_ret, success, "function returned incorrect value" ) + JSVERIFY( tc.exp_nelems, count, NULL ) + if (success == SUCCEED) { + FAIL_IF( parsed == NULL ) + for (elem_i = 0; elem_i < count; elem_i++) { + JSVERIFY_STR( tc.exp_elems[elem_i], parsed[elem_i], NULL ) + } + /* TEARDOWN */ + HDassert(parsed != NULL); + HDassert(cpy != NULL); + HDfree(parsed); + parsed = NULL; + HDfree(cpy); + cpy = NULL; + } else { + FAIL_IF( parsed != NULL ) + } /* if parse_tuple() == SUCCEED or no */ + + } /* for each testcase */ + + PASSED(); + return 0; + +error: + /*********** + * CLEANUP * + ***********/ + + if (parsed != NULL) HDfree(parsed); + if (cpy != NULL) HDfree(cpy); + + return 1; + +} /* test_parse_tuple */ + + +/*---------------------------------------------------------------------------- + * + * Function: test_populate_ros3_fa() + * + * Purpose: Verify behavior of `populate_ros3_fa()` + * + * Return: 0 if test passes + * 1 if failure + * + * Programmer: Jacob Smith + * 2017-11-13 + * + * Changes: None + * + *---------------------------------------------------------------------------- + */ +static unsigned +test_populate_ros3_fa(void) +{ +#ifdef H5_HAVE_ROS3_VFD + /************************* + * TEST-LOCAL STRUCTURES * + *************************/ + + /************************ + * TEST-LOCAL VARIABLES * + ************************/ + + hbool_t show_progress = FALSE; + int bad_version = 0xf87a; /* arbitrarily wrong version number */ +#endif /* H5_HAVE_ROS3_VFD */ + + TESTING("programmatic ros3 fapl population"); + +#ifndef H5_HAVE_ROS3_VFD + HDputs(" -SKIP-"); + HDputs(" Read-Only S3 VFD not enabled"); + HDfflush(stdout); + return 0; +#else +#if H5TOOLS_UTILS_TEST_DEBUG > 0 + show_progress = TRUE; +#endif /* H5TOOLS_UTILS_TEST_DEBUG */ + + HDassert(bad_version != H5FD_CURR_ROS3_FAPL_T_VERSION); + + /********* + * TESTS * + *********/ + + /* NULL fapl config pointer fails + */ + { + const char *values[] = {"x", "y", "z"}; + + if (show_progress) { HDprintf("NULL fapl pointer\n"); } + + JSVERIFY( 0, h5tools_populate_ros3_fapl(NULL, values), + "fapl pointer cannot be null" ) + } + + /* NULL values pointer yields default fapl + */ + { + H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"}; + + if (show_progress) { HDprintf("NULL values pointer\n"); } + + JSVERIFY( 1, h5tools_populate_ros3_fapl(&fa, NULL), + "NULL values pointer yields \"default\" fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( FALSE, fa.authenticate, NULL ) + JSVERIFY_STR( "", fa.aws_region, NULL ) + JSVERIFY_STR( "", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* all-empty values + * yields default fapl + */ + { + H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"}; + const char *values[] = {"", "", ""}; + + if (show_progress) { HDprintf("all empty values\n"); } + + JSVERIFY( 1, h5tools_populate_ros3_fapl(&fa, values), + "empty values yields \"default\" fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( FALSE, fa.authenticate, NULL ) + JSVERIFY_STR( "", fa.aws_region, NULL ) + JSVERIFY_STR( "", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* successfully set fapl with values + * excess value is ignored + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char *values[] = {"x", "y", "z", "a"}; + + if (show_progress) { HDprintf("successful full set\n"); } + + JSVERIFY( 1, h5tools_populate_ros3_fapl(&fa, values), + "four values" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( TRUE, fa.authenticate, NULL ) + JSVERIFY_STR( "x", fa.aws_region, NULL ) + JSVERIFY_STR( "y", fa.secret_id, NULL ) + JSVERIFY_STR( "z", fa.secret_key, NULL ) + } + + /* NULL region + * yeilds default fapl + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char *values[] = {NULL, "y", "z", NULL}; + + if (show_progress) { HDprintf("NULL region\n"); } + + JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values), + "could not fill fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( FALSE, fa.authenticate, NULL ) + JSVERIFY_STR( "", fa.aws_region, NULL ) + JSVERIFY_STR( "", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* empty region + * yeilds default fapl + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char *values[] = {"", "y", "z", NULL}; + + if (show_progress) { HDprintf("empty region; non-empty id, key\n"); } + + JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values), + "could not fill fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( FALSE, fa.authenticate, NULL ) + JSVERIFY_STR( "", fa.aws_region, NULL ) + JSVERIFY_STR( "", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* region overflow + * yeilds default fapl + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char *values[] = { + "somewhere over the rainbow not too high " \ + "there is another rainbow bounding some darkened sky", + "y", + "z"}; + + if (show_progress) { HDprintf("region overflow\n"); } + + HDassert(HDstrlen(values[0]) > H5FD_ROS3_MAX_REGION_LEN); + + JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values), + "could not fill fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( FALSE, fa.authenticate, NULL ) + JSVERIFY_STR( "", fa.aws_region, NULL ) + JSVERIFY_STR( "", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* NULL id + * yields default fapl + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char *values[] = {"x", NULL, "z", NULL}; + + if (show_progress) { HDprintf("NULL id\n"); } + + JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values), + "could not fill fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( FALSE, fa.authenticate, NULL ) + JSVERIFY_STR( "", fa.aws_region, NULL ) + JSVERIFY_STR( "", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* empty id (non-empty region, key) + * yeilds default fapl + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char *values[] = {"x", "", "z", NULL}; + + if (show_progress) { HDprintf("empty id; non-empty region and key\n"); } + + JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values), + "could not fill fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( FALSE, fa.authenticate, NULL ) + JSVERIFY_STR( "", fa.aws_region, NULL ) + JSVERIFY_STR( "", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* id overflow + * partial set: region + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char *values[] = { + "x", + "Why is it necessary to solve the problem? " \ + "What benefits will you receive by solving the problem? " \ + "What is the unknown? " \ + "What is it you don't yet understand? " \ + "What is the information you have? " \ + "What isn't the problem? " \ + "Is the information insufficient, redundant, or contradictory? " \ + "Should you draw a diagram or figure of the problem? " \ + "What are the boundaries of the problem? " \ + "Can you separate the various parts of the problem?", + "z"}; + + if (show_progress) { HDprintf("id overflow\n"); } + + HDassert(HDstrlen(values[1]) > H5FD_ROS3_MAX_SECRET_ID_LEN); + + JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values), + "could not fill fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( FALSE, fa.authenticate, NULL ) + JSVERIFY_STR( "x", fa.aws_region, NULL ) + JSVERIFY_STR( "", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* NULL key + * yields default fapl + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char *values[] = {"x", "y", NULL, NULL}; + + if (show_progress) { HDprintf("NULL key\n"); } + + JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values), + "could not fill fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( FALSE, fa.authenticate, NULL ) + JSVERIFY_STR( "", fa.aws_region, NULL ) + JSVERIFY_STR( "", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* empty key (non-empty region, id) + * yeilds authenticating fapl + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char *values[] = {"x", "y", "", NULL}; + + if (show_progress) { HDprintf("empty key; non-empty region and id\n"); } + + JSVERIFY( 1, h5tools_populate_ros3_fapl(&fa, values), + "could not fill fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( TRUE, fa.authenticate, NULL ) + JSVERIFY_STR( "x", fa.aws_region, NULL ) + JSVERIFY_STR( "y", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* empty key, region (non-empty id) + * yeilds default fapl + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char *values[] = {"", "y", "", NULL}; + + if (show_progress) { HDprintf("empty key and region; non-empty id\n"); } + + JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values), + "could not fill fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( FALSE, fa.authenticate, NULL ) + JSVERIFY_STR( "", fa.aws_region, NULL ) + JSVERIFY_STR( "", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* empty key, id (non-empty region) + * yeilds default fapl + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char *values[] = {"x", "", "", NULL}; + + if (show_progress) { HDprintf("empty key and id; non-empty region\n"); } + + JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values), + "could not fill fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( FALSE, fa.authenticate, NULL ) + JSVERIFY_STR( "", fa.aws_region, NULL ) + JSVERIFY_STR( "", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* key overflow + * partial set: region, id + */ + { + H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"}; + const char *values[] = { + "x", + "y", + "Why is it necessary to solve the problem? " \ + "What benefits will you receive by solving the problem? " \ + "What is the unknown? " \ + "What is it you don't yet understand? " \ + "What is the information you have? " \ + "What isn't the problem? " \ + "Is the information insufficient, redundant, or contradictory? " \ + "Should you draw a diagram or figure of the problem? " \ + "What are the boundaries of the problem? " \ + "Can you separate the various parts of the problem?"}; + + if (show_progress) { HDprintf("key overflow\n"); } + + HDassert(HDstrlen(values[2]) > H5FD_ROS3_MAX_SECRET_KEY_LEN); + + JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values), + "could not fill fapl" ) + JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL ) + JSVERIFY( FALSE, fa.authenticate, NULL ) + JSVERIFY_STR( "x", fa.aws_region, NULL ) + JSVERIFY_STR( "y", fa.secret_id, NULL ) + JSVERIFY_STR( "", fa.secret_key, NULL ) + } + + /* use case + */ + { + H5FD_ros3_fapl_t fa = {0, 0, "", "", ""}; + const char *values[] = { + "us-east-2", + "AKIAIMC3D3XLYXLN5COA", + "ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+" + }; + JSVERIFY( 1, + h5tools_populate_ros3_fapl(&fa, values), + "unable to set use case" ) + JSVERIFY( 1, fa.version, "version check" ) + JSVERIFY( 1, fa.authenticate, "should authenticate" ) + } + + PASSED(); + return 0; + +error : + /*********** + * CLEANUP * + ***********/ + + return 1; + +#endif /* H5_HAVE_ROS3_VFD */ + +} /* test_populate_ros3_fa */ + + +/*---------------------------------------------------------------------------- + * + * Function: test_set_configured_fapl() + * + * Purpose: Verify `h5tools_set_configured_fapl()` with ROS3 VFD + * + * Return: 0 if test passes + * 1 if failure + * + * Programmer: Jacob Smith + * 2018-07-12 + * + * Changes: None + * + *---------------------------------------------------------------------------- + */ +static unsigned +test_set_configured_fapl(void) +{ +#define UTIL_TEST_NOFAPL 1 +#define UTIL_TEST_DEFAULT 2 +#define UTIL_TEST_CREATE 3 + + /************************* + * TEST-LOCAL STRUCTURES * + *************************/ + typedef struct testcase { + const char message[88]; + int expected; + int fapl_choice; + const char vfdname[12]; + void *conf_fa; + } testcase; + + typedef struct other_fa_t { + int a; + int b; + int c; + } other_fa_t; + + /************************ + * TEST-LOCAL VARIABLES * + ************************/ + + hid_t fapl_id = -1; + other_fa_t wrong_fa = {0x432, 0xf82, 0x9093}; + H5FD_ros3_fapl_t ros3_anon_fa = {1, FALSE, "", "", ""}; + H5FD_ros3_fapl_t ros3_auth_fa = { + 1, /* fapl version */ + TRUE, /* authenticate */ + "us-east-1", /* aws region */ + "12345677890abcdef", /* simulate access key ID */ + "oiwnerwe9u0234nJw0-aoj+dsf", /* simulate secret key */ + }; + H5FD_hdfs_fapl_t hdfs_fa = { + 1, /* fapl version */ + "", /* namenode name */ + 0, /* namenode port */ + "", /* kerberos ticket cache */ + "", /* user name */ + 2048, /* stream buffer size */ + }; + unsigned n_cases = 7; /* number of common testcases */ + testcase cases[] = { + { "(common) should fail: no fapl id", + 0, + UTIL_TEST_NOFAPL, + "", + NULL, + }, + { "(common) should fail: no fapl id (with struct)", + 0, + UTIL_TEST_NOFAPL, + "", + &wrong_fa, + }, + { "(common) H5P_DEFAULT with no struct should succeed", + 1, + UTIL_TEST_DEFAULT, + "", + NULL, + }, + { "(common) H5P_DEFAULT with (ignored) struct should succeed", + 1, + UTIL_TEST_DEFAULT, + "", + &wrong_fa, + }, + { "(common) provided fapl entry should not fail", + 1, + UTIL_TEST_CREATE, + "", + NULL, + }, + { "(common) provided fapl entry should not fail; ignores struct", + 1, + UTIL_TEST_CREATE, + "", + &wrong_fa, + }, + { "(common) should fail: unrecoginzed vfd name", + 0, + UTIL_TEST_DEFAULT, + "unknown", + NULL, + }, + +#ifdef H5_HAVE_ROS3_VFD + /* WARNING: add number of ROS3 test cases after array definition + */ + { "(ROS3) should fail: no fapl id, no struct", + 0, + UTIL_TEST_NOFAPL, + "ros3", + NULL, + }, + { "(ROS3) should fail: no fapl id", + 0, + UTIL_TEST_NOFAPL, + "ros3", + &ros3_anon_fa, + }, + { "(ROS3) should fail: no struct", + 0, + UTIL_TEST_CREATE, + "ros3", + NULL, + }, + { "(ROS3) successful set", + 1, + UTIL_TEST_CREATE, + "ros3", + &ros3_anon_fa, + }, + { "(ROS3) should fail: attempt to set DEFAULT fapl", + 0, + UTIL_TEST_DEFAULT, + "ros3", + &ros3_anon_fa, + }, +#endif /* H5_HAVE_ROS3_VFD */ + +#ifdef H5_HAVE_LIBHDFS + /* WARNING: add number of HDFS test cases after array definition + */ + { "(HDFS) should fail: no fapl id, no struct", + 0, + UTIL_TEST_NOFAPL, + "hdfs", + NULL, + }, + { "(HDFS) should fail: no fapl id", + 0, + UTIL_TEST_NOFAPL, + "hdfs", + &hdfs_fa, + }, + { "(HDFS) should fail: no struct", + 0, + UTIL_TEST_CREATE, + "hdfs", + NULL, + }, + { "(HDFS) successful set", + 1, + UTIL_TEST_CREATE, + "hdfs", + &hdfs_fa, + }, + { "(HDFS) should fail: attempt to set DEFAULT fapl", + 0, + UTIL_TEST_DEFAULT, + "hdfs", + &hdfs_fa, + }, +#endif /* H5_HAVE_LIBHDFS */ + + }; /* testcases `cases` array */ + +#ifdef H5_HAVE_ROS3_VFD + n_cases += 5; +#endif /* H5_HAVE_ROS3_VFD */ + +#ifdef H5_HAVE_LIBHDFS + n_cases += 5; +#endif /* H5_HAVE_LIBHDFS */ + + TESTING("programmatic fapl set"); + + for (unsigned i = 0; i < n_cases; i++) { + int result; + testcase C = cases[i]; + + fapl_id = -1; + +#if UTIL_TEST_DEBUG + HDfprintf(stderr, "setup test %d\t%s\n", i, C.message); fflush(stderr); +#endif /* UTIL_TEST_DEBUG */ + + /* per-test setup */ + if (C.fapl_choice == UTIL_TEST_DEFAULT) { + fapl_id = H5P_DEFAULT; + } else if (C.fapl_choice == UTIL_TEST_CREATE) { + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + FAIL_IF( fapl_id < 0 ) + } + +#if UTIL_TEST_DEBUG + HDfprintf(stderr, "before test\n"); fflush(stderr); +#endif /* UTIL_TEST_DEBUG */ + + /* test */ + result = h5tools_set_configured_fapl( + fapl_id, + C.vfdname, + C.conf_fa); + JSVERIFY( result, C.expected, C.message ) + +#if UTIL_TEST_DEBUG + HDfprintf(stderr, "after test\n"); fflush(stderr); +#endif /* UTIL_TEST_DEBUG */ + + /* per-test-teardown */ + if (fapl_id > 0) { + FAIL_IF( FAIL == H5Pclose(fapl_id) ) + } + fapl_id = -1; + +#if UTIL_TEST_DEBUG + HDfprintf(stderr, "after cleanup\n"); fflush(stderr); +#endif /* UTIL_TEST_DEBUG */ + + } + +#if UTIL_TEST_DEBUG + HDfprintf(stderr, "after loop\n"); fflush(stderr); +#endif /* UTIL_TEST_DEBUG */ + + PASSED(); + return 0; + +error : + /*********** + * CLEANUP * + ***********/ + +#if UTIL_TEST_DEBUG + HDfprintf(stderr, "ERROR\n"); fflush(stderr); +#endif /* UTIL_TEST_DEBUG */ + + if (fapl_id > 0) { + (void)H5Pclose(fapl_id); + } + + return 1; + +#undef UTIL_TEST_NOFAPL +#undef UTIL_TEST_DEFAULT +#undef UTIL_TEST_CREATE +} /* test_set_configured_fapl */ + + +/*---------------------------------------------------------------------------- + * + * Function: main() + * + * Purpose: Run all test functions. + * + * Return: 0 iff all test pass + * 1 iff any failures + * + * Programmer: Jacob Smith + * 2017-11-10 + * + * Changes: None. + * + *---------------------------------------------------------------------------- + */ +int +main(void) +{ + unsigned nerrors = 0; + +#ifdef _H5TEST_ + h5reset(); /* h5test? */ +#endif /* _H5TEST_ */ + + HDfprintf(stdout, "Testing h5tools_utils corpus.\n"); + + nerrors += test_parse_tuple(); + nerrors += test_populate_ros3_fa(); + nerrors += test_set_configured_fapl(); + + if (nerrors > 0) { + HDfprintf(stdout, "***** %d h5tools_utils TEST%s FAILED! *****\n", + nerrors, + nerrors > 1 ? "S" : ""); + nerrors = 1; + } else { + HDfprintf(stdout, "All h5tools_utils tests passed\n"); + } + + return (int)nerrors; + +} /* main */ + + diff --git a/tools/misc/h5debug.c b/tools/misc/h5debug.c index 245b5a1..676e050 100644 --- a/tools/misc/h5debug.c +++ b/tools/misc/h5debug.c @@ -51,7 +51,7 @@ #define INDENT 3 #define VCOL 50 - + /*------------------------------------------------------------------------- * Function: get_H5B2_class * @@ -74,45 +74,45 @@ get_H5B2_class(const uint8_t *sig) const H5B2_class_t *cls; switch(subtype) { - case H5B2_TEST_ID: - cls = H5B2_TEST; - break; + case H5B2_TEST_ID: + cls = H5B2_TEST; + break; - case H5B2_FHEAP_HUGE_INDIR_ID: - cls = H5HF_HUGE_BT2_INDIR; - break; + case H5B2_FHEAP_HUGE_INDIR_ID: + cls = H5HF_HUGE_BT2_INDIR; + break; - case H5B2_FHEAP_HUGE_FILT_INDIR_ID: - cls = H5HF_HUGE_BT2_FILT_INDIR; - break; + case H5B2_FHEAP_HUGE_FILT_INDIR_ID: + cls = H5HF_HUGE_BT2_FILT_INDIR; + break; - case H5B2_FHEAP_HUGE_DIR_ID: - cls = H5HF_HUGE_BT2_DIR; - break; + case H5B2_FHEAP_HUGE_DIR_ID: + cls = H5HF_HUGE_BT2_DIR; + break; - case H5B2_FHEAP_HUGE_FILT_DIR_ID: - cls = H5HF_HUGE_BT2_FILT_DIR; - break; + case H5B2_FHEAP_HUGE_FILT_DIR_ID: + cls = H5HF_HUGE_BT2_FILT_DIR; + break; - case H5B2_GRP_DENSE_NAME_ID: - cls = H5G_BT2_NAME; - break; + case H5B2_GRP_DENSE_NAME_ID: + cls = H5G_BT2_NAME; + break; - case H5B2_GRP_DENSE_CORDER_ID: - cls = H5G_BT2_CORDER; - break; + case H5B2_GRP_DENSE_CORDER_ID: + cls = H5G_BT2_CORDER; + break; - case H5B2_SOHM_INDEX_ID: - cls = H5SM_INDEX; - break; + case H5B2_SOHM_INDEX_ID: + cls = H5SM_INDEX; + break; - case H5B2_ATTR_DENSE_NAME_ID: - cls = H5A_BT2_NAME; - break; + case H5B2_ATTR_DENSE_NAME_ID: + cls = H5A_BT2_NAME; + break; - case H5B2_ATTR_DENSE_CORDER_ID: - cls = H5A_BT2_CORDER; - break; + case H5B2_ATTR_DENSE_CORDER_ID: + cls = H5A_BT2_CORDER; + break; default: HDfprintf(stderr, "Unknown B-tree subtype %u\n", (unsigned)(subtype)); @@ -122,7 +122,7 @@ get_H5B2_class(const uint8_t *sig) return(cls); } /* end get_H5B2_class() */ - + /*------------------------------------------------------------------------- * Function: main * @@ -147,12 +147,12 @@ main(int argc, char *argv[]) uint8_t sig[H5F_SIGNATURE_LEN]; size_t u; H5E_auto2_t func; - void *edata; + void *edata; herr_t status = SUCCEED; if(argc == 1) { - HDfprintf(stderr, "Usage: %s filename [signature-addr [extra]]\n", argv[0]); - HDexit(1); + HDfprintf(stderr, "Usage: %s filename [signature-addr [extra]]\n", argv[0]); + HDexit(1); } /* end if */ /* Initialize the library */ @@ -224,13 +224,15 @@ main(int argc, char *argv[]) */ status = H5HL_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL); - } else if(!HDmemcmp (sig, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { - /* - * Debug a global heap collection. - */ - status = H5HG_debug (f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL); + } + else if(!HDmemcmp (sig, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + /* + * Debug a global heap collection. + */ + status = H5HG_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL); - } else if(!HDmemcmp(sig, H5G_NODE_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5G_NODE_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug a symbol table node. */ @@ -244,7 +246,8 @@ main(int argc, char *argv[]) status = H5G_node_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra); - } else if(!HDmemcmp(sig, H5B_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5B_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug a B-tree. B-trees are debugged through the B-tree * subclass. The subclass identifier is the byte immediately @@ -254,27 +257,29 @@ main(int argc, char *argv[]) unsigned ndims; switch(subtype) { - case H5B_SNODE_ID: - /* Check for extra parameters */ - if(extra == 0) { - HDfprintf(stderr, "\nWarning: Providing the group's local heap address will give more information\n"); - HDfprintf(stderr, "B-tree symbol table node usage:\n"); - HDfprintf(stderr, "\th5debug
\n\n"); - } /* end if */ - - status = H5G_node_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra); - break; + case H5B_SNODE_ID: + /* Check for extra parameters */ + if(extra == 0) { + HDfprintf(stderr, "\nWarning: Providing the group's local heap address will give more information\n"); + HDfprintf(stderr, "B-tree symbol table node usage:\n"); + HDfprintf(stderr, "\th5debug
\n\n"); + HDexit(4); + } /* end if */ - case H5B_CHUNK_ID: - /* Check for extra parameters */ - if(extra == 0) { - HDfprintf(stderr, "ERROR: Need number of dimensions of chunk in order to dump chunk B-tree node\n"); - HDfprintf(stderr, "B-tree chunked storage node usage:\n"); - HDfprintf(stderr, "\th5debug <# of dimensions>\n"); - HDexit(4); - } /* end if */ + status = H5G_node_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra); + break; + + case H5B_CHUNK_ID: + /* Check for extra parameters */ + if(extra == 0) { + HDfprintf(stderr, "ERROR: Need number of dimensions of chunk in order to dump chunk B-tree node\n"); + HDfprintf(stderr, "B-tree chunked storage node usage:\n"); + HDfprintf(stderr, "\th5debug <# of dimensions> ...\n"); + HDexit(4); + } /* end if */ - ndims = (unsigned)extra; + /* Build array of chunk dimensions */ + ndims = (unsigned)extra; status = H5D_btree_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, ndims); break; @@ -283,7 +288,8 @@ main(int argc, char *argv[]) HDexit(4); } - } else if(!HDmemcmp(sig, H5B2_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5B2_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug a v2 B-tree header. */ @@ -292,7 +298,8 @@ main(int argc, char *argv[]) status = H5B2_hdr_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, cls, (haddr_t)extra); - } else if(!HDmemcmp(sig, H5B2_INT_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5B2_INT_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug a v2 B-tree internal node. */ @@ -310,7 +317,8 @@ main(int argc, char *argv[]) status = H5B2_int_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, cls, extra, (unsigned)extra2, (unsigned)extra3, (haddr_t)extra4); - } else if(!HDmemcmp(sig, H5B2_LEAF_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5B2_LEAF_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug a v2 B-tree leaf node. */ @@ -327,13 +335,15 @@ main(int argc, char *argv[]) status = H5B2_leaf_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, cls, extra, (unsigned)extra2, (haddr_t)extra3); - } else if(!HDmemcmp(sig, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug a fractal heap header. */ status = H5HF_hdr_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL); - } else if(!HDmemcmp(sig, H5HF_DBLOCK_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5HF_DBLOCK_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug a fractal heap direct block. */ @@ -348,7 +358,8 @@ main(int argc, char *argv[]) status = H5HF_dblock_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra, (size_t)extra2); - } else if(!HDmemcmp(sig, H5HF_IBLOCK_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5HF_IBLOCK_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug a fractal heap indirect block. */ @@ -363,14 +374,16 @@ main(int argc, char *argv[]) status = H5HF_iblock_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra, (unsigned)extra2); - } else if(!HDmemcmp(sig, H5FS_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5FS_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug a free space header. */ status = H5FS_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL); - } else if(!HDmemcmp(sig, H5FS_SINFO_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5FS_SINFO_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug free space serialized sections. */ @@ -385,14 +398,16 @@ main(int argc, char *argv[]) status = H5FS_sects_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra, extra2); - } else if(!HDmemcmp(sig, H5SM_TABLE_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5SM_TABLE_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug shared message master table. */ status = H5SM_table_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, (unsigned) UFAIL, (unsigned) UFAIL); - } else if(!HDmemcmp(sig, H5SM_LIST_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5SM_LIST_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug shared message list index. */ @@ -407,33 +422,37 @@ main(int argc, char *argv[]) status = H5SM_list_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, (unsigned) extra, (size_t) extra2); - } else if(!HDmemcmp(sig, H5O_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + } + else if(!HDmemcmp(sig, H5EA_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* * Debug v2 object header (which have signatures). */ status = H5O_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL); - } else if(sig[0] == H5O_VERSION_1) { + } + else if(sig[0] == H5O_VERSION_1) { /* * This could be a v1 object header. Since they don't have a signature * it's a somewhat "ify" detection. */ status = H5O_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL); - } else { + } + else { /* * Got some other unrecognized signature. */ - printf("%-*s ", VCOL, "Signature:"); + HDprintf("%-*s ", VCOL, "Signature:"); for (u = 0; u < sizeof(sig); u++) { if (sig[u] > ' ' && sig[u] <= '~' && '\\' != sig[u]) HDputchar(sig[u]); else if ('\\' == sig[u]) { HDputchar('\\'); HDputchar('\\'); - } else - printf("\\%03o", sig[u]); + } + else + HDprintf("\\%03o", sig[u]); } HDputchar('\n'); diff --git a/tools/misc/h5mkgrp.c b/tools/misc/h5mkgrp.c index b7ff73c..feb60d4 100644 --- a/tools/misc/h5mkgrp.c +++ b/tools/misc/h5mkgrp.c @@ -251,7 +251,7 @@ main(int argc, const char *argv[]) /* Display some output if requested */ if(params.verbose) - printf("%s: Creating groups with latest version of the format\n", h5tools_getprogname()); + HDprintf("%s: Creating groups with latest version of the format\n", h5tools_getprogname()); } /* end if */ /* Attempt to open an existing HDF5 file first */ @@ -284,7 +284,7 @@ main(int argc, const char *argv[]) /* Display some output if requested */ if(params.verbose) - printf("%s: Creating parent groups\n", h5tools_getprogname()); + HDprintf("%s: Creating parent groups\n", h5tools_getprogname()); } /* end if */ /* Loop over creating requested groups */ @@ -305,7 +305,7 @@ main(int argc, const char *argv[]) /* Display some output if requested */ if(params.verbose) - printf("%s: created group '%s'\n", h5tools_getprogname(), params.groups[curr_group]); + HDprintf("%s: created group '%s'\n", h5tools_getprogname(), params.groups[curr_group]); } /* end for */ /* Close link creation property list */ diff --git a/tools/misc/h5repart.c b/tools/misc/h5repart.c index dea27d4..461f631 100644 --- a/tools/misc/h5repart.c +++ b/tools/misc/h5repart.c @@ -15,11 +15,11 @@ * Programmer: Robb Matzke * Wednesday, May 13, 1998 * - * Purpose: Repartitions a file family. This program can be used to - * split a single file into a family of files, join a family of - * files into a single file, or copy one family to another while - * changing the size of the family members. It can also be used - * to copy a single file to a single file with holes. + * Purpose: Repartitions a file family. This program can be used to + * split a single file into a family of files, join a family of + * files into a single file, or copy one family to another while + * changing the size of the family members. It can also be used + * to copy a single file to a single file with holes. */ /* See H5private.h for how to include system headers */ @@ -50,13 +50,13 @@ /*------------------------------------------------------------------------- - * Function: usage + * Function: usage * - * Purpose: Prints a usage message. + * Purpose: Prints a usage message. * - * Return: void + * Return: void * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Wednesday, May 13, 1998 * * Modifications: @@ -67,96 +67,97 @@ static void usage (const char *progname) { HDfprintf(stderr, "usage: %s [-v] [-V] [-[b|m] N[g|m|k]] [-family_to_sec2] SRC DST\n", - progname); + progname); HDfprintf(stderr, " -v Produce verbose output\n"); HDfprintf(stderr, " -V Print a version number and exit\n"); HDfprintf(stderr, " -b N The I/O block size, defaults to 1kB\n"); HDfprintf(stderr, " -m N The destination member size or 1GB\n"); HDfprintf(stderr, " -family_to_sec2 Change file driver from family to sec2\n"); HDfprintf(stderr, " SRC The name of the source file\n"); - HDfprintf(stderr, " DST The name of the destination files\n"); + HDfprintf(stderr, " DST The name of the destination files\n"); HDfprintf(stderr, "Sizes may be suffixed with `g' for GB, `m' for MB or " - "`k' for kB.\n"); + "`k' for kB.\n"); HDfprintf(stderr, "File family names include an integer printf " - "format such as `%%d'\n"); - exit (EXIT_FAILURE); + "format such as `%%d'\n"); + HDexit (EXIT_FAILURE); } - + /*------------------------------------------------------------------------- - * Function: get_size + * Function: get_size * - * Purpose: Reads a size option of the form `-XNS' where `X' is any - * letter, `N' is a multi-character positive decimal number, and - * `S' is an optional suffix letter in the set [GgMmk]. The - * option may also be split among two arguments as: `-X NS'. - * The input value of ARGNO is the argument number for the - * switch in the ARGV vector and ARGC is the number of entries - * in that vector. + * Purpose: Reads a size option of the form `-XNS' where `X' is any + * letter, `N' is a multi-character positive decimal number, and + * `S' is an optional suffix letter in the set [GgMmk]. The + * option may also be split among two arguments as: `-X NS'. + * The input value of ARGNO is the argument number for the + * switch in the ARGV vector and ARGC is the number of entries + * in that vector. * - * Return: Success: The value N multiplied according to the - * suffix S. On return ARGNO will be the number - * of the next argument to process. + * Return: Success: The value N multiplied according to the + * suffix S. On return ARGNO will be the number + * of the next argument to process. * - * Failure: Calls usage() which exits with a non-zero - * status. + * Failure: Calls usage() which exits with a non-zero + * status. * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Wednesday, May 13, 1998 - * - * Modifications: - * *------------------------------------------------------------------------- */ static off_t get_size (const char *progname, int *argno, int argc, char *argv[]) { - off_t retval=-1; - char *suffix; + off_t retval=-1; + char *suffix; if (isdigit ((int)(argv[*argno][2]))) { - retval = strtol (argv[*argno]+2, &suffix, 10); - (*argno)++; - } else if (argv[*argno][2] || *argno+1>=argc) { - usage (progname); - } else { - retval = strtol (argv[*argno+1], &suffix, 0); - if (suffix==argv[*argno+1]) usage (progname); - *argno += 2; + retval = HDstrtol(argv[*argno]+2, &suffix, 10); + (*argno)++; + } + else if (argv[*argno][2] || *argno+1>=argc) { + usage (progname); + } + else { + retval = HDstrtol(argv[*argno+1], &suffix, 0); + if (suffix == argv[*argno+1]) + usage (progname); + *argno += 2; } if (suffix && suffix[0] && !suffix[1]) { - switch (*suffix) { - case 'G': - case 'g': - retval *= 1024 * 1024 * 1024; - break; - case 'M': - case 'm': - retval *= 1024 * 1024; - break; - case 'k': - retval *= 1024; - break; - default: - usage (progname); - } - } else if (suffix && suffix[0]) { - usage (progname); + switch (*suffix) { + case 'G': + case 'g': + retval *= 1024 * 1024 * 1024; + break; + case 'M': + case 'm': + retval *= 1024 * 1024; + break; + case 'k': + retval *= 1024; + break; + default: + usage(progname); + } + } + else if (suffix && suffix[0]) { + usage (progname); } return retval; } - + /*------------------------------------------------------------------------- - * Function: main + * Function: main * - * Purpose: Split an hdf5 file + * Purpose: Split an hdf5 file * - * Return: Success: + * Return: Success: * - * Failure: + * Failure: * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Wednesday, May 13, 1998 * * Modifications: @@ -166,36 +167,36 @@ get_size (const char *progname, int *argno, int argc, char *argv[]) int main (int argc, char *argv[]) { - const char *prog_name; /*program name */ - size_t blk_size=1024; /*size of each I/O block */ - char *buf=NULL; /*I/O block buffer */ - size_t n, i; /*counters */ - ssize_t nio; /*I/O return value */ - int argno=1; /*program argument number */ - int src, dst=-1; /*source & destination files */ - int need_seek=FALSE; /*destination needs to seek? */ - int need_write; /*data needs to be written? */ - h5_stat_t sb; /*temporary file stat buffer */ - - int verbose=FALSE; /*display file names? */ - - const char *src_gen_name; /*general source name */ - char *src_name=NULL; /*source member name */ - - int src_is_family; /*is source name a family name? */ - int src_membno=0; /*source member number */ - - const char *dst_gen_name; /*general destination name */ - char *dst_name=NULL; /*destination member name */ - int dst_is_family; /*is dst name a family name? */ - int dst_membno=0; /*destination member number */ - - off_t left_overs=0; /*amount of zeros left over */ - off_t src_offset=0; /*offset in source member */ - off_t dst_offset=0; /*offset in destination member */ - off_t src_size; /*source logical member size */ - off_t src_act_size; /*source actual member size */ - off_t dst_size=1 GB; /*destination logical memb size */ + const char *prog_name; /*program name */ + size_t blk_size=1024; /*size of each I/O block */ + char *buf=NULL; /*I/O block buffer */ + size_t n, i; /*counters */ + ssize_t nio; /*I/O return value */ + int argno=1; /*program argument number */ + int src, dst=-1; /*source & destination files */ + int need_seek=FALSE; /*destination needs to seek? */ + int need_write; /*data needs to be written? */ + h5_stat_t sb; /*temporary file stat buffer */ + + int verbose=FALSE; /*display file names? */ + + const char *src_gen_name; /*general source name */ + char *src_name=NULL; /*source member name */ + + int src_is_family; /*is source name a family name? */ + int src_membno=0; /*source member number */ + + const char *dst_gen_name; /*general destination name */ + char *dst_name=NULL; /*destination member name */ + int dst_is_family; /*is dst name a family name? */ + int dst_membno=0; /*destination member number */ + + off_t left_overs=0; /*amount of zeros left over */ + off_t src_offset=0; /*offset in source member */ + off_t dst_offset=0; /*offset in destination member */ + off_t src_size; /*source logical member size */ + off_t src_act_size; /*source actual member size */ + off_t dst_size=1 GB; /*destination logical memb size */ hid_t fapl; /*file access property list */ hid_t file; hsize_t hdsize; /*destination logical memb size */ @@ -204,203 +205,218 @@ main (int argc, char *argv[]) /* * Get the program name from argv[0]. Use only the last component. */ - if ((prog_name=strrchr (argv[0], '/'))) prog_name++; - else prog_name = argv[0]; + if ((prog_name = HDstrrchr (argv[0], '/'))) + prog_name++; + else + prog_name = argv[0]; /* * Parse switches. */ - while (argno=argc) usage (prog_name); + if (argno >= argc) + usage(prog_name); src_gen_name = argv[argno++]; - sprintf (src_name, src_gen_name, src_membno); + HDsprintf(src_name, src_gen_name, src_membno); src_is_family = strcmp (src_name, src_gen_name); if ((src = HDopen(src_name, O_RDONLY, 0)) < 0) { - HDperror (src_name); - HDexit (EXIT_FAILURE); + HDperror(src_name); + HDexit(EXIT_FAILURE); } - if (HDfstat(src, &sb)<0) { - perror ("fstat"); - exit (EXIT_FAILURE); + if (HDfstat(src, &sb) < 0) { + HDperror("fstat"); + HDexit(EXIT_FAILURE); } src_size = src_act_size = sb.st_size; - if (verbose) HDfprintf (stderr, "< %s\n", src_name); + if (verbose) + HDfprintf(stderr, "< %s\n", src_name); /* * Get the name for the destination file and open the first member. */ - if (argno>=argc) usage (prog_name); + if (argno>=argc) + usage (prog_name); dst_gen_name = argv[argno++]; - sprintf (dst_name, dst_gen_name, dst_membno); - dst_is_family = strcmp (dst_name, dst_gen_name); + HDsprintf(dst_name, dst_gen_name, dst_membno); + dst_is_family = HDstrcmp(dst_name, dst_gen_name); - if ((dst = HDopen (dst_name, O_RDWR|O_CREAT|O_TRUNC, 0666)) < 0) { - HDperror (dst_name); - HDexit (EXIT_FAILURE); + if ((dst = HDopen(dst_name, O_RDWR|O_CREAT|O_TRUNC, 0666)) < 0) { + HDperror(dst_name); + HDexit(EXIT_FAILURE); } - if (verbose) HDfprintf (stderr, "> %s\n", dst_name); + if (verbose) + HDfprintf(stderr, "> %s\n", dst_name); /* No more arguments */ - if (argnosrc_size) { - HDfprintf (stderr, "%s: member truncated to %lu bytes\n", - src_name, (unsigned long)src_size); - } - src_offset = 0; - if (verbose) HDfprintf (stderr, "< %s\n", src_name); - } - - /* - * Update the destination offset, opening a new member if one will be - * needed. The first member is extended to the logical member size - * but other members might be smaller if they end with a hole. - */ - dst_offset = dst_offset + (off_t)n; - if (dst_is_family && dst_offset==dst_size) { - if (0==dst_membno) { - if (HDlseek (dst, dst_size-1, SEEK_SET)<0) { - perror ("HDHDlseek"); - exit (EXIT_FAILURE); - } - if (HDread (dst, buf, 1)<0) { - perror ("read"); - exit (EXIT_FAILURE); - } - if (HDlseek (dst, dst_size-1, SEEK_SET)<0) { - perror ("HDlseek"); - exit (EXIT_FAILURE); - } - if (HDwrite (dst, buf, 1)<0) { - perror ("write"); - exit (EXIT_FAILURE); - } - } - HDclose (dst); - sprintf (dst_name, dst_gen_name, ++dst_membno); - if ((dst = HDopen (dst_name, O_RDWR|O_CREAT|O_TRUNC, 0666)) < 0) { - HDperror (dst_name); - HDexit (EXIT_FAILURE); - } - dst_offset = 0; - need_seek = FALSE; - if (verbose) HDfprintf (stderr, "> %s\n", dst_name); - } + /* Read a block. The amount to read is the minimum of: + * 1. The I/O block size + * 2. What's left to write in the destination member + * 3. Left over zeros or what's left in the source member. + */ + n = blk_size; + if (dst_is_family) + n = (size_t)MIN((off_t)n, dst_size-dst_offset); + if (left_overs) { + n = (size_t)MIN((off_t)n, left_overs); + left_overs = left_overs - (off_t) n; + need_write = FALSE; + } + else if (src_offset < src_act_size) { + n = (size_t)MIN((off_t)n, src_act_size-src_offset); + if ((nio = HDread(src, buf, n)) < 0) { + HDperror("read"); + HDexit(EXIT_FAILURE); + } + else if ((size_t) nio != n) { + HDfprintf(stderr, "%s: short read\n", src_name); + HDexit(EXIT_FAILURE); + } + for (i = 0; i < n; i++) { + if (buf[i]) + break; + } + need_write = (i < n); + } + else { + n = 0; + left_overs = src_size - src_act_size; + need_write = FALSE; + } + + /* + * If the block contains non-zero data then write it to the + * destination, otherwise just remember that we'll have to do a seek + * later in the destination when we finally get non-zero data. + */ + if (need_write) { + if (need_seek && HDlseek (dst, dst_offset, SEEK_SET) < 0) { + HDperror("HDlseek"); + HDexit(EXIT_FAILURE); + } + if ((nio = HDwrite(dst, buf, n)) < 0) { + HDperror("write"); + HDexit(EXIT_FAILURE); + } + else if ((size_t) nio != n) { + HDfprintf(stderr, "%s: short write\n", dst_name); + HDexit(EXIT_FAILURE); + } + need_seek = FALSE; + } + else { + need_seek = TRUE; + } + + /* + * Update the source offset and open the next source family member if + * necessary. The source stream ends at the first member which + * cannot be opened because it doesn't exist. At the end of the + * source stream, update the destination offset and break out of the + * loop. The destination offset must be updated so we can fix + * trailing holes. + */ + src_offset = src_offset + (off_t) n; + if (src_offset == src_act_size) { + HDclose(src); + if (!src_is_family) { + dst_offset = dst_offset + (off_t) n; + break; + } + HDsprintf(src_name, src_gen_name, ++src_membno); + if ((src = HDopen(src_name, O_RDONLY)) < 0 && ENOENT == errno) { + dst_offset = dst_offset + (off_t) n; + break; + } + else if (src < 0) { + HDperror(src_name); + HDexit(EXIT_FAILURE); + } + if (HDfstat (src, &sb) < 0) { + HDperror("fstat"); + HDexit(EXIT_FAILURE); + } + src_act_size = sb.st_size; + if (src_act_size > src_size) { + HDfprintf(stderr, "%s: member truncated to %lu bytes\n", src_name, (unsigned long) src_size); + } + src_offset = 0; + if (verbose) + HDfprintf(stderr, "< %s\n", src_name); + } + + /* + * Update the destination offset, opening a new member if one will be + * needed. The first member is extended to the logical member size + * but other members might be smaller if they end with a hole. + */ + dst_offset = dst_offset + (off_t) n; + if (dst_is_family && dst_offset == dst_size) { + if (0 == dst_membno) { + if (HDlseek (dst, dst_size-1, SEEK_SET) < 0) { + HDperror("HDHDlseek"); + HDexit(EXIT_FAILURE); + } + if (HDread (dst, buf, 1) < 0) { + HDperror("read"); + HDexit(EXIT_FAILURE); + } + if (HDlseek (dst, dst_size-1, SEEK_SET) < 0) { + HDperror("HDlseek"); + HDexit(EXIT_FAILURE); + } + if (HDwrite (dst, buf, 1) < 0) { + HDperror("write"); + HDexit(EXIT_FAILURE); + } + } + HDclose(dst); + HDsprintf(dst_name, dst_gen_name, ++dst_membno); + if ((dst = HDopen(dst_name, O_RDWR | O_CREAT | O_TRUNC, 0666)) < 0) { + HDperror(dst_name); + HDexit(EXIT_FAILURE); + } + dst_offset = 0; + need_seek = FALSE; + if (verbose) + HDfprintf(stderr, "> %s\n", dst_name); + } } /* @@ -409,30 +425,30 @@ main (int argc, char *argv[]) * family has been truncated. */ if (need_seek) { - if (HDlseek (dst, dst_offset-1, SEEK_SET)<0) { - perror ("HDlseek"); - exit (EXIT_FAILURE); - } - if (HDread (dst, buf, 1)<0) { - perror ("read"); - exit (EXIT_FAILURE); - } - if (HDlseek (dst, dst_offset-1, SEEK_SET)<0) { - perror ("HDlseek"); - exit (EXIT_FAILURE); - } - if (HDwrite (dst, buf, 1)<0) { - perror ("write"); - exit (EXIT_FAILURE); - } + if (HDlseek (dst, dst_offset-1, SEEK_SET)<0) { + HDperror ("HDlseek"); + HDexit (EXIT_FAILURE); + } + if (HDread (dst, buf, 1)<0) { + HDperror ("read"); + HDexit (EXIT_FAILURE); + } + if (HDlseek (dst, dst_offset-1, SEEK_SET)<0) { + HDperror ("HDlseek"); + HDexit (EXIT_FAILURE); + } + if (HDwrite (dst, buf, 1)<0) { + HDperror ("write"); + HDexit (EXIT_FAILURE); + } } HDclose (dst); /* Modify family driver information saved in superblock through private property. * These private properties are for this tool only. */ - if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) { - perror ("H5Pcreate"); - exit (EXIT_FAILURE); + if ((fapl=H5Pcreate(H5P_FILE_ACCESS)) < 0) { + HDperror ("H5Pcreate"); + HDexit (EXIT_FAILURE); } if(family_to_sec2) { @@ -440,23 +456,24 @@ main (int argc, char *argv[]) * with sec2 driver. This property signals the library to ignore the family * driver information saved in the superblock. */ if(H5Pset(fapl, H5F_ACS_FAMILY_TO_SEC2_NAME, &family_to_sec2) < 0) { - perror ("H5Pset"); - exit (EXIT_FAILURE); + HDperror ("H5Pset"); + HDexit (EXIT_FAILURE); } - } else { + } + else { /* Modify family size saved in superblock through private property. It signals * library to save the new member size(specified in command line) in superblock. * This private property is for this tool only. */ if(H5Pset_fapl_family(fapl, H5F_FAMILY_DEFAULT, H5P_DEFAULT) < 0) { - perror ("H5Pset_fapl_family"); - exit (EXIT_FAILURE); + HDperror ("H5Pset_fapl_family"); + HDexit (EXIT_FAILURE); } /* Set the property of the new member size as hsize_t */ hdsize = (hsize_t)dst_size; if(H5Pset(fapl, H5F_ACS_FAMILY_NEWSIZE_NAME, &hdsize) < 0) { - perror ("H5Pset"); - exit (EXIT_FAILURE); + HDperror ("H5Pset"); + HDexit (EXIT_FAILURE); } } @@ -468,19 +485,19 @@ main (int argc, char *argv[]) * new file can only be a sec2 file, reopen the new file should fail. There's * nothing to do in this case. */ H5E_BEGIN_TRY { - file=H5Fopen(dst_gen_name, H5F_ACC_RDWR, fapl); + file = H5Fopen(dst_gen_name, H5F_ACC_RDWR, fapl); } H5E_END_TRY; - if(file>=0) { - if(H5Fclose(file)<0) { - perror ("H5Fclose"); - exit (EXIT_FAILURE); + if(file >= 0) { + if(H5Fclose(file) < 0) { + HDperror("H5Fclose"); + HDexit(EXIT_FAILURE); } /* end if */ } /* end if */ - if(H5Pclose(fapl)<0) { - perror ("H5Pclose"); - exit (EXIT_FAILURE); + if(H5Pclose(fapl) < 0) { + HDperror("H5Pclose"); + HDexit(EXIT_FAILURE); } /* end if */ /* Free resources and return */ diff --git a/tools/testfiles/h5dump-help.txt b/tools/testfiles/h5dump-help.txt index bc37a8f..17c7b9d 100644 --- a/tools/testfiles/h5dump-help.txt +++ b/tools/testfiles/h5dump-help.txt @@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --s3-cred= Supply S3 authentication information to "ros3" vfd. + :: "(,,)" + If absent or -> "(,,)", no authentication. + Has no effect is filedriver is not `ros3'. + --hdfs-attrs= Supply configuration information for HDFS file access. + For use with "--filedriver=hdfs" + :: (,, + ,, + ) + Any absent attribute will use a default value. --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/help-1.ls b/tools/testfiles/help-1.ls index 491f696..396bed3 100644 --- a/tools/testfiles/help-1.ls +++ b/tools/testfiles/help-1.ls @@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] -V, --version Print version number and exit --vfd=DRIVER Use the specified virtual file driver -x, --hexdump Show raw data in hexadecimal format + --s3-cred=C Supply S3 authentication information to "ros3" vfd. + Accepts tuple of "(,,)". + If absent or C->"(,,)", defaults to no-authentication. + Has no effect if vfd flag not set to "ros3". + --hdfs-attrs=A Supply configuration information to Hadoop VFD. + Accepts tuple of (,, + ...,,) + If absent or A == '(,,,,)', all default values are used. + Has no effect if vfd flag is not 'hdfs'. file/OBJECT Each object consists of an HDF5 file name optionally followed by a diff --git a/tools/testfiles/help-2.ls b/tools/testfiles/help-2.ls index 491f696..396bed3 100644 --- a/tools/testfiles/help-2.ls +++ b/tools/testfiles/help-2.ls @@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] -V, --version Print version number and exit --vfd=DRIVER Use the specified virtual file driver -x, --hexdump Show raw data in hexadecimal format + --s3-cred=C Supply S3 authentication information to "ros3" vfd. + Accepts tuple of "(,,)". + If absent or C->"(,,)", defaults to no-authentication. + Has no effect if vfd flag not set to "ros3". + --hdfs-attrs=A Supply configuration information to Hadoop VFD. + Accepts tuple of (,, + ...,,) + If absent or A == '(,,,,)', all default values are used. + Has no effect if vfd flag is not 'hdfs'. file/OBJECT Each object consists of an HDF5 file name optionally followed by a diff --git a/tools/testfiles/help-3.ls b/tools/testfiles/help-3.ls index 491f696..396bed3 100644 --- a/tools/testfiles/help-3.ls +++ b/tools/testfiles/help-3.ls @@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] -V, --version Print version number and exit --vfd=DRIVER Use the specified virtual file driver -x, --hexdump Show raw data in hexadecimal format + --s3-cred=C Supply S3 authentication information to "ros3" vfd. + Accepts tuple of "(,,)". + If absent or C->"(,,)", defaults to no-authentication. + Has no effect if vfd flag not set to "ros3". + --hdfs-attrs=A Supply configuration information to Hadoop VFD. + Accepts tuple of (,, + ...,,) + If absent or A == '(,,,,)', all default values are used. + Has no effect if vfd flag is not 'hdfs'. file/OBJECT Each object consists of an HDF5 file name optionally followed by a diff --git a/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl b/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl index bc37a8f..17c7b9d 100644 --- a/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl +++ b/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl @@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --s3-cred= Supply S3 authentication information to "ros3" vfd. + :: "(,,)" + If absent or -> "(,,)", no authentication. + Has no effect is filedriver is not `ros3'. + --hdfs-attrs= Supply configuration information for HDFS file access. + For use with "--filedriver=hdfs" + :: (,, + ,, + ) + Any absent attribute will use a default value. --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/pbits/tpbitsIncomplete.ddl b/tools/testfiles/pbits/tpbitsIncomplete.ddl index bc37a8f..17c7b9d 100644 --- a/tools/testfiles/pbits/tpbitsIncomplete.ddl +++ b/tools/testfiles/pbits/tpbitsIncomplete.ddl @@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --s3-cred= Supply S3 authentication information to "ros3" vfd. + :: "(,,)" + If absent or -> "(,,)", no authentication. + Has no effect is filedriver is not `ros3'. + --hdfs-attrs= Supply configuration information for HDFS file access. + For use with "--filedriver=hdfs" + :: (,, + ,, + ) + Any absent attribute will use a default value. --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/pbits/tpbitsLengthExceeded.ddl b/tools/testfiles/pbits/tpbitsLengthExceeded.ddl index bc37a8f..17c7b9d 100644 --- a/tools/testfiles/pbits/tpbitsLengthExceeded.ddl +++ b/tools/testfiles/pbits/tpbitsLengthExceeded.ddl @@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --s3-cred= Supply S3 authentication information to "ros3" vfd. + :: "(,,)" + If absent or -> "(,,)", no authentication. + Has no effect is filedriver is not `ros3'. + --hdfs-attrs= Supply configuration information for HDFS file access. + For use with "--filedriver=hdfs" + :: (,, + ,, + ) + Any absent attribute will use a default value. --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/pbits/tpbitsLengthPositive.ddl b/tools/testfiles/pbits/tpbitsLengthPositive.ddl index bc37a8f..17c7b9d 100644 --- a/tools/testfiles/pbits/tpbitsLengthPositive.ddl +++ b/tools/testfiles/pbits/tpbitsLengthPositive.ddl @@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --s3-cred= Supply S3 authentication information to "ros3" vfd. + :: "(,,)" + If absent or -> "(,,)", no authentication. + Has no effect is filedriver is not `ros3'. + --hdfs-attrs= Supply configuration information for HDFS file access. + For use with "--filedriver=hdfs" + :: (,, + ,, + ) + Any absent attribute will use a default value. --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/pbits/tpbitsMaxExceeded.ddl b/tools/testfiles/pbits/tpbitsMaxExceeded.ddl index bc37a8f..17c7b9d 100644 --- a/tools/testfiles/pbits/tpbitsMaxExceeded.ddl +++ b/tools/testfiles/pbits/tpbitsMaxExceeded.ddl @@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --s3-cred= Supply S3 authentication information to "ros3" vfd. + :: "(,,)" + If absent or -> "(,,)", no authentication. + Has no effect is filedriver is not `ros3'. + --hdfs-attrs= Supply configuration information for HDFS file access. + For use with "--filedriver=hdfs" + :: (,, + ,, + ) + Any absent attribute will use a default value. --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl b/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl index bc37a8f..17c7b9d 100644 --- a/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl +++ b/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl @@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --s3-cred= Supply S3 authentication information to "ros3" vfd. + :: "(,,)" + If absent or -> "(,,)", no authentication. + Has no effect is filedriver is not `ros3'. + --hdfs-attrs= Supply configuration information for HDFS file access. + For use with "--filedriver=hdfs" + :: (,, + ,, + ) + Any absent attribute will use a default value. --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/pbits/tpbitsOffsetNegative.ddl b/tools/testfiles/pbits/tpbitsOffsetNegative.ddl index bc37a8f..17c7b9d 100644 --- a/tools/testfiles/pbits/tpbitsOffsetNegative.ddl +++ b/tools/testfiles/pbits/tpbitsOffsetNegative.ddl @@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files -b B, --binary=B Binary file output, of form B -O F, --ddl=F Output ddl text into file F Use blank(empty) filename F to suppress ddl display + --s3-cred= Supply S3 authentication information to "ros3" vfd. + :: "(,,)" + If absent or -> "(,,)", no authentication. + Has no effect is filedriver is not `ros3'. + --hdfs-attrs= Supply configuration information for HDFS file access. + For use with "--filedriver=hdfs" + :: (,, + ,, + ) + Any absent attribute will use a default value. --------------- Object Options --------------- -a P, --attribute=P Print the specified attribute If an attribute name contains a slash (/), escape the diff --git a/tools/testfiles/textlinksrc-nodangle-1.ls b/tools/testfiles/textlinksrc-nodangle-1.ls index 491f696..396bed3 100644 --- a/tools/testfiles/textlinksrc-nodangle-1.ls +++ b/tools/testfiles/textlinksrc-nodangle-1.ls @@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] -V, --version Print version number and exit --vfd=DRIVER Use the specified virtual file driver -x, --hexdump Show raw data in hexadecimal format + --s3-cred=C Supply S3 authentication information to "ros3" vfd. + Accepts tuple of "(,,)". + If absent or C->"(,,)", defaults to no-authentication. + Has no effect if vfd flag not set to "ros3". + --hdfs-attrs=A Supply configuration information to Hadoop VFD. + Accepts tuple of (,, + ...,,) + If absent or A == '(,,,,)', all default values are used. + Has no effect if vfd flag is not 'hdfs'. file/OBJECT Each object consists of an HDF5 file name optionally followed by a diff --git a/tools/testfiles/tgroup-1.ls b/tools/testfiles/tgroup-1.ls index 491f696..396bed3 100644 --- a/tools/testfiles/tgroup-1.ls +++ b/tools/testfiles/tgroup-1.ls @@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...] -V, --version Print version number and exit --vfd=DRIVER Use the specified virtual file driver -x, --hexdump Show raw data in hexadecimal format + --s3-cred=C Supply S3 authentication information to "ros3" vfd. + Accepts tuple of "(,,)". + If absent or C->"(,,)", defaults to no-authentication. + Has no effect if vfd flag not set to "ros3". + --hdfs-attrs=A Supply configuration information to Hadoop VFD. + Accepts tuple of (,, + ...,,) + If absent or A == '(,,,,)', all default values are used. + Has no effect if vfd flag is not 'hdfs'. file/OBJECT Each object consists of an HDF5 file name optionally followed by a diff --git a/tools/testfiles/tudlink-2.ddl b/tools/testfiles/tudlink-2.ddl index 7f4281a..82b9f8e 100644 --- a/tools/testfiles/tudlink-2.ddl +++ b/tools/testfiles/tudlink-2.ddl @@ -1,6 +1,5 @@ HDF5 "tudlink.h5" { USERDEFINED_LINK "udlink2" { - USERDEFINED_LINK "udlink2" { LINKCLASS 187 } } -- cgit v0.12