summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAllen Byrne <byrn@hdfgroup.org>2019-09-23 20:30:07 (GMT)
committerAllen Byrne <byrn@hdfgroup.org>2019-09-23 20:30:07 (GMT)
commita4d6a270783d69c663357c31fcd6196b98706dfb (patch)
treece4cd3ff16aa0a3918c4e999d3b6ba63a35a41fe
parenta0947d808fb3b0af98032beb1a4f5353443c8aa3 (diff)
downloadhdf5-a4d6a270783d69c663357c31fcd6196b98706dfb.zip
hdf5-a4d6a270783d69c663357c31fcd6196b98706dfb.tar.gz
hdf5-a4d6a270783d69c663357c31fcd6196b98706dfb.tar.bz2
HDFFV-10903 merge S3 from dev
-rw-r--r--CMakeLists.txt25
-rw-r--r--MANIFEST34
-rw-r--r--config/cmake/ConfigureChecks.cmake38
-rw-r--r--config/cmake/FindHDFS.cmake70
-rw-r--r--config/cmake/H5pubconf.h.in31
-rw-r--r--config/cmake/HDFCompilerFlags.cmake84
-rw-r--r--config/cmake/libhdf5.settings.cmake.in36
-rwxr-xr-xconfig/cmake/scripts/CTestScript.cmake20
-rwxr-xr-xconfig/cmake/scripts/HDF5config.cmake101
-rw-r--r--config/cmake/scripts/HPC/bsub-HDF5options.cmake31
-rw-r--r--config/cmake/scripts/HPC/qsub-HDF5options.cmake42
-rw-r--r--config/cmake/scripts/HPC/raybsub-HDF5options.cmake32
-rw-r--r--config/cmake/scripts/HPC/sbatch-HDF5options.cmake43
-rw-r--r--config/cmake/wait_H5Tinit.cmake11
-rw-r--r--config/cmake_ext_mod/runTest.cmake9
-rw-r--r--config/toolchain/build32.cmake36
-rw-r--r--config/toolchain/crayle.cmake10
-rw-r--r--release_docs/README_HDF5_CMake23
-rw-r--r--release_docs/README_HPC206
-rw-r--r--release_docs/RELEASE.txt25
-rw-r--r--release_docs/USING_CMake_Examples.txt5
-rw-r--r--src/CMakeLists.txt261
-rw-r--r--src/H5FD.c1054
-rw-r--r--src/H5FDcore.h2
-rw-r--r--src/H5FDhdfs.c1876
-rw-r--r--src/H5FDhdfs.h123
-rw-r--r--src/H5FDprivate.h16
-rw-r--r--src/H5FDpublic.h73
-rw-r--r--src/H5FDros3.c1787
-rw-r--r--src/H5FDros3.h106
-rw-r--r--src/H5FDs3comms.c3593
-rw-r--r--src/H5FDs3comms.h610
-rw-r--r--src/Makefile.am14
-rw-r--r--src/Makefile.in11
-rw-r--r--src/libhdf5.settings.in88
-rw-r--r--test/CMakeLists.txt4
-rw-r--r--test/Makefile.am6
-rw-r--r--test/dtypes.c1307
-rw-r--r--test/extend.c22
-rw-r--r--test/filter_fail.c12
-rw-r--r--test/flush1.c207
-rw-r--r--test/flush2.c65
-rw-r--r--test/h5test.c39
-rw-r--r--test/hdfs.c1767
-rw-r--r--test/hyperslab.c12
-rw-r--r--test/istore.c17
-rw-r--r--test/links.c4
-rw-r--r--test/links_env.c12
-rw-r--r--test/mtime.c66
-rw-r--r--test/objcopy.c6
-rw-r--r--test/reserved.c80
-rw-r--r--test/ros3.c1937
-rw-r--r--test/s3comms.c2730
-rw-r--r--test/testframe.c5
-rw-r--r--test/vfd.c950
-rw-r--r--tools/CMakeLists.txt9
-rw-r--r--tools/h5copy/h5copy.c8
-rw-r--r--tools/h5copy/h5copygentest.c20
-rw-r--r--tools/h5diff/h5diffgentest.c162
-rw-r--r--tools/h5diff/ph5diff_main.c26
-rw-r--r--tools/h5dump/h5dump.c328
-rw-r--r--tools/h5dump/h5dump_ddl.c31
-rw-r--r--tools/h5dump/h5dump_defines.h24
-rw-r--r--tools/h5dump/h5dump_xml.c5
-rw-r--r--tools/h5dump/h5dumpgentest.c99
-rw-r--r--tools/h5import/h5importtest.c2
-rw-r--r--tools/h5jam/h5jam.c42
-rw-r--r--tools/h5jam/h5jamgentest.c12
-rw-r--r--tools/h5jam/h5unjam.c12
-rw-r--r--tools/h5ls/h5ls.c1216
-rw-r--r--tools/h5repack/h5repack.c64
-rw-r--r--tools/h5repack/h5repack.h28
-rw-r--r--tools/h5repack/h5repack_copy.c34
-rw-r--r--tools/h5repack/h5repack_filters.c16
-rw-r--r--tools/h5repack/h5repack_main.c4
-rw-r--r--tools/h5repack/h5repack_refs.c34
-rw-r--r--tools/h5repack/testfiles/h5repack-help.txt4
-rw-r--r--tools/h5stat/h5stat.c386
-rw-r--r--tools/h5stat/testfiles/h5stat_help1.ddl10
-rw-r--r--tools/h5stat/testfiles/h5stat_help2.ddl10
-rw-r--r--tools/h5stat/testfiles/h5stat_nofile.ddl10
-rw-r--r--tools/lib/h5diff_array.c11
-rw-r--r--tools/lib/h5tools_dump.c26
-rw-r--r--tools/lib/h5tools_utils.c485
-rw-r--r--tools/lib/h5tools_utils.h13
-rw-r--r--tools/libtest/CMakeLists.txt18
-rw-r--r--tools/libtest/CMakeTests.cmake49
-rw-r--r--tools/libtest/Makefile.am34
-rw-r--r--tools/libtest/h5tools_test_utils.c1265
-rw-r--r--tools/misc/h5debug.c173
-rw-r--r--tools/misc/h5mkgrp.c6
-rw-r--r--tools/misc/h5repart.c585
-rw-r--r--tools/testfiles/h5dump-help.txt10
-rw-r--r--tools/testfiles/help-1.ls9
-rw-r--r--tools/testfiles/help-2.ls9
-rw-r--r--tools/testfiles/help-3.ls9
-rw-r--r--tools/testfiles/pbits/tnofilename-with-packed-bits.ddl10
-rw-r--r--tools/testfiles/pbits/tpbitsIncomplete.ddl10
-rw-r--r--tools/testfiles/pbits/tpbitsLengthExceeded.ddl10
-rw-r--r--tools/testfiles/pbits/tpbitsLengthPositive.ddl10
-rw-r--r--tools/testfiles/pbits/tpbitsMaxExceeded.ddl10
-rw-r--r--tools/testfiles/pbits/tpbitsOffsetExceeded.ddl10
-rw-r--r--tools/testfiles/pbits/tpbitsOffsetNegative.ddl10
-rw-r--r--tools/testfiles/textlinksrc-nodangle-1.ls9
-rw-r--r--tools/testfiles/tgroup-1.ls9
-rw-r--r--tools/testfiles/tudlink-2.ddl1
106 files changed, 21731 insertions, 3430 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 4fe71ff..63a0dfd 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,6 +1,10 @@
cmake_minimum_required (VERSION 3.10)
project (HDF5 C)
+if(POLICY CMP0074)
+ cmake_policy(SET CMP0074 NEW)
+endif()
+
#-----------------------------------------------------------------------------
# Instructions for use : Normal Build
#
@@ -469,6 +473,27 @@ include (${HDF_RESOURCES_DIR}/HDFCompilerFlags.cmake)
set (CMAKE_MODULE_PATH ${HDF_RESOURCES_DIR} ${HDF_RESOURCES_EXT_DIR} ${CMAKE_MODULE_PATH})
#-----------------------------------------------------------------------------
+# Option to Enable HDFS
+#-----------------------------------------------------------------------------
+option (HDF5_ENABLE_HDFS "Enable HDFS" OFF)
+if (HDF5_ENABLE_HDFS)
+ find_package(JNI REQUIRED)
+ if (JNI_FOUND)
+ set (H5_HAVE_LIBJVM 1)
+ endif ()
+ find_package(HDFS REQUIRED)
+ if (HDFS_FOUND)
+ set (H5_HAVE_LIBHDFS 1)
+ set (H5_HAVE_HDFS_H 1)
+ if (NOT MSVC)
+ list (APPEND LINK_LIBS -pthread)
+ endif ()
+ else ()
+ message (FATAL_ERROR "Set to use libhdfs library, but could not find or use libhdfs. Please verify that the path to HADOOP_HOME is valid, and/or reconfigure without HDF5_ENABLE_HDFS")
+ endif ()
+endif ()
+
+#-----------------------------------------------------------------------------
# Option to Enable MPI Parallel
#-----------------------------------------------------------------------------
option (HDF5_ENABLE_PARALLEL "Enable parallel build (requires MPI)" OFF)
diff --git a/MANIFEST b/MANIFEST
index eeae62a..463042d 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -93,6 +93,17 @@
./bin/timekeeper _DO_NOT_DISTRIBUTE_
./bin/trace
./bin/yodconfigure
+./bin/batch/ctestP.lsf.in.cmake
+./bin/batch/ctestP.sl.in.cmake
+./bin/batch/ctestS.lsf.in.cmake
+./bin/batch/ctestS.sl.in.cmake
+./bin/batch/knl_ctestP.sl.in.cmake
+./bin/batch/knl_ctestS.sl.in.cmake
+./bin/batch/knl_H5detect.sl.in.cmake
+./bin/batch/ctest.qsub.in.cmake
+./bin/batch/ray_ctestP.lsf.in.cmake
+./bin/batch/ray_ctestS.lsf.in.cmake
+./bin/batch/raybsub
./config/COPYING
./config/BlankForm
@@ -466,6 +477,8 @@
./release_docs/INSTALL_parallel
./release_docs/INSTALL_Warnings.txt
./release_docs/INSTALL_Windows.txt
+./release_docs/README_HDF5_CMake
+./release_docs/README_HPC
./release_docs/RELEASE.txt
./release_docs/USING_HDF5_CMake.txt
./release_docs/USING_HDF5_VS.txt
@@ -591,9 +604,13 @@
./src/H5FDmpio.h
./src/H5FDmulti.c
./src/H5FDmulti.h
+./src/H5FDros3.c
+./src/H5FDros3.h
./src/H5FDpkg.h
./src/H5FDprivate.h
./src/H5FDpublic.h
+./src/H5FDs3comms.h
+./src/H5FDs3comms.c
./src/H5FDsec2.c
./src/H5FDsec2.h
./src/H5FDspace.c
@@ -921,6 +938,7 @@
./test/group_old.h5
./test/h5test.c
./test/h5test.h
+./test/hdfs.c
./test/hyperslab.c
./test/istore.c
./test/le_data.h5
@@ -940,7 +958,9 @@
./test/objcopy.c
./test/plugin.c
./test/reserved.c
+./test/ros3.c
./test/pool.c
+./test/s3comms.c
./test/set_extent.c
# ====distribute this for now. See HDFFV-8236====
./test/space_overflow.c
@@ -1191,6 +1211,10 @@
./tools/lib/io_timer.c
./tools/lib/io_timer.h
+./tools/libtest/Makefile.am
+./tools/libtest/Makefile.in
+./tools/libtest/h5tools_test_utils.c
+
./tools/misc/Makefile.am
./tools/misc/Makefile.in
./tools/misc/h5debug.c
@@ -2341,6 +2365,7 @@
./config/cmake/ConfigureChecks.cmake
./config/cmake/CPack.Info.plist.in
./config/cmake/CTestCustom.cmake
+./config/cmake/FindHDFS.cmake
./config/cmake/H5cxx_config.h.in
./config/cmake/H5pubconf.h.in
./config/cmake/hdf5-config.cmake.in
@@ -2357,6 +2382,7 @@
./config/cmake/README.txt.cmake.in
./config/cmake/userblockTest.cmake
./config/cmake/vfdTest.cmake
+./config/cmake/wait_H5Tinit.cmake
./config/cmake_ext_mod/ConfigureChecks.cmake
./config/cmake_ext_mod/CTestCustom.cmake
@@ -2458,6 +2484,8 @@
./tools/h5stat/CMakeLists.txt
./tools/h5stat/CMakeTests.cmake
./tools/lib/CMakeLists.txt
+./tools/libtest/CMakeLists.txt
+./tools/libtest/CMakeTests.cmake
./tools/misc/CMakeLists.txt
./tools/misc/CMakeTestsMkgrp.cmake
./tools/misc/CMakeTestsRepart.cmake
@@ -2470,3 +2498,9 @@
./config/cmake/scripts/CTestScript.cmake
./config/cmake/scripts/HDF5config.cmake
./config/cmake/scripts/HDF5options.cmake
+
+# CMake-specific HPC Scripts
+./config/cmake/scripts/HPC/sbatch-HDF5options.cmake
+./config/cmake/scripts/HPC/bsub-HDF5options.cmake
+./config/cmake/scripts/HPC/qsub-HDF5options.cmake
+./config/cmake/scripts/HPC/raybsub-HDF5options.cmake
diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake
index 3c23375..938042a 100644
--- a/config/cmake/ConfigureChecks.cmake
+++ b/config/cmake/ConfigureChecks.cmake
@@ -47,7 +47,7 @@ MARK_AS_ADVANCED (HDF5_METADATA_TRACE_FILE)
# conversions. If not, some hard conversions will still be prefered even
# though the data may be wrong (for example, some compilers don't
# support denormalized floating values) to maximize speed.
-#
+#-----------------------------------------------------------------------------
option (HDF5_WANT_DATA_ACCURACY "IF data accuracy is guaranteed during data conversions" ON)
if (HDF5_WANT_DATA_ACCURACY)
set (${HDF_PREFIX}_WANT_DATA_ACCURACY 1)
@@ -59,7 +59,7 @@ MARK_AS_ADVANCED (HDF5_WANT_DATA_ACCURACY)
# checked and data conversion exceptions are returned. This is mainly
# for the speed optimization of hard conversions. Soft conversions can
# actually benefit little.
-#
+#-----------------------------------------------------------------------------
option (HDF5_WANT_DCONV_EXCEPTION "exception handling functions is checked during data conversions" ON)
if (HDF5_WANT_DCONV_EXCEPTION)
set (${HDF_PREFIX}_WANT_DCONV_EXCEPTION 1)
@@ -68,7 +68,7 @@ MARK_AS_ADVANCED (HDF5_WANT_DCONV_EXCEPTION)
# ----------------------------------------------------------------------
# Check if they would like the function stack support compiled in
-#
+#-----------------------------------------------------------------------------
option (HDF5_ENABLE_CODESTACK "Enable the function stack tracing (for developer debugging)." OFF)
if (HDF5_ENABLE_CODESTACK)
set (${HDF_PREFIX}_HAVE_CODESTACK 1)
@@ -89,7 +89,7 @@ set (${HDF_PREFIX}_HAVE_TMPFILE 1)
# TODO --------------------------------------------------------------------------
# Should the Default Virtual File Driver be compiled?
# This is hard-coded now but option should added to match configure
-#
+#-----------------------------------------------------------------------------
set (${HDF_PREFIX}_DEFAULT_VFD H5FD_SEC2)
if (NOT DEFINED "${HDF_PREFIX}_DEFAULT_PLUGINDIR")
@@ -106,6 +106,7 @@ if (WINDOWS)
# Set the flag to indicate that the machine has window style pathname,
# that is, "drive-letter:\" (e.g. "C:") or "drive-letter:/" (e.g. "C:/").
# (This flag should be _unset_ for all machines, except for Windows)
+ #-----------------------------------------------------------------------
set (${HDF_PREFIX}_HAVE_WINDOW_PATH 1)
endif ()
@@ -168,6 +169,25 @@ if (NOT WINDOWS)
endif ()
endif ()
+#-----------------------------------------------------------------------------
+# Check if ROS3 driver can be built
+#-----------------------------------------------------------------------------
+option (HDF5_ENABLE_ROS3_VFD "Build the ROS3 Virtual File Driver" OFF)
+ if (HDF5_ENABLE_ROS3_VFD)
+ # CMake version 3.13 fixed FindCURL module
+ if(CMAKE_VERSION VERSION_LESS "3.13.0" AND WIN32)
+ MESSAGE(FATAL_ERROR "Windows builds for this option requires a minimum of CMake 3.13")
+ endif ()
+ find_package(CURL REQUIRED)
+ find_package(OpenSSL REQUIRED)
+ if (${CURL_FOUND} AND ${OPENSSL_FOUND})
+ set (${HDF_PREFIX}_HAVE_ROS3_VFD 1)
+ list (APPEND LINK_LIBS ${CURL_LIBRARIES} ${OPENSSL_LIBRARIES})
+ INCLUDE_DIRECTORIES (${CURL_INCLUDE_DIRS} ${OPENSSL_INCLUDE_DIR})
+ else ()
+ message (STATUS "The Read-Only S3 VFD was requested but cannot be built.\nPlease check that openssl and cURL are available on your\nsystem, and/or re-configure without option HDF5_ENABLE_ROS3_VFD.")
+ endif ()
+endif ()
#-----------------------------------------------------------------------------
# Macro to determine the various conversion capabilities
@@ -215,7 +235,7 @@ endmacro ()
# is 0x004733ce17af227f, not the same as the library's conversion to 0x004733ce17af2282.
# The machine's conversion gets the correct value. We define the macro and disable
# this kind of test until we figure out what algorithm they use.
-#
+#-----------------------------------------------------------------------------
H5ConversionTests (${HDF_PREFIX}_LDOUBLE_TO_LONG_SPECIAL "Checking IF your system converts long double to (unsigned) long values with special algorithm")
# ----------------------------------------------------------------------
# Set the flag to indicate that the machine is using a special algorithm
@@ -224,7 +244,7 @@ H5ConversionTests (${HDF_PREFIX}_LDOUBLE_TO_LONG_SPECIAL "Checking IF your syst
# when the bit sequences are 003fff..., 007fff..., 00ffff..., 01ffff...,
# ..., 7fffff..., the compiler uses a unknown algorithm. We define a
# macro and skip the test for now until we know about the algorithm.
-#
+#-----------------------------------------------------------------------------
H5ConversionTests (${HDF_PREFIX}_LONG_TO_LDOUBLE_SPECIAL "Checking IF your system can convert (unsigned) long to long double values with special algorithm")
# ----------------------------------------------------------------------
# Set the flag to indicate that the machine can accurately convert
@@ -234,7 +254,7 @@ H5ConversionTests (${HDF_PREFIX}_LONG_TO_LDOUBLE_SPECIAL "Checking IF your syste
# start to go wrong on these two machines. Adjusting it higher to
# 0x4351ccf385ebc8a0dfcc... or 0x4351ccf385ebc8a0ffcc... will make the converted
# values wildly wrong. This test detects this wrong behavior and disable the test.
-#
+#-----------------------------------------------------------------------------
H5ConversionTests (${HDF_PREFIX}_LDOUBLE_TO_LLONG_ACCURATE "Checking IF correctly converting long double to (unsigned) long long values")
# ----------------------------------------------------------------------
# Set the flag to indicate that the machine can accurately convert
@@ -242,9 +262,9 @@ H5ConversionTests (${HDF_PREFIX}_LDOUBLE_TO_LLONG_ACCURATE "Checking IF correctl
# all machines, except for Mac OS 10.4, when the bit sequences are 003fff...,
# 007fff..., 00ffff..., 01ffff..., ..., 7fffff..., the converted values are twice
# as big as they should be.
-#
+#-----------------------------------------------------------------------------
H5ConversionTests (${HDF_PREFIX}_LLONG_TO_LDOUBLE_CORRECT "Checking IF correctly converting (unsigned) long long to long double values")
# ----------------------------------------------------------------------
# Check if pointer alignments are enforced
-#
+#-----------------------------------------------------------------------------
H5ConversionTests (${HDF_PREFIX}_NO_ALIGNMENT_RESTRICTIONS "Checking IF alignment restrictions are strictly enforced")
diff --git a/config/cmake/FindHDFS.cmake b/config/cmake/FindHDFS.cmake
new file mode 100644
index 0000000..e401a94
--- /dev/null
+++ b/config/cmake/FindHDFS.cmake
@@ -0,0 +1,70 @@
+
+# DerivedFrom: https://github.com/cloudera/Impala/blob/cdh5-trunk/cmake_modules/FindHDFS.cmake
+# - Find HDFS (hdfs.h and libhdfs.so)
+# This module defines
+# Hadoop_VERSION, version string of ant if found
+# HDFS_INCLUDE_DIR, directory containing hdfs.h
+# HDFS_LIBRARIES, location of libhdfs.so
+# HDFS_FOUND, whether HDFS is found.
+
+exec_program($ENV{HADOOP_HOME}/bin/hadoop ARGS version OUTPUT_VARIABLE Hadoop_VERSION
+ RETURN_VALUE Hadoop_RETURN)
+
+# currently only looking in HADOOP_HOME
+find_path(HDFS_INCLUDE_DIR hdfs.h PATHS
+ $ENV{HADOOP_HOME}/include/
+ # make sure we don't accidentally pick up a different version
+ NO_DEFAULT_PATH
+)
+
+if ("${CMAKE_SIZEOF_VOID_P}" STREQUAL "8")
+ set(arch_hint "x64")
+elseif ("$ENV{LIB}" MATCHES "(amd64|ia64)")
+ set(arch_hint "x64")
+else ()
+ set(arch_hint "x86")
+endif()
+
+message(STATUS "Architecture: ${arch_hint}")
+
+if ("${arch_hint}" STREQUAL "x64")
+ set(HDFS_LIB_PATHS $ENV{HADOOP_HOME}/lib/native)
+else ()
+ set(HDFS_LIB_PATHS $ENV{HADOOP_HOME}/lib/native)
+endif ()
+
+message(STATUS "HDFS_LIB_PATHS: ${HDFS_LIB_PATHS}")
+
+find_library(HDFS_LIB NAMES hdfs PATHS
+ ${HDFS_LIB_PATHS}
+ # make sure we don't accidentally pick up a different version
+ NO_DEFAULT_PATH
+)
+
+if (HDFS_LIB)
+ set(HDFS_FOUND TRUE)
+ set(HDFS_LIBRARIES ${HDFS_LIB})
+ set(HDFS_STATIC_LIB ${HDFS_LIB_PATHS}/${CMAKE_STATIC_LIBRARY_PREFIX}hdfs${CMAKE_STATIC_LIBRARY_SUFFIX})
+
+ add_library(hdfs_static STATIC IMPORTED)
+ set_target_properties(hdfs_static PROPERTIES IMPORTED_LOCATION ${HDFS_STATIC_LIB})
+else ()
+ set(HDFS_FOUND FALSE)
+endif ()
+
+if (HDFS_FOUND)
+ if (NOT HDFS_FIND_QUIETLY)
+ message(STATUS "${Hadoop_VERSION}")
+ message(STATUS "HDFS_INCLUDE_DIR: ${HDFS_INCLUDE_DIR}")
+ message(STATUS "HDFS_LIBRARIES: ${HDFS_LIBRARIES}")
+ message(STATUS "hdfs_static: ${HDFS_STATIC_LIB}")
+ endif ()
+else ()
+ message(FATAL_ERROR "HDFS includes and libraries NOT found."
+ "(${HDFS_INCLUDE_DIR}, ${HDFS_LIB})")
+endif ()
+
+mark_as_advanced(
+ HDFS_LIBRARIES
+ HDFS_INCLUDE_DIR
+)
diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in
index 5906f34..15a16b4 100644
--- a/config/cmake/H5pubconf.h.in
+++ b/config/cmake/H5pubconf.h.in
@@ -78,6 +78,9 @@
/* Define if the function stack tracing code is to be compiled in */
#cmakedefine H5_HAVE_CODESTACK @H5_HAVE_CODESTACK@
+/* Define to 1 if you have the <curl/curl.h> header file. */
+#cmakedefine H5_HAVE_CURL_H @H5_HAVE_CURL_H@
+
/* Define if Darwin or Mac OS X */
#cmakedefine H5_HAVE_DARWIN @H5_HAVE_DARWIN@
@@ -156,6 +159,9 @@
/* Define to 1 if you have the `gettimeofday' function. */
#cmakedefine H5_HAVE_GETTIMEOFDAY @H5_HAVE_GETTIMEOFDAY@
+/* Define to 1 if you have the <hdfs.h> header file. */
+#cmakedefine H5_HAVE_HDFS_H @H5_HAVE_HDFS_H@
+
/* Define if the compiler understands inline */
#cmakedefine H5_HAVE_INLINE @H5_HAVE_INLINE@
@@ -172,12 +178,24 @@
/* Define to 1 if you have the <io.h> header file. */
#cmakedefine H5_HAVE_IO_H @H5_HAVE_IO_H@
+/* Define to 1 if you have the `crypto' library (-lcrypto). */
+#cmakedefine H5_HAVE_LIBCRYPTO @H5_HAVE_LIBCRYPTO@
+
+/* Define to 1 if you have the `curl' library (-lcurl). */
+#cmakedefine H5_HAVE_LIBCURL @H5_HAVE_LIBCURL@
+
/* Define to 1 if you have the `dl' library (-ldl). */
#cmakedefine H5_HAVE_LIBDL @H5_HAVE_LIBDL@
/* Define to 1 if you have the `dmalloc' library (-ldmalloc). */
#cmakedefine H5_HAVE_LIBDMALLOC @H5_HAVE_LIBDMALLOC@
+/* Proceed to build with libhdfs */
+#cmakedefine H5_HAVE_LIBHDFS @H5_HAVE_LIBHDFS@
+
+/* Define to 1 if you have the `jvm' library (-ljvm). */
+#cmakedefine H5_HAVE_LIBJVM @H5_HAVE_LIBJVM@
+
/* Define to 1 if you have the `m' library (-lm). */
#cmakedefine H5_HAVE_LIBM @H5_HAVE_LIBM@
@@ -235,6 +253,15 @@
/* Define if MPI_Info_c2f and MPI_Info_f2c exists */
#cmakedefine H5_HAVE_MPI_MULTI_LANG_Info @H5_HAVE_MPI_MULTI_LANG_Info@
+/* Define to 1 if you have the <openssl/evp.h> header file. */
+#cmakedefine H5_HAVE_OPENSSL_EVP_H @H5_HAVE_OPENSSL_EVP_H@
+
+/* Define to 1 if you have the <openssl/hmac.h> header file. */
+#cmakedefine H5_HAVE_OPENSSL_HMAC_H @H5_HAVE_OPENSSL_HMAC_H@
+
+/* Define to 1 if you have the <openssl/sha.h> header file. */
+#cmakedefine H5_HAVE_OPENSSL_SHA_H @H5_HAVE_OPENSSL_SHA_H@
+
/* Define if we have parallel support */
#cmakedefine H5_HAVE_PARALLEL @H5_HAVE_PARALLEL@
@@ -247,6 +274,10 @@
/* Define to 1 if you have the `rand_r' function. */
#cmakedefine H5_HAVE_RAND_R @H5_HAVE_RAND_R@
+/* Define whether the Read-Only S3 virtual file driver (VFD) should be
+ compiled */
+#cmakedefine H5_HAVE_ROS3_VFD @H5_HAVE_ROS3_VFD@
+
/* Define to 1 if you have the `round' function. */
#cmakedefine H5_HAVE_ROUND @H5_HAVE_ROUND@
diff --git a/config/cmake/HDFCompilerFlags.cmake b/config/cmake/HDFCompilerFlags.cmake
index 82d63af..adace89 100644
--- a/config/cmake/HDFCompilerFlags.cmake
+++ b/config/cmake/HDFCompilerFlags.cmake
@@ -143,7 +143,7 @@ if (NOT MSVC AND CMAKE_COMPILER_IS_GNUCC)
#
# Technically, variable-length arrays are part of the C99 standard, but
# we should approach them a bit cautiously... -QAK
- set (H5_CFLAGS1 "${H5_CFLAGS1} -Wlogical-op -Wlarger-than=2048 -Wvla")
+ set (H5_CFLAGS1 "${H5_CFLAGS1} -Wlogical-op -Wlarger-than=2560 -Wvla")
# Append more extra warning flags that only gcc 4.4+ know about
set (H5_CFLAGS1 "${H5_CFLAGS1} -Wsync-nand -Wframe-larger-than=16384 -Wpacked-bitfield-compat")
@@ -210,9 +210,10 @@ if (NOT MSVC AND CMAKE_COMPILER_IS_GNUCC)
# Append more extra warning flags that only gcc 9.x+ know about
if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0)
- set (H5_CFLAGS4 "${H5_CFLAGS4} Wattribute-alias=2 -Wmissing-profile")
+ set (H5_CFLAGS4 "${H5_CFLAGS4} -Wattribute-alias=2 -Wmissing-profile")
endif ()
-
+elseif (CMAKE_C_COMPILER_ID STREQUAL "PGI")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Minform=inform")
endif ()
#-----------------------------------------------------------------------------
@@ -339,3 +340,80 @@ endif ()
if (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_LOADED)
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fmessage-length=0")
endif ()
+
+#-----------------------------------------------------------------------------
+# Option for --enable-asserts
+# By default, CMake adds NDEBUG to CMAKE_${lang}_FLAGS for Release build types
+# This option will force/override the default setting for all configurations
+#-----------------------------------------------------------------------------
+#option (HDF5_ENABLE_ASSERTS "Determines whether NDEBUG is defined to control assertions." OFF)
+set (HDF5_ENABLE_ASSERTS "OFF" CACHE STRING "Determines whether NDEBUG is defined to control assertions (OFF NO YES)")
+set_property (CACHE HDF5_ENABLE_ASSERTS PROPERTY STRINGS OFF NO YES)
+if (HDF5_ENABLE_ASSERTS MATCHES "YES")
+ add_compile_options ("-UNDEBUG")
+elseif (HDF5_ENABLE_ASSERTS MATCHES "NO")
+ add_compile_options ("-DNDEBUG")
+endif ()
+MARK_AS_ADVANCED (HDF5_ENABLE_ASSERTS)
+
+#-----------------------------------------------------------------------------
+# Option for --enable-symbols
+# This option will force/override the default setting for all configurations
+#-----------------------------------------------------------------------------
+#option (HDF5_ENABLE_SYMBOLS "Add debug symbols to the library independent of the build mode and optimization level." OFF)
+set (HDF5_ENABLE_SYMBOLS "OFF" CACHE STRING "Add debug symbols to the library independent of the build mode and optimization level (OFF NO YES)")
+set_property (CACHE HDF5_ENABLE_SYMBOLS PROPERTY STRINGS OFF NO YES)
+if (HDF5_ENABLE_SYMBOLS MATCHES "YES")
+ if (CMAKE_C_COMPILER_ID STREQUAL "Intel")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g")
+ elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fno-omit-frame-pointer")
+ endif ()
+ if(CMAKE_CXX_COMPILER_LOADED)
+ if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
+ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g")
+ endif ()
+ endif ()
+elseif (HDF5_ENABLE_SYMBOLS MATCHES "NO")
+ if (CMAKE_C_COMPILER_ID STREQUAL "Intel")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-s")
+ elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -s")
+ endif ()
+ if(CMAKE_CXX_COMPILER_LOADED)
+ if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+ set (CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -Wl,-s")
+ elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -s")
+ endif ()
+ endif ()
+endif ()
+MARK_AS_ADVANCED (HDF5_ENABLE_SYMBOLS)
+
+#-----------------------------------------------------------------------------
+# Option for --enable-profiling
+# This option will force/override the default setting for all configurations
+#-----------------------------------------------------------------------------
+option (HDF5_ENABLE_PROFILING "Enable profiling flags independently from the build mode." OFF)
+if (HDF5_ENABLE_PROFILING)
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PROFILE_CFLAGS}")
+ if(CMAKE_CXX_COMPILER_LOADED)
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PROFILE_CXXFLAGS}")
+ endif ()
+endif ()
+MARK_AS_ADVANCED (HDF5_ENABLE_PROFILING)
+
+#-----------------------------------------------------------------------------
+# Option for --enable-optimization
+# This option will force/override the default setting for all configurations
+#-----------------------------------------------------------------------------
+option (HDF5_ENABLE_OPTIMIZATION "Enable optimization flags/settings independently from the build mode" OFF)
+if (HDF5_ENABLE_OPTIMIZATION)
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OPTIMIZE_CFLAGS}")
+ if(CMAKE_CXX_COMPILER_LOADED)
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OPTIMIZE_CXXFLAGS}")
+ endif ()
+endif ()
+MARK_AS_ADVANCED (HDF5_ENABLE_OPTIMIZATION)
diff --git a/config/cmake/libhdf5.settings.cmake.in b/config/cmake/libhdf5.settings.cmake.in
index c3efbdd..7bfadb1 100644
--- a/config/cmake/libhdf5.settings.cmake.in
+++ b/config/cmake/libhdf5.settings.cmake.in
@@ -10,32 +10,40 @@ General Information:
Host system: @CMAKE_HOST_SYSTEM@
Uname information: @CMAKE_SYSTEM_NAME@
Byte sex: @BYTESEX@
- Libraries: @BUILD_NAME_EXT@
Installation point: @CMAKE_INSTALL_PREFIX@
Compiling Options:
------------------
- Compilation Mode: @CMAKE_BUILD_TYPE@ @CMAKE_C_COMPILER_VERSION@
- C Compiler: @CMAKE_C_COMPILER@
- CFLAGS: @CMAKE_C_FLAGS@
- H5_CFLAGS: @H5_CFLAGS@
- AM_CFLAGS: @AM_CFLAGS@
- CPPFLAGS: @CPPFLAGS@
- H5_CPPFLAGS: @H5_CPPFLAGS@
- AM_CPPFLAGS: @AM_CPPFLAGS@
- Shared C Library: @H5_ENABLE_SHARED_LIB@
- Static C Library: @H5_ENABLE_STATIC_LIB@
+ Build Mode: @CMAKE_BUILD_TYPE@
+ Debugging Symbols: @HDF5_ENABLE_SYMBOLS@
+ Asserts: @HDF5_ENABLE_ASSERTS@
+ Profiling: @HDF5_ENABLE_PROFILING@
+ Optimization Level: @HDF5_ENABLE_OPTIMIZATION@
+
+Linking Options:
+----------------
+ Libraries: @BUILD_NAME_EXT@
Statically Linked Executables: @BUILD_STATIC_EXECS@
LDFLAGS: @CMAKE_SHARED_LINKER_FLAGS@
+ H5_LDFLAGS: @H5_LDFLAGS@
AM_LDFLAGS: @AM_LDFLAGS@
Extra libraries: @LINK_LIBS@
Archiver: @CMAKE_AR@
Ranlib: @CMAKE_RANLIB@
- Debugged Packages: @DEBUG_PKG@
- API Tracing: @HDF5_ENABLE_TRACE@
Languages:
----------
+ C: YES
+ C Compiler: @CMAKE_C_COMPILER@ @CMAKE_C_COMPILER_VERSION@
+ CPPFLAGS: @CPPFLAGS@
+ H5_CPPFLAGS: @H5_CPPFLAGS@
+ AM_CPPFLAGS: @AM_CPPFLAGS@
+ CFLAGS: @CMAKE_C_FLAGS@
+ H5_CFLAGS: @H5_CFLAGS@
+ AM_CFLAGS: @AM_CFLAGS@
+ Shared C Library: @H5_ENABLE_SHARED_LIB@
+ Static C Library: @H5_ENABLE_STATIC_LIB@
+
Fortran: @HDF5_BUILD_FORTRAN@
@BUILD_FORTRAN_CONDITIONAL_TRUE@ Fortran Compiler: @CMAKE_Fortran_COMPILER@ @CMAKE_Fortran_COMPILER_VERSION@
@BUILD_FORTRAN_CONDITIONAL_TRUE@ Fortran 2003 Compiler: @HDF5_ENABLE_F2003@
@@ -65,6 +73,8 @@ Features:
I/O filters (external): @EXTERNAL_FILTERS@
MPE: @H5_HAVE_LIBLMPE@
Direct VFD: @H5_HAVE_DIRECT@
+ (Read-Only) S3 VFD: @H5_HAVE_ROS3_VFD@
+ (Read-Only) HDFS VFD: @H5_HAVE_LIBHDFS@
dmalloc: @H5_HAVE_LIBDMALLOC@
Clear file buffers before write: @HDF5_Enable_Clear_File_Buffers@
Using memory checker: @HDF5_ENABLE_USING_MEMCHECKER@
diff --git a/config/cmake/scripts/CTestScript.cmake b/config/cmake/scripts/CTestScript.cmake
index 8ff52ec..dc3939e 100755
--- a/config/cmake/scripts/CTestScript.cmake
+++ b/config/cmake/scripts/CTestScript.cmake
@@ -277,7 +277,15 @@ message (STATUS "Dashboard script configuration:\n${vars}\n")
ctest_test (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND ${ctest_test_args} RETURN_VALUE res)
else ()
file(STRINGS ${CTEST_BINARY_DIRECTORY}/Testing/TAG TAG_CONTENTS REGEX "^2([0-9]+)[-]([0-9]+)$")
- execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME})
+ if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub")
+ execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME})
+ else ()
+ if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "qsub")
+ execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME} ctestS.out)
+ else ()
+ execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME})
+ endif()
+ endif ()
message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml")
execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
while(result)
@@ -292,7 +300,15 @@ message (STATUS "Dashboard script configuration:\n${vars}\n")
file (RENAME ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/SerialTest.xml)
file (RENAME ${CTEST_BINARY_DIRECTORY}/Testing/Temporary/LastTest_${TAG_CONTENTS}.log ${CTEST_BINARY_DIRECTORY}/Testing/Temporary/LastTest_${TAG_CONTENTS}_Serial.log)
unset(result CACHE)
- execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
+ if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub")
+ execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
+ else ()
+ if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "qsub")
+ execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME} ctestP.out)
+ else ()
+ execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
+ endif ()
+ endif ()
message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml")
execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
while(result)
diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake
index 9d84ba6..185a86e 100755
--- a/config/cmake/scripts/HDF5config.cmake
+++ b/config/cmake/scripts/HDF5config.cmake
@@ -9,56 +9,62 @@
# If you do not have access to either file, you may request a copy from
# help@hdfgroup.org.
#
-#############################################################################################
-### ${CTEST_SCRIPT_ARG} is of the form OPTION=VALUE ###
+#############################################################################################
+### ${CTEST_SCRIPT_ARG} is of the form OPTION=VALUE ###
### BUILD_GENERATOR required [Unix, VS2017, VS201764, VS2015, VS201564, VS2013, VS201364] ###
-### ctest -S HDF518config.cmake,BUILD_GENERATOR=VS201764 -C Release -VV -O hdf518.log ###
-#############################################################################################
-
-cmake_minimum_required (VERSION 3.10)
-############################################################################
-# Usage:
-# ctest -S HDF518config.cmake,OPTION=VALUE -C Release -VV -O test.log
-# where valid options for OPTION are:
-# BUILD_GENERATOR - The cmake build generator:
-# Unix * Unix Makefiles
+### ctest -S HDF518config.cmake,BUILD_GENERATOR=VS201764 -C Release -VV -O hdf518.log ###
+#############################################################################################
+
+cmake_minimum_required (VERSION 3.10)
+############################################################################
+# Usage:
+# ctest -S HDF518config.cmake,OPTION=VALUE -C Release -VV -O test.log
+# where valid options for OPTION are:
+# BUILD_GENERATOR - The cmake build generator:
+# Unix * Unix Makefiles
# VS2017 * Visual Studio 15 2017
# VS201764 * Visual Studio 15 2017 Win64
-# VS2015 * Visual Studio 14 2015
-# VS201564 * Visual Studio 14 2015 Win64
-# VS2013 * Visual Studio 12 2013
-# VS201364 * Visual Studio 12 2013 Win64
-#
-# INSTALLDIR - root folder where hdf5 is installed
-# CTEST_CONFIGURATION_TYPE - Release, Debug, etc
-# CTEST_SOURCE_NAME - source folder
-##############################################################################
-
-set (CTEST_SOURCE_VERSION "1.8.22")
-set (CTEST_SOURCE_VERSEXT "-snap3")
-
-##############################################################################
-# handle input parameters to script.
-#BUILD_GENERATOR - which CMake generator to use, required
-#INSTALLDIR - HDF5-1.8 root folder
-#CTEST_CONFIGURATION_TYPE - Release, Debug, RelWithDebInfo
-#CTEST_SOURCE_NAME - name of source folder; HDF5-1.8.x
+# VS2015 * Visual Studio 14 2015
+# VS201564 * Visual Studio 14 2015 Win64
+# VS2013 * Visual Studio 12 2013
+# VS201364 * Visual Studio 12 2013 Win64
+#
+# INSTALLDIR - root folder where hdf5 is installed
+# CTEST_CONFIGURATION_TYPE - Release, Debug, etc
+# CTEST_SOURCE_NAME - source folder
+##############################################################################
+
+set (CTEST_SOURCE_VERSION "1.8.22")
+set (CTEST_SOURCE_VERSEXT "-snap3")
+
+##############################################################################
+# handle input parameters to script.
+#BUILD_GENERATOR - which CMake generator to use, required
+#INSTALLDIR - HDF5-1.8 root folder
+#CTEST_CONFIGURATION_TYPE - Release, Debug, RelWithDebInfo
+#CTEST_SOURCE_NAME - name of source folder; HDF5-1.8.x
#MODEL - CDash group name
+#HPC - run alternate configurations for HPC machines; sbatch, bsub, raybsub, qsub
#MPI - enable MPI
-if (DEFINED CTEST_SCRIPT_ARG)
- # transform ctest script arguments of the form
- # script.ctest,var1=value1,var2=value2
- # to variables with the respective names set to the respective values
- string (REPLACE "," ";" script_args "${CTEST_SCRIPT_ARG}")
- foreach (current_var ${script_args})
- if ("${current_var}" MATCHES "^([^=]+)=(.+)$")
- set ("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}")
- endif ()
- endforeach ()
-endif ()
-
-# build generator must be defined
-if (NOT DEFINED BUILD_GENERATOR)
+if (DEFINED CTEST_SCRIPT_ARG)
+ # transform ctest script arguments of the form
+ # script.ctest,var1=value1,var2=value2
+ # to variables with the respective names set to the respective values
+ string (REPLACE "," ";" script_args "${CTEST_SCRIPT_ARG}")
+ foreach (current_var ${script_args})
+ if ("${current_var}" MATCHES "^([^=]+)=(.+)$")
+ set ("${CMAKE_MATCH_1}" "${CMAKE_MATCH_2}")
+ endif ()
+ endforeach ()
+endif ()
+
+#HPC - run alternate configurations for HPC machines
+if (DEFINED HPC)
+ set (BUILD_GENERATOR "Unix")
+endif ()
+
+# build generator must be defined
+if (NOT DEFINED BUILD_GENERATOR)
message (FATAL_ERROR "BUILD_GENERATOR must be defined - Unix, VS2017, or VS201764, VS2015, VS201564, VS2013, VS201364")
endif ()
@@ -94,6 +100,7 @@ endif ()
###################################################################
######### Following describes compiler ############
+if (NOT DEFINED HPC)
if (NOT DEFINED BUILD_GENERATOR)
message (FATAL_ERROR "BUILD_GENERATOR must be defined - Unix, VS2017, or VS201764, VS2015, VS201564, VS2013, VS201364")
endif ()
@@ -169,6 +176,10 @@ endif ()
set (ENV{CXXFLAGS} "${RR_WARNINGS_CXX} ${RR_FLAGS_CXX}")
endif ()
endif ()
+else ()
+ set (CTEST_SITE "${SITE_OS_NAME}")
+ set (CTEST_CMAKE_GENERATOR "Unix Makefiles")
+endif ()
###################################################################
###################################################################
diff --git a/config/cmake/scripts/HPC/bsub-HDF5options.cmake b/config/cmake/scripts/HPC/bsub-HDF5options.cmake
new file mode 100644
index 0000000..7473e8a
--- /dev/null
+++ b/config/cmake/scripts/HPC/bsub-HDF5options.cmake
@@ -0,0 +1,31 @@
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+#############################################################################################
+#### Change default configuration of options in config/cmake/cacheinit.cmake file ###
+#### format: set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ") ###
+#############################################################################################
+if (DEFINED MPI)
+ # maximum parallel processor count for build and test ####
+ set (MAX_PROC_COUNT 8)
+endif()
+#############################################################################################
+### options to run test scripts in batch commands
+set (LOCAL_BATCH_SCRIPT_COMMAND "bsub")
+set (LOCAL_BATCH_TEST "TRUE")
+set (LOCAL_BATCH_SCRIPT_NAME "ctestS.lsf")
+set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "ctestP.lsf")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_TEST:BOOL=ON")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_NAME:STRING=${LOCAL_BATCH_SCRIPT_NAME}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_PARALLEL_NAME:STRING=${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_EXECUTABLE:STRING=srun")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_NUMPROC_FLAG:STRING=-n")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_MAX_NUMPROCS:STRING=6")
diff --git a/config/cmake/scripts/HPC/qsub-HDF5options.cmake b/config/cmake/scripts/HPC/qsub-HDF5options.cmake
new file mode 100644
index 0000000..34eba1b
--- /dev/null
+++ b/config/cmake/scripts/HPC/qsub-HDF5options.cmake
@@ -0,0 +1,42 @@
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+#############################################################################################
+#### Change default configuration of options in config/cmake/cacheinit.cmake file ###
+#### format: set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ") ###
+#############################################################################################
+if (DEFINED MPI)
+ # maximum parallel processor count for build and test ####
+ set (MAX_PROC_COUNT 8)
+endif()
+#############################################################################################
+### options to run test scripts in batch commands
+set (LOCAL_BATCH_SCRIPT_NAME "ctest.qsub")
+set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "ctest.qsub")
+if (DEFINED KNL)
+ ### some additions and alternatives to cross compile on haswell for knl
+ set (COMPUTENODE_HWCOMPILE_MODULE "craype-mic-knl")
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake")
+endif ()
+set (LOCAL_BATCH_SCRIPT_COMMAND "qsub")
+set (LOCAL_BATCH_TEST "TRUE")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_TEST:BOOL=ON")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_NAME:STRING=${LOCAL_BATCH_SCRIPT_NAME}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_PARALLEL_NAME:STRING=${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_EXECUTABLE:STRING=aprun")
+# Option to suppress writing job statistics; to avoid issues with h5diff comparisons.
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_PREFLAGS:STRING=-q")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_NUMPROC_FLAG:STRING=-n")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_MAX_NUMPROCS:STRING=6")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DACCOUNT_ID:STRING=${LOCAL_BATCH_SCRIPT_ARGS}")
+
+#############################################################################################
+#############################################################################################
diff --git a/config/cmake/scripts/HPC/raybsub-HDF5options.cmake b/config/cmake/scripts/HPC/raybsub-HDF5options.cmake
new file mode 100644
index 0000000..fa1ec4a
--- /dev/null
+++ b/config/cmake/scripts/HPC/raybsub-HDF5options.cmake
@@ -0,0 +1,32 @@
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+#############################################################################################
+#### Change default configuration of options in config/cmake/cacheinit.cmake file ###
+#### format: set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ") ###
+#############################################################################################
+if (DEFINED MPI)
+ # maximum parallel processor count for build and test ####
+ set (MAX_PROC_COUNT 8)
+endif()
+#############################################################################################
+### options to run test scripts in batch commands
+set (LOCAL_BATCH_SCRIPT_COMMAND "raybsub")
+set (LOCAL_BATCH_TEST "TRUE")
+set (LOCAL_BATCH_SCRIPT_NAME "ray_ctestS.lsf")
+set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "ray_ctestP.lsf")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_TEST:BOOL=ON")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_COMMAND:STRING=${LOCAL_BATCH_SCRIPT_COMMAND}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_NAME:STRING=${LOCAL_BATCH_SCRIPT_NAME}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_PARALLEL_NAME:STRING=${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_EXECUTABLE:STRING=mpirun")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_NUMPROC_FLAG:STRING=-np")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_MAX_NUMPROCS:STRING=6")
diff --git a/config/cmake/scripts/HPC/sbatch-HDF5options.cmake b/config/cmake/scripts/HPC/sbatch-HDF5options.cmake
new file mode 100644
index 0000000..3205a1c
--- /dev/null
+++ b/config/cmake/scripts/HPC/sbatch-HDF5options.cmake
@@ -0,0 +1,43 @@
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+#############################################################################################
+#### Change default configuration of options in config/cmake/cacheinit.cmake file ###
+#### format: set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ") ###
+#############################################################################################
+if (DEFINED MPI)
+ # maximum parallel processor count for build and test ####
+ set (MAX_PROC_COUNT 8)
+endif()
+#############################################################################################
+### options to run test scripts in batch commands
+if (DEFINED KNL)
+ ### some additions and alternatives to cross compile on haswell for knl
+ set (COMPILENODE_HWCOMPILE_MODULE "craype-haswell")
+ set (COMPUTENODE_HWCOMPILE_MODULE "craype-mic-knl")
+ set (LOCAL_BATCH_SCRIPT_NAME "knl_ctestS.sl")
+ set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "knl_ctestP.sl")
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake")
+else ()
+ set (LOCAL_BATCH_SCRIPT_NAME "ctestS.sl")
+ set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "ctestP.sl")
+endif ()
+set (LOCAL_BATCH_SCRIPT_COMMAND "sbatch")
+set (LOCAL_BATCH_TEST "TRUE")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_TEST:BOOL=ON")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_NAME:STRING=${LOCAL_BATCH_SCRIPT_NAME}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_PARALLEL_NAME:STRING=${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_EXECUTABLE:STRING=srun")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_NUMPROC_FLAG:STRING=-n")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_MAX_NUMPROCS:STRING=6")
+
+#############################################################################################
+#############################################################################################
diff --git a/config/cmake/wait_H5Tinit.cmake b/config/cmake/wait_H5Tinit.cmake
new file mode 100644
index 0000000..c0b4d06
--- /dev/null
+++ b/config/cmake/wait_H5Tinit.cmake
@@ -0,0 +1,11 @@
+cmake_minimum_required (VERSION 3.10)
+
+message(STATUS "Check for existence of ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c")
+execute_process(COMMAND ls ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c RESULT_VARIABLE H5TI_result OUTPUT_QUIET ERROR_QUIET)
+while(H5TI_result)
+ ctest_sleep(30)
+ message(STATUS "Checking again for existence of ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c")
+ execute_process(COMMAND ls ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c RESULT_VARIABLE H5TI_result OUTPUT_QUIET ERROR_QUIET)
+endwhile(H5TI_result)
+file (TOUCH "${HDF5_BINARY_DIR}/H5Tinit_created")
+message(STATUS "Found ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c")
diff --git a/config/cmake_ext_mod/runTest.cmake b/config/cmake_ext_mod/runTest.cmake
index e601653..6f633f3 100644
--- a/config/cmake_ext_mod/runTest.cmake
+++ b/config/cmake_ext_mod/runTest.cmake
@@ -343,6 +343,15 @@ if (TEST_GREP_COMPARE)
endif ()
endif ()
+# dump the output unless nodisplay option is set
+if (TEST_SKIP_COMPARE AND NOT TEST_NO_DISPLAY)
+ file (READ ${TEST_FOLDER}/${TEST_OUTPUT} TEST_STREAM)
+ execute_process (
+ COMMAND ${CMAKE_COMMAND} -E echo ${TEST_STREAM}
+ RESULT_VARIABLE TEST_RESULT
+ )
+endif ()
+
# everything went fine...
message (STATUS "${TEST_PROGRAM} Passed")
diff --git a/config/toolchain/build32.cmake b/config/toolchain/build32.cmake
index d078956..deb5899 100644
--- a/config/toolchain/build32.cmake
+++ b/config/toolchain/build32.cmake
@@ -3,6 +3,42 @@ if (WIN32)
set (CMAKE_GENERATOR_PLATFORM "x86")
elseif(APPLE)
set (CMAKE_OSX_ARCHITECTURES "i386")
+elseif(MINGW)
+ set (CMAKE_SYSTEM_NAME Windows)
+ set (CMAKE_C_COMPILER i686-w64-mingw32-gcc)
+ set (CMAKE_CXX_COMPILER i686-w64-mingw32-g++)
+ set (CMAKE_RC_COMPILER i686-w64-mingw32-windres)
+ set (CMAKE_Fortran_COMPILER i686-w64-mingw32-gfortran)
+
+ set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "c++ flags")
+ set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32" CACHE STRING "c flags")
+
+ set (LIB32 /usr/lib) # Fedora
+
+ if (EXISTS "/usr/lib32")
+ set (LIB32 /usr/lib32) # Arch, Solus
+ endif ()
+
+ set (CMAKE_SYSTEM_LIBRARY_PATH ${LIB32} CACHE STRING "system library search path" FORCE)
+ set (CMAKE_LIBRARY_PATH ${LIB32} CACHE STRING "library search path" FORCE)
+
+ # this is probably unlikely to be needed, but just in case
+ set (CMAKE_EXE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "executable linker flags" FORCE)
+ set (CMAKE_SHARED_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "shared library linker flags" FORCE)
+ set (CMAKE_MODULE_LINKER_FLAGS "-m32 -L${LIB32}" CACHE STRING "module linker flags" FORCE)
+
+ # on Fedora and Arch and similar, point pkgconfig at 32 bit .pc files. We have
+ # to include the regular system .pc files as well (at the end), because some
+ # are not always present in the 32 bit directory
+ if (EXISTS "${LIB32}/pkgconfig")
+ set (ENV{PKG_CONFIG_LIBDIR} ${LIB32}/pkgconfig:/usr/share/pkgconfig:/usr/lib/pkgconfig:/usr/lib64/pkgconfig)
+ endif ()
+
+ set (CMAKE_FIND_ROOT_PATH /usr/i686-w64-mingw32)
+ set (CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER)
+ set (CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY)
+ set (CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY)
+ set (CMAKE_CROSSCOMPILING_EMULATOR wine32)
else ()
set (CMAKE_SYSTEM_NAME Linux)
diff --git a/config/toolchain/crayle.cmake b/config/toolchain/crayle.cmake
new file mode 100644
index 0000000..bf7cf69
--- /dev/null
+++ b/config/toolchain/crayle.cmake
@@ -0,0 +1,10 @@
+# The following line will use cross-compiling
+set(CMAKE_SYSTEM_NAME Linux)
+
+set(CMAKE_COMPILER_VENDOR "CrayLinuxEnvironment")
+
+set(CMAKE_C_COMPILER cc)
+set(CMAKE_Fortran_COMPILER ftn)
+
+# the following is used if cross-compiling
+set(CMAKE_CROSSCOMPILING_EMULATOR "")
diff --git a/release_docs/README_HDF5_CMake b/release_docs/README_HDF5_CMake
new file mode 100644
index 0000000..9580fc7
--- /dev/null
+++ b/release_docs/README_HDF5_CMake
@@ -0,0 +1,23 @@
+This tar file contains
+
+ build-unix.sh script to build HDF5 with CMake on unix machines
+ build-unix-hpc.sh script to build HDF5 with CMake on unix machines and run
+ tests with batch scripts (sbatch).
+ CTestScript.cmake
+ HDF5config.cmake CMake scripts for building HDF5
+ HDF5options.cmake
+ hdf5-1.8.22 HDF5 1.8.22 source
+ SZip.tar.gz source for building SZIP
+ ZLib.tar.gz source for building ZLIB
+
+For more information about building HDF5 with CMake, see USING_HDF5_CMake.txt in
+hdf5-1.8.22/release_docs, or
+https://portal.hdfgroup.org/display/support/Building+HDF5+with+CMake.
+
+For more information about building HDF5 with CMake on HPC machines, including
+cross compiling on Cray XC40, see README_HPC in hdf5-1.8.22/release_docs.
+
+
+
+
+
diff --git a/release_docs/README_HPC b/release_docs/README_HPC
new file mode 100644
index 0000000..97f088c
--- /dev/null
+++ b/release_docs/README_HPC
@@ -0,0 +1,206 @@
+************************************************************************
+* Using CMake to build and test HDF5 source on HPC machines *
+************************************************************************
+
+ Contents
+
+Section I: Prerequisites
+Section II: Obtain HDF5 source
+Section III: Using ctest command to build and test
+Section IV: Cross compiling
+Section V: Manual alternatives
+Section VI: Other cross compiling options
+
+************************************************************************
+
+========================================================================
+I. Prerequisites
+========================================================================
+ 1. Create a working directory that is accessible from the compute nodes for
+ running tests; the working directory should be in a scratch space or a
+ parallel file system space since testing will use this space. Building
+ from HDF5 source in a 'home' directory typically results in test
+ failures and should be avoided.
+
+ 2. Load modules for desired compilers, module for cmake version 3.10 or greater,
+ and set any needed environment variables for compilers (i.e., CC, FC, CXX).
+ Unload any problematic modules (i.e., craype-hugepages2M).
+
+========================================================================
+II. Obtain HDF5 source
+========================================================================
+Obtain HDF5 source code from the HDF5 repository using a git command or
+from a release tar file in a working directory:
+
+ git clone https://git@bitbucket.hdfgroup.org/scm/hdffv/hdf5.git
+ [-b branch] [source directory]
+
+If no branch is specified, then the 'develop' version will be checked out.
+If no source directory is specified, then the source will be located in the
+'hdf5' directory. The Cmake scripts expect the source to be in a directory
+named hdf5-<version string>, where 'version string' uses the format '1.xx.xx'.
+For example, for the current 'develop' version, the "hdf5" directory should
+be renamed "hdf5-1.8.22", or for the first hdf5_1_8_22 pre-release version,
+it should be renamed "hdf5-1.8.22-pre1".
+
+If the version number is not known a priori, the version string
+can be obtained by running bin/h5vers in the top level directory of the source clone, and
+the source directory renamed 'hdf5-<version string>'.
+
+Release or snapshot tar files may also be extracted and used.
+
+========================================================================
+III. Using ctest command to build and test
+========================================================================
+
+The ctest command [1]:
+
+ ctest -S HDF5config.cmake,BUILD_GENERATOR=Unix -C Release -V -O hdf5.log
+
+will configure, build, test and package HDF5 from the downloaded source
+after the setup steps outlined below are followed.
+
+CMake option variables are available to allow running test programs in batch
+scripts on compute nodes and to cross-compile for compute node hardware using
+a cross-compiling emulator. The setup steps will make default settings for
+parallel or serial only builds available to the CMake command.
+
+ 1. For the current 'develop' version the "hdf5" directory should be renamed
+ "hdf5-1.8.22".
+
+ 2. Three cmake script files need to be copied to the working directory, or
+ have symbolic links to them, created in the working directory:
+
+ hdf5-1.8.22/config/cmake/scripts/HDF5config.cmake
+ hdf5-1.8.22/config/cmake/scripts/CTestScript.cmake
+ hdf5-1.8.22/config/cmake/scripts/HDF5options.cmake
+
+ should be copied to the working directory.
+
+ 3. The resulting contents of the working directory are then:
+
+ CTestScript.cmake
+ HDF5config.cmake
+ HDF5options.cmake
+ hdf5-1.8.22
+
+ Additionally, when the ctest command runs [1], it will add a build directory
+ in the working directory.
+
+ 4. The following options (among others) can be added to the ctest
+ command [1], following '-S HDF5config.cmake,' and separated by ',':
+
+ HPC=sbatch (or 'bsub' or 'raybsub') indicates which type of batch
+ files to use for running tests. If omitted, test
+ will run on the local machine or login node.
+
+ KNL=true to cross-compile for KNL compute nodes on CrayXC40
+ (see section IV)
+
+ MPI=true enables parallel, disables c++, java, and threadsafe
+
+ LOCAL_BATCH_SCRIPT_ARGS="--account=<account#>" to supply user account
+ information for batch jobs
+
+ The HPC options will add BUILD_GENERATOR=Unix for the three HPC options.
+ An example ctest command for a parallel build on a system using sbatch is
+
+ ctest -S HDF5config.cmake,HPC=sbatch,MPI=true -C Release -V -O hdf5.log
+
+ Adding the option 'KNL=true' to the above list will compile for KNL nodes,
+ for example, on 'mutrino' and other CrayXC40 machines.
+
+ Changing -V to -VV will produce more logging information in HDF5.log.
+
+ More detailed CMake information can be found in the HDF5 source in
+ release_docs/INSTALL_CMake.txt.
+
+========================================================================
+IV. Cross-compiling
+========================================================================
+For cross-compiling on Cray, set environment variables CC=cc, FC=ftn
+and CXX=CC (for c++) after all compiler modules are loaded since switching
+compiler modules may unset or reset these variables.
+
+CMake provides options for cross-compiling. To cross-compile for KNL hardware
+on mutrino and other CrayXC40 machines, add HPC=sbatch,KNL=true to the
+ctest command line. This will set the following options from the
+config/cmake/scripts/HPC/sbatch-HDF5options.cmake file:
+
+ set (COMPILENODE_HWCOMPILE_MODULE "craype-haswell")
+ set (COMPUTENODE_HWCOMPILE_MODULE "craype-mic-knl")
+ set (LOCAL_BATCH_SCRIPT_NAME "knl_ctestS.sl")
+ set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "knl_ctestP.sl")
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake")
+
+On the Cray XC40 the craype-haswell module is needed for configuring, and the
+craype-mic-knl module is needed for building to run on the KNL nodes. CMake
+with the above options will swap modules after configuring is complete,
+but before compiling programs for KNL.
+
+The sbatch script arguments for running jobs on KNL nodes may differ on CrayXC40
+machines other than mutrino. The batch scripts knl_ctestS.sl and knl_ctestP.sl
+have the correct arguments for mutrino: "#SBATCH -p knl -C quad,cache". For
+cori, another CrayXC40, that line is replaced by "#SBATCH -C knl,quad,cache".
+For cori (and other machines), the values in LOCAL_BATCH_SCRIPT_NAME and
+LOCAL_BATCH_SCRIPT_PARALLEL_NAME in the config/cmake/scripts/HPC/sbatch-HDF5options.cmake
+file can be replaced by cori_knl_ctestS.sl and cori_knl_ctestS.sl, or the lines
+can be edited in the batch files in hdf5-1.8.22/bin/batch.
+
+========================================================================
+V. Manual alternatives
+========================================================================
+If using ctest is undesirable, one can create a build directory and run the cmake
+configure command, for example
+
+"/projects/Mutrino/hpcsoft/cle6.0/common/cmake/3.10.2/bin/cmake"
+-C "<working directory>/hdf5-1.8.22/config/cmake/cacheinit.cmake"
+-DCMAKE_BUILD_TYPE:STRING=Release -DHDF5_BUILD_FORTRAN:BOOL=ON
+-DHDF5_BUILD_JAVA:BOOL=OFF
+-DCMAKE_INSTALL_PREFIX:PATH=<working directory>/HDF_Group/HDF5/1.8.22
+-DHDF5_ENABLE_Z_LIB_SUPPORT:BOOL=OFF -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF
+-DHDF5_ENABLE_PARALLEL:BOOL=ON -DHDF5_BUILD_CPP_LIB:BOOL=OFF
+-DHDF5_ENABLE_THREADSAFE:BOOL=OFF
+-DHDF5_PACKAGE_EXTLIBS:BOOL=ON -DLOCAL_BATCH_TEST:BOOL=ON
+-DMPIEXEC_EXECUTABLE:STRING=srun -DMPIEXEC_NUMPROC_FLAG:STRING=-n
+-DMPIEXEC_MAX_NUMPROCS:STRING=6
+-DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake
+-DLOCAL_BATCH_SCRIPT_NAME:STRING=knl_ctestS.sl
+-DLOCAL_BATCH_SCRIPT_PARALLEL_NAME:STRING=knl_ctestP.sl -DSITE:STRING=mutrino
+-DBUILDNAME:STRING=par-knl_GCC493-SHARED-Linux-4.4.156-94.61.1.16335.0.PTF.1107299-default-x86_64
+"-GUnix Makefiles" "" "<working directory>/hdf5-1.8.22"
+
+followed by make and batch jobs to run tests.
+
+To cross-compile on CrayXC40, run the configure command with the craype-haswell
+module loaded, then switch to the craype-mic-knl module for the build process.
+
+Tests on machines using slurm can be run with
+
+"sbatch -p knl -C quad,cache ctestS.sl"
+
+or
+
+"sbatch -p knl -C quad,cache ctestP.sl"
+
+for parallel builds.
+
+Tests on machines using LSF will typically use "bsub ctestS.lsf", etc.
+
+========================================================================
+VI. Other cross compiling options
+========================================================================
+Settings for two other cross-compiling options are also in the config/toolchain
+files which do not seem to be necessary with the Cray PrgEnv-* modules
+
+1. HDF5_USE_PREGEN. This option, along with the HDF5_USE_PREGEN_DIR CMake
+ variable would allow the use of an appropriate H5Tinit.c file with type
+ information generated on a compute node to be used when cross compiling
+ for those compute nodes. The use of the variables in lines 110 and 111
+ of HDF5options.cmake file seem to preclude needing this option with the
+ available Cray modules and CMake option.
+
+2. HDF5_BATCH_H5DETECT and associated CMake variables. This option when
+ properly configured will run H5detect in a batch job on a compute node
+ at the beginning of the CMake build process. It was also found to be
+ unnecessary with the available Cray modules and CMake options.
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 05fe857..23f3652 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -148,6 +148,31 @@ New Features
Library
-------
+ - Add S3 and HDFS VFDs to HDF5 maintenance
+
+ Fix windows requirements and java tests. Windows requires CMake 3.13.
+ Install openssl library (with dev files);
+ from "Shining Light Productions". msi package preferred.
+
+ PATH should have been updated with the installation dir.
+ set ENV variable OPENSSL_ROOT_DIR to the installation dir.
+ set ENV variable OPENSSL_CONF to the cfg file, likely %OPENSSL_ROOT_DIR%\bin\openssl.cfg
+ Install libcurl library (with dev files);
+ download the latest released version using git: https://github.com/curl/curl.git
+
+ Open a Visual Studio Command prompt
+ change to the libcurl root folder
+ run the "buildconf.bat" batch file
+ change to the winbuild directory
+ nmake /f Makefile.vc mode=dll MACHINE=x64
+ copy libcurl-vc-x64-release-dll-ipv6-sspi-winssl dir to C:\curl (installation dir)
+ set ENV variable CURL_ROOT to C:\curl (installation dir)
+ update PATH ENV variable to %CURL_ROOT%\bin (installation bin dir).
+ the aws credentials file should be in %USERPROFILE%\.aws folder
+ set the ENV variable "HDF5_ROS3_TEST_BUCKET_URL=https://s3.us-east-2.amazonaws.com/hdf5ros3"
+
+ (ADB - 2019/09/12, HDFFV-10854)
+
- Allow pre-generated H5Tinit.c and H5make_libsettings.c to be used.
Rather than always running H5detect and generating H5Tinit.c and
diff --git a/release_docs/USING_CMake_Examples.txt b/release_docs/USING_CMake_Examples.txt
index f98b39e..a5ababa 100644
--- a/release_docs/USING_CMake_Examples.txt
+++ b/release_docs/USING_CMake_Examples.txt
@@ -58,8 +58,9 @@ Default installation process:
with the CTEST_CONFIGURATION_TYPE script option. Note that this must
be the same as the value used with the -C command line option.
The default build configuration is defined to build and use static libraries.
- Shared libraries can be used with the STATIC_ONLY script option set to "NO".
- Other options can be changed by editing the HDF518_Examples.cmake file.
+
+ Shared libraries and other options can be changed by editing the
+ HDF5_Examples_options.cmake file.
If the defaults are okay, execute from this directory:
ctest -S HDF518_Examples.cmake -C Release -V -O test.log
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 86deb10..544859f 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -4,7 +4,7 @@ project (HDF5_SRC C)
#-----------------------------------------------------------------------------
# List Source Files
#-----------------------------------------------------------------------------
-set (H5_SRCS
+set (H5_SOURCES
${HDF5_SRC_DIR}/H5.c
${HDF5_SRC_DIR}/H5checksum.c
${HDF5_SRC_DIR}/H5dbg.c
@@ -20,9 +20,9 @@ set (H5_HDRS
#${HDF5_SRC_DIR}/H5version.h
#${HDF5_SRC_DIR}/H5overflow.h
)
-IDE_GENERATED_PROPERTIES ("H5" "${H5_HDRS}" "${H5_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5" "${H5_HDRS}" "${H5_SOURCES}" )
-set (H5A_SRCS
+set (H5A_SOURCES
${HDF5_SRC_DIR}/H5A.c
${HDF5_SRC_DIR}/H5Abtree2.c
${HDF5_SRC_DIR}/H5Adense.c
@@ -34,28 +34,28 @@ set (H5A_SRCS
set (H5A_HDRS
${HDF5_SRC_DIR}/H5Apublic.h
)
-IDE_GENERATED_PROPERTIES ("H5A" "${H5A_HDRS}" "${H5A_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5A" "${H5A_HDRS}" "${H5A_SOURCES}" )
-set (H5AC_SRCS
+set (H5AC_SOURCES
${HDF5_SRC_DIR}/H5AC.c
)
set (H5AC_HDRS
${HDF5_SRC_DIR}/H5ACpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5AC" "${H5AC_HDRS}" "${H5AC_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5AC" "${H5AC_HDRS}" "${H5AC_SOURCES}" )
-set (H5B_SRCS
+set (H5B_SOURCES
${HDF5_SRC_DIR}/H5B.c
${HDF5_SRC_DIR}/H5Bcache.c
${HDF5_SRC_DIR}/H5Bdbg.c
)
set (H5B_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5B" "${H5B_HDRS}" "${H5B_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5B" "${H5B_HDRS}" "${H5B_SOURCES}" )
-set (H5B2_SRCS
+set (H5B2_SOURCES
${HDF5_SRC_DIR}/H5B2.c
${HDF5_SRC_DIR}/H5B2cache.c
${HDF5_SRC_DIR}/H5B2dbg.c
@@ -66,10 +66,10 @@ set (H5B2_SRCS
)
set (H5B2_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5B2" "${H5B2_HDRS}" "${H5B2_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5B2" "${H5B2_HDRS}" "${H5B2_SOURCES}" )
-set (H5C_SRCS
+set (H5C_SOURCES
${HDF5_SRC_DIR}/H5C.c
)
set (H5C_HDRS
@@ -78,7 +78,7 @@ set (H5C_HDRS
IDE_GENERATED_PROPERTIES ("H5C" "${H5C_HDRS}" "${H5C_SOURCES}" )
-set (H5CS_SRCS
+set (H5CS_SOURCES
${HDF5_SRC_DIR}/H5CS.c
)
set (H5CS_HDRS
@@ -86,7 +86,7 @@ set (H5CS_HDRS
IDE_GENERATED_PROPERTIES ("H5CS" "${H5CS_HDRS}" "${H5CS_SOURCES}" )
-set (H5D_SRCS
+set (H5D_SOURCES
${HDF5_SRC_DIR}/H5D.c
${HDF5_SRC_DIR}/H5Dbtree.c
${HDF5_SRC_DIR}/H5Dchunk.c
@@ -109,9 +109,9 @@ set (H5D_SRCS
set (H5D_HDRS
${HDF5_SRC_DIR}/H5Dpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5D" "${H5D_HDRS}" "${H5D_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5D" "${H5D_HDRS}" "${H5D_SOURCES}" )
-set (H5E_SRCS
+set (H5E_SOURCES
${HDF5_SRC_DIR}/H5E.c
${HDF5_SRC_DIR}/H5Edeprec.c
${HDF5_SRC_DIR}/H5Eint.c
@@ -121,9 +121,9 @@ set (H5E_HDRS
${HDF5_SRC_DIR}/H5Epubgen.h
${HDF5_SRC_DIR}/H5Epublic.h
)
-IDE_GENERATED_PROPERTIES ("H5E" "${H5E_HDRS}" "${H5E_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5E" "${H5E_HDRS}" "${H5E_SOURCES}" )
-set (H5F_SRCS
+set (H5F_SOURCES
${HDF5_SRC_DIR}/H5F.c
${HDF5_SRC_DIR}/H5Faccum.c
${HDF5_SRC_DIR}/H5Fcwfs.c
@@ -144,18 +144,21 @@ set (H5F_SRCS
set (H5F_HDRS
${HDF5_SRC_DIR}/H5Fpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5F" "${H5F_HDRS}" "${H5F_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5F" "${H5F_HDRS}" "${H5F_SOURCES}" )
-set (H5FD_SRCS
+set (H5FD_SOURCES
${HDF5_SRC_DIR}/H5FD.c
${HDF5_SRC_DIR}/H5FDcore.c
${HDF5_SRC_DIR}/H5FDdirect.c
${HDF5_SRC_DIR}/H5FDfamily.c
+ ${HDF5_SRC_DIR}/H5FDhdfs.c
${HDF5_SRC_DIR}/H5FDint.c
${HDF5_SRC_DIR}/H5FDlog.c
${HDF5_SRC_DIR}/H5FDmpi.c
${HDF5_SRC_DIR}/H5FDmpio.c
${HDF5_SRC_DIR}/H5FDmulti.c
+ ${HDF5_SRC_DIR}/H5FDros3.c
+ ${HDF5_SRC_DIR}/H5FDs3comms.c
${HDF5_SRC_DIR}/H5FDsec2.c
${HDF5_SRC_DIR}/H5FDspace.c
${HDF5_SRC_DIR}/H5FDstdio.c
@@ -166,35 +169,38 @@ set (H5FD_HDRS
${HDF5_SRC_DIR}/H5FDcore.h
${HDF5_SRC_DIR}/H5FDdirect.h
${HDF5_SRC_DIR}/H5FDfamily.h
+ ${HDF5_SRC_DIR}/H5FDhdfs.h
${HDF5_SRC_DIR}/H5FDlog.h
${HDF5_SRC_DIR}/H5FDmpi.h
${HDF5_SRC_DIR}/H5FDmpio.h
${HDF5_SRC_DIR}/H5FDmulti.h
${HDF5_SRC_DIR}/H5FDpublic.h
+ ${HDF5_SRC_DIR}/H5FDros3.h
+ ${HDF5_SRC_DIR}/H5FDs3comms.c
${HDF5_SRC_DIR}/H5FDsec2.h
${HDF5_SRC_DIR}/H5FDstdio.h
${HDF5_SRC_DIR}/H5FDwindows.h
)
-IDE_GENERATED_PROPERTIES ("H5FD" "${H5FD_HDRS}" "${H5FD_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5FD" "${H5FD_HDRS}" "${H5FD_SOURCES}" )
-set (H5FL_SRCS
+set (H5FL_SOURCES
${HDF5_SRC_DIR}/H5FL.c
)
set (H5FL_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5FL" "${H5FL_HDRS}" "${H5FL_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5FL" "${H5FL_HDRS}" "${H5FL_SOURCES}" )
-set (H5FO_SRCS
+set (H5FO_SOURCES
${HDF5_SRC_DIR}/H5FO.c
)
set (H5FO_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5FO" "${H5FO_HDRS}" "${H5FO_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5FO" "${H5FO_HDRS}" "${H5FO_SOURCES}" )
-set (H5FS_SRCS
+set (H5FS_SOURCES
${HDF5_SRC_DIR}/H5FS.c
${HDF5_SRC_DIR}/H5FScache.c
${HDF5_SRC_DIR}/H5FSdbg.c
@@ -205,9 +211,9 @@ set (H5FS_SRCS
set (H5FS_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5FS" "${H5FS_HDRS}" "${H5FS_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5FS" "${H5FS_HDRS}" "${H5FS_SOURCES}" )
-set (H5G_SRCS
+set (H5G_SOURCES
${HDF5_SRC_DIR}/H5G.c
${HDF5_SRC_DIR}/H5Gbtree2.c
${HDF5_SRC_DIR}/H5Gcache.c
@@ -231,9 +237,9 @@ set (H5G_SRCS
set (H5G_HDRS
${HDF5_SRC_DIR}/H5Gpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5G" "${H5G_HDRS}" "${H5G_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5G" "${H5G_HDRS}" "${H5G_SOURCES}" )
-set (H5HF_SRCS
+set (H5HF_SOURCES
${HDF5_SRC_DIR}/H5HF.c
${HDF5_SRC_DIR}/H5HFbtree2.c
${HDF5_SRC_DIR}/H5HFcache.c
@@ -254,9 +260,9 @@ set (H5HF_SRCS
set (H5HF_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5HF" "${H5HF_HDRS}" "${H5HF_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5HF" "${H5HF_HDRS}" "${H5HF_SOURCES}" )
-set (H5HG_SRCS
+set (H5HG_SOURCES
${HDF5_SRC_DIR}/H5HG.c
${HDF5_SRC_DIR}/H5HGcache.c
${HDF5_SRC_DIR}/H5HGdbg.c
@@ -265,9 +271,9 @@ set (H5HG_SRCS
set (H5HG_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5HG" "${H5HG_HDRS}" "${H5HG_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5HG" "${H5HG_HDRS}" "${H5HG_SOURCES}" )
-set (H5HL_SRCS
+set (H5HL_SOURCES
${HDF5_SRC_DIR}/H5HL.c
${HDF5_SRC_DIR}/H5HLcache.c
${HDF5_SRC_DIR}/H5HLdbg.c
@@ -276,38 +282,38 @@ set (H5HL_SRCS
set (H5HL_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5HL" "${H5HL_HDRS}" "${H5HL_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5HL" "${H5HL_HDRS}" "${H5HL_SOURCES}" )
-set (H5HP_SRCS
+set (H5HP_SOURCES
${HDF5_SRC_DIR}/H5HP.c
)
set (H5HP_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5HP" "${H5HP_HDRS}" "${H5HP_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5HP" "${H5HP_HDRS}" "${H5HP_SOURCES}" )
-set (H5I_SRCS
+set (H5I_SOURCES
${HDF5_SRC_DIR}/H5I.c
${HDF5_SRC_DIR}/H5Itest.c
)
set (H5I_HDRS
${HDF5_SRC_DIR}/H5Ipublic.h
)
-IDE_GENERATED_PROPERTIES ("H5I" "${H5I_HDRS}" "${H5I_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5I" "${H5I_HDRS}" "${H5I_SOURCES}" )
-set (H5L_SRCS
+set (H5L_SOURCES
${HDF5_SRC_DIR}/H5L.c
${HDF5_SRC_DIR}/H5Lexternal.c
)
set (H5L_HDRS
${HDF5_SRC_DIR}/H5Lpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5L" "${H5L_HDRS}" "${H5L_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5L" "${H5L_HDRS}" "${H5L_SOURCES}" )
-set (H5MF_SRCS
+set (H5MF_SOURCES
${HDF5_SRC_DIR}/H5MF.c
${HDF5_SRC_DIR}/H5MFaggr.c
${HDF5_SRC_DIR}/H5MFdbg.c
@@ -316,28 +322,28 @@ set (H5MF_SRCS
set (H5MF_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5MF" "${H5MF_HDRS}" "${H5MF_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5MF" "${H5MF_HDRS}" "${H5MF_SOURCES}" )
-set (H5MM_SRCS
+set (H5MM_SOURCES
${HDF5_SRC_DIR}/H5MM.c
)
set (H5MM_HDRS
${HDF5_SRC_DIR}/H5MMpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5MM" "${H5MM_HDRS}" "${H5MM_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5MM" "${H5MM_HDRS}" "${H5MM_SOURCES}" )
-set (H5MP_SRCS
+set (H5MP_SOURCES
${HDF5_SRC_DIR}/H5MP.c
${HDF5_SRC_DIR}/H5MPtest.c
)
set (H5MP_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5MP" "${H5MP_HDRS}" "${H5MP_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5MP" "${H5MP_HDRS}" "${H5MP_SOURCES}" )
-set (H5O_SRCS
+set (H5O_SOURCES
${HDF5_SRC_DIR}/H5O.c
${HDF5_SRC_DIR}/H5Oainfo.c
${HDF5_SRC_DIR}/H5Oalloc.c
@@ -375,9 +381,9 @@ set (H5O_SRCS
set (H5O_HDRS
${HDF5_SRC_DIR}/H5Opublic.h
)
-IDE_GENERATED_PROPERTIES ("H5O" "${H5O_HDRS}" "${H5O_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5O" "${H5O_HDRS}" "${H5O_SOURCES}" )
-set (H5P_SRCS
+set (H5P_SOURCES
${HDF5_SRC_DIR}/H5P.c
${HDF5_SRC_DIR}/H5Pacpl.c
${HDF5_SRC_DIR}/H5Pdapl.c
@@ -400,9 +406,9 @@ set (H5P_SRCS
set (H5P_HDRS
${HDF5_SRC_DIR}/H5Ppublic.h
)
-IDE_GENERATED_PROPERTIES ("H5P" "${H5P_HDRS}" "${H5P_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5P" "${H5P_HDRS}" "${H5P_SOURCES}" )
-set (H5PL_SRCS
+set (H5PL_SOURCES
${HDF5_SRC_DIR}/H5PL.c
)
@@ -410,35 +416,36 @@ set (H5PL_HDRS
${HDF5_SRC_DIR}/H5PLextern.h
${HDF5_SRC_DIR}/H5PLpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5PL" "${H5PL_HDRS}" "${H5PL_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5PL" "${H5PL_HDRS}" "${H5PL_SOURCES}" )
-set (H5R_SRCS
+set (H5R_SOURCES
${HDF5_SRC_DIR}/H5R.c
${HDF5_SRC_DIR}/H5Rdeprec.c
)
set (H5R_HDRS
${HDF5_SRC_DIR}/H5Rpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5R" "${H5R_HDRS}" "${H5R_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5R" "${H5R_HDRS}" "${H5R_SOURCES}" )
-set (H5RC_SRCS
+set (H5RC_SOURCES
${HDF5_SRC_DIR}/H5RC.c
)
set (H5RC_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5RC" "${H5RC_HDRS}" "${H5RC_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5RC" "${H5RC_HDRS}" "${H5RC_SOURCES}" )
-set (H5RS_SRCS
+set (H5RS_SOURCES
${HDF5_SRC_DIR}/H5RS.c
)
set (H5RS_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5RS" "${H5RS_HDRS}" "${H5RS_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5RS" "${H5RS_HDRS}" "${H5RS_SOURCES}" )
-set (H5S_SRCS
+
+set (H5S_SOURCES
${HDF5_SRC_DIR}/H5S.c
${HDF5_SRC_DIR}/H5Sall.c
${HDF5_SRC_DIR}/H5Sdbg.c
@@ -453,18 +460,18 @@ set (H5S_SRCS
set (H5S_HDRS
${HDF5_SRC_DIR}/H5Spublic.h
)
-IDE_GENERATED_PROPERTIES ("H5S" "${H5S_HDRS}" "${H5S_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5S" "${H5S_HDRS}" "${H5S_SOURCES}" )
-set (H5SL_SRCS
+set (H5SL_SOURCES
${HDF5_SRC_DIR}/H5SL.c
)
set (H5SL_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5SL" "${H5SL_HDRS}" "${H5SL_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5SL" "${H5SL_HDRS}" "${H5SL_SOURCES}" )
-set (H5SM_SRCS
+set (H5SM_SOURCES
${HDF5_SRC_DIR}/H5SM.c
${HDF5_SRC_DIR}/H5SMbtree2.c
${HDF5_SRC_DIR}/H5SMcache.c
@@ -474,18 +481,18 @@ set (H5SM_SRCS
set (H5SM_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5SM" "${H5SM_HDRS}" "${H5SM_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5SM" "${H5SM_HDRS}" "${H5SM_SOURCES}" )
-set (H5ST_SRCS
+set (H5ST_SOURCES
${HDF5_SRC_DIR}/H5ST.c
)
set (H5ST_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5ST" "${H5ST_HDRS}" "${H5ST_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5ST" "${H5ST_HDRS}" "${H5ST_SOURCES}" )
-set (H5T_SRCS
+set (H5T_SOURCES
${HDF5_SRC_DIR}/H5T.c
${HDF5_SRC_DIR}/H5Tarray.c
${HDF5_SRC_DIR}/H5Tbit.c
@@ -514,34 +521,34 @@ set (H5T_SRCS
set (H5T_HDRS
${HDF5_SRC_DIR}/H5Tpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5T" "${H5T_HDRS}" "${H5T_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5T" "${H5T_HDRS}" "${H5T_SOURCES}" )
-set (H5TS_SRCS
+set (H5TS_SOURCES
${HDF5_SRC_DIR}/H5TS.c
)
set (H5TS_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5TS" "${H5TS_HDRS}" "${H5TS_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5TS" "${H5TS_HDRS}" "${H5TS_SOURCES}" )
-set (H5VM_SRCS
+set (H5VM_SOURCES
${HDF5_SRC_DIR}/H5VM.c
)
set (H5VM_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5VM" "${H5VM_HDRS}" "${H5VM_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5VM" "${H5VM_HDRS}" "${H5VM_SOURCES}" )
-set (H5WB_SRCS
+set (H5WB_SOURCES
${HDF5_SRC_DIR}/H5WB.c
)
set (H5WB_HDRS
)
-IDE_GENERATED_PROPERTIES ("H5WB" "${H5WB_HDRS}" "${H5WB_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5WB" "${H5WB_HDRS}" "${H5WB_SOURCES}" )
-set (H5Z_SRCS
+set (H5Z_SOURCES
${HDF5_SRC_DIR}/H5Z.c
${HDF5_SRC_DIR}/H5Zdeflate.c
${HDF5_SRC_DIR}/H5Zfletcher32.c
@@ -560,48 +567,48 @@ endif ()
set (H5Z_HDRS
${HDF5_SRC_DIR}/H5Zpublic.h
)
-IDE_GENERATED_PROPERTIES ("H5Z" "${H5Z_HDRS}" "${H5Z_SRCS}" )
+IDE_GENERATED_PROPERTIES ("H5Z" "${H5Z_HDRS}" "${H5Z_SOURCES}" )
set (common_SRCS
- ${H5_SRCS}
- ${H5A_SRCS}
- ${H5AC_SRCS}
- ${H5B_SRCS}
- ${H5B2_SRCS}
- ${H5C_SRCS}
- ${H5CS_SRCS}
- ${H5D_SRCS}
- ${H5E_SRCS}
- ${H5F_SRCS}
- ${H5FD_SRCS}
- ${H5FL_SRCS}
- ${H5FO_SRCS}
- ${H5FS_SRCS}
- ${H5G_SRCS}
- ${H5HF_SRCS}
- ${H5HG_SRCS}
- ${H5HL_SRCS}
- ${H5HP_SRCS}
- ${H5I_SRCS}
- ${H5L_SRCS}
- ${H5MF_SRCS}
- ${H5MM_SRCS}
- ${H5MP_SRCS}
- ${H5O_SRCS}
- ${H5P_SRCS}
- ${H5PL_SRCS}
- ${H5R_SRCS}
- ${H5RC_SRCS}
- ${H5RS_SRCS}
- ${H5S_SRCS}
- ${H5SL_SRCS}
- ${H5SM_SRCS}
- ${H5ST_SRCS}
- ${H5T_SRCS}
- ${H5TS_SRCS}
- ${H5VM_SRCS}
- ${H5WB_SRCS}
- ${H5Z_SRCS}
+ ${H5_SOURCES}
+ ${H5A_SOURCES}
+ ${H5AC_SOURCES}
+ ${H5B_SOURCES}
+ ${H5B2_SOURCES}
+ ${H5C_SOURCES}
+ ${H5CS_SOURCES}
+ ${H5D_SOURCES}
+ ${H5E_SOURCES}
+ ${H5F_SOURCES}
+ ${H5FD_SOURCES}
+ ${H5FL_SOURCES}
+ ${H5FO_SOURCES}
+ ${H5FS_SOURCES}
+ ${H5G_SOURCES}
+ ${H5HF_SOURCES}
+ ${H5HG_SOURCES}
+ ${H5HL_SOURCES}
+ ${H5HP_SOURCES}
+ ${H5I_SOURCES}
+ ${H5L_SOURCES}
+ ${H5MF_SOURCES}
+ ${H5MM_SOURCES}
+ ${H5MP_SOURCES}
+ ${H5O_SOURCES}
+ ${H5P_SOURCES}
+ ${H5PL_SOURCES}
+ ${H5R_SOURCES}
+ ${H5RC_SOURCES}
+ ${H5RS_SOURCES}
+ ${H5S_SOURCES}
+ ${H5SL_SOURCES}
+ ${H5SM_SOURCES}
+ ${H5ST_SOURCES}
+ ${H5T_SOURCES}
+ ${H5TS_SOURCES}
+ ${H5VM_SOURCES}
+ ${H5WB_SOURCES}
+ ${H5Z_SOURCES}
)
set (H5_PUBLIC_HEADERS
@@ -797,6 +804,27 @@ if (BUILD_SHARED_LIBS)
file (MAKE_DIRECTORY "${HDF5_BINARY_DIR}/shared")
endif ()
+if (LOCAL_BATCH_TEST)
+ if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub")
+ configure_file (
+ ${HDF5_SOURCE_DIR}/bin/batch/${LOCAL_BATCH_SCRIPT_COMMAND}
+ ${HDF5_BINARY_DIR}/${LOCAL_BATCH_SCRIPT_COMMAND} ESCAPE_QUOTES @ONLY
+ )
+ endif ()
+ if (LOCAL_BATCH_SCRIPT_NAME)
+ configure_file (
+ ${HDF5_SOURCE_DIR}/bin/batch/${LOCAL_BATCH_SCRIPT_NAME}.in.cmake
+ ${HDF5_BINARY_DIR}/${LOCAL_BATCH_SCRIPT_NAME} ESCAPE_QUOTES @ONLY
+ )
+ endif ()
+ if (LOCAL_BATCH_SCRIPT_PARALLEL_NAME)
+ configure_file (
+ ${HDF5_SOURCE_DIR}/bin/batch/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}.in.cmake
+ ${HDF5_BINARY_DIR}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME} ESCAPE_QUOTES @ONLY
+ )
+ endif ()
+endif ()
+
if (NOT EXISTS "${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c")
add_executable (H5detect ${HDF5_SRC_DIR}/H5detect.c)
target_include_directories (H5detect PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
@@ -941,7 +969,6 @@ if (NOT ONLY_SHARED_LIBS)
PUBLIC
${HDF_EXTRA_C_FLAGS}
${HDF_EXTRA_FLAGS}
- $<IF:$<CONFIG:Debug>,DEBUG,NDEBUG>
PRIVATE
$<$<BOOL:${HDF5_ENABLE_TRACE}>:H5_DEBUG_API> # Enable tracing of the API
$<$<BOOL:${HDF5_ENABLE_DEBUG_APIS}>:H5Z_DEBUG;H5T_DEBUG;H5ST_DEBUG;H5S_DEBUG;H5O_DEBUG;H5I_DEBUG;H5HL_DEBUG;H5F_DEBUG;H5D_DEBUG;H5B2_DEBUG;H5AC_DEBUG>
@@ -971,6 +998,7 @@ if (BUILD_SHARED_LIBS)
add_library (${HDF5_LIBSH_TARGET} SHARED ${common_SRCS} ${shared_gen_SRCS} ${H5_PUBLIC_HEADERS} ${H5_PRIVATE_HEADERS} ${H5_GENERATED_HEADERS})
target_include_directories (${HDF5_LIBSH_TARGET}
PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>"
+ PUBLIC "$<$<BOOL:${HDF5_ENABLE_HDFS}>:${HDFS_INCLUDE_DIR}>"
INTERFACE "$<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include>"
)
target_compile_definitions(${HDF5_LIBSH_TARGET}
@@ -978,7 +1006,6 @@ if (BUILD_SHARED_LIBS)
"H5_BUILT_AS_DYNAMIC_LIB"
${HDF_EXTRA_C_FLAGS}
${HDF_EXTRA_FLAGS}
- $<IF:$<CONFIG:Debug>,DEBUG,NDEBUG>
PRIVATE
$<$<BOOL:${HDF5_ENABLE_THREADSAFE}>:H5_HAVE_THREADSAFE>
$<$<BOOL:${HDF5_ENABLE_TRACE}>:H5_DEBUG_API> # Enable tracing of the API
diff --git a/src/H5FD.c b/src/H5FD.c
index 63b7dfc..11ef24a 100644
--- a/src/H5FD.c
+++ b/src/H5FD.c
@@ -15,7 +15,7 @@
* Programmer: Robb Matzke <matzke@llnl.gov>
* Monday, July 26, 1999
*
- * Purpose: The Virtual File Layer as described in documentation.
+ * Purpose: The Virtual File Layer as described in documentation.
* This is the greatest common denominator for all types of
* storage access whether a file, memory, network, etc. This
* layer usually just dispatches the request to an actual
@@ -26,35 +26,35 @@
/* Module Setup */
/****************/
-#define H5F_PACKAGE /*suppress error about including H5Fpkg */
-#define H5FD_PACKAGE /*suppress error about including H5FDpkg */
+#define H5F_PACKAGE /*suppress error about including H5Fpkg */
+#define H5FD_PACKAGE /*suppress error about including H5FDpkg */
/* Interface initialization */
-#define H5_INTERFACE_INIT_FUNC H5FD_init_interface
+#define H5_INTERFACE_INIT_FUNC H5FD_init_interface
/***********/
/* Headers */
/***********/
-#include "H5private.h" /* Generic Functions */
-#include "H5Dprivate.h" /* Datasets */
-#include "H5Eprivate.h" /* Error handling */
-#include "H5Fpkg.h" /* File access */
-#include "H5FDpkg.h" /* File Drivers */
-#include "H5FDcore.h" /* Files stored entirely in memory */
-#include "H5FDfamily.h" /* File families */
-#include "H5FDlog.h" /* sec2 driver with I/O logging (for debugging) */
-#include "H5FDmpi.h" /* MPI-based file drivers */
-#include "H5FDmulti.h" /* Usage-partitioned file family */
-#include "H5FDsec2.h" /* POSIX unbuffered file I/O */
-#include "H5FDstdio.h" /* Standard C buffered I/O */
+#include "H5private.h" /* Generic Functions */
+#include "H5Dprivate.h" /* Datasets */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Fpkg.h" /* File access */
+#include "H5FDpkg.h" /* File Drivers */
+#include "H5FDcore.h" /* Files stored entirely in memory */
+#include "H5FDfamily.h" /* File families */
+#include "H5FDlog.h" /* sec2 driver with I/O logging (for debugging) */
+#include "H5FDmpi.h" /* MPI-based file drivers */
+#include "H5FDmulti.h" /* Usage-partitioned file family */
+#include "H5FDsec2.h" /* POSIX unbuffered file I/O */
+#include "H5FDstdio.h" /* Standard C buffered I/O */
#ifdef H5_HAVE_WINDOWS
#include "H5FDwindows.h" /* Windows buffered I/O */
#endif
-#include "H5FDdirect.h" /* Direct file I/O */
-#include "H5Iprivate.h" /* IDs */
-#include "H5MMprivate.h" /* Memory management */
-#include "H5Pprivate.h" /* Property lists */
+#include "H5FDdirect.h" /* Direct file I/O */
+#include "H5Iprivate.h" /* IDs */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5Pprivate.h" /* Property lists */
/****************/
/* Local Macros */
@@ -113,23 +113,23 @@ static unsigned long H5FD_file_serial_no_g;
/* File driver ID class */
static const H5I_class_t H5I_VFL_CLS[1] = {{
- H5I_VFL, /* ID class value */
- H5I_CLASS_REUSE_IDS, /* Class flags */
- 0, /* # of reserved IDs for class */
- (H5I_free_t)H5FD_free_cls /* Callback routine for closing objects of this class */
+ H5I_VFL, /* ID class value */
+ H5I_CLASS_REUSE_IDS, /* Class flags */
+ 0, /* # of reserved IDs for class */
+ (H5I_free_t)H5FD_free_cls /* Callback routine for closing objects of this class */
}};
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_init
+ * Function: H5FD_init
*
- * Purpose: Initialize the interface from some other package.
+ * Purpose: Initialize the interface from some other package.
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, January 3, 2007
*
*-------------------------------------------------------------------------
@@ -146,17 +146,17 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_init() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_init_interface
+ * Function: H5FD_init_interface
*
- * Purpose: Initialize the virtual file layer.
+ * Purpose: Initialize the virtual file layer.
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, July 26, 1999
*
* Modifications:
@@ -171,7 +171,7 @@ H5FD_init_interface(void)
FUNC_ENTER_NOAPI_NOINIT
if(H5I_register_type(H5I_VFL_CLS) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize interface")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "unable to initialize interface")
/* Reset the file serial numbers */
H5FD_file_serial_no_g = 0;
@@ -180,21 +180,21 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_init_interface() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_term_interface
+ * Function: H5FD_term_interface
*
- * Purpose: Terminate this interface: free all memory and reset global
- * variables to their initial values. Release all ID groups
- * associated with this interface.
+ * Purpose: Terminate this interface: free all memory and reset global
+ * variables to their initial values. Release all ID groups
+ * associated with this interface.
*
- * Return: Success: Positive if anything was done that might
- * have affected other interfaces; zero
- * otherwise.
+ * Return: Success: Positive if anything was done that might
+ * have affected other interfaces; zero
+ * otherwise.
*
- * Failure: Never fails.
+ * Failure: Never fails.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Friday, February 19, 1999
*
* Modifications:
@@ -204,13 +204,13 @@ done:
int
H5FD_term_interface(void)
{
- int n = 0;
+ int n = 0;
FUNC_ENTER_NOAPI_NOINIT_NOERR
if(H5_interface_initialize_g) {
- if(H5I_nmembers(H5I_VFL) > 0) {
- (void)H5I_clear_type(H5I_VFL, FALSE, FALSE);
+ if(H5I_nmembers(H5I_VFL) > 0) {
+ (void)H5I_clear_type(H5I_VFL, FALSE, FALSE);
/* Reset the VFL drivers, if they've been closed */
if(H5I_nmembers(H5I_VFL)==0) {
@@ -232,33 +232,33 @@ H5FD_term_interface(void)
} /* end if */
n++; /*H5I*/
- } /* end if */
+ } /* end if */
else {
/* Destroy the VFL driver id group */
- (void)H5I_dec_type_ref(H5I_VFL);
+ (void)H5I_dec_type_ref(H5I_VFL);
n++; /*H5I*/
- /* Mark closed */
- H5_interface_initialize_g = 0;
- } /* end else */
+ /* Mark closed */
+ H5_interface_initialize_g = 0;
+ } /* end else */
} /* end if */
FUNC_LEAVE_NOAPI(n)
} /* end H5FD_term_interface() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_free_cls
+ * Function: H5FD_free_cls
*
- * Purpose: Frees a file driver class struct and returns an indication of
- * success. This function is used as the free callback for the
- * virtual file layer object identifiers (cf H5FD_init_interface).
+ * Purpose: Frees a file driver class struct and returns an indication of
+ * success. This function is used as the free callback for the
+ * virtual file layer object identifiers (cf H5FD_init_interface).
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, July 26, 1999
*
* Modifications:
@@ -275,22 +275,22 @@ H5FD_free_cls(H5FD_class_t *cls)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FD_free_cls() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDregister
+ * Function: H5FDregister
*
- * Purpose: Registers a new file driver as a member of the virtual file
- * driver class. Certain fields of the class struct are
- * required and that is checked here so it doesn't have to be
- * checked every time the field is accessed.
+ * Purpose: Registers a new file driver as a member of the virtual file
+ * driver class. Certain fields of the class struct are
+ * required and that is checked here so it doesn't have to be
+ * checked every time the field is accessed.
*
- * Return: Success: A file driver ID which is good until the
- * library is closed or the driver is
- * unregistered.
+ * Return: Success: A file driver ID which is good until the
+ * library is closed or the driver is
+ * unregistered.
*
- * Failure: A negative value.
+ * Failure: A negative value.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, July 26, 1999
*
* Modifications:
@@ -303,26 +303,26 @@ H5FD_free_cls(H5FD_class_t *cls)
hid_t
H5FDregister(const H5FD_class_t *cls)
{
- hid_t ret_value;
- H5FD_mem_t type;
+ hid_t ret_value;
+ H5FD_mem_t type;
FUNC_ENTER_API(FAIL)
H5TRACE1("i", "*x", cls);
/* Check arguments */
if(!cls)
- HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "null class pointer is disallowed")
+ HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "null class pointer is disallowed")
if(!cls->open || !cls->close)
- HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`open' and/or `close' methods are not defined")
+ HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`open' and/or `close' methods are not defined")
if(!cls->get_eoa || !cls->set_eoa)
- HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`get_eoa' and/or `set_eoa' methods are not defined")
+ HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`get_eoa' and/or `set_eoa' methods are not defined")
if(!cls->get_eof)
- HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`get_eof' method is not defined")
+ HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`get_eof' method is not defined")
if(!cls->read || !cls->write)
- HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`read' and/or `write' method is not defined")
+ HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL, "`read' and/or `write' method is not defined")
for (type=H5FD_MEM_DEFAULT; type<H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t,type))
- if(cls->fl_map[type]<H5FD_MEM_NOLIST || cls->fl_map[type]>=H5FD_MEM_NTYPES)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid free-list mapping")
+ if(cls->fl_map[type]<H5FD_MEM_NOLIST || cls->fl_map[type]>=H5FD_MEM_NTYPES)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid free-list mapping")
/* Create the new class ID */
if((ret_value=H5FD_register(cls, sizeof(H5FD_class_t), TRUE)) < 0)
@@ -332,22 +332,22 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDregister() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_register
+ * Function: H5FD_register
*
- * Purpose: Registers a new file driver as a member of the virtual file
- * driver class. Certain fields of the class struct are
- * required and that is checked here so it doesn't have to be
- * checked every time the field is accessed.
+ * Purpose: Registers a new file driver as a member of the virtual file
+ * driver class. Certain fields of the class struct are
+ * required and that is checked here so it doesn't have to be
+ * checked every time the field is accessed.
*
- * Return: Success: A file driver ID which is good until the
- * library is closed or the driver is
- * unregistered.
+ * Return: Success: A file driver ID which is good until the
+ * library is closed or the driver is
+ * unregistered.
*
- * Failure: A negative value.
+ * Failure: A negative value.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, July 26, 1999
*
* Modifications:
@@ -363,10 +363,10 @@ done:
hid_t
H5FD_register(const void *_cls, size_t size, hbool_t app_ref)
{
- const H5FD_class_t *cls = (const H5FD_class_t *)_cls;
- H5FD_class_t *saved = NULL;
- H5FD_mem_t type;
- hid_t ret_value;
+ const H5FD_class_t *cls = (const H5FD_class_t *)_cls;
+ H5FD_class_t *saved = NULL;
+ H5FD_mem_t type;
+ hid_t ret_value;
FUNC_ENTER_NOAPI(FAIL)
@@ -381,7 +381,7 @@ H5FD_register(const void *_cls, size_t size, hbool_t app_ref)
/* Copy the class structure so the caller can reuse or free it */
if(NULL == (saved = (H5FD_class_t *)H5MM_malloc(size)))
- HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for file driver class struct")
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for file driver class struct")
HDmemcpy(saved, cls, size);
/* Create the new class ID */
@@ -396,20 +396,20 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_register() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDunregister
+ * Function: H5FDunregister
*
- * Purpose: Removes a driver ID from the library. This in no way affects
- * file access property lists which have been defined to use
- * this driver or files which are already opened under this
- * driver.
+ * Purpose: Removes a driver ID from the library. This in no way affects
+ * file access property lists which have been defined to use
+ * this driver or files which are already opened under this
+ * driver.
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, July 26, 1999
*
*-------------------------------------------------------------------------
@@ -424,33 +424,33 @@ H5FDunregister(hid_t driver_id)
/* Check arguments */
if(NULL == H5I_object_verify(driver_id, H5I_VFL))
- HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file driver")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file driver")
/* The H5FD_class_t struct will be freed by this function */
if(H5I_dec_app_ref(driver_id) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "unable to unregister file driver")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTDEC, FAIL, "unable to unregister file driver")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDunregister() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_get_class
+ * Function: H5FD_get_class
*
- * Purpose: Obtains a pointer to the driver struct containing all the
- * callback pointers, etc. The PLIST_ID argument can be a file
- * access property list, a data transfer property list, or a
- * file driver identifier.
+ * Purpose: Obtains a pointer to the driver struct containing all the
+ * callback pointers, etc. The PLIST_ID argument can be a file
+ * access property list, a data transfer property list, or a
+ * file driver identifier.
*
- * Return: Success: Ptr to the driver information. The pointer is
- * only valid as long as the driver remains
- * registered or some file or property list
- * exists which references the driver.
+ * Return: Success: Ptr to the driver information. The pointer is
+ * only valid as long as the driver remains
+ * registered or some file or property list
+ * exists which references the driver.
*
- * Failure: NULL
+ * Failure: NULL
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Friday, August 20, 1999
*
*-------------------------------------------------------------------------
@@ -458,12 +458,12 @@ done:
H5FD_class_t *
H5FD_get_class(hid_t id)
{
- H5FD_class_t *ret_value = NULL;
+ H5FD_class_t *ret_value = NULL;
FUNC_ENTER_NOAPI(NULL)
if(H5I_VFL == H5I_get_type(id))
- ret_value = (H5FD_class_t *)H5I_object(id);
+ ret_value = (H5FD_class_t *)H5I_object(id);
else {
H5P_genplist_t *plist; /* Property list pointer */
hid_t driver_id = -1;
@@ -485,19 +485,19 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_get_class() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_sb_size
+ * Function: H5FD_sb_size
*
- * Purpose: Obtains the number of bytes required to store the driver file
- * access data in the HDF5 superblock.
+ * Purpose: Obtains the number of bytes required to store the driver file
+ * access data in the HDF5 superblock.
*
- * Return: Success: Number of bytes required.
+ * Return: Success: Number of bytes required.
*
- * Failure: 0 if an error occurs or if the driver has no
- * data to store in the superblock.
+ * Failure: 0 if an error occurs or if the driver has no
+ * data to store in the superblock.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, August 16, 1999
*
* Modifications:
@@ -507,35 +507,35 @@ done:
hsize_t
H5FD_sb_size(H5FD_t *file)
{
- hsize_t ret_value=0;
+ hsize_t ret_value=0;
FUNC_ENTER_NOAPI(0)
HDassert(file && file->cls);
if(file->cls->sb_size)
- ret_value = (file->cls->sb_size)(file);
+ ret_value = (file->cls->sb_size)(file);
done:
FUNC_LEAVE_NOAPI(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_sb_encode
+ * Function: H5FD_sb_encode
*
- * Purpose: Encode driver-specific data into the output arguments. The
- * NAME is a nine-byte buffer which should get an
- * eight-character driver name and/or version followed by a null
- * terminator. The BUF argument is a buffer to receive the
- * encoded driver-specific data. The size of the BUF array is
- * the size returned by the H5FD_sb_size() call.
+ * Purpose: Encode driver-specific data into the output arguments. The
+ * NAME is a nine-byte buffer which should get an
+ * eight-character driver name and/or version followed by a null
+ * terminator. The BUF argument is a buffer to receive the
+ * encoded driver-specific data. The size of the BUF array is
+ * the size returned by the H5FD_sb_size() call.
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, August 16, 1999
*
* Modifications:
@@ -552,22 +552,22 @@ H5FD_sb_encode(H5FD_t *file, char *name/*out*/, uint8_t *buf)
HDassert(file && file->cls);
if(file->cls->sb_encode &&
(file->cls->sb_encode)(file, name/*out*/, buf/*out*/) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_encode request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_encode request failed")
done:
FUNC_LEAVE_NOAPI(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_sb_decode
+ * Function: H5FD_sb_decode
*
- * Purpose: Decodes the driver information block.
+ * Purpose: Decodes the driver information block.
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, August 16, 1999
*
*-------------------------------------------------------------------------
@@ -581,25 +581,25 @@ H5FD_sb_decode(H5FD_t *file, const char *name, const uint8_t *buf)
HDassert(file && file->cls);
if(file->cls->sb_decode && (file->cls->sb_decode)(file, name, buf) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_decode request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver sb_decode request failed")
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_sb_decode() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_pl_copy
+ * Function: H5FD_pl_copy
*
- * Purpose: Copies the driver-specific part of the a property list.
+ * Purpose: Copies the driver-specific part of the a property list.
* This is common code, used by both the dataset transfer and
* file access property list routines.
*
- * Return: Success: non-negative
+ * Return: Success: non-negative
*
- * Failure: negative
+ * Failure: negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, October 23, 2003
*
* Modifications:
@@ -638,18 +638,18 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_pl_copy() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_pl_close
+ * Function: H5FD_pl_close
*
- * Purpose: Closes a driver for a property list
+ * Purpose: Closes a driver for a property list
* This is common code, used by both the dataset transfer and
* file access property list routines.
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, October 23, 2003
*
*-------------------------------------------------------------------------
@@ -663,11 +663,11 @@ H5FD_pl_close(hid_t driver_id, herr_t (*free_func)(void *), void *pl)
/* Allow driver to free or do it ourselves */
if(pl && free_func) {
- if((free_func)(pl) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver free request failed")
+ if((free_func)(pl) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "driver free request failed")
} /* end if */
else
- H5MM_xfree(pl);
+ H5MM_xfree(pl);
/* Decrement reference count for driver */
if(H5I_dec_ref(driver_id) < 0)
@@ -677,27 +677,27 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_pl_close() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_fapl_get
+ * Function: H5FD_fapl_get
*
- * Purpose: Gets the file access property list associated with a file.
- * Usually the file will copy what it needs from the original
- * file access property list when the file is created. The
- * purpose of this function is to create a new file access
- * property list based on the settings in the file, which may
- * have been modified from the original file access property
- * list.
+ * Purpose: Gets the file access property list associated with a file.
+ * Usually the file will copy what it needs from the original
+ * file access property list when the file is created. The
+ * purpose of this function is to create a new file access
+ * property list based on the settings in the file, which may
+ * have been modified from the original file access property
+ * list.
*
- * Return: Success: Pointer to a new file access property list
- * with all members copied. If the file is
- * closed then this property list lives on, and
- * vice versa.
+ * Return: Success: Pointer to a new file access property list
+ * with all members copied. If the file is
+ * closed then this property list lives on, and
+ * vice versa.
*
- * Failure: NULL, including when the file has no
- * properties.
+ * Failure: NULL, including when the file has no
+ * properties.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Friday, August 13, 1999
*
* Modifications:
@@ -707,30 +707,30 @@ done:
void *
H5FD_fapl_get(H5FD_t *file)
{
- void *ret_value=NULL;
+ void *ret_value=NULL;
FUNC_ENTER_NOAPI(NULL)
HDassert(file);
if(file->cls->fapl_get)
- ret_value = (file->cls->fapl_get)(file);
+ ret_value = (file->cls->fapl_get)(file);
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_fapl_get() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_fapl_open
+ * Function: H5FD_fapl_open
*
- * Purpose: Mark a driver as used by a file access property list
+ * Purpose: Mark a driver as used by a file access property list
*
- * Return: Success: non-negative
+ * Return: Success: non-negative
*
- * Failure: negative
+ * Failure: negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, October 23, 2003
*
* Modifications:
@@ -766,18 +766,18 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_fapl_open() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_fapl_copy
+ * Function: H5FD_fapl_copy
*
- * Purpose: Copies the driver-specific part of the file access property
- * list.
+ * Purpose: Copies the driver-specific part of the file access property
+ * list.
*
- * Return: Success: non-negative
+ * Return: Success: non-negative
*
- * Failure: negative
+ * Failure: negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Tuesday, August 3, 1999
*
*-------------------------------------------------------------------------
@@ -792,7 +792,7 @@ H5FD_fapl_copy(hid_t driver_id, const void *old_fapl, void **copied_fapl)
/* Check args */
if(NULL == (driver = (H5FD_class_t *)H5I_object(driver_id)))
- HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a driver ID")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a driver ID")
/* Copy the file access property list */
if(H5FD_pl_copy(driver->fapl_copy, driver->fapl_size, old_fapl, copied_fapl) < 0)
@@ -802,16 +802,16 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_fapl_close
+ * Function: H5FD_fapl_close
*
- * Purpose: Closes a driver for a dataset transfer property list
+ * Purpose: Closes a driver for a dataset transfer property list
*
- * Return: Success: non-negative
- * Failure: negative
+ * Return: Success: non-negative
+ * Failure: negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Tuesday, August 3, 1999
*
* Modifications:
@@ -821,7 +821,7 @@ done:
herr_t
H5FD_fapl_close(hid_t driver_id, void *fapl)
{
- H5FD_class_t *driver = NULL;
+ H5FD_class_t *driver = NULL;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
@@ -840,56 +840,56 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_fapl_close() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDopen
- *
- * Purpose: Opens a file named NAME for the type(s) of access described
- * by the bit vector FLAGS according to a file access property
- * list FAPL_ID (which may be the constant H5P_DEFAULT). The
- * file should expect to handle format addresses in the range [0,
- * MAXADDR] (if MAXADDR is the undefined address then the caller
- * doesn't care about the address range).
- *
- * Possible values for the FLAGS bits are:
- *
- * H5F_ACC_RDWR: Open the file for read and write access. If
- * this bit is not set then open the file for
- * read only access. It is permissible to open a
- * file for read and write access when only read
- * access is requested by the library (the
- * library will never attempt to write to a file
- * which it opened with only read access).
- *
- * H5F_ACC_CREATE: Create the file if it doesn't already exist.
- * However, see H5F_ACC_EXCL below.
- *
- * H5F_ACC_TRUNC: Truncate the file if it already exists. This
- * is equivalent to deleting the file and then
- * creating a new empty file.
- *
- * H5F_ACC_EXCL: When used with H5F_ACC_CREATE, if the file
- * already exists then the open should fail.
- * Note that this is unsupported/broken with
- * some file drivers (e.g., sec2 across nfs) and
- * will contain a race condition when used to
- * perform file locking.
- *
- * The MAXADDR is the maximum address which will be requested by
- * the library during an allocation operation. Usually this is
- * the same value as the MAXADDR field of the class structure,
- * but it can be smaller if the driver is being used under some
- * other driver.
- *
- * Note that when the driver `open' callback gets control that
- * the public part of the file struct (the H5FD_t part) will be
- * incomplete and will be filled in after that callback returns.
- *
- * Return: Success: Pointer to a new file driver struct.
- *
- * Failure: NULL
- *
- * Programmer: Robb Matzke
+ * Function: H5FDopen
+ *
+ * Purpose: Opens a file named NAME for the type(s) of access described
+ * by the bit vector FLAGS according to a file access property
+ * list FAPL_ID (which may be the constant H5P_DEFAULT). The
+ * file should expect to handle format addresses in the range [0,
+ * MAXADDR] (if MAXADDR is the undefined address then the caller
+ * doesn't care about the address range).
+ *
+ * Possible values for the FLAGS bits are:
+ *
+ * H5F_ACC_RDWR: Open the file for read and write access. If
+ * this bit is not set then open the file for
+ * read only access. It is permissible to open a
+ * file for read and write access when only read
+ * access is requested by the library (the
+ * library will never attempt to write to a file
+ * which it opened with only read access).
+ *
+ * H5F_ACC_CREATE: Create the file if it doesn't already exist.
+ * However, see H5F_ACC_EXCL below.
+ *
+ * H5F_ACC_TRUNC: Truncate the file if it already exists. This
+ * is equivalent to deleting the file and then
+ * creating a new empty file.
+ *
+ * H5F_ACC_EXCL: When used with H5F_ACC_CREATE, if the file
+ * already exists then the open should fail.
+ * Note that this is unsupported/broken with
+ * some file drivers (e.g., sec2 across nfs) and
+ * will contain a race condition when used to
+ * perform file locking.
+ *
+ * The MAXADDR is the maximum address which will be requested by
+ * the library during an allocation operation. Usually this is
+ * the same value as the MAXADDR field of the class structure,
+ * but it can be smaller if the driver is being used under some
+ * other driver.
+ *
+ * Note that when the driver `open' callback gets control that
+ * the public part of the file struct (the H5FD_t part) will be
+ * incomplete and will be filled in after that callback returns.
+ *
+ * Return: Success: Pointer to a new file driver struct.
+ *
+ * Failure: NULL
+ *
+ * Programmer: Robb Matzke
* Tuesday, July 27, 1999
*
* Modifications:
@@ -899,7 +899,7 @@ done:
H5FD_t *
H5FDopen(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
{
- H5FD_t *ret_value=NULL;
+ H5FD_t *ret_value=NULL;
FUNC_ENTER_API(NULL)
H5TRACE4("*x", "*sIuia", name, flags, fapl_id, maxaddr);
@@ -912,50 +912,50 @@ H5FDopen(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a file access property list")
if(NULL==(ret_value=H5FD_open(name, flags, fapl_id, maxaddr)))
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "unable to open file")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "unable to open file")
done:
FUNC_LEAVE_API(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_open
+ * Function: H5FD_open
*
- * Purpose: Private version of H5FDopen()
+ * Purpose: Private version of H5FDopen()
*
- * Return: Success: Pointer to a new file driver struct
+ * Return: Success: Pointer to a new file driver struct
*
- * Failure: NULL
+ * Failure: NULL
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, August 4, 1999
*
* Modifications:
*
- * Raymond Lu
- * Tuesday, Oct 23, 2001
- * Changed the file access list to the new generic property
- * list.
+ * Raymond Lu
+ * Tuesday, Oct 23, 2001
+ * Changed the file access list to the new generic property
+ * list.
*
*-------------------------------------------------------------------------
*/
H5FD_t *
H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
{
- H5FD_class_t *driver; /* VFD for file */
- H5FD_t *file = NULL; /* VFD file struct */
+ H5FD_class_t *driver; /* VFD for file */
+ H5FD_t *file = NULL; /* VFD file struct */
hid_t driver_id = -1; /* VFD ID */
H5P_genplist_t *plist; /* Property list pointer */
unsigned long driver_flags = 0; /* File-inspecific driver feature flags */
H5FD_file_image_info_t file_image_info; /* Initial file image */
- H5FD_t *ret_value; /* Return value */
+ H5FD_t *ret_value; /* Return value */
FUNC_ENTER_NOAPI(NULL)
/* Sanity check */
if(0 == maxaddr)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "zero format address range")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "zero format address range")
/* Get file access property list */
if(NULL == (plist = (H5P_genplist_t *)H5I_object(fapl_id)))
@@ -967,9 +967,9 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
/* Get driver info */
if(NULL == (driver = (H5FD_class_t *)H5I_object(driver_id)))
- HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "invalid driver ID in file access property list")
+ HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, NULL, "invalid driver ID in file access property list")
if(NULL == driver->open)
- HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "file driver has no `open' method")
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "file driver has no `open' method")
/* Query driver flag */
H5FD_driver_query(driver, &driver_flags);
@@ -988,7 +988,7 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
if(HADDR_UNDEF == maxaddr)
maxaddr = driver->maxaddr;
if(NULL == (file = (driver->open)(name, flags, fapl_id, maxaddr)))
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "open failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, NULL, "open failed")
/*
* Fill in public fields. We must increment the reference count on the
@@ -1027,20 +1027,20 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_open() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDclose
+ * Function: H5FDclose
*
* Purpose: Closes the file by calling the driver `close' callback, which
- * should free all driver-private data and free the file struct.
- * Note that the public part of the file struct (the H5FD_t part)
- * will be all zero during the driver close callback like during
- * the `open' callback.
+ * should free all driver-private data and free the file struct.
+ * Note that the public part of the file struct (the H5FD_t part)
+ * will be all zero during the driver close callback like during
+ * the `open' callback.
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Tuesday, July 27, 1999
*
*-------------------------------------------------------------------------
@@ -1054,25 +1054,25 @@ H5FDclose(H5FD_t *file)
H5TRACE1("e", "*x", file);
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
if(H5FD_close(file) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close file")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close file")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDclose() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_close
+ * Function: H5FD_close
*
- * Purpose: Private version of H5FDclose()
+ * Purpose: Private version of H5FDclose()
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, August 4, 1999
*
*-------------------------------------------------------------------------
@@ -1105,25 +1105,25 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_close() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDcmp
+ * Function: H5FDcmp
*
- * Purpose: Compare the keys of two files using the file driver callback
- * if the files belong to the same driver, otherwise sort the
- * files by driver class pointer value.
+ * Purpose: Compare the keys of two files using the file driver callback
+ * if the files belong to the same driver, otherwise sort the
+ * files by driver class pointer value.
*
- * Return: Success: A value like strcmp()
+ * Return: Success: A value like strcmp()
*
- * Failure: Must never fail. If both file handles are
- * invalid then they compare equal. If one file
- * handle is invalid then it compares less than
- * the other. If both files belong to the same
- * driver and the driver doesn't provide a
- * comparison callback then the file pointers
- * themselves are compared.
+ * Failure: Must never fail. If both file handles are
+ * invalid then they compare equal. If one file
+ * handle is invalid then it compares less than
+ * the other. If both files belong to the same
+ * driver and the driver doesn't provide a
+ * comparison callback then the file pointers
+ * themselves are compared.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Tuesday, July 27, 1999
*
* Modifications:
@@ -1133,7 +1133,7 @@ done:
int
H5FDcmp(const H5FD_t *f1, const H5FD_t *f2)
{
- int ret_value;
+ int ret_value;
FUNC_ENTER_API(-1) /*return value is arbitrary*/
H5TRACE2("Is", "*x*x", f1, f2);
@@ -1144,17 +1144,17 @@ done:
FUNC_LEAVE_API(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_cmp
+ * Function: H5FD_cmp
*
- * Purpose: Private version of H5FDcmp()
+ * Purpose: Private version of H5FDcmp()
*
- * Return: Success: A value like strcmp()
+ * Return: Success: A value like strcmp()
*
- * Failure: Must never fail.
+ * Failure: Must never fail.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, August 4, 1999
*
* Modifications:
@@ -1164,7 +1164,7 @@ done:
int
H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2)
{
- int ret_value;
+ int ret_value;
FUNC_ENTER_NOAPI(-1) /*return value is arbitrary*/
@@ -1181,11 +1181,11 @@ H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2)
/* Files are same driver; no cmp callback */
if(!f1->cls->cmp) {
- if(f1<f2)
+ if(f1<f2)
HGOTO_DONE(-1)
- if(f1>f2)
+ if(f1>f2)
HGOTO_DONE(1)
- HGOTO_DONE(0)
+ HGOTO_DONE(0)
}
ret_value = (f1->cls->cmp)(f1, f2);
@@ -1194,17 +1194,17 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDquery
+ * Function: H5FDquery
*
- * Purpose: Query a VFL driver for its feature flags. (listed in H5FDpublic.h)
+ * Purpose: Query a VFL driver for its feature flags. (listed in H5FDpublic.h)
*
- * Return: Success: non-negative
+ * Return: Success: non-negative
*
- * Failure: negative
+ * Failure: negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Friday, August 25, 2000
*
* Modifications:
@@ -1214,7 +1214,7 @@ done:
int
H5FDquery(const H5FD_t *f, unsigned long *flags/*out*/)
{
- int ret_value;
+ int ret_value;
FUNC_ENTER_API(FAIL)
H5TRACE2("Is", "*xx", f, flags);
@@ -1228,17 +1228,17 @@ done:
FUNC_LEAVE_API(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_query
+ * Function: H5FD_query
*
- * Purpose: Private version of H5FDquery()
+ * Purpose: Private version of H5FDquery()
*
- * Return: Success: non-negative
+ * Return: Success: non-negative
*
- * Failure: negative
+ * Failure: negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Friday, August 25, 2000
*
*-------------------------------------------------------------------------
@@ -1246,7 +1246,7 @@ done:
static int
H5FD_query(const H5FD_t *f, unsigned long *flags/*out*/)
{
- int ret_value = 0; /* Return value */
+ int ret_value = 0; /* Return value */
FUNC_ENTER_NOAPI_NOINIT_NOERR
@@ -1262,7 +1262,7 @@ H5FD_query(const H5FD_t *f, unsigned long *flags/*out*/)
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_query() */
-
+
/*-------------------------------------------------------------------------
* Function: H5FD_driver_query
*
@@ -1292,48 +1292,48 @@ H5FD_driver_query(const H5FD_class_t *driver, unsigned long *flags/*out*/)
/* Check for the driver to query and then query it */
if(driver->query)
ret_value = (driver->query)(NULL, flags);
- else
+ else
*flags = 0;
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_driver_query() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDalloc
+ * Function: H5FDalloc
*
- * Purpose: Allocates SIZE bytes of memory from the FILE. The memory will
- * be used according to the allocation class TYPE. First we try
- * to satisfy the request from one of the free lists, according
- * to the free list map provided by the driver. The free list
- * array has one entry for each request type and the value of
- * that array element can be one of four possibilities:
+ * Purpose: Allocates SIZE bytes of memory from the FILE. The memory will
+ * be used according to the allocation class TYPE. First we try
+ * to satisfy the request from one of the free lists, according
+ * to the free list map provided by the driver. The free list
+ * array has one entry for each request type and the value of
+ * that array element can be one of four possibilities:
*
- * It can be the constant H5FD_MEM_DEFAULT (or zero) which
- * indicates that the identity mapping is used. In other
- * words, the request type maps to its own free list.
+ * It can be the constant H5FD_MEM_DEFAULT (or zero) which
+ * indicates that the identity mapping is used. In other
+ * words, the request type maps to its own free list.
*
- * It can be the request type itself, which has the same
- * effect as the H5FD_MEM_DEFAULT value above.
+ * It can be the request type itself, which has the same
+ * effect as the H5FD_MEM_DEFAULT value above.
*
- * It can be the ID for another request type, which
- * indicates that the free list for the specified type
- * should be used instead.
+ * It can be the ID for another request type, which
+ * indicates that the free list for the specified type
+ * should be used instead.
*
- * It can be the constant H5FD_MEM_NOLIST which means that
- * no free list should be used for this type of request.
+ * It can be the constant H5FD_MEM_NOLIST which means that
+ * no free list should be used for this type of request.
*
- * If the request cannot be satisfied from a free list then
- * either the driver's `alloc' callback is invoked (if one was
- * supplied) or the end-of-address marker is extended. The
- * `alloc' callback is always called with the same arguments as
- * the H5FDalloc().
+ * If the request cannot be satisfied from a free list then
+ * either the driver's `alloc' callback is invoked (if one was
+ * supplied) or the end-of-address marker is extended. The
+ * `alloc' callback is always called with the same arguments as
+ * the H5FDalloc().
*
- * Return: Success: The format address of the new file memory.
+ * Return: Success: The format address of the new file memory.
*
- * Failure: The undefined address HADDR_UNDEF
+ * Failure: The undefined address HADDR_UNDEF
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Tuesday, July 27, 1999
*
*-------------------------------------------------------------------------
@@ -1341,18 +1341,18 @@ H5FD_driver_query(const H5FD_class_t *driver, unsigned long *flags/*out*/)
haddr_t
H5FDalloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size)
{
- haddr_t ret_value = HADDR_UNDEF;
+ haddr_t ret_value = HADDR_UNDEF;
FUNC_ENTER_API(HADDR_UNDEF)
H5TRACE4("a", "*xMtih", file, type, dxpl_id, size);
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer")
if(type < H5FD_MEM_DEFAULT || type >= H5FD_MEM_NTYPES)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid request type")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid request type")
if(size == 0)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "zero-size request")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "zero-size request")
if(H5P_DEFAULT == dxpl_id)
dxpl_id = H5P_DATASET_XFER_DEFAULT;
else
@@ -1361,7 +1361,7 @@ H5FDalloc(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, hsize_t size)
/* Do the real work */
if(HADDR_UNDEF == (ret_value = H5FD_alloc_real(file, dxpl_id, type, size, NULL, NULL)))
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "unable to allocate file memory")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "unable to allocate file memory")
/* (Note compensating for base address subtraction in internal routine) */
ret_value += file->base_addr;
@@ -1370,22 +1370,22 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDalloc() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDfree
+ * Function: H5FDfree
*
- * Purpose: Frees format addresses starting with ADDR and continuing for
- * SIZE bytes in the file FILE. The type of space being freed is
- * specified by TYPE, which is mapped to a free list as
- * described for the H5FDalloc() function above. If the request
- * doesn't map to a free list then either the application `free'
- * callback is invoked (if defined) or the memory is leaked.
+ * Purpose: Frees format addresses starting with ADDR and continuing for
+ * SIZE bytes in the file FILE. The type of space being freed is
+ * specified by TYPE, which is mapped to a free list as
+ * described for the H5FDalloc() function above. If the request
+ * doesn't map to a free list then either the application `free'
+ * callback is invoked (if defined) or the memory is leaked.
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, July 28, 1999
*
* Modifications:
@@ -1420,17 +1420,17 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDfree() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDget_eoa
+ * Function: H5FDget_eoa
*
- * Purpose: Returns the address of the first byte after the last
- * allocated memory in the file.
+ * Purpose: Returns the address of the first byte after the last
+ * allocated memory in the file.
*
- * Return: Success: First byte after allocated memory.
- * Failure: HADDR_UNDEF
+ * Return: Success: First byte after allocated memory.
+ * Failure: HADDR_UNDEF
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Friday, July 30, 1999
*
*-------------------------------------------------------------------------
@@ -1438,20 +1438,20 @@ done:
haddr_t
H5FDget_eoa(H5FD_t *file, H5FD_mem_t type)
{
- haddr_t ret_value;
+ haddr_t ret_value;
FUNC_ENTER_API(HADDR_UNDEF)
H5TRACE2("a", "*xMt", file, type);
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer")
if(type < H5FD_MEM_DEFAULT || type >= H5FD_MEM_NTYPES)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file type")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file type")
/* The real work */
if(HADDR_UNDEF == (ret_value = H5FD_get_eoa(file, type)))
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eoa request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eoa request failed")
/* (Note compensating for base address subtraction in internal routine) */
ret_value += file->base_addr;
@@ -1460,27 +1460,27 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDget_eoa() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDset_eoa
+ * Function: H5FDset_eoa
*
- * Purpose: Set the end-of-address marker for the file. The ADDR is the
- * address of the first byte past the last allocated byte of the
- * file. This function is called from two places:
+ * Purpose: Set the end-of-address marker for the file. The ADDR is the
+ * address of the first byte past the last allocated byte of the
+ * file. This function is called from two places:
*
- * It is called after an existing file is opened in order to
- * "allocate" enough space to read the superblock and then
- * to "allocate" the entire hdf5 file based on the contents
- * of the superblock.
+ * It is called after an existing file is opened in order to
+ * "allocate" enough space to read the superblock and then
+ * to "allocate" the entire hdf5 file based on the contents
+ * of the superblock.
*
- * It is called during file memory allocation if the
- * allocation request cannot be satisfied from the free list
- * and the driver didn't supply an allocation callback.
+ * It is called during file memory allocation if the
+ * allocation request cannot be satisfied from the free list
+ * and the driver didn't supply an allocation callback.
*
- * Return: Success: Non-negative
- * Failure: Negative, no side effect
+ * Return: Success: Non-negative
+ * Failure: Negative, no side effect
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Friday, July 30, 1999
*
*-------------------------------------------------------------------------
@@ -1495,42 +1495,42 @@ H5FDset_eoa(H5FD_t *file, H5FD_mem_t type, haddr_t addr)
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
if(type < H5FD_MEM_DEFAULT || type >= H5FD_MEM_NTYPES)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file type")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file type")
if(!H5F_addr_defined(addr) || addr > file->maxaddr)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid end-of-address value")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid end-of-address value")
/* The real work */
/* (Note compensating for base address addition in internal routine) */
if(H5FD_set_eoa(file, type, addr - file->base_addr) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file set eoa request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, FAIL, "file set eoa request failed")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDset_eoa() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDget_eof
+ * Function: H5FDget_eof
*
- * Purpose: Returns the end-of-file address, which is the greater of the
- * end-of-format address and the actual EOF marker. This
- * function is called after an existing file is opened in order
- * for the library to learn the true size of the underlying file
- * and to determine whether the hdf5 data has been truncated.
+ * Purpose: Returns the end-of-file address, which is the greater of the
+ * end-of-format address and the actual EOF marker. This
+ * function is called after an existing file is opened in order
+ * for the library to learn the true size of the underlying file
+ * and to determine whether the hdf5 data has been truncated.
*
- * It is also used when a file is first opened to learn whether
- * the file is empty or not.
+ * It is also used when a file is first opened to learn whether
+ * the file is empty or not.
*
- * It is permissible for the driver to return the maximum address
- * for the file size if the file is not empty.
+ * It is permissible for the driver to return the maximum address
+ * for the file size if the file is not empty.
*
- * Return: Success: The EOF address.
+ * Return: Success: The EOF address.
*
- * Failure: HADDR_UNDEF
+ * Failure: HADDR_UNDEF
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, July 29, 1999
*
* Modifications:
@@ -1540,18 +1540,18 @@ done:
haddr_t
H5FDget_eof(H5FD_t *file)
{
- haddr_t ret_value;
+ haddr_t ret_value;
FUNC_ENTER_API(HADDR_UNDEF)
H5TRACE1("a", "*x", file);
/* Check arguments */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, HADDR_UNDEF, "invalid file pointer")
/* The real work */
if(HADDR_UNDEF == (ret_value = H5FD_get_eof(file)))
- HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eof request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTINIT, HADDR_UNDEF, "file get eof request failed")
/* (Note compensating for base address subtraction in internal routine) */
ret_value += file->base_addr;
@@ -1560,16 +1560,16 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDget_eof() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_get_maxaddr
+ * Function: H5FD_get_maxaddr
*
- * Purpose: Private version of H5FDget_eof()
+ * Purpose: Private version of H5FDget_eof()
*
- * Return: Success: The maximum address allowed in the file.
- * Failure: HADDR_UNDEF
+ * Return: Success: The maximum address allowed in the file.
+ * Failure: HADDR_UNDEF
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, January 3, 2008
*
*-------------------------------------------------------------------------
@@ -1590,16 +1590,16 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_get_maxaddr() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_get_feature_flags
+ * Function: H5FD_get_feature_flags
*
- * Purpose: Retrieve the feature flags for the VFD
+ * Purpose: Retrieve the feature flags for the VFD
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Tuesday, January 8, 2008
*
*-------------------------------------------------------------------------
@@ -1618,16 +1618,16 @@ H5FD_get_feature_flags(const H5FD_t *file, unsigned long *feature_flags)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FD_get_feature_flags() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_get_fs_type_map
+ * Function: H5FD_get_fs_type_map
*
- * Purpose: Retrieve the free space type mapping for the VFD
+ * Purpose: Retrieve the free space type mapping for the VFD
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, January 17, 2008
*
*-------------------------------------------------------------------------
@@ -1657,29 +1657,29 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_get_fs_type_map() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDread
+ * Function: H5FDread
*
- * Purpose: Reads SIZE bytes from FILE beginning at address ADDR
- * according to the data transfer property list DXPL_ID (which may
- * be the constant H5P_DEFAULT). The result is written into the
- * buffer BUF.
+ * Purpose: Reads SIZE bytes from FILE beginning at address ADDR
+ * according to the data transfer property list DXPL_ID (which may
+ * be the constant H5P_DEFAULT). The result is written into the
+ * buffer BUF.
*
- * Return: Success: Non-negative. The read result is written into
- * the BUF buffer which should be allocated by
- * the caller.
+ * Return: Success: Non-negative. The read result is written into
+ * the BUF buffer which should be allocated by
+ * the caller.
*
- * Failure: Negative. The contents of BUF is undefined.
+ * Failure: Negative. The contents of BUF is undefined.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, July 29, 1999
*
*-------------------------------------------------------------------------
*/
herr_t
H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
- void *buf/*out*/)
+ void *buf/*out*/)
{
H5P_genplist_t *dxpl; /* DXPL object */
herr_t ret_value = SUCCEED; /* Return value */
@@ -1689,7 +1689,7 @@ H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
/* Get the default dataset transfer property list if the user didn't provide one */
if(H5P_DEFAULT == dxpl_id)
@@ -1698,7 +1698,7 @@ H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size
if(TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list")
if(!buf)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null result buffer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null result buffer")
/* Get the DXPL plist object for DXPL ID */
if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
@@ -1707,33 +1707,33 @@ H5FDread(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size
/* Do the real work */
/* (Note compensating for base address addition in internal routine) */
if(H5FD_read(file, dxpl, type, addr - file->base_addr, size, buf) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file read request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "file read request failed")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDread() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDwrite
+ * Function: H5FDwrite
*
- * Purpose: Writes SIZE bytes to FILE beginning at address ADDR according
- * to the data transfer property list DXPL_ID (which may be the
- * constant H5P_DEFAULT). The bytes to be written come from the
- * buffer BUF.
+ * Purpose: Writes SIZE bytes to FILE beginning at address ADDR according
+ * to the data transfer property list DXPL_ID (which may be the
+ * constant H5P_DEFAULT). The bytes to be written come from the
+ * buffer BUF.
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, July 29, 1999
*
*-------------------------------------------------------------------------
*/
herr_t
H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t size,
- const void *buf)
+ const void *buf)
{
H5P_genplist_t *dxpl; /* DXPL object */
herr_t ret_value = SUCCEED; /* Return value */
@@ -1743,7 +1743,7 @@ H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t siz
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
/* Get the default dataset transfer property list if the user didn't provide one */
if(H5P_DEFAULT == dxpl_id)
dxpl_id = H5P_DATASET_XFER_DEFAULT;
@@ -1751,7 +1751,7 @@ H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t siz
if(TRUE != H5P_isa_class(dxpl_id, H5P_DATASET_XFER))
HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list")
if(!buf)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null buffer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "null buffer")
/* Get the DXPL plist object for DXPL ID */
if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id)))
@@ -1760,24 +1760,24 @@ H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, size_t siz
/* The real work */
/* (Note compensating for base address addition in internal routine) */
if(H5FD_write(file, dxpl, type, addr - file->base_addr, size, buf) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "file write request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_WRITEERROR, FAIL, "file write request failed")
done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDwrite() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDflush
+ * Function: H5FDflush
*
- * Purpose: Notify driver to flush all cached data. If the driver has no
- * flush method then nothing happens.
+ * Purpose: Notify driver to flush all cached data. If the driver has no
+ * flush method then nothing happens.
*
- * Return: Success: Non-negative
+ * Return: Success: Non-negative
*
- * Failure: Negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, July 29, 1999
*
* Modifications:
@@ -1796,7 +1796,7 @@ H5FDflush(H5FD_t *file, hid_t dxpl_id, unsigned closing)
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
if(H5P_DEFAULT == dxpl_id)
dxpl_id = H5P_DATASET_XFER_DEFAULT;
else
@@ -1805,22 +1805,22 @@ H5FDflush(H5FD_t *file, hid_t dxpl_id, unsigned closing)
/* Do the real work */
if(H5FD_flush(file, dxpl_id, closing) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTFLUSH, FAIL, "file flush request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTFLUSH, FAIL, "file flush request failed")
done:
FUNC_LEAVE_API(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_flush
+ * Function: H5FD_flush
*
- * Purpose: Private version of H5FDflush()
+ * Purpose: Private version of H5FDflush()
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, August 4, 1999
*
*-------------------------------------------------------------------------
@@ -1841,16 +1841,16 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_flush() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FDtruncate
+ * Function: H5FDtruncate
*
- * Purpose: Notify driver to truncate the file back to the allocated size.
+ * Purpose: Notify driver to truncate the file back to the allocated size.
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, January 31, 2008
*
*-------------------------------------------------------------------------
@@ -1865,7 +1865,7 @@ H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing)
/* Check args */
if(!file || !file->cls)
- HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid file pointer")
if(H5P_DEFAULT == dxpl_id)
dxpl_id = H5P_DATASET_XFER_DEFAULT;
else
@@ -1874,22 +1874,22 @@ H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing)
/* Do the real work */
if(H5FD_truncate(file, dxpl_id, closing) < 0)
- HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "file flush request failed")
+ HGOTO_ERROR(H5E_VFL, H5E_CANTUPDATE, FAIL, "file flush request failed")
done:
FUNC_LEAVE_API(ret_value)
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_truncate
+ * Function: H5FD_truncate
*
- * Purpose: Private version of H5FDtruncate()
+ * Purpose: Private version of H5FDtruncate()
*
- * Return: Success: Non-negative
- * Failure: Negative
+ * Return: Success: Non-negative
+ * Failure: Negative
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, January 31, 2008
*
*-------------------------------------------------------------------------
@@ -1910,18 +1910,18 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_truncate() */
-
+
/*-------------------------------------------------------------------------
- * Function: H5FD_get_fileno
+ * Function: H5FD_get_fileno
*
- * Purpose: Quick and dirty routine to retrieve the file's 'fileno' value
+ * Purpose: Quick and dirty routine to retrieve the file's 'fileno' value
* (Mainly added to stop non-file routines from poking about in the
* H5FD_t data structure)
*
- * Return: Non-negative on success/Negative on failure
+ * Return: Non-negative on success/Negative on failure
*
- * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu>
- * March 27, 2002
+ * Programmer: Quincey Koziol <koziol@ncsa.uiuc.edu>
+ * March 27, 2002
*
*-------------------------------------------------------------------------
*/
@@ -1939,7 +1939,7 @@ H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FD_get_fileno() */
-
+
/*--------------------------------------------------------------------------
* Function: H5FDget_vfd_handle
*
@@ -1973,7 +1973,7 @@ done:
FUNC_LEAVE_API(ret_value)
} /* end H5FDget_vfd_handle() */
-
+
/*--------------------------------------------------------------------------
* Function: H5FD_get_vfd_handle
*
@@ -1998,7 +1998,7 @@ H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void **file_handle)
HDassert(file_handle);
if(NULL == file->cls->get_handle)
- HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "file driver has no `get_vfd_handle' method")
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "file driver has no `get_vfd_handle' method")
if((file->cls->get_handle)(file, fapl, file_handle) < 0)
HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get file handle for file driver")
@@ -2006,7 +2006,7 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5FD_get_vfd_handle() */
-
+
/*--------------------------------------------------------------------------
* Function: H5FD_set_base_addr
*
@@ -2033,14 +2033,14 @@ H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr)
FUNC_LEAVE_NOAPI(SUCCEED)
} /* end H5FD_set_base_addr() */
-
+
/*--------------------------------------------------------------------------
* Function: H5FD_get_base_addr
*
* Purpose: Get the base address for the file
*
- * Return: Success: The absolute base address of the file
- * Failure: The undefined address (HADDR_UNDEF)
+ * Return: Success: The absolute base address of the file
+ * Failure: The undefined address (HADDR_UNDEF)
*
* Programmer: Quincey Koziol
* Sept. 10, 2009
@@ -2058,3 +2058,37 @@ H5FD_get_base_addr(const H5FD_t *file)
FUNC_LEAVE_NOAPI(file->base_addr)
} /* end H5FD_get_base_addr() */
+/*-------------------------------------------------------------------------
+* Function: H5FDdriver_query
+*
+* Purpose: Similar to H5FD_query(), but intended for cases when we don't
+* have a file available (e.g. before one is opened). Since we
+* can't use the file to get the driver, the driver ID is passed
+* in as a parameter.
+*
+* Return: SUCCEED/FAIL
+*
+*-------------------------------------------------------------------------
+*/
+herr_t
+H5FDdriver_query(hid_t driver_id, unsigned long *flags/*out*/)
+{
+ H5FD_class_t *driver = NULL; /* Pointer to VFD class struct */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE2("e", "ix", driver_id, flags);
+
+ /* Check arguments */
+ if(NULL == flags)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "flags parameter cannot be NULL")
+
+ /* Check for the driver to query and then query it */
+ if (NULL == (driver = (H5FD_class_t *)H5I_object_verify(driver_id, H5I_VFL)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "not a VFL ID")
+ if (H5FD_driver_query(driver, flags) < 0)
+ HGOTO_ERROR(H5E_VFL, H5E_BADVALUE, FAIL, "driver flag query failed")
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* end H5FDdriver_query() */
diff --git a/src/H5FDcore.h b/src/H5FDcore.h
index 3ee61b6..f2b0fb0 100644
--- a/src/H5FDcore.h
+++ b/src/H5FDcore.h
@@ -15,7 +15,7 @@
* Programmer: Robb Matzke <matzke@llnl.gov>
* Monday, August 2, 1999
*
- * Purpose: The public header file for the sec2 driver.
+ * Purpose: The public header file for the core driver.
*/
#ifndef H5FDcore_H
#define H5FDcore_H
diff --git a/src/H5FDhdfs.c b/src/H5FDhdfs.c
new file mode 100644
index 0000000..59d954f
--- /dev/null
+++ b/src/H5FDhdfs.c
@@ -0,0 +1,1876 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Read-Only HDFS Virtual File Driver (VFD) *
+ * Copyright (c) 2018, The HDF Group. *
+ * *
+ * All rights reserved. *
+ * *
+ * NOTICE: *
+ * All information contained herein is, and remains, the property of The HDF *
+ * Group. The intellectual and technical concepts contained herein are *
+ * proprietary to The HDF Group. Dissemination of this information or *
+ * reproduction of this material is strictly forbidden unless prior written *
+ * permission is obtained from The HDF Group. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Programmer: Jacob Smith
+ * 2018-04-23
+ *
+ * Purpose: Provide read-only access to files on the Hadoop Distributed
+ * File System (HDFS).
+ */
+
+/* Interface initialization */
+#define H5_INTERFACE_INIT_FUNC H5FD_hdfs_init_interface
+
+
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Fprivate.h" /* File access */
+#include "H5FDprivate.h" /* File drivers */
+#include "H5FDhdfs.h" /* hdfs file driver */
+#include "H5FLprivate.h" /* Free Lists */
+#include "H5Iprivate.h" /* IDs */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5Pprivate.h" /* Property lists */
+
+#ifdef H5_HAVE_LIBHDFS
+#include "hdfs.h"
+#endif
+
+/* toggle function call prints: 1 turns on */
+#define HDFS_DEBUG 0
+
+/* toggle stats collection and reporting */
+#define HDFS_STATS 0
+
+/* The driver identification number, initialized at runtime */
+static hid_t H5FD_HDFS_g = 0;
+
+#if HDFS_STATS
+
+/* arbitrarily large value, such that any reasonable size read will be "less"
+ * than this value and set a true minimum
+ * not 0 because that may be a valid recorded minimum in degenerate cases
+ */
+#define HDFS_STATS_STARTING_MIN 0xfffffffful
+
+/* Configuration definitions for stats collection and breakdown
+ *
+ * 2^10 = 1024
+ * Reads up to 1024 bytes (1 kB) fall in bin 0
+ * 2^(10+(1*16)) = 2^26 = 64MB
+ * Reads of 64MB or greater fall in "overflow" bin[BIN_COUNT]
+ */
+#define HDFS_STATS_BASE 2
+#define HDFS_STATS_INTERVAL 1
+#define HDFS_STATS_START_POWER 10
+#define HDFS_STATS_BIN_COUNT 16 /* MUST BE GREATER THAN 0 */
+
+
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Calculate `BASE ^ (START_POWER + (INTERVAL * bin_i))`
+ * Stores result at `(unsigned long long *) out_ptr`.
+ * Used in computing boundaries between stats bins.
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ */
+#define HDFS_STATS_POW(bin_i, out_ptr) { \
+ unsigned long long donotshadowresult = 1; \
+ unsigned donotshadowindex = 0; \
+ for (donotshadowindex = 0; \
+ donotshadowindex < (((bin_i) * HDFS_STATS_INTERVAL) + \
+ HDFS_STATS_START_POWER); \
+ donotshadowindex++) \
+ { \
+ donotshadowresult *= HDFS_STATS_BASE; \
+ } \
+ *(out_ptr) = donotshadowresult; \
+}
+
+/* array to hold pre-computed boundaries for stats bins */
+static unsigned long long hdfs_stats_boundaries[HDFS_STATS_BIN_COUNT];
+
+
+/***************************************************************************
+ *
+ * Structure: hdfs_statsbin
+ *
+ * Purpose:
+ *
+ * Structure for storing per-file hdfs VFD usage statistics.
+ *
+ *
+ *
+ * `count` (unsigned long long)
+ *
+ * Number of reads with size in this bin's range.
+ *
+ * `bytes` (unsigned long long)
+ *
+ * Total number of bytes read through this bin.
+ *
+ * `min` (unsigned long long)
+ *
+ * Smallest read size in this bin.
+ *
+ * `max` (unsigned long long)
+ *
+ * Largest read size in this bin.
+ *
+ *
+ *
+ * Programmer: Jacob Smith
+ *
+ * Changes: None
+ *
+ ***************************************************************************/
+typedef struct {
+ unsigned long long count;
+ unsigned long long bytes;
+ unsigned long long min;
+ unsigned long long max;
+} hdfs_statsbin;
+
+#endif /* HDFS_STATS */
+
+#ifdef H5_HAVE_LIBHDFS
+
+/* "unique" identifier for `hdfs_t` structures.
+ * Randomly generated by unweighted dice rolls.
+ */
+#define HDFS_HDFST_MAGIC 0x1AD5DE84
+
+
+/***************************************************************************
+ *
+ * Structure: hdfs_t
+ *
+ * Purpose:
+ *
+ * Contain/retain information associated with a file hosted on Hadoop
+ * Distributed File System (HDFS). Instantiated and populated via
+ * `H5FD_hdfs_handle_open()` and cleaned up via `H5FD_hdfs_handle_close()`.
+ *
+ *
+ *
+ * `magic` (unisgned long)
+ *
+ * Number to indicate that this structure is of the promised
+ * type and should still be valid; should be HDFS_HDFST_MAGIC throughout
+ * the lifespan of the structure. Upon deletion of the structure, the
+ * programmer should set magic to anything but HDFS_HDFST_MAGIC, to
+ * indicate that the structure is to no longer be trusted.
+ *
+ * `filesystem` (hdfsFS)
+ *
+ * A libhdfs file system handle.
+ *
+ * `fileinfo` (hdfsFileInfo*)
+ *
+ * A pointer to a libhdfs file info structure.
+ *
+ * `file` (hdfsFile)
+ *
+ * A libhdfs file handle.
+ *
+ *
+ *
+ * Programmer: Jacob Smith
+ * May 2018
+ *
+ * Changes: None
+ *
+ ***************************************************************************
+ */
+typedef struct {
+ unsigned long magic;
+ hdfsFS filesystem;
+ hdfsFileInfo *fileinfo;
+ hdfsFile file;
+} hdfs_t;
+
+
+/*--------------------------------------------------------------------------
+ * Function: H5FD_hdfs_handle_open
+ *
+ * Purpose: Create a HDFS file handle, 'opening' the target file.
+ *
+ * Return: Success: Pointer to HDFS container/handle of opened file.
+ * Failure: NULL
+ *
+ * Programmer: Gerd Herber
+ * May 2018
+ *
+ * Changes: None.
+ *--------------------------------------------------------------------------
+ */
+static hdfs_t *
+H5FD_hdfs_handle_open(
+ const char *path,
+ const char *namenode_name,
+ const int32_t namenode_port,
+ const char *user_name,
+ const char *kerberos_ticket_cache,
+ const int32_t stream_buffer_size)
+{
+ struct hdfsBuilder *builder = NULL;
+ hdfs_t *handle = NULL;
+ hdfs_t *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "called H5FD_hdfs_handle_open.\n");
+#endif
+
+ if (path == NULL || path[0] == '\0') {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "path cannot be null.\n")
+ }
+ if (namenode_name == NULL /* || namenode_name[0] == '\0' */ ) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "namenode name cannot be null.\n")
+ }
+ if (namenode_port < 0 || namenode_port > 65535) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "namenode port must be non-negative and <= 65535.\n")
+ }
+ if (stream_buffer_size < 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "buffer size must non-negative.\n")
+ }
+
+ handle = (hdfs_t *)H5MM_malloc(sizeof(hdfs_t));
+ if (handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
+ "could not malloc space for handle.\n")
+ }
+
+ handle->magic = (unsigned long)HDFS_HDFST_MAGIC;
+ handle->filesystem = NULL; /* TODO: not a pointer; NULL may cause bug */
+ handle->fileinfo = NULL;
+ handle->file = NULL; /* TODO: not a pointer; NULL may cause bug */
+
+ builder = hdfsNewBuilder();
+ if (!builder) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "(hdfs) failed to create builder")
+ }
+ hdfsBuilderSetNameNode(builder, namenode_name);
+ hdfsBuilderSetNameNodePort(builder, (tPort)namenode_port);
+ if (user_name != NULL && user_name[0] != '\0') {
+ hdfsBuilderSetUserName(builder, user_name);
+ }
+ if (kerberos_ticket_cache != NULL && kerberos_ticket_cache[0] != '\0') {
+ hdfsBuilderSetKerbTicketCachePath(builder, kerberos_ticket_cache);
+ }
+ /* Call to `hdfsBuilderConnect` releases builder, regardless of success. */
+ handle->filesystem = hdfsBuilderConnect(builder);
+ if (!handle->filesystem) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "(hdfs) could not connect to default namenode")
+ }
+ handle->fileinfo = hdfsGetPathInfo(handle->filesystem, path);
+ if (!handle->fileinfo) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "hdfsGetPathInfo failed")
+ }
+ handle->file = hdfsOpenFile(
+ handle->filesystem,
+ path,
+ O_RDONLY,
+ stream_buffer_size,
+ 0,
+ 0);
+ if (!handle->file) {
+ HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL,
+ "(hdfs) could not open")
+ }
+
+ ret_value = handle;
+
+done:
+ if (ret_value == NULL && handle != NULL) {
+ /* error; clean up */
+ HDassert(handle->magic == HDFS_HDFST_MAGIC);
+ handle->magic++;
+ if (handle->file != NULL) {
+ if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) {
+ HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL,
+ "unable to close hdfs file handle")
+ }
+ }
+ if (handle->fileinfo != NULL) {
+ hdfsFreeFileInfo(handle->fileinfo, 1);
+ }
+ if (handle->filesystem != NULL) {
+ if (FAIL == (hdfsDisconnect(handle->filesystem))) {
+ HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL,
+ "unable to disconnect from hdfs")
+ }
+ }
+ H5MM_xfree(handle);
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5FD_hdfs_handle_open() */
+
+
+/*--------------------------------------------------------------------------
+ * Function: H5FD_hdfs_handle_close
+ *
+ * Purpose: 'Close' an HDFS file container/handle, releasing underlying
+ * resources.
+ *
+ * Return: Success: `SUCCEED` (0)
+ * Failure: `FAIL` (-1)
+ *
+ * Programmer: Gerd Herber
+ * May 2018
+ *
+ * Changes: None.
+ *--------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_handle_close(hdfs_t *handle)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "called H5FD_hdfs_close.\n");
+#endif
+
+ if (handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle cannot be null.\n")
+ }
+ if (handle->magic != HDFS_HDFST_MAGIC) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has invalid magic.\n")
+ }
+
+ handle->magic++;
+ if (handle->file != NULL) {
+ if (FAIL == (hdfsCloseFile(handle->filesystem, handle->file))) {
+ HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL,
+ "unable to close hdfs file handle")
+ }
+ }
+ if (handle->fileinfo != NULL) {
+ hdfsFreeFileInfo(handle->fileinfo, 1);
+ }
+ if (handle->filesystem != NULL) {
+ if (FAIL == (hdfsDisconnect(handle->filesystem))) {
+ HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL,
+ "unable to disconnect hdfs file system")
+ }
+ }
+
+ H5MM_xfree(handle);
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5FD_hdfs_close() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/***************************************************************************
+ *
+ * Structure: H5FD_hdfs_t
+ *
+ * Purpose:
+ *
+ * H5FD_hdfs_t is a structure used to store all information needed to
+ * maintain R/O access to a single HDF5 file in an HDFS file system.
+ * This structure is created when such a file is "opened" and
+ * discarded when it is "closed".
+ *
+ *
+ * `pub` (H5FD_t)
+ *
+ * Instance of H5FD_t which contains all fields common to all VFDs.
+ * It must be the first item in this structure, since at higher levels,
+ * this structure will be treated as an instance of H5FD_t.
+ *
+ * `fa` (H5FD_hdfs_fapl_t)
+ *
+ * Instance of `H5FD_hdfs_fapl_t` containing the HDFS configuration data
+ * needed to "open" the HDF5 file.
+ *
+ * `eoa` (haddr_t)
+ *
+ * End of addressed space in file. After open, it should always
+ * equal the file size.
+ *
+ * `hdfs_handle` (hdfs_t *)
+ *
+ * Instance of HDFS Request handle associated with the target resource.
+ * Responsible for communicating with remote host and presenting file
+ * contents as indistinguishable from a file on the local filesystem.
+ *
+ * *** present only if HDFS_SATS is flagged to enable stats collection ***
+ *
+ * `meta` (hdfs_statsbin[])
+ * `raw` (hdfs_statsbin[])
+ *
+ * Only present if hdfs stats collection is enabled.
+ *
+ * Arrays of `hdfs_statsbin` structures to record raw- and metadata reads.
+ *
+ * Records count and size of reads performed by the VFD, and is used to
+ * print formatted usage statistics to stdout upon VFD shutdown.
+ *
+ * Reads of each raw- and metadata type are recorded in an individual bin
+ * determined by the size of the read. The last bin of each type is
+ * reserved for "big" reads, with no defined upper bound.
+ *
+ * *** end HDFS_STATS ***
+ *
+ *
+ *
+ * Programmer: Jacob Smith
+ *
+ * Changes: None.
+ *
+ ***************************************************************************
+ */
+typedef struct H5FD_hdfs_t {
+ H5FD_t pub;
+ H5FD_hdfs_fapl_t fa;
+ haddr_t eoa;
+#ifdef H5_HAVE_LIBHDFS
+ hdfs_t *hdfs_handle;
+#endif
+#if HDFS_STATS
+ hdfs_statsbin meta[HDFS_STATS_BIN_COUNT + 1];
+ hdfs_statsbin raw[HDFS_STATS_BIN_COUNT + 1];
+#endif
+} H5FD_hdfs_t;
+
+/*
+ * These macros check for overflow of various quantities. These macros
+ * assume that HDoff_t is signed and haddr_t and size_t are unsigned.
+ *
+ * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t'
+ * is too large to be represented by the second argument
+ * of the file seek function.
+ * Only included if HDFS code should compile.
+ *
+ */
+#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1)
+#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR))
+
+/* Prototypes */
+static void *H5FD_hdfs_fapl_get(H5FD_t *_file);
+static void *H5FD_hdfs_fapl_copy(const void *_old_fa);
+static herr_t H5FD_hdfs_fapl_free(void *_fa);
+static H5FD_t *H5FD_hdfs_open(const char *name, unsigned flags, hid_t fapl_id,
+ haddr_t maxaddr);
+static herr_t H5FD_hdfs_close(H5FD_t *_file);
+static int H5FD_hdfs_cmp(const H5FD_t *_f1, const H5FD_t *_f2);
+static herr_t H5FD_hdfs_query(const H5FD_t *_f1, unsigned long *flags);
+static haddr_t H5FD_hdfs_get_eoa(const H5FD_t *_file, H5FD_mem_t type);
+static herr_t H5FD_hdfs_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr);
+static haddr_t H5FD_hdfs_get_eof(const H5FD_t *_file, H5FD_mem_t type);
+static herr_t H5FD_hdfs_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle);
+static herr_t H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr,
+ size_t size, void *buf);
+static herr_t H5FD_hdfs_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id, haddr_t addr,
+ size_t size, const void *buf);
+static herr_t H5FD_hdfs_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
+static herr_t H5FD_hdfs_lock(H5FD_t *_file, hbool_t rw);
+static herr_t H5FD_hdfs_unlock(H5FD_t *_file);
+static herr_t H5FD_hdfs_validate_config(const H5FD_hdfs_fapl_t * fa);
+
+static const H5FD_class_t H5FD_hdfs_g = {
+ "hdfs", /* name */
+ MAXADDR, /* maxaddr */
+ H5F_CLOSE_WEAK, /* fc_degree */
+ NULL, /* sb_size */
+ NULL, /* sb_encode */
+ NULL, /* sb_decode */
+ sizeof(H5FD_hdfs_fapl_t), /* fapl_size */
+ H5FD_hdfs_fapl_get, /* fapl_get */
+ H5FD_hdfs_fapl_copy, /* fapl_copy */
+ H5FD_hdfs_fapl_free, /* fapl_free */
+ 0, /* dxpl_size */
+ NULL, /* dxpl_copy */
+ NULL, /* dxpl_free */
+ H5FD_hdfs_open, /* open */
+ H5FD_hdfs_close, /* close */
+ H5FD_hdfs_cmp, /* cmp */
+ H5FD_hdfs_query, /* query */
+ NULL, /* get_type_map */
+ NULL, /* alloc */
+ NULL, /* free */
+ H5FD_hdfs_get_eoa, /* get_eoa */
+ H5FD_hdfs_set_eoa, /* set_eoa */
+ H5FD_hdfs_get_eof, /* get_eof */
+ H5FD_hdfs_get_handle, /* get_handle */
+ H5FD_hdfs_read, /* read */
+ H5FD_hdfs_write, /* write */
+ NULL, /* flush */
+ H5FD_hdfs_truncate, /* truncate */
+ H5FD_hdfs_lock, /* lock */
+ H5FD_hdfs_unlock, /* unlock */
+ H5FD_FLMAP_DICHOTOMY /* fl_map */
+};
+
+#ifdef H5_HAVE_LIBHDFS
+/* Declare a free list to manage the H5FD_hdfs_t struct */
+H5FL_DEFINE_STATIC(H5FD_hdfs_t);
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_init_interface
+ *
+ * Purpose: Initializes any interface-specific data or routines.
+ *
+ * Return: Success: The driver ID for the hdfs driver.
+ * Failure: Negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_init_interface(void)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ FUNC_LEAVE_NOAPI(H5FD_hdfs_init())
+} /* H5FD_hdfs_init_interface() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_init
+ *
+ * Purpose: Initialize this driver by registering the driver with the
+ * library.
+ *
+ * Return: Success: The driver ID for the hdfs driver.
+ * Failure: Negative
+ *
+ * Programmer: Robb Matzke
+ * Thursday, July 29, 1999
+ *
+ *-------------------------------------------------------------------------
+ */
+hid_t
+H5FD_hdfs_init(void)
+{
+ hid_t ret_value = H5I_INVALID_HID; /* Return value */
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_init() called.\n");
+#endif
+
+ if (H5I_VFL != H5I_get_type(H5FD_HDFS_g))
+ H5FD_HDFS_g = H5FD_register( &H5FD_hdfs_g, sizeof(H5FD_class_t), FALSE);
+
+#if HDFS_STATS
+ /* pre-compute statsbin boundaries
+ */
+ for (unsigned bin_i = 0; bin_i < HDFS_STATS_BIN_COUNT; bin_i++) {
+ unsigned long long value = 0;
+ HDFS_STATS_POW(bin_i, &value)
+ hdfs_stats_boundaries[bin_i] = value;
+ }
+#endif
+
+ ret_value = H5FD_HDFS_g;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_init() */
+
+
+/*---------------------------------------------------------------------------
+ * Function: H5FD_log_term
+ *
+ * Purpose: Shut down the VFD
+ *
+ * Returns: <none>
+ *
+ * Programmer: Jacob Smith 2018
+ *
+ *---------------------------------------------------------------------------
+ */
+void
+H5FD_hdfs_term(void)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_term() called.\n");
+#endif
+
+ /* Reset VFL ID */
+ H5FD_HDFS_g = 0;
+
+ FUNC_LEAVE_NOAPI_VOID
+} /* end H5FD_hdfs_term() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pset_fapl_hdfs
+ *
+ * Purpose: Modify the file access property list to use the H5FD_HDFS
+ * driver defined in this source file. All driver specfic
+ * properties are passed in as a pointer to a suitably
+ * initialized instance of H5FD_hdfs_fapl_t
+ *
+ * Return: SUCCEED/FAIL
+ *
+ * Programmer: Jacob Smith 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa)
+{
+ H5P_genplist_t *plist = NULL; /* Property list pointer */
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE2("e", "i*x", fapl_id, fa);
+
+ HDassert(fa != NULL);
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5Pset_fapl_hdfs() called.\n");
+#endif
+
+ if(NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list")
+
+ if (FAIL == H5FD_hdfs_validate_config(fa))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,"invalid hdfs config")
+
+ ret_value = H5P_set_driver(plist, H5FD_HDFS, (void *)fa);
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* H5Pset_fapl_hdfs() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_open()
+ *
+ * Purpose: Create and/or opens a file as an HDF5 file.
+ *
+ * Any flag except H5F_ACC_RDONLY will cause an error.
+ *
+ * Return: Success: A pointer to a new file data structure.
+ * The public fields will be initialized by the caller, which is
+ * always H5FD_open().
+ * Failure: NULL
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+static H5FD_t *
+H5FD_hdfs_open(const char *path, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
+{
+ H5FD_hdfs_t *file = NULL;
+ hdfs_t *handle = NULL;
+ H5FD_hdfs_fapl_t fa;
+ H5FD_t *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_open() called.\n");
+#endif /* HDFS_DEBUG */
+
+ /* Sanity check on file offsets */
+ HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t));
+
+ /* Check arguments */
+ if(!path || !*path)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name")
+ if(0 == maxaddr || HADDR_UNDEF == maxaddr)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr")
+ if(ADDR_OVERFLOW(maxaddr))
+ HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr")
+
+ if(flags != H5F_ACC_RDONLY)
+ HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, "only Read-Only access allowed")
+ if(fapl_id == H5P_DEFAULT || fapl_id == H5P_FILE_ACCESS_DEFAULT)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "fapl cannot be H5P_DEFAULT")
+ if(FAIL == H5Pget_fapl_hdfs(fapl_id, &fa))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't get property list")
+
+ handle = H5FD_hdfs_handle_open(path, fa.namenode_name, fa.namenode_port, fa.user_name, fa.kerberos_ticket_cache, fa.stream_buffer_size);
+
+ if(handle == NULL)
+ HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "could not open")
+
+ HDassert(handle->magic == HDFS_HDFST_MAGIC);
+
+ /* Create the new file struct */
+ if(NULL == (file = H5FL_CALLOC(H5FD_hdfs_t)))
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct")
+
+ file->hdfs_handle = handle;
+ HDmemcpy(&(file->fa), &fa, sizeof(H5FD_hdfs_fapl_t));
+
+#if HDFS_STATS
+ if (FAIL == hdfs_reset_stats(file))
+ HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics")
+#endif /* HDFS_STATS */
+
+ ret_value = (H5FD_t*)file;
+
+done:
+ if(NULL == ret_value) {
+ if (handle != NULL) {
+ if (FAIL == H5FD_hdfs_handle_close(handle))
+ HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close HDFS file handle")
+ }
+ if (file != NULL) {
+ file = H5FL_FREE(H5FD_hdfs_t, file);
+ }
+ } /* end if null return value (error) */
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5FD_hdfs_open() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static H5FD_t *
+H5FD_hdfs_open(const char H5_ATTR_UNUSED *path, unsigned H5_ATTR_UNUSED flags, hid_t H5_ATTR_UNUSED fapl_id, haddr_t H5_ATTR_UNUSED maxaddr)
+{
+ H5FD_t *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, NULL, "Illegal open of unsupported virtual file (hdfs)");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_open() */
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_close
+ *
+ * Purpose: Closes an HDF5 file.
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL, file not closed.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static herr_t
+H5FD_hdfs_close(H5FD_t *_file)
+{
+ H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_close() called.\n");
+#endif
+
+ /* Sanity checks */
+ HDassert(file != NULL);
+ HDassert(file->hdfs_handle != NULL);
+ HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC);
+
+ /* Close the underlying request handle */
+ if (file->hdfs_handle != NULL)
+ if (FAIL == H5FD_hdfs_handle_close(file->hdfs_handle)) {
+ HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "unable to close HDFS file handle")
+ }
+
+#if HDFS_STATS
+ /* TODO: mechanism to re-target stats printout */
+ if (FAIL == hdfs_fprint_stats(stdout, file))
+ HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL, "problem while writing file statistics")
+#endif /* HDFS_STATS */
+
+ /* Release the file info */
+ file = H5FL_FREE(H5FD_hdfs_t, file);
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_close() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static herr_t
+H5FD_hdfs_close(H5FD_t H5_ATTR_UNUSED *_file)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL, "Illegal close of unsupported Virtual File (hdfs)")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_close() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_cmp()
+ *
+ * Purpose: Compares two files using this driver by their HDFS-provided file info,
+ * field-by-field.
+ *
+ * Return: Equivalent: 0
+ * Not Equivalent: -1
+ *
+ * Programmer: Gerd Herber
+ * May 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static int
+H5FD_hdfs_cmp(const H5FD_t *_f1, const H5FD_t *_f2)
+{
+ const H5FD_hdfs_t *f1 = (const H5FD_hdfs_t *)_f1;
+ const H5FD_hdfs_t *f2 = (const H5FD_hdfs_t *)_f2;
+ hdfsFileInfo *finfo1 = NULL;
+ hdfsFileInfo *finfo2 = NULL;
+ int ret_value = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_cmp() called.\n");
+#endif /* HDFS_DEBUG */
+
+ HDassert(f1->hdfs_handle != NULL);
+ HDassert(f2->hdfs_handle != NULL);
+ HDassert(f1->hdfs_handle->magic == HDFS_HDFST_MAGIC);
+ HDassert(f2->hdfs_handle->magic == HDFS_HDFST_MAGIC);
+
+ finfo1 = f1->hdfs_handle->fileinfo;
+ finfo2 = f2->hdfs_handle->fileinfo;
+ HDassert(finfo1 != NULL);
+ HDassert(finfo2 != NULL);
+
+ if (finfo1->mKind != finfo2->mKind) { HGOTO_DONE(-1); }
+ if (finfo1->mName != finfo2->mName) { HGOTO_DONE(-1); }
+ if (finfo1->mLastMod != finfo2->mLastMod) { HGOTO_DONE(-1); }
+ if (finfo1->mSize != finfo2->mSize) { HGOTO_DONE(-1); }
+ if (finfo1->mReplication != finfo2->mReplication) { HGOTO_DONE(-1); }
+ if (finfo1->mBlockSize != finfo2->mBlockSize) { HGOTO_DONE(-1); }
+ if (strcmp(finfo1->mOwner, finfo2->mOwner)) { HGOTO_DONE(-1); }
+ if (strcmp(finfo1->mGroup, finfo2->mGroup)) { HGOTO_DONE(-1); }
+ if (finfo1->mPermissions != finfo2->mPermissions) { HGOTO_DONE(-1); }
+ if (finfo1->mLastAccess != finfo2->mLastAccess) { HGOTO_DONE(-1); }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_cmp() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static int
+H5FD_hdfs_cmp(
+ const H5FD_t H5_ATTR_UNUSED *_f1,
+ const H5FD_t H5_ATTR_UNUSED *_f2)
+{
+ int ret_value = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_cmp() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_query
+ *
+ * Purpose: Set the flags that this VFL driver is capable of supporting.
+ * (listed in H5FDpublic.h)
+ *
+ * Note that since the HDFS VFD is read only, most flags
+ * are irrelevant.
+ *
+ * The term "set" is highly misleading...
+ * stores/copies the supported flags in the out-pointer `flags`.
+ *
+ * Return: SUCCEED (Can't fail)
+ *
+ * Programmer: John Mainzer
+ * 9/11/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_query(const H5FD_t H5_ATTR_UNUSED *_file, unsigned long *flags) /* out variable */
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_query() called.\n");
+#endif
+
+ if(flags) {
+ *flags = 0;
+ *flags |= H5FD_FEAT_DATA_SIEVE;
+ }
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5FD_hdfs_query() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_get_eoa()
+ *
+ * Purpose: Gets the end-of-address marker for the file. The EOA marker
+ * is the first address past the last byte allocated in the
+ * format address space.
+ *
+ * Return: The end-of-address marker.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static haddr_t
+H5FD_hdfs_get_eoa(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type)
+{
+ const H5FD_hdfs_t *file = (const H5FD_hdfs_t *)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n");
+#endif
+
+ FUNC_LEAVE_NOAPI(file->eoa)
+} /* end H5FD_hdfs_get_eoa() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static haddr_t
+H5FD_hdfs_get_eoa( const H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_get_eoa() called.\n");
+#endif
+
+ FUNC_LEAVE_NOAPI(0)
+} /* end H5FD_hdfs_get_eoa() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_set_eoa()
+ *
+ * Purpose: Set the end-of-address marker for the file.
+ *
+ * Return: SUCCEED (Can't fail)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-03
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static herr_t
+H5FD_hdfs_set_eoa(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t addr)
+{
+ H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n");
+#endif
+
+ file->eoa = addr;
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5FD_hdfs_set_eoa() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static herr_t
+H5FD_hdfs_set_eoa(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, haddr_t H5_ATTR_UNUSED addr)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_set_eoa() called.\n");
+#endif
+
+ FUNC_LEAVE_NOAPI(FAIL)
+} /* H5FD_hdfs_set_eoa() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_get_eof()
+ *
+ * Purpose: Returns the end-of-file marker.
+ *
+ * Return: EOF: the first address past the end of the "file", either the
+ * filesystem file or the HDF5 file.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static haddr_t
+H5FD_hdfs_get_eof(const H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type)
+{
+ const H5FD_hdfs_t *file = (const H5FD_hdfs_t *)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n");
+#endif
+
+ HDassert(file->hdfs_handle != NULL);
+ HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC);
+
+ FUNC_LEAVE_NOAPI((size_t) file->hdfs_handle->fileinfo->mSize)
+} /* end H5FD_hdfs_get_eof() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static haddr_t
+H5FD_hdfs_get_eof(const H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_get_eof() called.\n");
+#endif
+
+ FUNC_LEAVE_NOAPI((size_t)0)
+} /* end H5FD_hdfs_get_eof() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_get_handle
+ *
+ * Purpose: Returns the HDFS handle (hdfs_t) of hdfs file driver.
+ *
+ * Returns: SUCCEED/FAIL
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static herr_t
+H5FD_hdfs_get_handle(H5FD_t *_file, hid_t H5_ATTR_UNUSED fapl, void **file_handle)
+{
+ H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n");
+#endif /* HDFS_DEBUG */
+
+ if(!file_handle)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid")
+
+ *file_handle = file->hdfs_handle;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_get_handle() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static herr_t
+H5FD_hdfs_get_handle(H5FD_t H5_ATTR_UNUSED *_file, hid_t H5_ATTR_UNUSED fapl, void H5_ATTR_UNUSED **file_handle)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_get_handle() called.\n");
+#endif /* HDFS_DEBUG */
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "Illegal get-handle of unsupported virtual file (hdfs)");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_get_handle() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_read
+ *
+ * Purpose: Reads SIZE bytes of data from FILE beginning at address ADDR
+ * into buffer BUF according to data transfer properties in DXPL_ID.
+ *
+ * Return: Success: SUCCEED
+ * - Result is stored in caller-supplied buffer BUF.
+ * Failure: FAIL
+ * - Unable to complete read.
+ * - Contents of buffer `buf` are undefined.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-??
+ *
+ *-------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_LIBHDFS
+
+static herr_t
+H5FD_hdfs_read(H5FD_t *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id,
+ haddr_t addr, size_t size, void *buf /*out*/)
+{
+ H5FD_hdfs_t *file = (H5FD_hdfs_t *)_file;
+ herr_t ret_value = SUCCEED;
+ size_t filesize = 0;
+#if HDFS_STATS
+ /* working variables for storing stats */
+ hdfs_statsbin *bin = NULL;
+ unsigned bin_i = 0;
+#endif /* HDFS_STATS */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_read() called.\n");
+#endif /* HDFS_DEBUG */
+
+ HDassert(file != NULL);
+ HDassert(file->hdfs_handle != NULL);
+ HDassert(file->hdfs_handle->magic == HDFS_HDFST_MAGIC);
+ HDassert(buf != NULL);
+
+ filesize = (size_t) file->hdfs_handle->fileinfo->mSize;
+
+ if ((addr > filesize) || ((addr + size) > filesize))
+ HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address")
+
+ if (FAIL == hdfsPread(file->hdfs_handle->filesystem, file->hdfs_handle->file, (tOffset)addr, buf, (tSize)size))
+ HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read")
+
+#if HDFS_STATS
+
+ /* Find which "bin" this read fits in. Can be "overflow" bin.
+ */
+ for (bin_i = 0; bin_i < HDFS_STATS_BIN_COUNT; bin_i++) {
+ if ((unsigned long long)size < hdfs_stats_boundaries[bin_i]) {
+ break;
+ }
+ }
+ bin = (type == H5FD_MEM_DRAW)
+ ? &file->raw[bin_i]
+ : &file->meta[bin_i];
+
+ /* Store collected stats in appropriate bin */
+ if (bin->count == 0) {
+ bin->min = size;
+ bin->max = size;
+ }
+ else {
+ if (size < bin->min) { bin->min = size; }
+ if (size > bin->max) { bin->max = size; }
+ }
+ bin->count++;
+ bin->bytes += (unsigned long long)size;
+
+#endif /* HDFS_STATS */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_read() */
+
+#else /* H5_HAVE_LIBHDFS not defined */
+
+static herr_t
+H5FD_hdfs_read(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id,
+ haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED size, void H5_ATTR_UNUSED *buf)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_read() called.\n");
+#endif /* HDFS_DEBUG */
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL,"Illegal get-handle of unsupported virtual file (hdfs)");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_read() */
+
+#endif /* H5_HAVE_LIBHDFS */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_write
+ *
+ * Purpose: Write bytes to file.
+ * UNSUPPORTED IN READ-ONLY HDFS VFD.
+ *
+ * Return: FAIL (Not possible with Read-Only S3 file.)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_write(H5FD_t H5_ATTR_UNUSED *_file, H5FD_mem_t H5_ATTR_UNUSED type, hid_t H5_ATTR_UNUSED dxpl_id,
+ haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED size, const void H5_ATTR_UNUSED *buf)
+{
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_write() called.\n");
+#endif
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "cannot write to read-only file.")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_write() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_truncate
+ *
+ * Purpose: Makes sure that the true file size is the same (or larger)
+ * than the end-of-address.
+ *
+ * NOT POSSIBLE ON READ-ONLY S3 FILES.
+ *
+ * Return: FAIL (Not possible on Read-Only S3 files.)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_truncate(H5FD_t H5_ATTR_UNUSED *_file, hid_t H5_ATTR_UNUSED dxpl_id, hbool_t H5_ATTR_UNUSED closing)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5FD_hdfs_truncate() called.\n");
+#endif
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL, "cannot truncate read-only file.")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_hdfs_truncate() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_lock
+ *
+ * Purpose: Place an advisory lock on a file.
+ * No effect on Read-Only S3 file.
+ *
+ * Suggestion: remove lock/unlock from class
+ * would result in error at H5FD_[un]lock() (H5FD.c)
+ *
+ * Return: SUCCEED (No-op always succeeds)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-03
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_lock(
+ H5FD_t H5_ATTR_UNUSED *_file,
+ hbool_t H5_ATTR_UNUSED rw)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5FD_hdfs_lock() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_unlock
+ *
+ * Purpose: Remove the existing lock on the file.
+ * No effect on Read-Only S3 file.
+ *
+ * Return: SUCCEED (No-op always succeeds)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-03
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_unlock(H5FD_t H5_ATTR_UNUSED *_file)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5FD_hdfs_unlock() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_validate_config()
+ *
+ * Purpose: Test to see if the supplied instance of H5FD_hdfs_fapl_t
+ * contains internally consistant data. Return SUCCEED if so,
+ * and FAIL otherwise.
+ *
+ * Note the difference between internally consistant and
+ * correct. As we will have to try to access the target
+ * object to determine whether the supplied data is correct,
+ * we will settle for internal consistancy at this point
+ *
+ * Return: SUCCEED if instance of H5FD_hdfs_fapl_t contains internally
+ * consistant data, FAIL otherwise.
+ *
+ * Programmer: Jacob Smith
+ * 9/10/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_validate_config(const H5FD_hdfs_fapl_t * fa)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ HDassert(fa != NULL);
+
+ if(fa->version != H5FD__CURR_HDFS_FAPL_T_VERSION)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Unknown H5FD_hdfs_fapl_t version");
+
+ if(fa->namenode_port > 65535)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Invalid namenode port number");
+ if(fa->namenode_port < 0)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Invalid namenode port number");
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_validate_config() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_fapl_hdfs
+ *
+ * Purpose: Returns information about the hdfs file access property
+ * list though the function arguments.
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ * Programmer: John Mainzer
+ * 9/10/17
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out)
+{
+ const H5FD_hdfs_fapl_t *fa = NULL;
+ H5P_genplist_t *plist = NULL;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE2("e", "i*x", fapl_id, fa_out);
+
+#if HDFS_DEBUG
+ HDfprintf(stdout, "H5Pget_fapl_hdfs() called.\n");
+#endif
+
+ if (fa_out == NULL)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL")
+ plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS);
+ if (plist == NULL)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list")
+ if (H5FD_HDFS != H5P_get_driver(plist))
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver")
+
+ fa = (const H5FD_hdfs_fapl_t *)H5P_get_driver_info(plist);
+ if (fa == NULL)
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info")
+
+ /* Copy the hdfs fapl data out */
+ HDmemcpy(fa_out, fa, sizeof(H5FD_hdfs_fapl_t));
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* H5Pget_fapl_hdfs() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_fapl_get
+ *
+ * Purpose: Gets a file access property list which could be used to
+ * create an identical file.
+ *
+ * Return: Success: Ptr to new file access property list value.
+ * Failure: NULL
+ *
+ * Programmer: John Mainzer
+ * 9/8/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static void *
+H5FD_hdfs_fapl_get(H5FD_t *_file)
+{
+ H5FD_hdfs_t *file = (H5FD_hdfs_t*)_file;
+ H5FD_hdfs_fapl_t *fa = NULL;
+ void *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ fa = (H5FD_hdfs_fapl_t *)H5MM_calloc(sizeof(H5FD_hdfs_fapl_t));
+ if(fa == NULL)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
+
+ /* Copy the fields of the structure */
+ HDmemcpy(fa, &(file->fa), sizeof(H5FD_hdfs_fapl_t));
+
+ ret_value = fa;
+
+done:
+ if (ret_value == NULL && fa != NULL) {
+ H5MM_xfree(fa); /* clean up on error */
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_fapl_get() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_fapl_copy
+ *
+ * Purpose: Copies the hdfs-specific file access properties.
+ *
+ * Return: Success: Ptr to a new property list
+ * Failure: NULL
+ *
+ * Programmer: John Mainzer
+ * 9/8/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static void *
+H5FD_hdfs_fapl_copy(const void *_old_fa)
+{
+ const H5FD_hdfs_fapl_t *old_fa = (const H5FD_hdfs_fapl_t*)_old_fa;
+ H5FD_hdfs_fapl_t *new_fa = NULL;
+ void *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ new_fa = (H5FD_hdfs_fapl_t *)H5MM_malloc(sizeof(H5FD_hdfs_fapl_t));
+ if(new_fa == NULL)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
+
+ HDmemcpy(new_fa, old_fa, sizeof(H5FD_hdfs_fapl_t));
+ ret_value = new_fa;
+
+done:
+ if (ret_value == NULL && new_fa != NULL)
+ H5MM_xfree(new_fa); /* clean up on error */
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_hdfs_fapl_copy() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_hdfs_fapl_free
+ *
+ * Purpose: Frees the hdfs-specific file access properties.
+ *
+ * Return: SUCCEED (cannot fail)
+ *
+ * Programmer: John Mainzer
+ * 9/8/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_hdfs_fapl_free(void *_fa)
+{
+ H5FD_hdfs_fapl_t *fa = (H5FD_hdfs_fapl_t*)_fa;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(fa != NULL); /* sanity check */
+
+ H5MM_xfree(fa);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* H5FD_hdfs_fapl_free() */
+
+
+#if HDFS_STATS
+
+/*----------------------------------------------------------------------------
+ * Function: hdfs_reset_stats()
+ *
+ * Purpose: Reset the stats collection elements in this virtual file structure.
+ * Clears any set data in stats bins; initializes/zeroes values.
+ *
+ * Return: - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - Occurs if the file is invalid somehow
+ *
+ * Programmer: Jacob Smith
+ * 2017-12-08
+ *
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+hdfs_reset_stats(H5FD_hdfs_t *file)
+{
+ unsigned i = 0;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if HDFS_DEBUG
+ HDprintf("hdfs_reset_stats() called\n");
+#endif
+
+ if (file == NULL)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file was null")
+
+ for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) {
+ file->raw[i].bytes = 0;
+ file->raw[i].count = 0;
+ file->raw[i].min = (unsigned long long)HDFS_STATS_STARTING_MIN;
+ file->raw[i].max = 0;
+
+ file->meta[i].bytes = 0;
+ file->meta[i].count = 0;
+ file->meta[i].min = (unsigned long long)HDFS_STATS_STARTING_MIN;
+ file->meta[i].max = 0;
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* hdfs_reset_stats */
+#endif /* HDFS_STATS */
+
+
+#if HDFS_STATS
+
+/*----------------------------------------------------------------------------
+ * Function: hdfs_fprint_stats()
+ *
+ * Purpose: Tabulate and pretty-print statistics for this virtual file.
+ *
+ * Should be called upon file close.
+ *
+ * Shows number of reads and bytes read, broken down by
+ * "raw" (H5FD_MEM_DRAW)
+ * or "meta" (any other flag)
+ *
+ * Prints filename and listing of total number of reads and bytes read,
+ * both as a grand total and separate meta- and rawdata reads.
+ *
+ * If any reads were done, prints out two tables:
+ *
+ * 1. overview of raw- and metadata reads
+ * - min (smallest size read)
+ * - average of size read
+ * - k,M,G suffixes by powers of 1024 (2^10)
+ * - max (largest size read)
+ * 2. tabulation of "bins", sepraring reads into exponentially-larger
+ * ranges of size.
+ * - columns for number of reads, total bytes, and average size, with
+ * separate sub-colums for raw- and metadata reads.
+ * - each row represents one bin, identified by the top of its range
+ *
+ * Bin ranges can be modified with pound-defines at the top of this file.
+ *
+ * Bins without any reads in their bounds are not printed.
+ *
+ * An "overflow" bin is also present, to catch "big" reads.
+ *
+ * Output for all bins (and range ceiling and average size report)
+ * is divied by powers of 1024. By corollary, four digits before the decimal
+ * is valid.
+ *
+ * - 41080 bytes is represented by 40.177k, not 41.080k
+ * - 1004.831M represents approx. 1052642000 bytes
+ *
+ * Return: - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - occurs if the file passed in is invalid
+ * - TODO: if stream is invalid? how can we check this?
+ *
+ * Programmer: Jacob Smith
+ *
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+hdfs_fprint_stats(FILE *stream, const H5FD_hdfs_t *file)
+{
+ herr_t ret_value = SUCCEED;
+ parsed_url_t *purl = NULL;
+ unsigned i = 0;
+ unsigned long count_meta = 0;
+ unsigned long count_raw = 0;
+ double average_meta = 0.0;
+ double average_raw = 0.0;
+ unsigned long long min_meta = (unsigned long long)HDFS_STATS_STARTING_MIN;
+ unsigned long long min_raw = (unsigned long long)HDFS_STATS_STARTING_MIN;
+ unsigned long long max_meta = 0;
+ unsigned long long max_raw = 0;
+ unsigned long long bytes_raw = 0;
+ unsigned long long bytes_meta = 0;
+ double re_dub = 0.0; /* re-usable double variable */
+ unsigned suffix_i = 0;
+ const char suffixes[] = { ' ', 'K', 'M', 'G', 'T', 'P' };
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ if (stream == NULL)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file stream cannot be null" )
+ if (file == NULL)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file cannot be null")
+ if (file->hdfs_handle == NULL)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hdfs handle cannot be null")
+ if (file->hdfs_handle->magic != HDFS_HDFST_MAGIC)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "hdfs handle has invalid magic")
+
+ /*******************
+ * AGGREGATE STATS *
+ *******************/
+
+ for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) {
+ const hdfs_statsbin *r = &file->raw[i];
+ const hdfs_statsbin *m = &file->meta[i];
+
+ if (m->min < min_meta) { min_meta = m->min; }
+ if (r->min < min_raw) { min_raw = r->min; }
+ if (m->max > max_meta) { max_meta = m->max; }
+ if (r->max > max_raw) { max_raw = r->max; }
+
+ count_raw += r->count;
+ count_meta += m->count;
+ bytes_raw += r->bytes;
+ bytes_meta += m->bytes;
+ }
+ if (count_raw > 0) {
+ average_raw = (double)bytes_raw / (double)count_raw;
+ }
+ if (count_meta > 0) {
+ average_meta = (double)bytes_meta / (double)count_meta;
+ }
+
+ /******************
+ * PRINT OVERVIEW *
+ ******************/
+
+ HDfprintf(stream, "TOTAL READS: %llu (%llu meta, %llu raw)\n",
+ count_raw + count_meta, count_meta, count_raw);
+ HDfprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n",
+ bytes_raw + bytes_meta, bytes_meta, bytes_raw);
+
+ if (count_raw + count_meta == 0) {
+ goto done;
+ }
+
+ /*************************
+ * PRINT AGGREGATE STATS *
+ *************************/
+
+ HDfprintf(stream, "SIZES meta raw\n");
+ HDfprintf(stream, " min ");
+ if (count_meta == 0) {
+ HDfprintf(stream, " 0.000 ");
+ }
+ else {
+ re_dub = (double)min_meta;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]);
+ }
+
+ if (count_raw == 0) {
+ HDfprintf(stream, " 0.000 \n");
+ }
+ else {
+ re_dub = (double)min_raw;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]);
+ }
+
+ HDfprintf(stream, " avg ");
+ re_dub = (double)average_meta;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]);
+
+ re_dub = (double)average_raw;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]);
+
+ HDfprintf(stream, " max ");
+ re_dub = (double)max_meta;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]);
+
+ re_dub = (double)max_raw;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]);
+
+ /******************************
+ * PRINT INDIVIDUAL BIN STATS *
+ ******************************/
+
+ HDfprintf(stream,
+ "BINS # of reads total bytes average size\n");
+ HDfprintf(stream,
+ " up-to meta raw meta raw meta raw\n");
+
+ for (i = 0; i <= HDFS_STATS_BIN_COUNT; i++) {
+ const hdfs_statsbin *m;
+ const hdfs_statsbin *r;
+ unsigned long long range_end = 0;
+ char bm_suffix = ' '; /* bytes-meta */
+ double bm_val = 0.0;
+ char br_suffix = ' '; /* bytes-raw */
+ double br_val = 0.0;
+ char am_suffix = ' '; /* average-meta */
+ double am_val = 0.0;
+ char ar_suffix = ' '; /* average-raw */
+ double ar_val = 0.0;
+
+ m = &file->meta[i];
+ r = &file->raw[i];
+ if (r->count == 0 && m->count == 0) {
+ continue;
+ }
+
+ range_end = hdfs_stats_boundaries[i];
+
+ if (i == HDFS_STATS_BIN_COUNT) {
+ range_end = hdfs_stats_boundaries[i-1];
+ HDfprintf(stream, ">");
+ } else {
+ HDfprintf(stream, " ");
+ }
+
+ bm_val = (double)m->bytes;
+ for (suffix_i = 0; bm_val >= 1024.0; suffix_i++) {
+ bm_val /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ bm_suffix = suffixes[suffix_i];
+
+ br_val = (double)r->bytes;
+ for (suffix_i = 0; br_val >= 1024.0; suffix_i++) {
+ br_val /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ br_suffix = suffixes[suffix_i];
+
+ if (m->count > 0) {
+ am_val = (double)(m->bytes) / (double)(m->count);
+ }
+ for (suffix_i = 0; am_val >= 1024.0; suffix_i++) {
+ am_val /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ am_suffix = suffixes[suffix_i];
+
+ if (r->count > 0) {
+ ar_val = (double)(r->bytes) / (double)(r->count);
+ }
+ for (suffix_i = 0; ar_val >= 1024.0; suffix_i++) {
+ ar_val /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+ ar_suffix = suffixes[suffix_i];
+
+ re_dub = (double)range_end;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++) {
+ re_dub /= 1024.0;
+ }
+ HDassert(suffix_i < sizeof(suffixes));
+
+ HDfprintf(
+ stream,
+ " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n",
+ re_dub, suffixes[suffix_i], /* bin ceiling */
+ m->count, /* metadata reads */
+ r->count, /* rawdata reads */
+ bm_val, bm_suffix, /* metadata bytes */
+ br_val, br_suffix, /* rawdata bytes */
+ am_val, am_suffix, /* metadata average */
+ ar_val, ar_suffix); /* rawdata average */
+ fflush(stream);
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* hdfs_fprint_stats */
+#endif /* HDFS_STATS */
+
diff --git a/src/H5FDhdfs.h b/src/H5FDhdfs.h
new file mode 100644
index 0000000..328d36a
--- /dev/null
+++ b/src/H5FDhdfs.h
@@ -0,0 +1,123 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Read-Only HDFS Virtual File Driver (VFD) *
+ * Copyright (c) 2018, The HDF Group. *
+ * *
+ * All rights reserved. *
+ * *
+ * NOTICE: *
+ * All information contained herein is, and remains, the property of The HDF *
+ * Group. The intellectual and technical concepts contained herein are *
+ * proprietary to The HDF Group. Dissemination of this information or *
+ * reproduction of this material is strictly forbidden unless prior written *
+ * permission is obtained from The HDF Group. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Programmer: Jacob Smith
+ * 2018-04-23
+ *
+ * Purpose: The public header file for the hdfs driver.
+ */
+
+#ifndef H5FDhdfs_H
+#define H5FDhdfs_H
+
+#define H5FD_HDFS (H5FD_hdfs_init())
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/****************************************************************************
+ *
+ * Structure: H5FD_hdfs_fapl_t
+ *
+ * Purpose:
+ *
+ * H5FD_hdfs_fapl_t is a public structure that is used to pass
+ * configuration information to the appropriate HDFS VFD via the FAPL.
+ * A pointer to an instance of this structure is a parameter to
+ * H5Pset_fapl_hdfs() and H5Pget_fapl_hdfs().
+ *
+ *
+ *
+ * `version` (int32_t)
+ *
+ * Version number of the `H5FD_hdfs_fapl_t` structure. Any instance passed
+ * to the above calls must have a recognized version number, or an error
+ * will be flagged.
+ *
+ * This field should be set to `H5FD__CURR_HDFS_FAPL_T_VERSION`.
+ *
+ * `namenode_name` (const char[])
+ *
+ * Name of "Name Node" to access as the HDFS server.
+ *
+ * Must not be longer than `H5FD__HDFS_NODE_NAME_SPACE`.
+ *
+ * TBD: Can be NULL.
+ *
+ * `namenode_port` (int32_t) TBD
+ *
+ * Port number to use to connect with Name Node.
+ *
+ * TBD: If 0, uses a default port.
+ *
+ * `kerberos_ticket_cache` (const char[])
+ *
+ * Path to the location of the Kerberos authentication cache.
+ *
+ * Must not be longer than `H5FD__HDFS_KERB_CACHE_PATH_SPACE`.
+ *
+ * TBD: Can be NULL.
+ *
+ * `user_name` (const char[])
+ *
+ * Username to use when accessing file.
+ *
+ * Must not be longer than `H5FD__HDFS_USER_NAME_SPACE`.
+ *
+ * TBD: Can be NULL.
+ *
+ * `stream_buffer_size` (int32_t)
+ *
+ * Size (in bytes) of the file read stream buffer.
+ *
+ * TBD: If -1, relies on a default value.
+ *
+ *
+ *
+ * Programmer: Jacob Smith
+ * 2018-04-23
+ *
+ * Changes: None
+ *
+ ****************************************************************************/
+
+#define H5FD__CURR_HDFS_FAPL_T_VERSION 1
+
+#define H5FD__HDFS_NODE_NAME_SPACE 128
+#define H5FD__HDFS_USER_NAME_SPACE 128
+#define H5FD__HDFS_KERB_CACHE_PATH_SPACE 128
+
+typedef struct H5FD_hdfs_fapl_t {
+ int32_t version;
+ char namenode_name[H5FD__HDFS_NODE_NAME_SPACE + 1];
+ int32_t namenode_port;
+ char user_name[H5FD__HDFS_USER_NAME_SPACE + 1];
+ char kerberos_ticket_cache[H5FD__HDFS_KERB_CACHE_PATH_SPACE + 1];
+ int32_t stream_buffer_size;
+} H5FD_hdfs_fapl_t;
+
+H5_DLL hid_t H5FD_hdfs_init(void);
+H5_DLL void H5FD_hdfs_term(void);
+H5_DLL herr_t H5Pget_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa_out);
+H5_DLL herr_t H5Pset_fapl_hdfs(hid_t fapl_id, H5FD_hdfs_fapl_t *fa);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* ifndef H5FDhdfs_H */
+
+
diff --git a/src/H5FDprivate.h b/src/H5FDprivate.h
index 9568f46..8c7f414 100644
--- a/src/H5FDprivate.h
+++ b/src/H5FDprivate.h
@@ -22,13 +22,13 @@
#include "H5FDpublic.h"
/* Private headers needed by this file */
-#include "H5Pprivate.h" /* Property lists */
+#include "H5Pprivate.h" /* Property lists */
/*
* The MPI drivers are needed because there are
* places where we check for things that aren't handled by these drivers.
*/
-#include "H5FDmpi.h" /* MPI-based file drivers */
+#include "H5FDmpi.h" /* MPI-based file drivers */
/**************************/
@@ -88,6 +88,12 @@ typedef struct {
} \
}
+/* Define structure to hold driver ID & info for FAPLs */
+typedef struct {
+ hid_t driver_id; /* Driver's ID */
+ const void *driver_info; /* Driver info, for open callbacks */
+} H5FD_driver_prop_t;
+
/*****************************/
/* Library Private Variables */
@@ -113,7 +119,7 @@ H5_DLL herr_t H5FD_fapl_open(struct H5P_genplist_t *plist, hid_t driver_id, cons
H5_DLL herr_t H5FD_fapl_close(hid_t driver_id, void *fapl);
H5_DLL hid_t H5FD_register(const void *cls, size_t size, hbool_t app_ref);
H5_DLL H5FD_t *H5FD_open(const char *name, unsigned flags, hid_t fapl_id,
- haddr_t maxaddr);
+ haddr_t maxaddr);
H5_DLL herr_t H5FD_close(H5FD_t *file);
H5_DLL int H5FD_cmp(const H5FD_t *f1, const H5FD_t *f2);
H5_DLL haddr_t H5FD_alloc(H5FD_t *file, hid_t dxpl_id, H5FD_mem_t type, struct H5F_t *f,
@@ -134,6 +140,8 @@ H5_DLL herr_t H5FD_write(H5FD_t *file, const H5P_genplist_t *dxpl, H5FD_mem_t ty
haddr_t addr, size_t size, const void *buf);
H5_DLL herr_t H5FD_flush(H5FD_t *file, hid_t dxpl_id, unsigned closing);
H5_DLL herr_t H5FD_truncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing);
+H5_DLL herr_t H5FD_lock(H5FD_t *file, hbool_t rw);
+H5_DLL herr_t H5FD_unlock(H5FD_t *file);
H5_DLL herr_t H5FD_get_fileno(const H5FD_t *file, unsigned long *filenum);
H5_DLL herr_t H5FD_get_vfd_handle(H5FD_t *file, hid_t fapl, void** file_handle);
H5_DLL herr_t H5FD_set_base_addr(H5FD_t *file, haddr_t base_addr);
@@ -145,7 +153,7 @@ H5_DLL haddr_t H5FD_get_base_addr(const H5FD_t *file);
H5_DLL haddr_t H5FD_mpi_MPIOff_to_haddr(MPI_Offset mpi_off);
H5_DLL herr_t H5FD_mpi_haddr_to_MPIOff(haddr_t addr, MPI_Offset *mpi_off/*out*/);
H5_DLL herr_t H5FD_mpi_comm_info_dup(MPI_Comm comm, MPI_Info info,
- MPI_Comm *comm_new, MPI_Info *info_new);
+ MPI_Comm *comm_new, MPI_Info *info_new);
H5_DLL herr_t H5FD_mpi_comm_info_free(MPI_Comm *comm, MPI_Info *info);
#ifdef NOT_YET
H5_DLL herr_t H5FD_mpio_wait_for_left_neighbor(H5FD_t *file);
diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h
index a52c57a..3faaf33 100644
--- a/src/H5FDpublic.h
+++ b/src/H5FDpublic.h
@@ -19,13 +19,13 @@
#define _H5FDpublic_H
#include "H5public.h"
-#include "H5Fpublic.h" /*for H5F_close_degree_t */
+#include "H5Fpublic.h" /*for H5F_close_degree_t */
#define H5_HAVE_VFL 1 /*define a convenient app feature test*/
#define H5FD_VFD_DEFAULT 0 /* Default VFL driver value */
/* Types of allocation requests: see H5Fpublic.h */
-typedef enum H5F_mem_t H5FD_mem_t;
+typedef enum H5F_mem_t H5FD_mem_t;
/* Map "fractal heap" header blocks to 'ohdr' type file memory, since its
* a fair amount of work to add a new kind of file memory and they are similar
@@ -79,42 +79,42 @@ typedef enum H5F_mem_t H5FD_mem_t;
* want to make most efficient reuse of freed memory. The use of the
* H5FD_MEM_SUPER free list is arbitrary.
*/
-#define H5FD_FLMAP_SINGLE { \
- H5FD_MEM_SUPER, /*default*/ \
- H5FD_MEM_SUPER, /*super*/ \
- H5FD_MEM_SUPER, /*btree*/ \
- H5FD_MEM_SUPER, /*draw*/ \
- H5FD_MEM_SUPER, /*gheap*/ \
- H5FD_MEM_SUPER, /*lheap*/ \
- H5FD_MEM_SUPER /*ohdr*/ \
+#define H5FD_FLMAP_SINGLE { \
+ H5FD_MEM_SUPER, /*default*/ \
+ H5FD_MEM_SUPER, /*super*/ \
+ H5FD_MEM_SUPER, /*btree*/ \
+ H5FD_MEM_SUPER, /*draw*/ \
+ H5FD_MEM_SUPER, /*gheap*/ \
+ H5FD_MEM_SUPER, /*lheap*/ \
+ H5FD_MEM_SUPER /*ohdr*/ \
}
/*
* A free-list map which segregates requests into `raw' or `meta' data
* pools.
*/
-#define H5FD_FLMAP_DICHOTOMY { \
- H5FD_MEM_SUPER, /*default*/ \
- H5FD_MEM_SUPER, /*super*/ \
- H5FD_MEM_SUPER, /*btree*/ \
- H5FD_MEM_DRAW, /*draw*/ \
- H5FD_MEM_DRAW, /*gheap*/ \
- H5FD_MEM_SUPER, /*lheap*/ \
- H5FD_MEM_SUPER /*ohdr*/ \
+#define H5FD_FLMAP_DICHOTOMY { \
+ H5FD_MEM_SUPER, /*default*/ \
+ H5FD_MEM_SUPER, /*super*/ \
+ H5FD_MEM_SUPER, /*btree*/ \
+ H5FD_MEM_DRAW, /*draw*/ \
+ H5FD_MEM_DRAW, /*gheap*/ \
+ H5FD_MEM_SUPER, /*lheap*/ \
+ H5FD_MEM_SUPER /*ohdr*/ \
}
/*
* The default free list map which causes each request type to use it's own
* free-list.
*/
-#define H5FD_FLMAP_DEFAULT { \
- H5FD_MEM_DEFAULT, /*default*/ \
- H5FD_MEM_DEFAULT, /*super*/ \
- H5FD_MEM_DEFAULT, /*btree*/ \
- H5FD_MEM_DEFAULT, /*draw*/ \
- H5FD_MEM_DEFAULT, /*gheap*/ \
- H5FD_MEM_DEFAULT, /*lheap*/ \
- H5FD_MEM_DEFAULT /*ohdr*/ \
+#define H5FD_FLMAP_DEFAULT { \
+ H5FD_MEM_DEFAULT, /*default*/ \
+ H5FD_MEM_DEFAULT, /*super*/ \
+ H5FD_MEM_DEFAULT, /*btree*/ \
+ H5FD_MEM_DEFAULT, /*draw*/ \
+ H5FD_MEM_DEFAULT, /*gheap*/ \
+ H5FD_MEM_DEFAULT, /*lheap*/ \
+ H5FD_MEM_DEFAULT /*ohdr*/ \
}
@@ -187,7 +187,7 @@ typedef enum H5F_mem_t H5FD_mem_t;
* instead of the default H5D_ALLOC_TIME_LATE
*/
#define H5FD_FEAT_ALLOCATE_EARLY 0x00000200
- /*
+ /*
* Defining the H5FD_FEAT_ALLOW_FILE_IMAGE for a VFL driver means that
* the driver is able to use a file image in the fapl as the initial
* contents of a file.
@@ -245,9 +245,9 @@ typedef struct H5FD_class_t {
/* A free list is a singly-linked list of address/size pairs. */
typedef struct H5FD_free_t {
- haddr_t addr;
- hsize_t size;
- struct H5FD_free_t *next;
+ haddr_t addr;
+ hsize_t size;
+ struct H5FD_free_t *next;
} H5FD_free_t;
/*
@@ -270,7 +270,7 @@ struct H5FD_t {
/* Define enum for the source of file image callbacks */
typedef enum {
H5FD_FILE_IMAGE_OP_NO_OP,
- H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET,
+ H5FD_FILE_IMAGE_OP_PROPERTY_LIST_SET,
H5FD_FILE_IMAGE_OP_PROPERTY_LIST_COPY,
H5FD_FILE_IMAGE_OP_PROPERTY_LIST_GET,
H5FD_FILE_IMAGE_OP_PROPERTY_LIST_CLOSE,
@@ -281,13 +281,13 @@ typedef enum {
/* Define structure to hold file image callbacks */
typedef struct {
- void *(*image_malloc)(size_t size, H5FD_file_image_op_t file_image_op,
+ void *(*image_malloc)(size_t size, H5FD_file_image_op_t file_image_op,
void *udata);
void *(*image_memcpy)(void *dest, const void *src, size_t size,
H5FD_file_image_op_t file_image_op, void *udata);
- void *(*image_realloc)(void *ptr, size_t size,
+ void *(*image_realloc)(void *ptr, size_t size,
H5FD_file_image_op_t file_image_op, void *udata);
- herr_t (*image_free)(void *ptr, H5FD_file_image_op_t file_image_op,
+ herr_t (*image_free)(void *ptr, H5FD_file_image_op_t file_image_op,
void *udata);
void *(*udata_copy)(void *udata);
herr_t (*udata_free)(void *udata);
@@ -319,6 +319,11 @@ H5_DLL herr_t H5FDwrite(H5FD_t *file, H5FD_mem_t type, hid_t dxpl_id,
haddr_t addr, size_t size, const void *buf);
H5_DLL herr_t H5FDflush(H5FD_t *file, hid_t dxpl_id, unsigned closing);
H5_DLL herr_t H5FDtruncate(H5FD_t *file, hid_t dxpl_id, hbool_t closing);
+/* H5_DLL herr_t H5FDlock(H5FD_t *file, hbool_t rw);
+H5_DLL herr_t H5FDunlock(H5FD_t *file); */
+/* Allows querying a VFD ID for features before the file is opened */
+H5_DLL herr_t H5FDdriver_query(hid_t driver_id, unsigned long *flags/*out*/);
+
#ifdef __cplusplus
}
diff --git a/src/H5FDros3.c b/src/H5FDros3.c
new file mode 100644
index 0000000..5456a1b
--- /dev/null
+++ b/src/H5FDros3.c
@@ -0,0 +1,1787 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Read-Only S3 Virtual File Driver (VFD)
+ *
+ * Programmer: Jacob Smith <jake.smith@hdfgroup.org>
+ * 2017-10-13
+ *
+ * Purpose: Provide read-only access to files hosted on Amazon's S3 service.
+ * Relies on "s3comms" utility layer to implement the AWS REST API.
+ */
+
+/* Interface initialization */
+#define H5_INTERFACE_INIT_FUNC H5FD_ros3_init_interface
+
+
+#include "H5private.h" /* Generic Functions */
+#include "H5Eprivate.h" /* Error handling */
+#include "H5Fprivate.h" /* File access */
+#include "H5FDprivate.h" /* File drivers */
+#include "H5FDros3.h" /* ros3 file driver */
+#include "H5FLprivate.h" /* Free Lists */
+#include "H5Iprivate.h" /* IDs */
+#include "H5MMprivate.h" /* Memory management */
+#include "H5Pprivate.h" /* Property lists */
+#include "H5FDs3comms.h" /* S3 Communications */
+
+#ifdef H5_HAVE_ROS3_VFD
+
+/* toggle function call prints: 1 turns on
+ */
+#define ROS3_DEBUG 0
+
+/* toggle stats collection and reporting
+ */
+#define ROS3_STATS 0
+
+/* The driver identification number, initialized at runtime
+ */
+static hid_t H5FD_ROS3_g = 0;
+
+#if ROS3_STATS
+
+/* arbitrarily large value, such that any reasonable size read will be "less"
+ * than this value and set a true minimum
+ * not 0 because that may be a valid recorded minimum in degenerate cases
+ */
+#define ROS3_STATS_STARTING_MIN 0xfffffffful
+
+/* Configuration definitions for stats collection and breakdown
+ *
+ * 2^10 = 1024
+ * Reads up to 1024 bytes (1 kB) fall in bin 0
+ * 2^(10+(1*16)) = 2^26 = 64MB
+ * Reads of 64MB or greater fall in "overflow" bin[BIN_COUNT]
+ */
+#define ROS3_STATS_BASE 2
+#define ROS3_STATS_INTERVAL 1
+#define ROS3_STATS_START_POWER 10
+#define ROS3_STATS_BIN_COUNT 16 /* MUST BE GREATER THAN 0 */
+
+
+/*
+ * Calculate `BASE ^ (START_POWER + (INTERVAL * bin_i))`
+ * Stores result at `(unsigned long long *) out_ptr`.
+ * Used in computing boundaries between stats bins.
+ */
+#define ROS3_STATS_POW(bin_i, out_ptr) { \
+ unsigned long long donotshadowresult = 1; \
+ unsigned donotshadowindex = 0; \
+ for (donotshadowindex = 0; \
+ donotshadowindex < (((bin_i) * ROS3_STATS_INTERVAL) + \
+ ROS3_STATS_START_POWER); \
+ donotshadowindex++) \
+ { \
+ donotshadowresult *= ROS3_STATS_BASE; \
+ } \
+ *(out_ptr) = donotshadowresult; \
+}
+
+/* array to hold pre-computed boundaries for stats bins
+ */
+static unsigned long long ros3_stats_boundaries[ROS3_STATS_BIN_COUNT];
+
+/***************************************************************************
+ *
+ * Structure: ros3_statsbin
+ *
+ * Purpose:
+ *
+ * Structure for storing per-file ros3 VFD usage statistics.
+ *
+ *
+ *
+ * `count` (unsigned long long)
+ *
+ * Number of reads with size in this bin's range.
+ *
+ * `bytes` (unsigned long long)
+ *
+ * Total number of bytes read through this bin.
+ *
+ * `min` (unsigned long long)
+ *
+ * Smallest read size in this bin.
+ *
+ * `max` (unsigned long long)
+ *
+ * Largest read size in this bin.
+ *
+ *
+ *
+ * Programmer: Jacob Smith
+ *
+ ***************************************************************************/
+typedef struct {
+ unsigned long long count;
+ unsigned long long bytes;
+ unsigned long long min;
+ unsigned long long max;
+} ros3_statsbin;
+
+#endif /* ROS3_STATS */
+
+/***************************************************************************
+ *
+ * Structure: H5FD_ros3_t
+ *
+ * Purpose:
+ *
+ * H5FD_ros3_t is a structure used to store all information needed to
+ * maintain R/O access to a single HDF5 file that has been stored as a
+ * S3 object. This structure is created when such a file is "opened" and
+ * discarded when it is "closed".
+ *
+ * Presents an S3 object as a file to the HDF5 library.
+ *
+ *
+ *
+ * `pub` (H5FD_t)
+ *
+ * Instance of H5FD_t which contains all fields common to all VFDs.
+ * It must be the first item in this structure, since at higher levels,
+ * this structure will be treated as an instance of H5FD_t.
+ *
+ * `fa` (H5FD_ros3_fapl_t)
+ *
+ * Instance of `H5FD_ros3_fapl_t` containing the S3 configuration data
+ * needed to "open" the HDF5 file.
+ *
+ * `eoa` (haddr_t)
+ *
+ * End of addressed space in file. After open, it should always
+ * equal the file size.
+ *
+ * `s3r_handle` (s3r_t *)
+ *
+ * Instance of S3 Request handle associated with the target resource.
+ * Responsible for communicating with remote host and presenting file
+ * contents as indistinguishable from a file on the local filesystem.
+ *
+ * *** present only if ROS3_SATS is flagged to enable stats collection ***
+ *
+ * `meta` (ros3_statsbin[])
+ * `raw` (ros3_statsbin[])
+ *
+ * Only present if ros3 stats collection is enabled.
+ *
+ * Arrays of `ros3_statsbin` structures to record raw- and metadata reads.
+ *
+ * Records count and size of reads performed by the VFD, and is used to
+ * print formatted usage statistics to stdout upon VFD shutdown.
+ *
+ * Reads of each raw- and metadata type are recorded in an individual bin
+ * determined by the size of the read. The last bin of each type is
+ * reserved for "big" reads, with no defined upper bound.
+ *
+ * *** end ROS3_STATS ***
+ *
+ *
+ *
+ * Programmer: Jacob Smith
+ *
+ ***************************************************************************/
+typedef struct H5FD_ros3_t {
+ H5FD_t pub;
+ H5FD_ros3_fapl_t fa;
+ haddr_t eoa;
+ s3r_t *s3r_handle;
+#if ROS3_STATS
+ ros3_statsbin meta[ROS3_STATS_BIN_COUNT + 1];
+ ros3_statsbin raw[ROS3_STATS_BIN_COUNT + 1];
+#endif
+} H5FD_ros3_t;
+
+/*
+ * These macros check for overflow of various quantities. These macros
+ * assume that HDoff_t is signed and haddr_t and size_t are unsigned.
+ *
+ * ADDR_OVERFLOW: Checks whether a file address of type `haddr_t'
+ * is too large to be represented by the second argument
+ * of the file seek function.
+ * Only included if it may be used -- ROS3 VFD is enabled.
+ *
+ */
+#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1)
+#define ADDR_OVERFLOW(A) (HADDR_UNDEF==(A) || ((A) & ~(haddr_t)MAXADDR))
+
+/* Prototypes */
+static H5FD_t *H5FD_ros3_open(const char *name, unsigned flags, hid_t fapl_id,
+ haddr_t maxaddr);
+static herr_t H5FD_ros3_close(H5FD_t *_file);
+static int H5FD_ros3_cmp(const H5FD_t *_f1, const H5FD_t *_f2);
+static herr_t H5FD_ros3_query(const H5FD_t *_f1, unsigned long *flags);
+static haddr_t H5FD_ros3_get_eoa(const H5FD_t *_file, H5FD_mem_t type);
+static herr_t H5FD_ros3_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr);
+static haddr_t H5FD_ros3_get_eof(const H5FD_t *_file, H5FD_mem_t type);
+static herr_t H5FD_ros3_get_handle(H5FD_t *_file, hid_t fapl, void** file_handle);
+static herr_t H5FD_ros3_read(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id,
+ haddr_t addr, size_t size, void *buf);
+static herr_t H5FD_ros3_write(H5FD_t *_file, H5FD_mem_t type, hid_t fapl_id,
+ haddr_t addr, size_t size, const void *buf);
+static herr_t H5FD_ros3_truncate(H5FD_t *_file, hid_t dxpl_id, hbool_t closing);
+static herr_t H5FD_ros3_lock(H5FD_t *_file, hbool_t rw);
+static herr_t H5FD_ros3_unlock(H5FD_t *_file);
+static herr_t H5FD_ros3_validate_config(const H5FD_ros3_fapl_t * fa);
+static void *H5FD_ros3_fapl_get(H5FD_t *_file);
+static void *H5FD_ros3_fapl_copy(const void *_old_fa);
+static herr_t H5FD_ros3_fapl_free(void *_fa);
+
+static const H5FD_class_t H5FD_ros3_g = {
+ "ros3", /* name */
+ MAXADDR, /* maxaddr */
+ H5F_CLOSE_WEAK, /* fc_degree */
+ H5FD_ros3_term, /* terminate */
+ NULL, /* sb_size */
+ NULL, /* sb_encode */
+ NULL, /* sb_decode */
+ sizeof(H5FD_ros3_fapl_t), /* fapl_size */
+ H5FD_ros3_fapl_get, /* fapl_get */
+ H5FD_ros3_fapl_copy, /* fapl_copy */
+ H5FD_ros3_fapl_free, /* fapl_free */
+ 0, /* dxpl_size */
+ NULL, /* dxpl_copy */
+ NULL, /* dxpl_free */
+ H5FD_ros3_open, /* open */
+ H5FD_ros3_close, /* close */
+ H5FD_ros3_cmp, /* cmp */
+ H5FD_ros3_query, /* query */
+ NULL, /* get_type_map */
+ NULL, /* alloc */
+ NULL, /* free */
+ H5FD_ros3_get_eoa, /* get_eoa */
+ H5FD_ros3_set_eoa, /* set_eoa */
+ H5FD_ros3_get_eof, /* get_eof */
+ H5FD_ros3_get_handle, /* get_handle */
+ H5FD_ros3_read, /* read */
+ H5FD_ros3_write, /* write */
+ NULL, /* flush */
+ H5FD_ros3_truncate, /* truncate */
+ H5FD_ros3_lock, /* lock */
+ H5FD_ros3_unlock, /* unlock */
+ H5FD_FLMAP_DICHOTOMY /* fl_map */
+};
+
+/* Declare a free list to manage the H5FD_ros3_t struct */
+H5FL_DEFINE_STATIC(H5FD_ros3_t);
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_init_interface
+ *
+ * Purpose: Initializes any interface-specific data or routines.
+ *
+ * Return: Success: The driver ID for the ros3 driver.
+ * Failure: Negative
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_init_interface(void)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ FUNC_LEAVE_NOAPI(H5FD_ros3_init())
+} /* H5FD_ros3_init_interface() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_init
+ *
+ * Purpose: Initialize this driver by registering the driver with the
+ * library.
+ *
+ * Return: Success: The driver ID for the ros3 driver.
+ * Failure: Negative
+ *
+ * Programmer: Jacob Smith 2017
+ *
+ *-------------------------------------------------------------------------
+ */
+hid_t
+H5FD_ros3_init(void)
+{
+ hid_t ret_value = H5I_INVALID_HID;
+
+ FUNC_ENTER_NOAPI(FAIL)
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_init() called.\n");
+#endif
+
+ if (H5I_VFL != H5I_get_type(H5FD_ROS3_g))
+ H5FD_ROS3_g = H5FD_register(&H5FD_ros3_g, sizeof(H5FD_class_t), FALSE);
+
+#if ROS3_STATS
+ /* pre-compute statsbin boundaries
+ */
+ for (unsigned bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) {
+ unsigned long long value = 0;
+ ROS3_STATS_POW(bin_i, &value)
+ ros3_stats_boundaries[bin_i] = value;
+ }
+#endif
+
+ /* Set return value */
+ ret_value = H5FD_ROS3_g;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_ros3_init() */
+
+
+/*---------------------------------------------------------------------------
+ * Function: H5FD_ros3_term
+ *
+ * Purpose: Shut down the VFD
+ *
+ * Returns: SUCCEED (Can't fail)
+ *
+ * Programmer: Jacob Smith 2017
+ *
+ *---------------------------------------------------------------------------
+ */
+void
+H5FD_ros3_term(void)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_term() called.\n");
+#endif
+
+ /* Reset VFL ID */
+ H5FD_ROS3_g = 0;
+
+ FUNC_LEAVE_NOAPI_VOID
+} /* end H5FD_ros3_term() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pset_fapl_ros3
+ *
+ * Purpose: Modify the file access property list to use the H5FD_ROS3
+ * driver defined in this source file. All driver specfic
+ * properties are passed in as a pointer to a suitably
+ * initialized instance of H5FD_ros3_fapl_t
+ *
+ * Return: SUCCEED/FAIL
+ *
+ * Programmer: John Mainzer
+ * 9/10/17
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa)
+{
+ H5P_genplist_t *plist = NULL; /* Property list pointer */
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE2("e", "i*x", fapl_id, fa);
+
+ HDassert(fa != NULL);
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5Pset_fapl_ros3() called.\n");
+#endif
+
+ if(NULL == (plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access property list")
+
+ if (FAIL == H5FD_ros3_validate_config(fa))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid ros3 config")
+
+ ret_value = H5P_set_driver(plist, H5FD_ROS3, (void *)fa);
+
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* end H5Pset_fapl_ros3() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_open()
+ *
+ * Purpose: Create and/or opens a file as an HDF5 file.
+ * Any flag except H5F_ACC_RDONLY will cause an error.
+ *
+ * Name (as received from `H5FD_open()`) must conform to web url:
+ * NAME :: HTTP "://" DOMAIN [PORT] ["/" [URI] [QUERY] ]
+ * HTTP :: "http" [ "s" ]
+ * DOMAIN :: e.g., "mybucket.host.org"
+ * PORT :: ":" <number> (e.g., ":9000" )
+ * URI :: <string> (e.g., "path/to/resource.hd5" )
+ * QUERY :: "?" <string> (e.g., "arg1=param1&arg2=param2")
+ *
+ * Return: Success: A pointer to a new file data structure.
+ * The public fields will be initialized by the caller, which is
+ * always H5FD_open().
+ * Failure: NULL
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+static H5FD_t *
+H5FD_ros3_open(const char *url, unsigned flags, hid_t fapl_id, haddr_t maxaddr)
+{
+ H5FD_ros3_t *file = NULL;
+ struct tm *now = NULL;
+ char iso8601now[ISO8601_SIZE];
+ unsigned char signing_key[SHA256_DIGEST_LENGTH];
+ s3r_t *handle = NULL;
+ H5FD_ros3_fapl_t fa;
+ H5FD_t *ret_value = NULL;
+
+
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_open() called.\n");
+#endif
+
+ /* Sanity check on file offsets */
+ HDcompile_assert(sizeof(HDoff_t) >= sizeof(size_t));
+
+ /* Check arguments */
+ if(!url || !*url)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid file name")
+ if(0 == maxaddr || HADDR_UNDEF == maxaddr)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "bogus maxaddr")
+ if(ADDR_OVERFLOW(maxaddr))
+ HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, NULL, "bogus maxaddr")
+ if(flags != H5F_ACC_RDONLY)
+ HGOTO_ERROR(H5E_ARGS, H5E_UNSUPPORTED, NULL, "only Read-Only access allowed")
+
+ if (FAIL == H5Pget_fapl_ros3(fapl_id, &fa))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "can't get property list")
+
+ if (CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "unable to initialize curl global (placeholder flags)")
+
+ /* open file; procedure depends on whether or not the fapl instructs to
+ * authenticate requests or not.
+ */
+ if (fa.authenticate == TRUE) {
+ /* compute signing key (part of AWS/S3 REST API)
+ * can be re-used by user/key for 7 days after creation.
+ * find way to re-use/share
+ */
+ now = gmnow();
+ HDassert( now != NULL );
+ if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while writing iso8601 timestamp")
+ if (FAIL == H5FD_s3comms_signing_key(signing_key,
+ (const char *)fa.secret_key,
+ (const char *)fa.aws_region,
+ (const char *)iso8601now) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "problem while computing signing key")
+ }
+
+ handle = H5FD_s3comms_s3r_open(
+ url,
+ (const char *)fa.aws_region,
+ (const char *)fa.secret_id,
+ (const unsigned char *)signing_key);
+ }
+ else {
+ handle = H5FD_s3comms_s3r_open(url, NULL, NULL, NULL);
+ } /* if/else should authenticate */
+
+ if (handle == NULL) {
+ /* If we want to check CURL's say on the matter in a controlled
+ * fashion, this is the place to do it, but would need to make a
+ * few minor changes to s3comms `s3r_t` and `s3r_read()`.
+ */
+ HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, NULL, "could not open");
+ }
+
+ /* create new file struct
+ */
+ file = H5FL_CALLOC(H5FD_ros3_t);
+ if (file == NULL)
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate file struct")
+
+ file->s3r_handle = handle;
+ HDmemcpy(&(file->fa), &fa, sizeof(H5FD_ros3_fapl_t));
+
+#if ROS3_STATS
+ if (FAIL == ros3_reset_stats(file))
+ HGOTO_ERROR(H5E_INTERNAL, H5E_UNINITIALIZED, NULL, "unable to reset file statistics")
+#endif /* ROS3_STATS */
+
+ ret_value = (H5FD_t*)file;
+
+done:
+ if (ret_value == NULL) {
+ if (handle != NULL) {
+ if (FAIL == H5FD_s3comms_s3r_close(handle))
+ HDONE_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, NULL, "unable to close s3 file handle")
+ }
+ if (file != NULL) {
+ file = H5FL_FREE(H5FD_ros3_t, file);
+ }
+ curl_global_cleanup(); /* early cleanup because open failed */
+ } /* end if null return value (error) */
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_ros3_open() */
+
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_close()
+ *
+ * Purpose:
+ *
+ * Close an HDF5 file.
+ *
+ * Return:
+ *
+ * SUCCEED/FAIL
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_close(H5FD_t H5_ATTR_UNUSED *_file)
+{
+ H5FD_ros3_t *file = (H5FD_ros3_t *)_file;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_close() called.\n");
+#endif
+
+ /* Sanity checks
+ */
+ HDassert(file != NULL);
+ HDassert(file->s3r_handle != NULL);
+
+ /* Close the underlying request handle
+ */
+ if (FAIL == H5FD_s3comms_s3r_close(file->s3r_handle)) {
+ HGOTO_ERROR(H5E_VFL, H5E_CANTCLOSEFILE, FAIL,
+ "unable to close S3 request handle")
+ }
+
+#if ROS3_STATS
+ /* TODO: mechanism to re-target stats printout */
+ if (ros3_fprint_stats(stdout, file) == FAIL) {
+ HGOTO_ERROR(H5E_INTERNAL, H5E_ERROR, FAIL,
+ "problem while writing file statistics")
+ }
+#endif /* ROS3_STATS */
+
+ /* Release the file info
+ */
+ file = H5FL_FREE(H5FD_ros3_t, file);
+
+done:
+ curl_global_cleanup(); /* cleanup to answer init on open */
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_close() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_cmp()
+ *
+ * Purpose:
+ *
+ * Compares two files belonging to this driver using an arbitrary
+ * (but consistent) ordering:
+ *
+ * + url scheme
+ * + url host
+ * + url port
+ * + url path
+ * + url query
+ * + fapl aws_region
+ * + fapl secret_id
+ * + fapl secret_key
+ *
+ * tl;dr -> check URL, check crentials
+ *
+ * Return:
+ *
+ * - Equivalent: 0
+ * - Not Equivalent: -1
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-06
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5FD_ros3_cmp(
+ const H5FD_t *_f1,
+ const H5FD_t *_f2)
+{
+ const H5FD_ros3_t *f1 = (const H5FD_ros3_t *)_f1;
+ const H5FD_ros3_t *f2 = (const H5FD_ros3_t *)_f2;
+ const parsed_url_t *purl1 = NULL;
+ const parsed_url_t *purl2 = NULL;
+ int ret_value = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_cmp() called.\n");
+#endif
+
+ HDassert(f1->s3r_handle != NULL);
+ HDassert(f2->s3r_handle != NULL);
+
+ purl1 = (const parsed_url_t *)f1->s3r_handle->purl;
+ purl2 = (const parsed_url_t *)f2->s3r_handle->purl;
+ HDassert(purl1 != NULL);
+ HDassert(purl2 != NULL);
+ HDassert(purl1->scheme != NULL);
+ HDassert(purl2->scheme != NULL);
+ HDassert(purl1->host != NULL);
+ HDassert(purl2->host != NULL);
+
+ /* URL: SCHEME */
+ if (HDstrcmp(purl1->scheme, purl2->scheme)) {
+ HGOTO_DONE(-1);
+ }
+
+ /* URL: HOST */
+ if (HDstrcmp(purl1->host, purl2->host)) {
+ HGOTO_DONE(-1);
+ }
+
+ /* URL: PORT */
+ if (purl1->port && purl2->port) {
+ if (HDstrcmp(purl1->port, purl2->port)) {
+ HGOTO_DONE(-1);
+ }
+ }
+ else
+ if (purl1->port) {
+ HGOTO_DONE(-1);
+ }
+ else
+ if (purl2->port) {
+ HGOTO_DONE(-1);
+ }
+
+ /* URL: PATH */
+ if (purl1->path && purl2->path) {
+ if (HDstrcmp(purl1->path, purl2->path)) {
+ HGOTO_DONE(-1);
+ }
+ }
+ else
+ if (purl1->path && !purl2->path) {
+ HGOTO_DONE(-1);
+ }
+ else
+ if (purl2->path && !purl1->path) {
+ HGOTO_DONE(-1);
+ }
+
+ /* URL: QUERY */
+ if (purl1->query && purl2->query) {
+ if (HDstrcmp(purl1->query, purl2->query)) {
+ HGOTO_DONE(-1);
+ }
+ }
+ else
+ if (purl1->query && !purl2->query) {
+ HGOTO_DONE(-1);
+ }
+ else
+ if (purl2->query && !purl1->query) {
+ HGOTO_DONE(-1);
+ }
+
+ /* FAPL: AWS_REGION */
+ if (f1->fa.aws_region[0] != '\0' && f1->fa.aws_region[0] != '\0') {
+ if (HDstrcmp(f1->fa.aws_region, f2->fa.aws_region)) {
+ HGOTO_DONE(-1);
+ }
+ }
+ else
+ if (f1->fa.aws_region[0] != '\0') {
+ HGOTO_DONE(-1);
+ }
+ else
+ if (f2->fa.aws_region[0] != '\0') {
+ HGOTO_DONE(-1);
+ }
+
+ /* FAPL: SECRET_ID */
+ if (f1->fa.secret_id[0] != '\0' && f1->fa.secret_id[0] != '\0') {
+ if (HDstrcmp(f1->fa.secret_id, f2->fa.secret_id)) {
+ HGOTO_DONE(-1);
+ }
+ }
+ else
+ if (f1->fa.secret_id[0] != '\0') {
+ HGOTO_DONE(-1);
+ }
+ else
+ if (f2->fa.secret_id[0] != '\0') {
+ HGOTO_DONE(-1);
+ }
+
+ /* FAPL: SECRET_KEY */
+ if (f1->fa.secret_key[0] != '\0' && f1->fa.secret_key[0] != '\0') {
+ if (HDstrcmp(f1->fa.secret_key, f2->fa.secret_key)) {
+ HGOTO_DONE(-1);
+ }
+ }
+ else
+ if (f1->fa.secret_key[0] != '\0') {
+ HGOTO_DONE(-1);
+ }
+ else
+ if (f2->fa.secret_key[0] != '\0') {
+ HGOTO_DONE(-1);
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5FD_ros3_cmp() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_query
+ *
+ * Purpose: Set the flags that this VFL driver is capable of supporting.
+ * (listed in H5FDpublic.h)
+ *
+ * Note that since the ROS3 VFD is read only, most flags
+ * are irrelevant.
+ *
+ * The term "set" is highly misleading...
+ * stores/copies the supported flags in the out-pointer `flags`.
+ *
+ * Return: SUCCEED (Can't fail)
+ *
+ * Programmer: John Mainzer
+ * 9/11/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_query(const H5FD_t H5_ATTR_UNUSED *_file,
+ unsigned long *flags /* out */)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_query() called.\n");
+#endif
+
+ /* Set the VFL feature flags that this driver supports */
+ if (flags) {
+ *flags = 0;
+ /* OK to perform data sieving for faster raw data reads & writes */
+ *flags |= H5FD_FEAT_DATA_SIEVE;
+ } /* end if */
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+
+} /* H5FD_ros3_query() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_get_eoa()
+ *
+ * Purpose:
+ *
+ * Gets the end-of-address marker for the file. The EOA marker
+ * is the first address past the last byte allocated in the
+ * format address space.
+ *
+ * Return:
+ *
+ * The end-of-address marker.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+static haddr_t
+H5FD_ros3_get_eoa(const H5FD_t *_file,
+ H5FD_mem_t H5_ATTR_UNUSED type)
+{
+ const H5FD_ros3_t *file = (const H5FD_ros3_t *)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_get_eoa() called.\n");
+#endif
+
+ FUNC_LEAVE_NOAPI(file->eoa)
+
+} /* end H5FD_ros3_get_eoa() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_set_eoa()
+ *
+ * Purpose:
+ *
+ * Set the end-of-address marker for the file.
+ *
+ * Return:
+ *
+ * SUCCEED (can't fail)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-03
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_set_eoa(H5FD_t *_file,
+ H5FD_mem_t H5_ATTR_UNUSED type,
+ haddr_t addr)
+{
+ H5FD_ros3_t *file = (H5FD_ros3_t *)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_set_eoa() called.\n");
+#endif
+
+ file->eoa = addr;
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+
+} /* H5FD_ros3_set_eoa() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_get_eof()
+ *
+ * Purpose:
+ *
+ * Returns the end-of-file marker.
+ *
+ * Return:
+ *
+ * EOF: the first address past the end of the "file", either the
+ * filesystem file or the HDF5 file.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+static haddr_t
+H5FD_ros3_get_eof(const H5FD_t *_file,
+ H5FD_mem_t H5_ATTR_UNUSED type)
+{
+ const H5FD_ros3_t *file = (const H5FD_ros3_t *)_file;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_get_eof() called.\n");
+#endif
+
+ FUNC_LEAVE_NOAPI(H5FD_s3comms_s3r_get_filesize(file->s3r_handle))
+
+} /* end H5FD_ros3_get_eof() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_get_handle()
+ *
+ * Purpose:
+ *
+ * Returns the S3 Request handle (s3r_t) of ros3 file driver.
+ *
+ * Returns:
+ *
+ * SUCCEED/FAIL
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-02
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_get_handle(H5FD_t *_file,
+ hid_t H5_ATTR_UNUSED fapl,
+ void **file_handle)
+{
+ H5FD_ros3_t *file = (H5FD_ros3_t *)_file;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_get_handle() called.\n");
+#endif
+
+ if (!file_handle) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "file handle not valid")
+ }
+
+ *file_handle = file->s3r_handle;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_get_handle() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_read()
+ *
+ * Purpose
+ *
+ * Reads SIZE bytes of data from FILE beginning at address ADDR
+ * into buffer BUF according to data transfer properties in DXPL_ID.
+ *
+ * Return:
+ *
+ * Success: `SUCCEED`
+ * - Result is stored in caller-supplied buffer BUF.
+ * Failure: `FAIL`
+ * - Unable to complete read.
+ * - Contents of buffer `buf` are undefined.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-??
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_read(H5FD_t *_file,
+ H5FD_mem_t H5_ATTR_UNUSED type,
+ hid_t H5_ATTR_UNUSED dxpl_id,
+ haddr_t addr, /* start offset */
+ size_t size, /* length of read */
+ void *buf) /* out */
+{
+ H5FD_ros3_t *file = (H5FD_ros3_t *)_file;
+ size_t filesize = 0;
+ herr_t ret_value = SUCCEED;
+#if ROS3_STATS
+ /* working variables for storing stats */
+ ros3_statsbin *bin = NULL;
+ unsigned bin_i = 0;
+#endif /* ROS3_STATS */
+
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_read() called.\n");
+#endif
+
+ HDassert(file != NULL);
+ HDassert(file->s3r_handle != NULL);
+ HDassert(buf != NULL);
+
+ filesize = H5FD_s3comms_s3r_get_filesize(file->s3r_handle);
+
+ if ((addr > filesize) || ((addr + size) > filesize)) {
+ HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "range exceeds file address")
+ }
+
+ if (H5FD_s3comms_s3r_read(file->s3r_handle, addr, size, buf) == FAIL) {
+ HGOTO_ERROR(H5E_VFL, H5E_READERROR, FAIL, "unable to execute read")
+ }
+
+#if ROS3_STATS
+
+ /* Find which "bin" this read fits in. Can be "overflow" bin.
+ */
+ for (bin_i = 0; bin_i < ROS3_STATS_BIN_COUNT; bin_i++) {
+ if ((unsigned long long)size < ros3_stats_boundaries[bin_i]) {
+ break;
+ }
+ }
+ bin = (type == H5FD_MEM_DRAW)
+ ? &file->raw[bin_i]
+ : &file->meta[bin_i];
+
+ /* Store collected stats in appropriate bin
+ */
+ if (bin->count == 0) {
+ bin->min = size;
+ bin->max = size;
+ }
+ else {
+ if (size < bin->min) {
+ bin->min = size;
+ }
+ if (size > bin->max) {
+ bin->max = size;
+ }
+ }
+ bin->count++;
+ bin->bytes += (unsigned long long)size;
+
+#endif /* ROS3_STATS */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_read() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_write()
+ *
+ * Purpose:
+ *
+ * Write bytes to file.
+ * UNSUPPORTED IN READ-ONLY ROS3 VFD.
+ *
+ * Return:
+ *
+ * FAIL (Not possible with Read-Only S3 file.)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_write(H5FD_t H5_ATTR_UNUSED *_file,
+ H5FD_mem_t H5_ATTR_UNUSED type,
+ hid_t H5_ATTR_UNUSED dxpl_id,
+ haddr_t H5_ATTR_UNUSED addr,
+ size_t H5_ATTR_UNUSED size,
+ const void H5_ATTR_UNUSED *buf)
+{
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_write() called.\n");
+#endif
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL,
+ "cannot write to read-only file.")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5FD_ros3_write() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_truncate()
+ *
+ * Purpose:
+ *
+ * Makes sure that the true file size is the same (or larger)
+ * than the end-of-address.
+ *
+ * NOT POSSIBLE ON READ-ONLY S3 FILES.
+ *
+ * Return:
+ *
+ * FAIL (Not possible on Read-Only S3 files.)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_truncate(H5FD_t H5_ATTR_UNUSED *_file,
+ hid_t H5_ATTR_UNUSED dxpl_id,
+ hbool_t H5_ATTR_UNUSED closing)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5FD_ros3_truncate() called.\n");
+#endif
+
+ HGOTO_ERROR(H5E_VFL, H5E_UNSUPPORTED, FAIL,
+ "cannot truncate read-only file.")
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_truncate() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_lock()
+ *
+ * Purpose:
+ *
+ * Place an advisory lock on a file.
+ * No effect on Read-Only S3 file.
+ *
+ * Suggestion: remove lock/unlock from class
+ * > would result in error at H5FD_[un]lock() (H5FD.c)
+ *
+ * Return:
+ *
+ * SUCCEED (No-op always succeeds)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-03
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_lock(H5FD_t H5_ATTR_UNUSED *_file,
+ hbool_t H5_ATTR_UNUSED rw)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+ FUNC_LEAVE_NOAPI(SUCCEED)
+
+} /* end H5FD_ros3_lock() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: H5FD_ros3_unlock()
+ *
+ * Purpose:
+ *
+ * Remove the existing lock on the file.
+ * No effect on Read-Only S3 file.
+ *
+ * Return:
+ *
+ * SUCCEED (No-op always succeeds)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-03
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_unlock(H5FD_t H5_ATTR_UNUSED *_file)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+ FUNC_LEAVE_NOAPI(SUCCEED)
+
+} /* end H5FD_ros3_unlock() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_validate_config()
+ *
+ * Purpose: Test to see if the supplied instance of H5FD_ros3_fapl_t
+ * contains internally consistant data. Return SUCCEED if so,
+ * and FAIL otherwise.
+ *
+ * Note the difference between internally consistant and
+ * correct. As we will have to try to access the target
+ * object to determine whether the supplied data is correct,
+ * we will settle for internal consistancy at this point
+ *
+ * Return: SUCCEED if instance of H5FD_ros3_fapl_t contains internally
+ * consistant data, FAIL otherwise.
+ *
+ * Programmer: Jacob Smith
+ * 9/10/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_validate_config(const H5FD_ros3_fapl_t * fa)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ HDassert(fa != NULL);
+
+ if ( fa->version != H5FD_CURR_ROS3_FAPL_T_VERSION ) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "Unknown H5FD_ros3_fapl_t version");
+ }
+
+ /* if set to authenticate, region and id cannot be empty strings
+ */
+ if (fa->authenticate == TRUE) {
+ if ((fa->aws_region[0] == '\0') ||
+ (fa->secret_id[0] == '\0'))
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "Inconsistent authentication information");
+ }
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_validate_config() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_fapl_ros3
+ *
+ * Purpose: Returns information about the ros3 file access property
+ * list though the function arguments.
+ *
+ * Return: Success: Non-negative
+ *
+ * Failure: Negative
+ *
+ * Programmer: John Mainzer
+ * 9/10/17
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_fapl_ros3(hid_t fapl_id,
+ H5FD_ros3_fapl_t *fa_out)
+{
+ const H5FD_ros3_fapl_t *fa = NULL;
+ H5P_genplist_t *plist = NULL;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE2("e", "i*x", fapl_id, fa_out);
+
+#if ROS3_DEBUG
+ HDfprintf(stdout, "H5Pget_fapl_ros3() called.\n");
+#endif
+
+ if (fa_out == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "fa_out is NULL")
+ }
+
+ plist = H5P_object_verify(fapl_id, H5P_FILE_ACCESS);
+ if (plist == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file access list")
+ }
+
+ if (H5FD_ROS3 != H5P_get_driver(plist)) {
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "incorrect VFL driver")
+ }
+
+ fa = (const H5FD_ros3_fapl_t *)H5P_get_driver_info(plist);
+ if (fa == NULL) {
+ HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "bad VFL driver info")
+ }
+
+ /* Copy the ros3 fapl data out */
+ HDmemcpy(fa_out, fa, sizeof(H5FD_ros3_fapl_t));
+
+done:
+ FUNC_LEAVE_API(ret_value)
+
+} /* end H5Pget_fapl_ros3() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_fapl_get
+ *
+ * Purpose: Gets a file access property list which could be used to
+ * create an identical file.
+ *
+ * Return: Success: Ptr to new file access property list value.
+ *
+ * Failure: NULL
+ *
+ * Programmer: John Mainzer
+ * 9/8/17
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static void *
+H5FD_ros3_fapl_get(H5FD_t *_file)
+{
+ H5FD_ros3_t *file = (H5FD_ros3_t*)_file;
+ H5FD_ros3_fapl_t *fa = NULL;
+ void *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ fa = (H5FD_ros3_fapl_t *)H5MM_calloc(sizeof(H5FD_ros3_fapl_t));
+ if (fa == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL,
+ "memory allocation failed")
+ }
+
+ /* Copy the fields of the structure */
+ HDmemcpy(fa, &(file->fa), sizeof(H5FD_ros3_fapl_t));
+
+ /* Set return value */
+ ret_value = fa;
+
+done:
+ if (ret_value == NULL) {
+ if (fa != NULL) {
+ H5MM_xfree(fa);
+ }
+ }
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_fapl_get() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_fapl_copy
+ *
+ * Purpose: Copies the ros3-specific file access properties.
+ *
+ * Return: Success: Ptr to a new property list
+ *
+ * Failure: NULL
+ *
+ * Programmer: John Mainzer
+ * 9/8/17
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static void *
+H5FD_ros3_fapl_copy(const void *_old_fa)
+{
+ const H5FD_ros3_fapl_t *old_fa = (const H5FD_ros3_fapl_t*)_old_fa;
+ H5FD_ros3_fapl_t *new_fa = NULL;
+ void *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ new_fa = (H5FD_ros3_fapl_t *)H5MM_malloc(sizeof(H5FD_ros3_fapl_t));
+ if (new_fa == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL,
+ "memory allocation failed");
+ }
+
+ HDmemcpy(new_fa, old_fa, sizeof(H5FD_ros3_fapl_t));
+ ret_value = new_fa;
+
+done:
+ if (ret_value == NULL) {
+ if (new_fa != NULL) {
+ H5MM_xfree(new_fa);
+ }
+ }
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* end H5FD_ros3_fapl_copy() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_ros3_fapl_free
+ *
+ * Purpose: Frees the ros3-specific file access properties.
+ *
+ * Return: SUCCEED (cannot fail)
+ *
+ * Programmer: John Mainzer
+ * 9/8/17
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_ros3_fapl_free(void *_fa)
+{
+ H5FD_ros3_fapl_t *fa = (H5FD_ros3_fapl_t*)_fa;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ HDassert(fa != NULL); /* sanity check */
+
+ H5MM_xfree(fa);
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+
+} /* end H5FD_ros3_fapl_free() */
+
+#if ROS3_STATS
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: ros3_reset_stats()
+ *
+ * Purpose:
+ *
+ * Reset the stats collection elements in this virtual file structure.
+ *
+ * Clears any set data in stats bins; initializes/zeroes values.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - Occurs if the file is invalid somehow
+ *
+ * Programmer: Jacob Smith
+ * 2017-12-08
+ *
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+ros3_reset_stats(H5FD_ros3_t *file)
+{
+ unsigned i = 0;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if ROS3_DEBUG
+ HDprintf("ros3_reset_stats() called\n");
+#endif
+
+ if (file == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "file was null");
+ }
+
+ for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) {
+ file->raw[i].bytes = 0;
+ file->raw[i].count = 0;
+ file->raw[i].min = (unsigned long long)ROS3_STATS_STARTING_MIN;
+ file->raw[i].max = 0;
+
+ file->meta[i].bytes = 0;
+ file->meta[i].count = 0;
+ file->meta[i].min = (unsigned long long)ROS3_STATS_STARTING_MIN;
+ file->meta[i].max = 0;
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+
+} /* end ros3_reset_stats() */
+
+#endif /* ROS3_STATS */
+#if ROS3_STATS
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: ros3_fprint_stats()
+ *
+ * Purpose:
+ *
+ * Tabulate and pretty-print statistics for this virtual file.
+ *
+ * Should be called upon file close.
+ *
+ * Shows number of reads and bytes read, broken down by
+ * "raw" (H5FD_MEM_DRAW)
+ * or "meta" (any other flag)
+ *
+ * Prints filename and listing of total number of reads and bytes read,
+ * both as a grand total and separate meta- and rawdata reads.
+ *
+ * If any reads were done, prints out two tables:
+ *
+ * 1. overview of raw- and metadata reads
+ * - min (smallest size read)
+ * - average of size read
+ * - k,M,G suffixes by powers of 1024 (2^10)
+ * - max (largest size read)
+ * 2. tabulation of "bins", sepraring reads into exponentially-larger
+ * ranges of size.
+ * - columns for number of reads, total bytes, and average size, with
+ * separate sub-colums for raw- and metadata reads.
+ * - each row represents one bin, identified by the top of its range
+ *
+ * Bin ranges can be modified with pound-defines at the top of this file.
+ *
+ * Bins without any reads in their bounds are not printed.
+ *
+ * An "overflow" bin is also present, to catch "big" reads.
+ *
+ * Output for all bins (and range ceiling and average size report)
+ * is divied by powers of 1024. By corollary, four digits before the decimal
+ * is valid.
+ *
+ * - 41080 bytes is represented by 40.177k, not 41.080k
+ * - 1004.831M represents approx. 1052642000 bytes
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - occurs if the file passed in is invalid
+ * - TODO: if stream is invalid? how can we check this?
+ *
+ * Programmer: Jacob Smith
+ *
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+ros3_fprint_stats(FILE *stream,
+ const H5FD_ros3_t *file)
+{
+ herr_t ret_value = SUCCEED;
+ parsed_url_t *purl = NULL;
+ unsigned i = 0;
+ unsigned long count_meta = 0;
+ unsigned long count_raw = 0;
+ double average_meta = 0.0;
+ double average_raw = 0.0;
+ unsigned long long min_meta = (unsigned long long)ROS3_STATS_STARTING_MIN;
+ unsigned long long min_raw = (unsigned long long)ROS3_STATS_STARTING_MIN;
+ unsigned long long max_meta = 0;
+ unsigned long long max_raw = 0;
+ unsigned long long bytes_raw = 0;
+ unsigned long long bytes_meta = 0;
+ double re_dub = 0.0; /* re-usable double variable */
+ unsigned suffix_i = 0;
+ const char suffixes[] = { ' ', 'K', 'M', 'G', 'T', 'P' };
+
+
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+ if (stream == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "file stream cannot be null" );
+ }
+ if (file == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "file cannot be null");
+ }
+ if (file->s3r_handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "s3 request handle cannot be null");
+ }
+ if (file->s3r_handle->purl == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "parsed url structure cannot be null");
+ }
+ purl = file->s3r_handle->purl;
+
+ /******************
+ * PRINT FILENAME *
+ ******************/
+
+ HDfprintf(stream, "stats for %s://%s", purl->scheme, purl->host);
+ if (purl->port != NULL && purl->port[0] != '\0')
+ HDfprintf(stream, ":%s", purl->port);
+ if (purl->query != NULL && purl->query[0] != '\0') {
+ if (purl->path != NULL && purl->path[0] != '\0')
+ HDfprintf(stream, "/%s", purl->path);
+ else
+ HDfprintf(stream, "/");
+ HDfprintf(stream, "?%s", purl->query);
+ } else if (purl->path != NULL && purl->path[0] != '\0') {
+ HDfprintf(stream, "/%s", purl->path);
+ }
+ HDfprintf(stream, "\n");
+
+ /*******************
+ * AGGREGATE STATS *
+ *******************/
+
+ for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) {
+ const ros3_statsbin *r = &file->raw[i];
+ const ros3_statsbin *m = &file->meta[i];
+
+ if (m->min < min_meta) min_meta = m->min;
+ if (r->min < min_raw) min_raw = r->min;
+ if (m->max > max_meta) max_meta = m->max;
+ if (r->max > max_raw) max_raw = r->max;
+
+ count_raw += r->count;
+ count_meta += m->count;
+ bytes_raw += r->bytes;
+ bytes_meta += m->bytes;
+ }
+ if (count_raw > 0)
+ average_raw = (double)bytes_raw / (double)count_raw;
+ if (count_meta > 0)
+ average_meta = (double)bytes_meta / (double)count_meta;
+
+ /******************
+ * PRINT OVERVIEW *
+ ******************/
+
+ HDfprintf(stream, "TOTAL READS: %llu (%llu meta, %llu raw)\n",
+ count_raw + count_meta, count_meta, count_raw);
+ HDfprintf(stream, "TOTAL BYTES: %llu (%llu meta, %llu raw)\n",
+ bytes_raw + bytes_meta, bytes_meta, bytes_raw);
+
+ if (count_raw + count_meta == 0)
+ goto done;
+
+ /*************************
+ * PRINT AGGREGATE STATS *
+ *************************/
+
+ HDfprintf(stream, "SIZES meta raw\n");
+ HDfprintf(stream, " min ");
+ if (count_meta == 0) {
+ HDfprintf(stream, " 0.000 ");
+ } else {
+ re_dub = (double)min_meta;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]);
+ }
+
+ if (count_raw == 0) {
+ HDfprintf(stream, " 0.000 \n");
+ } else {
+ re_dub = (double)min_raw;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]);
+ }
+
+ HDfprintf(stream, " avg ");
+ re_dub = (double)average_meta;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]);
+
+ re_dub = (double)average_raw;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]);
+
+ HDfprintf(stream, " max ");
+ re_dub = (double)max_meta;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c ", re_dub, suffixes[suffix_i]);
+
+ re_dub = (double)max_raw;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ HDfprintf(stream, "%8.3lf%c\n", re_dub, suffixes[suffix_i]);
+
+ /******************************
+ * PRINT INDIVIDUAL BIN STATS *
+ ******************************/
+
+ HDfprintf(stream,
+ "BINS # of reads total bytes average size\n");
+ HDfprintf(stream,
+ " up-to meta raw meta raw meta raw\n");
+
+ for (i = 0; i <= ROS3_STATS_BIN_COUNT; i++) {
+ const ros3_statsbin *m;
+ const ros3_statsbin *r;
+ unsigned long long range_end = 0;
+ char bm_suffix = ' '; /* bytes-meta */
+ double bm_val = 0.0;
+ char br_suffix = ' '; /* bytes-raw */
+ double br_val = 0.0;
+ char am_suffix = ' '; /* average-meta */
+ double am_val = 0.0;
+ char ar_suffix = ' '; /* average-raw */
+ double ar_val = 0.0;
+
+ m = &file->meta[i];
+ r = &file->raw[i];
+ if (r->count == 0 && m->count == 0)
+ continue;
+
+ range_end = ros3_stats_boundaries[i];
+
+ if (i == ROS3_STATS_BIN_COUNT) {
+ range_end = ros3_stats_boundaries[i-1];
+ HDfprintf(stream, ">");
+ } else {
+ HDfprintf(stream, " ");
+ }
+
+ bm_val = (double)m->bytes;
+ for (suffix_i = 0; bm_val >= 1024.0; suffix_i++)
+ bm_val /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ bm_suffix = suffixes[suffix_i];
+
+ br_val = (double)r->bytes;
+ for (suffix_i = 0; br_val >= 1024.0; suffix_i++)
+ br_val /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ br_suffix = suffixes[suffix_i];
+
+ if (m->count > 0)
+ am_val = (double)(m->bytes) / (double)(m->count);
+ for (suffix_i = 0; am_val >= 1024.0; suffix_i++)
+ am_val /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ am_suffix = suffixes[suffix_i];
+
+ if (r->count > 0)
+ ar_val = (double)(r->bytes) / (double)(r->count);
+ for (suffix_i = 0; ar_val >= 1024.0; suffix_i++)
+ ar_val /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+ ar_suffix = suffixes[suffix_i];
+
+ re_dub = (double)range_end;
+ for (suffix_i = 0; re_dub >= 1024.0; suffix_i++)
+ re_dub /= 1024.0;
+ HDassert(suffix_i < sizeof(suffixes));
+
+ HDfprintf(stream,
+ " %8.3f%c %7d %7d %8.3f%c %8.3f%c %8.3f%c %8.3f%c\n",
+ re_dub, suffixes[suffix_i], /* bin ceiling */
+ m->count, /* metadata reads */
+ r->count, /* rawdata reads */
+ bm_val, bm_suffix, /* metadata bytes */
+ br_val, br_suffix, /* rawdata bytes */
+ am_val, am_suffix, /* metadata average */
+ ar_val, ar_suffix); /* rawdata average */
+
+ fflush(stream);
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+
+} /* ros3_fprint_stats */
+#endif /* ROS3_STATS */
+
+#endif /* H5_HAVE_ROS3_VFD */
+
diff --git a/src/H5FDros3.h b/src/H5FDros3.h
new file mode 100644
index 0000000..8ecd7f0
--- /dev/null
+++ b/src/H5FDros3.h
@@ -0,0 +1,106 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Read-Only S3 Virtual File Driver (VFD)
+ *
+ * Programmer: John Mainzer
+ * 2017-10-10
+ *
+ * Purpose: The public header file for the ros3 driver.
+ */
+#ifndef H5FDros3_H
+#define H5FDros3_H
+
+#ifdef H5_HAVE_ROS3_VFD
+#define H5FD_ROS3 (H5FD_ros3_init())
+#else
+#define H5FD_ROS3 (H5I_INVALID_HID)
+#endif /* H5_HAVE_ROS3_VFD */
+
+#ifdef H5_HAVE_ROS3_VFD
+
+/****************************************************************************
+ *
+ * Structure: H5FD_ros3_fapl_t
+ *
+ * Purpose:
+ *
+ * H5FD_ros3_fapl_t is a public structure that is used to pass S3
+ * authentication data to the appropriate S3 VFD via the FAPL. A pointer
+ * to an instance of this structure is a parameter to H5Pset_fapl_ros3()
+ * and H5Pget_fapl_ros3().
+ *
+ *
+ *
+ * `version` (int32_t)
+ *
+ * Version number of the H5FD_ros3_fapl_t structure. Any instance passed
+ * to the above calls must have a recognized version number, or an error
+ * will be flagged.
+ *
+ * This field should be set to H5FD_CURR_ROS3_FAPL_T_VERSION.
+ *
+ * `authenticate` (hbool_t)
+ *
+ * Flag TRUE or FALSE whether or not requests are to be authenticated
+ * with the AWS4 algorithm.
+ * If TRUE, `aws_region`, `secret_id`, and `secret_key` must be populated.
+ * If FALSE, those three components are unused.
+ *
+ * `aws_region` (char[])
+ *
+ * String: name of the AWS "region" of the host, e.g. "us-east-1".
+ *
+ * `secret_id` (char[])
+ *
+ * String: "Access ID" for the resource.
+ *
+ * `secret_key` (char[])
+ *
+ * String: "Secret Access Key" associated with the ID and resource.
+ *
+ ****************************************************************************/
+
+#define H5FD_CURR_ROS3_FAPL_T_VERSION 1
+
+#define H5FD_ROS3_MAX_REGION_LEN 32
+#define H5FD_ROS3_MAX_SECRET_ID_LEN 128
+#define H5FD_ROS3_MAX_SECRET_KEY_LEN 128
+
+typedef struct H5FD_ros3_fapl_t {
+ int32_t version;
+ hbool_t authenticate;
+ char aws_region[H5FD_ROS3_MAX_REGION_LEN + 1];
+ char secret_id[H5FD_ROS3_MAX_SECRET_ID_LEN + 1];
+ char secret_key[H5FD_ROS3_MAX_SECRET_KEY_LEN + 1];
+} H5FD_ros3_fapl_t;
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+H5_DLL hid_t H5FD_ros3_init(void);
+H5_DLL void H5FD_ros3_term(void);
+H5_DLL herr_t H5Pget_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa_out);
+H5_DLL herr_t H5Pset_fapl_ros3(hid_t fapl_id, H5FD_ros3_fapl_t *fa);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H5_HAVE_ROS3_VFD */
+
+#endif /* ifndef H5FDros3_H */
+
+
diff --git a/src/H5FDs3comms.c b/src/H5FDs3comms.c
new file mode 100644
index 0000000..f08e9d5
--- /dev/null
+++ b/src/H5FDs3comms.c
@@ -0,0 +1,3593 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*****************************************************************************
+ * Read-Only S3 Virtual File Driver (VFD)
+ *
+ * Source for S3 Communications module
+ *
+ * ***NOT A FILE DRIVER***
+ *
+ * Provide functions and structures required for interfacing with Amazon
+ * Simple Storage Service (S3).
+ *
+ * Provide S3 object access as if it were a local file.
+ *
+ * Connect to remote host, send and receive HTTP requests and responses
+ * as part of the AWS REST API, authenticating requests as appropriate.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-30
+ *
+ *****************************************************************************/
+
+/****************/
+/* Module Setup */
+/****************/
+
+/***********/
+/* Headers */
+/***********/
+
+#include "H5private.h" /* generic functions */
+#include "H5Eprivate.h" /* error handling */
+#include "H5MMprivate.h" /* memory management */
+#include "H5FDs3comms.h" /* S3 Communications */
+
+/****************/
+/* Local Macros */
+/****************/
+
+#ifdef H5_HAVE_ROS3_VFD
+
+/* toggle debugging (enable with 1)
+ */
+#define S3COMMS_DEBUG 0
+
+/* manipulate verbosity of CURL output
+ * operates separately from S3COMMS_DEBUG
+ *
+ * 0 -> no explicit curl output
+ * 1 -> on error, print failure info to stderr
+ * 2 -> in addition to above, print information for all performs; sets all
+ * curl handles with CURLOPT_VERBOSE
+ */
+#define S3COMMS_CURL_VERBOSITY 0
+
+/* size to allocate for "bytes=<first_byte>[-<last_byte>]" HTTP Range value
+ */
+#define S3COMMS_MAX_RANGE_STRING_SIZE 128
+
+
+/******************/
+/* Local Typedefs */
+/******************/
+
+/********************/
+/* Local Structures */
+/********************/
+
+/* struct s3r_datastruct
+ * Structure passed to curl write callback
+ * pointer to data region and record of bytes written (offset)
+ */
+struct s3r_datastruct {
+ unsigned long magic;
+ char *data;
+ size_t size;
+};
+#define S3COMMS_CALLBACK_DATASTRUCT_MAGIC 0x28c2b2ul
+
+/********************/
+/* Local Prototypes */
+/********************/
+
+size_t curlwritecallback(char *ptr,
+ size_t size,
+ size_t nmemb,
+ void *userdata);
+
+herr_t H5FD_s3comms_s3r_getsize(s3r_t *handle);
+
+/*********************/
+/* Package Variables */
+/*********************/
+
+/*****************************/
+/* Library Private Variables */
+/*****************************/
+
+/*******************/
+/* Local Variables */
+/*******************/
+
+/*************/
+/* Functions */
+/*************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: curlwritecallback()
+ *
+ * Purpose:
+ *
+ * Function called by CURL to write received data.
+ *
+ * Writes bytes to `userdata`.
+ *
+ * Internally manages number of bytes processed.
+ *
+ * Return:
+ *
+ * - Number of bytes processed.
+ * - Should equal number of bytes passed to callback.
+ * - Failure will result in curl error: CURLE_WRITE_ERROR.
+ *
+ * Programmer: Jacob Smith
+ * 2017-08-17
+ *
+ *----------------------------------------------------------------------------
+ */
+size_t
+curlwritecallback(char *ptr,
+ size_t size,
+ size_t nmemb,
+ void *userdata)
+{
+ struct s3r_datastruct *sds = (struct s3r_datastruct *)userdata;
+ size_t product = (size * nmemb);
+ size_t written = 0;
+
+ if (sds->magic != S3COMMS_CALLBACK_DATASTRUCT_MAGIC) {
+ return written;
+ }
+
+ if (size > 0) {
+ HDmemcpy(&(sds->data[sds->size]), ptr, product);
+ sds->size += product;
+ written = product;
+ }
+
+ return written;
+
+} /* end curlwritecallback() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_hrb_node_set()
+ *
+ * Purpose:
+ *
+ * Create, insert, modify, and remove elements in a field node list.
+ *
+ * `name` cannot be null; will return FAIL and list will be unaltered.
+ *
+ * Entries are accessed via the lowercase representation of their name:
+ * "Host", "host", and "hOSt" would all access the same node,
+ * but name's case is relevant in HTTP request output.
+ *
+ * List pointer `L` must always point to either of :
+ * - header node with lowest alphabetical order (by lowername)
+ * - NULL, if list is empty
+ *
+ * Types of operations:
+ *
+ * - CREATE
+ * - If `L` is NULL and `name` and `value` are not NULL,
+ * a new node is created at `L`, starting a list.
+ * - MODIFY
+ * - If a node is found with a matching lowercase name and `value`
+ * is not NULL, the existing name, value, and cat values are released
+ * and replaced with the new data.
+ * - No modifications are made to the list pointers.
+ * - REMOVE
+ * - If `value` is NULL, will attempt to remove node with matching
+ * lowercase name.
+ * - If no match found, returns FAIL and list is not modified.
+ * - When removing a node, all its resources is released.
+ * - If removing the last node in the list, list pointer is set to NULL.
+ * - INSERT
+ * - If no nodes exists with matching lowercase name and `value`
+ * is not NULL, a new node is created, inserted into list
+ * alphabetically by lowercase name.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - List was successfully modified
+ * - FAILURE: `FAIL`
+ * - Unable to perform operation
+ * - Forbidden (attempting to remove absent node, e.g.)
+ * - Internal error
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-22
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_hrb_node_set(
+ hrb_node_t **L,
+ const char *name,
+ const char *value)
+{
+ size_t i = 0;
+ char *valuecpy = NULL;
+ char *namecpy = NULL;
+ size_t namelen = 0;
+ char *lowername = NULL;
+ char *nvcat = NULL;
+ hrb_node_t *node_ptr = NULL;
+ hrb_node_t *new_node = NULL;
+ hbool_t is_looking = TRUE;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_hrb_node_set.\n");
+ HDprintf("NAME: %s\n", name);
+ HDprintf("VALUE: %s\n", value);
+ HDprintf("LIST:\n->");
+ for (node_ptr = (*L); node_ptr != NULL; node_ptr = node_ptr->next) {
+ HDfprintf(stdout, "{%s}\n->", node_ptr->cat);
+ }
+ HDprintf("(null)\n");
+ fflush(stdout);
+ node_ptr = NULL;
+#endif
+
+ if (name == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to operate on null name.\n");
+ }
+ namelen = HDstrlen(name);
+
+ /***********************
+ * PREPARE ALL STRINGS *
+ **********************/
+
+ /* copy and lowercase name
+ */
+ lowername = (char *)H5MM_malloc(sizeof(char) * (namelen + 1));
+ if (lowername == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
+ "cannot make space for lowercase name copy.\n");
+ }
+ for (i = 0; i < namelen; i++) {
+ lowername[i] = (char)tolower((int)name[i]);
+ }
+ lowername[namelen] = 0;
+
+ /* If value supplied, copy name, value, and concatenated "name: value".
+ * If NULL, we will be removing a node or doing nothing, so no need for
+ * copies
+ */
+ if (value != NULL) {
+ int ret = 0;
+ size_t valuelen = HDstrlen(value);
+ size_t catlen = namelen + valuelen + 2; /* +2 from ": " */
+ size_t catwrite = catlen + 3; /* 3 not 1 to quiet compiler warning */
+
+
+ namecpy = (char *)H5MM_malloc(sizeof(char) * (namelen + 1));
+ if (namecpy == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
+ "cannot make space for name copy.\n");
+ }
+ HDmemcpy(namecpy, name, (namelen + 1));
+
+ valuecpy = (char *)H5MM_malloc(sizeof(char) * (valuelen + 1));
+ if (valuecpy == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
+ "cannot make space for value copy.\n");
+ }
+ HDmemcpy(valuecpy, value, (valuelen + 1));
+
+ nvcat = (char *)H5MM_malloc(sizeof(char) * catwrite);
+ if (nvcat == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
+ "cannot make space for concatenated string.\n");
+ }
+ ret = HDsnprintf(nvcat, catwrite, "%s: %s", name, value);
+ if (ret < 0 || (size_t)ret > catlen) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot concatenate `%s: %s", name, value);
+ }
+ HDassert( catlen == HDstrlen(nvcat) );
+
+ /* create new_node, should we need it
+ */
+ new_node = (hrb_node_t *)H5MM_malloc(sizeof(hrb_node_t));
+ if (new_node == NULL) {
+ HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL,
+ "cannot make space for new set.\n");
+ }
+
+ new_node->magic = S3COMMS_HRB_NODE_MAGIC;
+ new_node->name = NULL;
+ new_node->value = NULL;
+ new_node->cat = NULL;
+ new_node->lowername = NULL;
+ new_node->next = NULL;
+ }
+
+ /***************
+ * ACT ON LIST *
+ ***************/
+
+ if (*L == NULL) {
+ if (value == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "trying to remove node from empty list");
+ }
+ else {
+#if S3COMMS_DEBUG
+HDprintf("CREATE NEW\n"); fflush(stdout);
+#endif
+ /*******************
+ * CREATE NEW LIST *
+ *******************/
+
+ new_node->cat = nvcat;
+ new_node->name = namecpy;
+ new_node->lowername = lowername;
+ new_node->value = valuecpy;
+
+ *L = new_node;
+ goto done; /* bypass further seeking */
+ }
+ }
+
+ /* sanity-check pointer passed in
+ */
+ HDassert( (*L) != NULL );
+ HDassert( (*L)->magic == S3COMMS_HRB_NODE_MAGIC );
+ node_ptr = (*L);
+
+ /* Check whether to modify/remove first node in list
+ */
+ if (strcmp(lowername, node_ptr->lowername) == 0) {
+
+ is_looking = FALSE;
+
+ if (value == NULL) {
+#if S3COMMS_DEBUG
+HDprintf("REMOVE HEAD\n"); fflush(stdout);
+#endif
+ /***************
+ * REMOVE HEAD *
+ ***************/
+
+ *L = node_ptr->next;
+
+#if S3COMMS_DEBUG
+HDprintf("FREEING CAT (node)\n"); fflush(stdout);
+#endif
+ H5MM_xfree(node_ptr->cat);
+#if S3COMMS_DEBUG
+HDprintf("FREEING LOWERNAME (node)\n"); fflush(stdout);
+#endif
+ H5MM_xfree(node_ptr->lowername);
+#if S3COMMS_DEBUG
+HDprintf("FREEING NAME (node)\n"); fflush(stdout);
+#endif
+ H5MM_xfree(node_ptr->name);
+#if S3COMMS_DEBUG
+HDprintf("FREEING VALUE (node)\n"); fflush(stdout);
+#endif
+ H5MM_xfree(node_ptr->value);
+#if S3COMMS_DEBUG
+HDprintf("MAGIC OK? %s\n",
+ (node_ptr->magic == S3COMMS_HRB_NODE_MAGIC) ? "YES" : "NO");
+fflush(stdout);
+#endif
+ HDassert( node_ptr->magic == S3COMMS_HRB_NODE_MAGIC );
+ node_ptr->magic += 1ul;
+#if S3COMMS_DEBUG
+HDprintf("FREEING POINTER\n"); fflush(stdout);
+#endif
+ H5MM_xfree(node_ptr);
+
+#if S3COMMS_DEBUG
+HDprintf("FREEING WORKING LOWERNAME\n"); fflush(stdout);
+#endif
+ H5MM_xfree(lowername); lowername = NULL;
+ }
+ else {
+#if S3COMMS_DEBUG
+HDprintf("MODIFY HEAD\n"); fflush(stdout);
+#endif
+ /***************
+ * MODIFY HEAD *
+ ***************/
+
+ H5MM_xfree(node_ptr->cat);
+ H5MM_xfree(node_ptr->name);
+ H5MM_xfree(node_ptr->value);
+
+ node_ptr->name = namecpy;
+ node_ptr->value = valuecpy;
+ node_ptr->cat = nvcat;
+
+ H5MM_xfree(lowername);
+ lowername = NULL;
+ new_node->magic += 1ul;
+ H5MM_xfree(new_node);
+ new_node = NULL;
+ }
+ }
+ else
+ if (strcmp(lowername, node_ptr->lowername) < 0) {
+
+ is_looking = FALSE;
+
+ if (value == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "trying to remove a node 'before' head");
+ }
+ else {
+#if S3COMMS_DEBUG
+HDprintf("PREPEND NEW HEAD\n"); fflush(stdout);
+#endif
+ /*******************
+ * INSERT NEW HEAD *
+ *******************/
+
+ new_node->name = namecpy;
+ new_node->value = valuecpy;
+ new_node->lowername = lowername;
+ new_node->cat = nvcat;
+ new_node->next = node_ptr;
+ *L = new_node;
+ }
+ }
+
+ /***************
+ * SEARCH LIST *
+ ***************/
+
+ while (is_looking) {
+ if (node_ptr->next == NULL) {
+
+ is_looking = FALSE;
+
+ if (value == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "trying to remove absent node");
+ }
+ else {
+#if S3COMMS_DEBUG
+HDprintf("APPEND A NODE\n"); fflush(stdout);
+#endif
+ /*******************
+ * APPEND NEW NODE *
+ *******************/
+
+ HDassert( strcmp(lowername, node_ptr->lowername) > 0 );
+ new_node->name = namecpy;
+ new_node->value = valuecpy;
+ new_node->lowername = lowername;
+ new_node->cat = nvcat;
+ node_ptr->next = new_node;
+ }
+ }
+ else
+ if (strcmp(lowername, node_ptr->next->lowername) < 0) {
+
+ is_looking = FALSE;
+
+ if (value == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "trying to remove absent node");
+ }
+ else {
+#if S3COMMS_DEBUG
+HDprintf("INSERT A NODE\n"); fflush(stdout);
+#endif
+ /*******************
+ * INSERT NEW NODE *
+ *******************/
+
+ HDassert( strcmp(lowername, node_ptr->lowername) > 0 );
+ new_node->name = namecpy;
+ new_node->value = valuecpy;
+ new_node->lowername = lowername;
+ new_node->cat = nvcat;
+ new_node->next = node_ptr->next;
+ node_ptr->next = new_node;
+ }
+ }
+ else
+ if (strcmp(lowername, node_ptr->next->lowername) == 0) {
+
+ is_looking = FALSE;
+
+ if (value == NULL) {
+ /*****************
+ * REMOVE A NODE *
+ *****************/
+
+ hrb_node_t *tmp = node_ptr->next;
+ node_ptr->next = tmp->next;
+
+#if S3COMMS_DEBUG
+HDprintf("REMOVE A NODE\n"); fflush(stdout);
+#endif
+ H5MM_xfree(tmp->cat);
+ H5MM_xfree(tmp->lowername);
+ H5MM_xfree(tmp->name);
+ H5MM_xfree(tmp->value);
+
+ HDassert( tmp->magic == S3COMMS_HRB_NODE_MAGIC );
+ tmp->magic += 1ul;
+ H5MM_xfree(tmp);
+
+ H5MM_xfree(lowername);
+ lowername = NULL;
+ }
+ else {
+#if S3COMMS_DEBUG
+HDprintf("MODIFY A NODE\n"); fflush(stdout);
+#endif
+ /*****************
+ * MODIFY A NODE *
+ *****************/
+
+ node_ptr = node_ptr->next;
+ H5MM_xfree(node_ptr->name);
+ H5MM_xfree(node_ptr->value);
+ H5MM_xfree(node_ptr->cat);
+
+ HDassert( new_node->magic == S3COMMS_HRB_NODE_MAGIC );
+ new_node->magic += 1ul;
+ H5MM_xfree(new_node);
+ H5MM_xfree(lowername);
+ new_node = NULL;
+ lowername = NULL;
+
+ node_ptr->name = namecpy;
+ node_ptr->value = valuecpy;
+ node_ptr->cat = nvcat;
+ }
+ }
+ else {
+ /****************
+ * KEEP LOOKING *
+ ****************/
+
+ node_ptr = node_ptr->next;
+ }
+ } /* end while is_looking */
+
+done:
+ if (ret_value == FAIL) {
+ /* clean up
+ */
+ if (nvcat != NULL) { H5MM_xfree(nvcat); }
+ if (namecpy != NULL) { H5MM_xfree(namecpy); }
+ if (lowername != NULL) { H5MM_xfree(lowername); }
+ if (valuecpy != NULL) { H5MM_xfree(valuecpy); }
+ if (new_node != NULL) {
+ HDassert( new_node->magic == S3COMMS_HRB_NODE_MAGIC );
+ new_node->magic += 1ul;
+ H5MM_xfree(new_node);
+ }
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD_s3comms_hrb_node_set() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_hrb_destroy()
+ *
+ * Purpose:
+ *
+ * Destroy and free resources _directly_ associated with an HTTP Buffer.
+ *
+ * Takes a pointer to pointer to the buffer structure.
+ * This allows for the pointer itself to be NULLed from within the call.
+ *
+ * If buffer or buffer pointer is NULL, there is no effect.
+ *
+ * Headers list at `first_header` is not touched.
+ *
+ * - Programmer should re-use or destroy `first_header` pointer
+ * (hrb_node_t *) as suits their purposes.
+ * - Recommend fetching prior to destroy()
+ * e.g., `reuse_node = hrb_to_die->first_header; destroy(hrb_to_die);`
+ * or maintaining an external reference.
+ * - Destroy node/list separately as appropriate
+ * - Failure to account for this will result in a memory leak.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - successfully released buffer resources
+ * - if `buf` is NULL or `*buf` is NULL, no effect
+ * - FAILURE: `FAIL`
+ * - `buf->magic != S3COMMS_HRB_MAGIC`
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-21
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_hrb_destroy(hrb_t **_buf)
+{
+ hrb_t *buf = NULL;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_hrb_destroy.\n");
+#endif
+
+ if (_buf != NULL && *_buf != NULL) {
+ buf = *_buf;
+ if (buf->magic != S3COMMS_HRB_MAGIC) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "pointer's magic does not match.\n");
+ }
+
+ H5MM_xfree(buf->verb);
+ H5MM_xfree(buf->version);
+ H5MM_xfree(buf->resource);
+ buf->magic += 1ul;
+ H5MM_xfree(buf);
+ *_buf = NULL;
+ } /* end if `_buf` has some value */
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_s3comms_hrb_destroy() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_hrb_init_request()
+ *
+ * Purpose:
+ *
+ * Create a new HTTP Request Buffer
+ *
+ * All non-null arguments should be null-terminated strings.
+ *
+ * If `verb` is NULL, defaults to "GET".
+ * If `http_version` is NULL, defaults to "HTTP/1.1".
+ *
+ * `resource` cannot be NULL; should be string beginning with slash
+ * character ('/').
+ *
+ * All strings are copied into the structure, making them safe from
+ * modification in source strings.
+ *
+ * Return:
+ *
+ * - SUCCESS: pointer to new `hrb_t`
+ * - FAILURE: `NULL`
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-21
+ *
+ *----------------------------------------------------------------------------
+ */
+hrb_t *
+H5FD_s3comms_hrb_init_request(const char *_verb,
+ const char *_resource,
+ const char *_http_version)
+{
+ hrb_t *request = NULL;
+ char *res = NULL;
+ size_t reslen = 0;
+ hrb_t *ret_value = NULL;
+ char *verb = NULL;
+ size_t verblen = 0;
+ char *vrsn = NULL;
+ size_t vrsnlen = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_hrb_init_request.\n");
+#endif
+
+ if (_resource == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "resource string cannot be null.\n");
+ }
+
+ /* populate valid NULLs with defaults
+ */
+ if (_verb == NULL) {
+ _verb = "GET";
+ }
+
+ if (_http_version == NULL) {
+ _http_version = "HTTP/1.1";
+ }
+
+ /* malloc space for and prepare structure
+ */
+ request = (hrb_t *)H5MM_malloc(sizeof(hrb_t));
+ if (request == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
+ "no space for request structure");
+ }
+ request->magic = S3COMMS_HRB_MAGIC;
+ request->body = NULL;
+ request->body_len = 0;
+ request->first_header = NULL;
+
+
+
+ /* malloc and copy strings for the structure
+ */
+ reslen = HDstrlen(_resource);
+
+ if (_resource[0] == '/') {
+ res = (char *)H5MM_malloc(sizeof(char) * (reslen+1));
+ if (res == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
+ "no space for resource string");
+ }
+ HDmemcpy(res, _resource, (reslen+1));
+ }
+ else {
+ res = (char *)H5MM_malloc(sizeof(char) * (reslen+2));
+ if (res == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
+ "no space for resource string");
+ }
+ *res = '/';
+ HDmemcpy((&res[1]), _resource, (reslen+1));
+ HDassert( (reslen+1) == HDstrlen(res) );
+ } /* end if (else resource string not starting with '/') */
+
+ verblen = HDstrlen(_verb) + 1;
+ verb = (char *)H5MM_malloc(sizeof(char) * verblen);
+ if (verb == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "no space for verb string");
+ }
+ HDstrncpy(verb, _verb, verblen);
+
+ vrsnlen = HDstrlen(_http_version) + 1;
+ vrsn = (char *)H5MM_malloc(sizeof(char) * vrsnlen);
+ if (vrsn == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "no space for http-version string");
+ }
+ HDstrncpy(vrsn, _http_version, vrsnlen);
+
+
+
+ /* place new copies into structure
+ */
+ request->resource = res;
+ request->verb = verb;
+ request->version = vrsn;
+
+ ret_value = request;
+
+done:
+ /* if there is an error, clean up after ourselves
+ */
+ if (ret_value == NULL) {
+ if (request != NULL) H5MM_xfree(request);
+ if (vrsn != NULL) H5MM_xfree(vrsn);
+ if (verb != NULL) H5MM_xfree(verb);
+ if (res != NULL) H5MM_xfree(res);
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_s3comms_hrb_init_request() */
+
+
+/****************************************************************************
+ * S3R FUNCTIONS
+ ****************************************************************************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_s3r_close()
+ *
+ * Purpose:
+ *
+ * Close communications through given S3 Request Handle (`s3r_t`)
+ * and clean up associated resources.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - fails if handle is null or has invalid magic number
+ *
+ *
+ * Programmer: Jacob Smith
+ * 2017-08-31
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_s3r_close(s3r_t *handle)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_s3r_close.\n");
+#endif
+
+ if (handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle cannot be null.\n");
+ }
+ if (handle->magic != S3COMMS_S3R_MAGIC) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has invalid magic.\n");
+ }
+
+ curl_easy_cleanup(handle->curlhandle);
+
+ H5MM_xfree(handle->secret_id);
+ H5MM_xfree(handle->region);
+ H5MM_xfree(handle->signing_key);
+
+ HDassert( handle->httpverb != NULL );
+ H5MM_xfree(handle->httpverb);
+
+ if (FAIL == H5FD_s3comms_free_purl(handle->purl)) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to release parsed url structure")
+ }
+
+ H5MM_xfree(handle);
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_s3comms_s3r_close */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_s3r_get_filesize()
+ *
+ * Purpose:
+ *
+ * Retrieve the filesize of an open request handle.
+ *
+ * Wrapper "getter" to hide implementation details.
+ *
+ *
+ * Return:
+ *
+ * - SUCCESS: size of file, in bytes, if handle is valid.
+ * - FAILURE: 0, if handle is NULL or undefined.
+ *
+ * Programmer: Jacob Smith 2017-01-14
+ *
+ *----------------------------------------------------------------------------
+ */
+size_t
+H5FD_s3comms_s3r_get_filesize(s3r_t *handle)
+{
+ size_t ret_value = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+ if (handle != NULL) {
+ ret_value = handle->filesize;
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_s3comms_s3r_get_filesize */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_s3r_getsize()
+ *
+ * Purpose:
+ *
+ * Get the number of bytes of handle's target resource.
+ *
+ * Sets handle and curlhandle with to enact an HTTP HEAD request on file,
+ * and parses received headers to extract "Content-Length" from response
+ * headers, storing file size at `handle->filesize`.
+ *
+ * Critical step in opening (initiating) an `s3r_t` handle.
+ *
+ * Wraps `s3r_read()`.
+ * Sets curlhandle to write headers to a temporary buffer (using extant
+ * write callback) and provides no buffer for body.
+ *
+ * Upon exit, unsets HTTP HEAD settings from curl handle, returning to
+ * initial state. In event of error, curl handle state is undefined and is
+ * not to be trusted.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ *
+ * Programmer: Jacob Smith
+ * 2017-08-23
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_s3r_getsize(s3r_t *handle)
+{
+ uintmax_t content_length = 0;
+ CURL *curlh = NULL;
+ char *end = NULL;
+ char *headerresponse = NULL;
+ struct s3r_datastruct sds = {
+ S3COMMS_CALLBACK_DATASTRUCT_MAGIC,
+ NULL,
+ 0 };
+ char *start = NULL;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_s3r_getsize.\n");
+#endif
+
+ if (handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle cannot be null.\n");
+ }
+ if (handle->magic != S3COMMS_S3R_MAGIC) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has invalid magic.\n");
+ }
+ if (handle->curlhandle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has bad (null) curlhandle.\n")
+ }
+
+ /********************
+ * PREPARE FOR HEAD *
+ ********************/
+
+ curlh = handle->curlhandle;
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_NOBODY,
+ 1L) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "error while setting CURL option (CURLOPT_NOBODY). "
+ "(placeholder flags)");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_HEADERDATA,
+ &sds) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "error while setting CURL option (CURLOPT_HEADERDATA). "
+ "(placeholder flags)");
+ }
+
+ HDassert( handle->httpverb == NULL );
+ handle->httpverb = (char *)H5MM_malloc(sizeof(char) * 16);
+ if (handle->httpverb == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "unable to allocate space for S3 request HTTP verb");
+ }
+ HDmemcpy(handle->httpverb, "HEAD", 5);
+
+ headerresponse = (char *)H5MM_malloc(sizeof(char) * CURL_MAX_HTTP_HEADER);
+ if (headerresponse == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "unable to allocate space for curl header response");
+ }
+ sds.data = headerresponse;
+
+ /*******************
+ * PERFORM REQUEST *
+ *******************/
+
+ /* these parameters fetch the entire file,
+ * but, with a NULL destination and NOBODY and HEADERDATA supplied above,
+ * only http metadata will be sent by server and recorded by s3comms
+ */
+ if (FAIL ==
+ H5FD_s3comms_s3r_read(handle, 0, 0, NULL) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem in reading during getsize.\n");
+ }
+
+ if (sds.size > CURL_MAX_HTTP_HEADER) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "HTTP metadata buffer overrun\n");
+ } else if (sds.size == 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "No HTTP metadata\n");
+#if S3COMMS_DEBUG
+ } else {
+ HDfprintf(stderr, "GETSIZE: OK\n");
+#endif
+ }
+
+
+ /******************
+ * PARSE RESPONSE *
+ ******************/
+
+ start = strstr(headerresponse,
+ "\r\nContent-Length: ");
+ if (start == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not find \"Content-Length\" in response.\n");
+ }
+
+ /* move "start" to beginning of value in line; find end of line
+ */
+ start = start + HDstrlen("\r\nContent-Length: ");
+ end = strstr(start, "\r\n");
+ if (end == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not find end of content length line");
+ }
+
+ /* place null terminator at end of numbers
+ */
+ *end = '\0';
+
+ content_length = strtoumax((const char *)start,
+ NULL,
+ 0);
+
+ if (UINTMAX_MAX > SIZE_MAX && content_length > SIZE_MAX) {
+ HGOTO_ERROR(H5E_ARGS, H5E_OVERFLOW, FAIL, "content_length overflows size_t\n");
+ }
+
+ if (content_length == 0 ||
+ errno == ERANGE) /* errno set by strtoumax*/
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not convert found \"Content-Length\" response (\"%s\")",
+ start); /* range is null-terminated, remember */
+ }
+
+ handle->filesize = (size_t)content_length;
+
+ /**********************
+ * UNDO HEAD SETTINGS *
+ **********************/
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_NOBODY,
+ NULL) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "error while setting CURL option (CURLOPT_NOBODY). "
+ "(placeholder flags)");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_HEADERDATA,
+ NULL) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "error while setting CURL option (CURLOPT_HEADERDATA). "
+ "(placeholder flags)");
+ }
+
+done:
+ H5MM_xfree(headerresponse);
+ sds.magic += 1; /* set to bad magic */
+
+ FUNC_LEAVE_NOAPI(ret_value);
+
+} /* H5FD_s3comms_s3r_getsize */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_s3r_open()
+ *
+ * Purpose:
+ *
+ * Logically 'open' a file hosted on S3.
+ *
+ * - create new Request Handle
+ * - copy supplied url
+ * - copy authentication info if supplied
+ * - create CURL handle
+ * - fetch size of file
+ * - connect with server and execute HEAD request
+ * - return request handle ready for reads
+ *
+ * To use 'default' port to connect, `port` should be 0.
+ *
+ * To prevent AWS4 authentication, pass null pointer to `region`, `id`,
+ * and `signing_key`.
+ *
+ * Uses `H5FD_s3comms_parse_url()` to validate and parse url input.
+ *
+ * Return:
+ *
+ * - SUCCESS: Pointer to new request handle.
+ * - FAILURE: NULL
+ * - occurs if:
+ * - authentication strings are inconsistent
+ * - must _all_ be null, or have at least `region` and `id`
+ * - url is NULL (no filename)
+ * - unable to parse url (malformed?)
+ * - error while performing `getsize()`
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-01
+ *
+ *----------------------------------------------------------------------------
+ */
+s3r_t *
+H5FD_s3comms_s3r_open(const char *url,
+ const char *region,
+ const char *id,
+ const unsigned char *signing_key)
+{
+ size_t tmplen = 0;
+ CURL *curlh = NULL;
+ s3r_t *handle = NULL;
+ parsed_url_t *purl = NULL;
+ s3r_t *ret_value = NULL;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_s3r_open.\n");
+#endif
+
+
+
+ if (url == NULL || url[0] == '\0') {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "url cannot be null.\n");
+ }
+
+ if (FAIL == H5FD_s3comms_parse_url(url, &purl)) {
+ /* probably a malformed url, but could be internal error */
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCREATE, NULL,
+ "unable to create parsed url structure");
+ }
+ HDassert( purl != NULL ); /* if above passes, this must be true */
+ HDassert( purl->magic == S3COMMS_PARSED_URL_MAGIC );
+
+ handle = (s3r_t *)H5MM_malloc(sizeof(s3r_t));
+ if (handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, NULL,
+ "could not malloc space for handle.\n");
+ }
+
+ handle->magic = S3COMMS_S3R_MAGIC;
+ handle->purl = purl;
+ handle->filesize = 0;
+ handle->region = NULL;
+ handle->secret_id = NULL;
+ handle->signing_key = NULL;
+ handle->httpverb = NULL;
+
+ /*************************************
+ * RECORD AUTHENTICATION INFORMATION *
+ *************************************/
+
+ if ((region != NULL && *region != '\0') ||
+ (id != NULL && *id != '\0') ||
+ (signing_key != NULL && *signing_key != '\0'))
+ {
+ /* if one exists, all three must exist
+ */
+ if (region == NULL || region[0] == '\0') {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "region cannot be null.\n");
+ }
+ if (id == NULL || id[0] == '\0') {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "secret id cannot be null.\n");
+ }
+ if (signing_key == NULL || signing_key[0] == '\0') {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "signing key cannot be null.\n");
+ }
+
+ /* copy strings
+ */
+ tmplen = HDstrlen(region) + 1;
+ handle->region = (char *)H5MM_malloc(sizeof(char) * tmplen);
+ if (handle->region == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "could not malloc space for handle region copy.\n");
+ }
+ HDmemcpy(handle->region, region, tmplen);
+
+ tmplen = HDstrlen(id) + 1;
+ handle->secret_id = (char *)H5MM_malloc(sizeof(char) * tmplen);
+ if (handle->secret_id == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "could not malloc space for handle ID copy.\n");
+ }
+ HDmemcpy(handle->secret_id, id, tmplen);
+
+ tmplen = SHA256_DIGEST_LENGTH;
+ handle->signing_key =
+ (unsigned char *)H5MM_malloc(sizeof(unsigned char) * tmplen);
+ if (handle->signing_key == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "could not malloc space for handle key copy.\n");
+ }
+ HDmemcpy(handle->signing_key, signing_key, tmplen);
+ } /* if authentication information provided */
+
+ /************************
+ * INITIATE CURL HANDLE *
+ ************************/
+
+ curlh = curl_easy_init();
+
+ if (curlh == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "problem creating curl easy handle!\n");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_HTTPGET,
+ 1L) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "error while setting CURL option (CURLOPT_HTTPGET). "
+ "(placeholder flags)");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_HTTP_VERSION,
+ CURL_HTTP_VERSION_1_1) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "error while setting CURL option (CURLOPT_HTTP_VERSION). "
+ "(placeholder flags)");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_FAILONERROR,
+ 1L) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "error while setting CURL option (CURLOPT_FAILONERROR). "
+ "(placeholder flags)");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_WRITEFUNCTION,
+ curlwritecallback) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "error while setting CURL option (CURLOPT_WRITEFUNCTION). "
+ "(placeholder flags)");
+ }
+
+ if ( CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_URL,
+ url) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "error while setting CURL option (CURLOPT_URL). "
+ "(placeholder flags)");
+ }
+
+#if S3COMMS_CURL_VERBOSITY > 1
+ /* CURL will print (to stdout) information for each operation
+ */
+ curl_easy_setopt(curlh, CURLOPT_VERBOSE, 1L);
+#endif
+
+ handle->curlhandle = curlh;
+
+ /*******************
+ * OPEN CONNECTION *
+ * * * * * * * * * *
+ * GET FILE SIZE *
+ *******************/
+
+ if (FAIL ==
+ H5FD_s3comms_s3r_getsize(handle) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "problem in H5FD_s3comms_s3r_getsize.\n");
+ }
+
+ /*********************
+ * FINAL PREPARATION *
+ *********************/
+
+ HDassert( handle->httpverb != NULL );
+ HDmemcpy(handle->httpverb, "GET", 4);
+
+ ret_value = handle;
+
+done:
+ if (ret_value == NULL) {
+ if (curlh != NULL) {
+ curl_easy_cleanup(curlh);
+ }
+ if (FAIL == H5FD_s3comms_free_purl(purl)) {
+ HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, NULL,
+ "unable to free parsed url structure")
+ }
+ if (handle != NULL) {
+ H5MM_xfree(handle->region);
+ H5MM_xfree(handle->secret_id);
+ H5MM_xfree(handle->signing_key);
+ if (handle->httpverb != NULL) {
+ H5MM_xfree(handle->httpverb);
+ }
+ H5MM_xfree(handle);
+ }
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+
+} /* H5FD_s3comms_s3r_open */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_s3r_read()
+ *
+ * Purpose:
+ *
+ * Read file pointed to by request handle, writing specified
+ * `offset` .. `offset + len` bytes to buffer `dest`.
+ *
+ * If `len` is 0, reads entirety of file starting at `offset`.
+ * If `offset` and `len` are both 0, reads entire file.
+ *
+ * If `offset` or `offset+len` is greater than the file size, read is
+ * aborted and returns `FAIL`.
+ *
+ * Uses configured "curl easy handle" to perform request.
+ *
+ * In event of error, buffer should remain unaltered.
+ *
+ * If handle is set to authorize a request, creates a new (temporary)
+ * HTTP Request object (hrb_t) for generating requisite headers,
+ * which is then translated to a `curl slist` and set in the curl handle
+ * for the request.
+ *
+ * `dest` _may_ be NULL, but no body data will be recorded.
+ *
+ * - In general practice, NULL should never be passed in as `dest`.
+ * - NULL `dest` passed in by internal function `s3r_getsize()`, in
+ * conjunction with CURLOPT_NOBODY to preempt transmission of file data
+ * from server.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ *
+ * Programmer: Jacob Smith
+ * 2017-08-22
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_s3r_read(s3r_t *handle,
+ haddr_t offset,
+ size_t len,
+ void *dest)
+{
+ CURL *curlh = NULL;
+ CURLcode p_status = CURLE_OK;
+ struct curl_slist *curlheaders = NULL;
+ hrb_node_t *headers = NULL;
+ hrb_node_t *node = NULL;
+ struct tm *now = NULL;
+ char *rangebytesstr = NULL;
+ hrb_t *request = NULL;
+ int ret = 0; /* working variable to check */
+ /* return value of HDsnprintf */
+ struct s3r_datastruct *sds = NULL;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_s3r_read.\n");
+#endif
+
+ /**************************************
+ * ABSOLUTELY NECESSARY SANITY-CHECKS *
+ **************************************/
+
+ if (handle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle cannot be null.\n");
+ }
+ if (handle->magic != S3COMMS_S3R_MAGIC) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has invalid magic.\n");
+ }
+ if (handle->curlhandle == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has bad (null) curlhandle.\n")
+ }
+ if (handle->purl == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle has bad (null) url.\n")
+ }
+ HDassert( handle->purl->magic == S3COMMS_PARSED_URL_MAGIC );
+ if (offset > handle->filesize || (len + offset) > handle->filesize) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to read past EoF")
+ }
+
+ curlh = handle->curlhandle;
+
+ /*********************
+ * PREPARE WRITEDATA *
+ *********************/
+
+ if (dest != NULL) {
+ sds = (struct s3r_datastruct *)H5MM_malloc(
+ sizeof(struct s3r_datastruct));
+ if (sds == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "could not malloc destination datastructure.\n");
+ }
+
+ sds->magic = S3COMMS_CALLBACK_DATASTRUCT_MAGIC;
+ sds->data = (char *)dest;
+ sds->size = 0;
+ if (CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_WRITEDATA,
+ sds) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_UNINITIALIZED, FAIL,
+ "error while setting CURL option (CURLOPT_WRITEDATA). "
+ "(placeholder flags)");
+ }
+ }
+
+ /*********************
+ * FORMAT HTTP RANGE *
+ *********************/
+
+ if (len > 0) {
+ rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \
+ (S3COMMS_MAX_RANGE_STRING_SIZE+1) );
+ if (rangebytesstr == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "could not malloc range format string.\n");
+ }
+ ret = HDsnprintf(rangebytesstr,
+ (S3COMMS_MAX_RANGE_STRING_SIZE),
+ "bytes="H5_PRINTF_HADDR_FMT"-"H5_PRINTF_HADDR_FMT,
+ offset,
+ offset + len - 1);
+ if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to format HTTP Range value");
+ } else if (offset > 0) {
+ rangebytesstr = (char *)H5MM_malloc(sizeof(char) * \
+ (S3COMMS_MAX_RANGE_STRING_SIZE+1));
+ if (rangebytesstr == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "could not malloc range format string.\n");
+ }
+ ret = HDsnprintf(rangebytesstr,
+ (S3COMMS_MAX_RANGE_STRING_SIZE),
+ "bytes="H5_PRINTF_HADDR_FMT"-",
+ offset);
+ if (ret <= 0 || ret >= S3COMMS_MAX_RANGE_STRING_SIZE)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to format HTTP Range value");
+ }
+
+ /*******************
+ * COMPILE REQUEST *
+ *******************/
+
+ if (handle->signing_key == NULL) {
+ /* Do not authenticate.
+ */
+ if (rangebytesstr != NULL) {
+ /* Pass in range directly
+ */
+ char *bytesrange_ptr = NULL; /* pointer past "bytes=" portion */
+
+ bytesrange_ptr = strchr(rangebytesstr, '=');
+ HDassert( bytesrange_ptr != NULL );
+ bytesrange_ptr++; /* move to first char past '=' */
+ HDassert( *bytesrange_ptr != '\0' );
+
+ if (CURLE_OK !=
+ curl_easy_setopt(curlh,
+ CURLOPT_RANGE,
+ bytesrange_ptr) )
+ {
+ HGOTO_ERROR(H5E_VFL, H5E_UNINITIALIZED, FAIL,
+ "error while setting CURL option (CURLOPT_RANGE). ");
+ }
+ }
+ } else {
+ /* authenticate request
+ */
+ char authorization[512+1];
+ /* 512 := approximate max length...
+ * 67 <len("AWS4-HMAC-SHA256 Credential=///s3/aws4_request,"
+ * "SignedHeaders=,Signature=")>
+ * + 8 <yyyyMMDD>
+ * + 64 <hex(sha256())>
+ * + 128 <max? len(secret_id)>
+ * + 20 <max? len(region)>
+ * + 128 <max? len(signed_headers)>
+ */
+ char buffer1[512+1]; /* -> Canonical Request -> Signature */
+ char buffer2[256+1]; /* -> String To Sign -> Credential */
+ char iso8601now[ISO8601_SIZE];
+ char signed_headers[48+1];
+ /* should be large enough for nominal listing:
+ * "host;range;x-amz-content-sha256;x-amz-date"
+ * + '\0', with "range;" possibly absent
+ */
+
+ /* zero start of strings */
+ authorization[0] = 0;
+ buffer1[0] = 0;
+ buffer2[0] = 0;
+ iso8601now[0] = 0;
+ signed_headers[0] = 0;
+
+ /**** VERIFY INFORMATION EXISTS ****/
+
+ if (handle->region == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle must have non-null region.\n");
+ }
+ if (handle->secret_id == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle must have non-null secret_id.\n");
+ }
+ if (handle->signing_key == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle must have non-null signing_key.\n");
+ }
+ if (handle->httpverb == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle must have non-null httpverb.\n");
+ }
+ if (handle->purl->host == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle must have non-null host.\n");
+ }
+ if (handle->purl->path == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "handle must have non-null resource.\n");
+ }
+
+ /**** CREATE HTTP REQUEST STRUCTURE (hrb_t) ****/
+
+ request = H5FD_s3comms_hrb_init_request(
+ (const char *)handle->httpverb,
+ (const char *)handle->purl->path,
+ "HTTP/1.1");
+ if (request == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not allocate hrb_t request.\n");
+ }
+ HDassert( request->magic == S3COMMS_HRB_MAGIC );
+
+ now = gmnow();
+ if (ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1)) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not format ISO8601 time.\n");
+ }
+
+ if (FAIL ==
+ H5FD_s3comms_hrb_node_set(
+ &headers,
+ "x-amz-date",
+ (const char *)iso8601now) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to set x-amz-date header")
+ }
+ if (headers == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem building headers list. "
+ "(placeholder flags)\n");
+ }
+ HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC );
+
+ if (FAIL ==
+ H5FD_s3comms_hrb_node_set(
+ &headers,
+ "x-amz-content-sha256",
+ (const char *)EMPTY_SHA256) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to set x-amz-content-sha256 header")
+ }
+ if (headers == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem building headers list. "
+ "(placeholder flags)\n");
+ }
+ HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC );
+
+ if (rangebytesstr != NULL) {
+ if (FAIL ==
+ H5FD_s3comms_hrb_node_set(
+ &headers,
+ "Range",
+ (const char *)rangebytesstr) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to set range header")
+ }
+ if (headers == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem building headers list. "
+ "(placeholder flags)\n");
+ }
+ HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC );
+ }
+
+ if (FAIL ==
+ H5FD_s3comms_hrb_node_set(
+ &headers,
+ "Host",
+ (const char *)handle->purl->host) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to set host header")
+ }
+ if (headers == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem building headers list. "
+ "(placeholder flags)\n");
+ }
+ HDassert( headers->magic == S3COMMS_HRB_NODE_MAGIC );
+
+ request->first_header = headers;
+
+ /**** COMPUTE AUTHORIZATION ****/
+
+ if (FAIL == /* buffer1 -> canonical request */
+ H5FD_s3comms_aws_canonical_request(
+ buffer1,
+ 512,
+ signed_headers,
+ 48,
+ request) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "(placeholder flags)\n");
+ }
+ if ( FAIL == /* buffer2->string-to-sign */
+ H5FD_s3comms_tostringtosign(buffer2,
+ buffer1,
+ iso8601now,
+ handle->region) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "(placeholder flags)\n");
+ }
+ if (FAIL == /* buffer1 -> signature */
+ H5FD_s3comms_HMAC_SHA256(handle->signing_key,
+ SHA256_DIGEST_LENGTH,
+ buffer2,
+ HDstrlen(buffer2),
+ buffer1) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "(placeholder flags)\n");
+ }
+
+ iso8601now[8] = 0; /* trim to yyyyMMDD */
+ ret = S3COMMS_FORMAT_CREDENTIAL(buffer2,
+ handle->secret_id,
+ iso8601now,
+ handle->region,
+ "s3");
+ if (ret == 0 || ret >= S3COMMS_MAX_CREDENTIAL_SIZE)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to format aws4 credential string");
+
+ ret = HDsnprintf(
+ authorization,
+ 512,
+ "AWS4-HMAC-SHA256 Credential=%s,SignedHeaders=%s,Signature=%s",
+ buffer2,
+ signed_headers,
+ buffer1);
+ if (ret <= 0 || ret >= 512) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to format aws4 authorization string");
+ }
+
+ /* append authorization header to http request buffer
+ */
+ if (H5FD_s3comms_hrb_node_set(
+ &headers,
+ "Authorization",
+ (const char *)authorization)
+ == FAIL)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to set Authorization header")
+ }
+ if (headers == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem building headers list. "
+ "(placeholder flags)\n");
+ }
+
+ /* update hrb's "first header" pointer
+ */
+ request->first_header = headers;
+
+ /**** SET CURLHANDLE HTTP HEADERS FROM GENERATED DATA ****/
+
+ node = request->first_header;
+ while (node != NULL) {
+ HDassert( node->magic == S3COMMS_HRB_NODE_MAGIC );
+ curlheaders = curl_slist_append(curlheaders,
+ (const char *)node->cat);
+ if (curlheaders == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not append header to curl slist. "
+ "(placeholder flags)\n");
+ }
+ node = node->next;
+ }
+
+ /* sanity-check
+ */
+ if (curlheaders == NULL) {
+ /* above loop was probably never run */
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "curlheaders was never populated.\n");
+ }
+
+ /* finally, set http headers in curl handle
+ */
+ if (curl_easy_setopt(
+ curlh,
+ CURLOPT_HTTPHEADER,
+ curlheaders)
+ != CURLE_OK)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "error while setting CURL option "
+ "(CURLOPT_HTTPHEADER). (placeholder flags)");
+ }
+
+ } /* end if should authenticate (info provided) */
+
+ /*******************
+ * PERFORM REQUEST *
+ *******************/
+
+#if S3COMMS_CURL_VERBOSITY > 0
+ /* In event of error, print detailed information to stderr
+ * This is not the default behavior.
+ */
+ {
+ long int httpcode = 0;
+ char curlerrbuf[CURL_ERROR_SIZE];
+ curlerrbuf[0] = '\0';
+
+ if (CURLE_OK !=
+ curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, curlerrbuf) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem setting error buffer")
+ }
+
+ p_status = curl_easy_perform(curlh);
+
+ if (p_status != CURLE_OK) {
+ if (CURLE_OK !=
+ curl_easy_getinfo(curlh, CURLINFO_RESPONSE_CODE, &httpcode) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem getting response code")
+ }
+ HDfprintf(stderr, "CURL ERROR CODE: %d\nHTTP CODE: %d\n",
+ p_status, httpcode);
+ HDfprintf(stderr, "%s\n", curl_easy_strerror(p_status));
+ HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL,
+ "problem while performing request.\n");
+ }
+ if (CURLE_OK !=
+ curl_easy_setopt(curlh, CURLOPT_ERRORBUFFER, NULL) )
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem unsetting error buffer")
+ }
+ } /* verbose error reporting */
+#else
+ p_status = curl_easy_perform(curlh);
+
+ if (p_status != CURLE_OK) {
+ HGOTO_ERROR(H5E_VFL, H5E_CANTOPENFILE, FAIL,
+ "curl cannot perform request\n")
+ }
+#endif
+
+#if S3COMMS_DEBUG
+ if (dest != NULL) {
+ HDfprintf(stderr, "len: %d\n", (int)len);
+ HDfprintf(stderr, "CHECKING FOR BUFFER OVERFLOW\n");
+ if (sds == NULL) {
+ HDfprintf(stderr, "sds is NULL!\n");
+ }
+ else {
+ HDfprintf(stderr, "sds: 0x%lx\n", (long long)sds);
+ HDfprintf(stderr, "sds->size: %d\n", (int)sds->size);
+ if (len > sds->size) {
+ HDfprintf(stderr, "buffer overwrite\n");
+ }
+ }
+ }
+ else {
+ HDfprintf(stderr, "performed on entire file\n");
+ }
+#endif
+
+done:
+ /* clean any malloc'd resources
+ */
+ if (curlheaders != NULL) {
+ curl_slist_free_all(curlheaders);
+ curlheaders = NULL;
+ }
+ if (rangebytesstr != NULL) {
+ H5MM_xfree(rangebytesstr);
+ rangebytesstr = NULL;
+ }
+ if (sds != NULL) {
+ H5MM_xfree(sds);
+ sds = NULL;
+ }
+ if (request != NULL) {
+ while (headers != NULL)
+ if (FAIL ==
+ H5FD_s3comms_hrb_node_set(&headers, headers->name, NULL))
+ {
+ HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot release header node")
+ }
+ HDassert( NULL == headers );
+ if (FAIL == H5FD_s3comms_hrb_destroy(&request)) {
+ HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot release header request structure")
+ }
+ HDassert( NULL == request );
+ }
+
+ if (curlh != NULL) {
+ /* clear any Range */
+ if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_RANGE, NULL) ) {
+ HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot unset CURLOPT_RANGE")
+ }
+
+ /* clear headers */
+ if (CURLE_OK != curl_easy_setopt(curlh, CURLOPT_HTTPHEADER, NULL) ) {
+ HDONE_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot unset CURLOPT_HTTPHEADER")
+ }
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* H5FD_s3comms_s3r_read */
+
+
+/****************************************************************************
+ * MISCELLANEOUS FUNCTIONS
+ ****************************************************************************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: gmnow()
+ *
+ * Purpose:
+ *
+ * Get the output of `time.h`'s `gmtime()` call while minimizing setup
+ * clutter where important.
+ *
+ * Return:
+ *
+ * Pointer to resulting `struct tm`,as created by gmtime(time_t * T).
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-12
+ *
+ *----------------------------------------------------------------------------
+ */
+struct tm *
+gmnow(void)
+{
+ time_t now;
+ time_t *now_ptr = &now;
+ struct tm *ret_value = NULL;
+
+ /* Doctor assert, checks against error in time() */
+ if ( (time_t)(-1) != time(now_ptr) ) {
+ ret_value = gmtime(now_ptr);
+ }
+
+ HDassert( ret_value != NULL );
+
+ return ret_value;
+} /* end gmnow() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_aws_canonical_request()
+ *
+ * Purpose:
+ *
+ * Compose AWS "Canonical Request" (and signed headers string)
+ * as defined in the REST API documentation.
+ *
+ * Both destination strings are null-terminated.
+ *
+ * Destination string arguments must be provided with adequate space.
+ *
+ * Canonical Request format:
+ *
+ * <HTTP VERB>"\n"
+ * <resource path>"\n"
+ * <query string>"\n"
+ * <header1>"\n" (`lowercase(name)`":"`trim(value)`)
+ * <header2>"\n"
+ * ... (headers sorted by name)
+ * <header_n>"\n"
+ * "\n"
+ * <signed headers>"\n" (`lowercase(header 1 name)`";"`header 2 name`;...)
+ * <hex-string of sha256sum of body> ("e3b0c4429...", e.g.)
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - writes canonical request to respective `...dest` strings
+ * - FAILURE: `FAIL`
+ * - one or more input argument was NULL
+ * - internal error
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-04
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_aws_canonical_request(
+ char *canonical_request_dest,
+ int _cr_size,
+ char *signed_headers_dest,
+ int _sh_size,
+ hrb_t *http_request)
+{
+ hrb_node_t *node = NULL;
+ const char *query_params = ""; /* unused at present */
+ herr_t ret_value = SUCCEED;
+ int ret = 0;
+ size_t cr_size = (size_t)_cr_size;
+ size_t sh_size = (size_t)_sh_size;
+ size_t cr_len = 0; /* working length of canonical request str */
+ size_t sh_len = 0; /* working length of signed headers str */
+ char tmpstr[256+1];
+ tmpstr[256] = 0; /* terminating NULL */
+
+ /* "query params" refers to the optional element in the URL, e.g.
+ * http://bucket.aws.com/myfile.txt?max-keys=2&prefix=J
+ * ^-----------------^
+ *
+ * Not handled/implemented as of 2017-10-xx.
+ * Element introduced as empty placeholder and reminder.
+ * Further research to be done if this is ever relevant for the
+ * VFD use-cases.
+ */
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_aws_canonical_request.\n");
+#endif
+
+ if (http_request == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "hrb object cannot be null.\n");
+ }
+ HDassert( http_request->magic == S3COMMS_HRB_MAGIC );
+
+ if (canonical_request_dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "canonical request destination cannot be null.\n");
+ }
+
+ if (signed_headers_dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "signed headers destination cannot be null.\n");
+ }
+
+ /* HTTP verb, resource path, and query string lines
+ */
+ cr_len = (HDstrlen(http_request->verb) +
+ HDstrlen(http_request->resource) +
+ HDstrlen(query_params) +
+ (size_t)3); /* three newline chars */
+ if (cr_len >= cr_size) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "not enough space in canonical request");
+ }
+ ret = HDsnprintf( /* TODO: compiler warning */
+ canonical_request_dest,
+ (cr_size-1),
+ "%s\n%s\n%s\n",
+ http_request->verb,
+ http_request->resource,
+ query_params);
+ if (ret < 0 || (size_t)ret >= cr_size) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to compose canonical request first line");
+ }
+
+ /* write in canonical headers, building signed headers concurrently
+ */
+ node = http_request->first_header; /* assumed sorted */
+ while (node != NULL) {
+
+ HDassert(node->magic == S3COMMS_HRB_NODE_MAGIC);
+
+ ret = HDsnprintf(
+ tmpstr,
+ 256,
+ "%s:%s\n",
+ node->lowername,
+ node->value);
+ if (ret < 0 || ret >= 256) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to concatenate HTTP header %s:%s",
+ node->lowername,
+ node->value);
+ }
+ cr_len += HDstrlen(tmpstr);
+ if (cr_len + 1 > cr_size) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "not enough space in canonical request");
+ }
+ strcat(canonical_request_dest, tmpstr);
+
+ ret = HDsnprintf(
+ tmpstr,
+ 256,
+ "%s;",
+ node->lowername);
+ if (ret < 0 || ret >= 256) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to append semicolon to lowername %s",
+ node->lowername);
+ }
+ sh_len += HDstrlen(tmpstr);
+ if (sh_len + 1 > sh_size) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "not enough space in signed headers");
+ }
+ strcat(signed_headers_dest, tmpstr);
+
+ node = node->next;
+ } /* end while node is not NULL */
+
+ /* remove tailing ';' from signed headers sequence
+ */
+ signed_headers_dest[HDstrlen(signed_headers_dest) - 1] = '\0';
+
+ /* append signed headers and payload hash
+ * NOTE: at present, no HTTP body is handled, per the nature of
+ * requests/range-gets
+ */
+ strcat(canonical_request_dest, "\n");
+ strcat(canonical_request_dest, signed_headers_dest);
+ strcat(canonical_request_dest, "\n");
+ strcat(canonical_request_dest, EMPTY_SHA256);
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD_s3comms_aws_canonical_request() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_bytes_to_hex()
+ *
+ * Purpose:
+ *
+ * Produce human-readable hex string [0-9A-F] from sequence of bytes.
+ *
+ * For each byte (char), writes two-character hexadecimal representation.
+ *
+ * No null-terminator appended.
+ *
+ * Assumes `dest` is allocated to enough size (msg_len * 2).
+ *
+ * Fails if either `dest` or `msg` are null.
+ *
+ * `msg_len` message length of 0 has no effect.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - hex string written to `dest` (not null-terminated)
+ * - FAILURE: `FAIL`
+ * - `dest == NULL`
+ * - `msg == NULL`
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-12
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_bytes_to_hex(
+ char *dest,
+ const unsigned char *msg,
+ size_t msg_len,
+ hbool_t lowercase)
+{
+ size_t i = 0;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_bytes_to_hex.\n");
+#endif
+
+ if (dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "hex destination cannot be null.\n")
+ }
+ if (msg == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "bytes sequence cannot be null.\n")
+ }
+
+ for (i = 0; i < msg_len; i++) {
+ int chars_written = HDsnprintf(&(dest[i * 2]),
+ 3, /* 'X', 'X', '\n' */
+ (lowercase == TRUE) ? "%02x" : "%02X",
+ msg[i]);
+ if (chars_written != 2) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem while writing hex chars for %c",
+ msg[i]);
+ }
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+
+} /* end H5FD_s3comms_bytes_to_hex() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_free_purl()
+ *
+ * Purpose:
+ *
+ * Release resources from a parsed_url_t pointer.
+ *
+ * If pointer is null, nothing happens.
+ *
+ * Return:
+ *
+ * `SUCCEED` (never fails)
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-01
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_free_purl(parsed_url_t *purl)
+{
+ FUNC_ENTER_NOAPI_NOINIT_NOERR
+
+#if S3COMMS_DEBUG
+ HDprintf("called H5FD_s3comms_free_purl.\n");
+#endif
+
+ if (purl != NULL) {
+ HDassert( purl->magic == S3COMMS_PARSED_URL_MAGIC );
+ if (purl->scheme != NULL) { H5MM_xfree(purl->scheme); }
+ if (purl->host != NULL) { H5MM_xfree(purl->host); }
+ if (purl->port != NULL) { H5MM_xfree(purl->port); }
+ if (purl->path != NULL) { H5MM_xfree(purl->path); }
+ if (purl->query != NULL) { H5MM_xfree(purl->query); }
+ purl->magic += 1ul;
+ H5MM_xfree(purl);
+ }
+
+ FUNC_LEAVE_NOAPI(SUCCEED)
+} /* end H5FD_s3comms_free_purl() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_HMAC_SHA256()
+ *
+ * Purpose:
+ *
+ * Generate Hash-based Message Authentication Checksum using the SHA-256
+ * hashing algorithm.
+ *
+ * Given a key, message, and respective lengths (to accommodate null
+ * characters in either), generate _hex string_ of authentication checksum
+ * and write to `dest`.
+ *
+ * `dest` must be at least `SHA256_DIGEST_LENGTH * 2` characters in size.
+ * Not enforceable by this function.
+ * `dest` will _not_ be null-terminated by this function.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - hex string written to `dest` (not null-terminated)
+ * - FAILURE: `FAIL`
+ * - `dest == NULL`
+ * - error while generating hex string output
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-??
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_HMAC_SHA256(
+ const unsigned char *key,
+ size_t key_len,
+ const char *msg,
+ size_t msg_len,
+ char *dest)
+{
+ unsigned char md[SHA256_DIGEST_LENGTH];
+ unsigned int md_len = SHA256_DIGEST_LENGTH;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_HMAC_SHA256.\n");
+#endif
+
+ if (dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "destination cannot be null.");
+ }
+
+ HMAC(EVP_sha256(),
+ key,
+ (int)key_len,
+ (const unsigned char *)msg,
+ msg_len,
+ md,
+ &md_len);
+
+ if (H5FD_s3comms_bytes_to_hex(
+ dest,
+ (const unsigned char *)md,
+ (size_t)md_len,
+ true)
+ == FAIL)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not convert to hex string.");
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* H5FD_s3comms_HMAC_SHA256 */
+
+
+/*-----------------------------------------------------------------------------
+ *
+ * Function: H5FD__s3comms_load_aws_creds_from_file()
+ *
+ * Purpose:
+ *
+ * Extract AWS configuration information from a target file.
+ *
+ * Given a file and a profile name, e.g. "ros3_vfd_test", attempt to locate
+ * that region in the file. If not found, returns in error and output
+ * pointers are not modified.
+ *
+ * If the profile label is found, attempts to locate and parse configuration
+ * data, stopping at the first line where:
+ * + reached end of file
+ * + line does not start with a recognized setting name
+ *
+ * Following AWS documentation, looks for any of:
+ * + aws_access_key_id
+ * + aws_secret_access_key
+ * + region
+ *
+ * To be valid, the setting must begin the line with one of the keywords,
+ * followed immediately by an equals sign '=', and have some data before
+ * newline at end of line.
+ * + `spam=eggs` would be INVALID because name is unrecognized
+ * + `region = us-east-2` would be INVALID because of spaces
+ * + `region=` would be INVALID because no data.
+ *
+ * Upon successful parsing of a setting line, will store the result in the
+ * corresponding output pointer. If the output pointer is NULL, will skip
+ * any matching setting line while parsing -- useful to prevent overwrite
+ * when reading from multiple files.
+ *
+ * Return:
+ *
+ * + SUCCESS: `SUCCEED`
+ * + no error. settings may or may not have been loaded.
+ * + FAILURE: `FAIL`
+ * + internal error occurred.
+ * + -1 :: unable to format profile label
+ * + -2 :: profile name/label not found in file
+ * + -3 :: some other error
+ *
+ * Programmer: Jacob Smith
+ * 2018-02-27
+ *
+ *-----------------------------------------------------------------------------
+ */
+static herr_t
+H5FD__s3comms_load_aws_creds_from_file(
+ FILE *file,
+ const char *profile_name,
+ char *key_id,
+ char *access_key,
+ char *aws_region)
+{
+ char profile_line[32];
+ char buffer[128];
+ const char *setting_names[] = {
+ "region",
+ "aws_access_key_id",
+ "aws_secret_access_key",
+ };
+ char * const setting_pointers[] = {
+ aws_region,
+ key_id,
+ access_key,
+ };
+ unsigned setting_count = 3;
+ herr_t ret_value = SUCCEED;
+ unsigned buffer_i = 0;
+ unsigned setting_i = 0;
+ int found_setting = 0;
+ char *line_buffer = &(buffer[0]);
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called load_aws_creds_from_file.\n");
+#endif
+
+ /* format target line for start of profile */
+ if (32 < HDsnprintf(profile_line, 32, "[%s]", profile_name)) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
+ "unable to format profile label")
+ }
+
+ /* look for start of profile */
+ do {
+ /* clear buffer */
+ for (buffer_i=0; buffer_i < 128; buffer_i++) {
+ buffer[buffer_i] = 0;
+ }
+
+ line_buffer = fgets(line_buffer, 128, file);
+ if (line_buffer == NULL) { /* reached end of file */
+ goto done;
+ }
+ } while (strncmp(line_buffer, profile_line, HDstrlen(profile_line)));
+
+ /* extract credentials from lines */
+ do {
+ /* clear buffer */
+ for (buffer_i=0; buffer_i < 128; buffer_i++) {
+ buffer[buffer_i] = 0;
+ }
+
+ /* collect a line from file */
+ line_buffer = fgets(line_buffer, 128, file);
+ if (line_buffer == NULL) {
+ goto done; /* end of file */
+ }
+
+ /* loop over names to see if line looks like assignment */
+ for (setting_i = 0; setting_i < setting_count; setting_i++) {
+ size_t setting_name_len = 0;
+ const char *setting_name = NULL;
+ char line_prefix[128];
+
+ setting_name = setting_names[setting_i];
+ setting_name_len = HDstrlen(setting_name);
+ if (HDsnprintf(line_prefix, 128, "%s=", setting_name) < 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
+ "unable to format line prefix")
+ }
+
+ /* found a matching name? */
+ if (!HDstrncmp(line_buffer, line_prefix, setting_name_len + 1)) {
+ found_setting = 1;
+
+ /* skip NULL destination buffer */
+ if (setting_pointers[setting_i] == NULL) {
+ break;
+ }
+
+ /* advance to end of name in string */
+ do {
+ line_buffer++;
+ } while (*line_buffer != 0 && *line_buffer != '=');
+
+ if (*line_buffer == 0 || *(line_buffer+1) == 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "incomplete assignment in file")
+ }
+ line_buffer++; /* was pointing at '='; advance */
+
+ /* copy line buffer into out pointer */
+ if (HDstrncpy(
+ setting_pointers[setting_i],
+ (const char *)line_buffer,
+ HDstrlen(line_buffer))
+ == NULL)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
+ "unable to copy line into pointer")
+ }
+
+ /* "trim" tailing whitespace by replacing with null terminator*/
+ buffer_i = 0;
+ while (!isspace(setting_pointers[setting_i][buffer_i])) {
+ buffer_i++;
+ }
+ setting_pointers[setting_i][buffer_i] = '\0';
+
+ break; /* have read setting; don't compare with others */
+ } /* end if possible name match */
+ } /* end for each setting name */
+ } while (found_setting);
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD__s3comms_load_aws_creds_from_file() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_load_aws_profile()
+ *
+ * Purpose :
+ *
+ * Read aws profile elements from standard location on system and store
+ * settings in memory.
+ *
+ * Looks for both `~/.aws/config` and `~/.aws/credentials`, the standard
+ * files for AWS tools. If a file exists (can be opened), looks for the
+ * given profile name and reads the settings into the relevant buffer.
+ *
+ * Any setting duplicated in both files will be set to that from
+ * `credentials`.
+ *
+ * Settings are stored in the supplied buffers as null-terminated strings.
+ *
+ * Return:
+ *
+ * + SUCCESS: `SUCCEED` (0)
+ * + no error occurred and all settings were populated
+ * + FAILURE: `FAIL` (-1)
+ * + internal error occurred
+ * + unable to locate profile
+ * + region, key id, and secret key were not all found and set
+ *
+ * Programmer: Jacob Smith
+ * 2018-02-27
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_load_aws_profile(const char *profile_name,
+ char *key_id_out,
+ char *secret_access_key_out,
+ char *aws_region_out)
+{
+ herr_t ret_value = SUCCEED;
+ FILE *credfile = NULL;
+ char awspath[117];
+ char filepath[128];
+ int ret = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_load_aws_profile.\n");
+#endif
+
+#ifdef H5_HAVE_WIN32_API
+ ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("USERPROFILE")) ;
+#else
+ ret = HDsnprintf(awspath, 117, "%s/.aws/", getenv("HOME")) ;
+#endif
+ if (ret < 0 || (size_t)ret >= 117) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
+ "unable to format home-aws path")
+ }
+ ret = HDsnprintf(filepath, 128, "%s%s", awspath, "credentials");
+ if (ret < 0 || (size_t)ret >= 128) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
+ "unable to format credentials path")
+ }
+
+ credfile = fopen(filepath, "r");
+ if (credfile != NULL) {
+ if (H5FD__s3comms_load_aws_creds_from_file(
+ credfile,
+ profile_name,
+ key_id_out,
+ secret_access_key_out,
+ aws_region_out)
+ == FAIL)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to load from aws credentials")
+ }
+ if (fclose(credfile) == EOF) {
+ HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL,
+ "unable to close credentials file")
+ }
+ credfile = NULL;
+ } /* end if credential file opened */
+
+ ret = HDsnprintf(filepath, 128, "%s%s", awspath, "config");
+ if (ret < 0 || (size_t)ret >= 128) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTCOPY, FAIL,
+ "unable to format config path")
+ }
+ credfile = fopen(filepath, "r");
+ if (credfile != NULL) {
+ if (H5FD__s3comms_load_aws_creds_from_file(
+ credfile,
+ profile_name,
+ (*key_id_out == 0) ? key_id_out : NULL,
+ (*secret_access_key_out == 0) ? secret_access_key_out : NULL,
+ (*aws_region_out == 0) ? aws_region_out : NULL)
+ == FAIL)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to load from aws config")
+ }
+ if (fclose(credfile) == EOF) {
+ HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL,
+ "unable to close config file")
+ }
+ credfile = NULL;
+ } /* end if credential file opened */
+
+ /* fail if not all three settings were loaded */
+ if (*key_id_out == 0 ||
+ *secret_access_key_out == 0 ||
+ *aws_region_out == 0)
+ {
+ ret_value = FAIL;
+ }
+
+done:
+ if (credfile != NULL) {
+ if (fclose(credfile) == EOF) {
+ HDONE_ERROR(H5E_ARGS, H5E_ARGS, FAIL,
+ "problem error-closing aws configuration file")
+ }
+ }
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD_s3comms_load_aws_profile() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_nlowercase()
+ *
+ * Purpose:
+ *
+ * From string starting at `s`, write `len` characters to `dest`,
+ * converting all to lowercase.
+ *
+ * Behavior is undefined if `s` is NULL or `len` overruns the allocated
+ * space of either `s` or `dest`.
+ *
+ * Provided as convenience.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - upon completion, `dest` is populated
+ * - FAILURE: `FAIL`
+ * - `dest == NULL`
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-18
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_nlowercase(
+ char *dest,
+ const char *s,
+ size_t len)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_nlowercase.\n");
+#endif
+
+ if (dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "destination cannot be null.\n");
+ }
+
+ if (len > 0) {
+ HDmemcpy(dest, s, len);
+ do {
+ len--;
+ dest[len] = (char)tolower( (int)dest[len] );
+ } while (len > 0);
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD_s3comms_nlowercase() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_parse_url()
+ *
+ * Purpose:
+ *
+ * Parse URL-like string and stuff URL components into
+ * `parsed_url` structure, if possible.
+ *
+ * Expects null-terminated string of format:
+ * SCHEME "://" HOST [":" PORT ] ["/" [ PATH ] ] ["?" QUERY]
+ * where SCHEME :: "[a-zA-Z/.-]+"
+ * PORT :: "[0-9]"
+ *
+ * Stores resulting structure in argument pointer `purl`, if successful,
+ * creating and populating new `parsed_url_t` structure pointer.
+ * Empty or absent elements are NULL in new purl structure.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - `purl` pointer is populated
+ * - FAILURE: `FAIL`
+ * - unable to parse
+ * - `purl` is unaltered (probably NULL)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-30
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_parse_url(
+ const char *str,
+ parsed_url_t **_purl)
+{
+ parsed_url_t *purl = NULL; /* pointer to new structure */
+ const char *tmpstr = NULL; /* working pointer in string */
+ const char *curstr = str; /* "start" pointer in string */
+ long int len = 0; /* substring length */
+ long int urllen = 0; /* length of passed-in url string */
+ unsigned int i = 0;
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER_NOAPI_NOINIT;
+
+#if S3COMMS_DEBUG
+ HDprintf("called H5FD_s3comms_parse_url.\n");
+#endif
+
+ if (str == NULL || *str == '\0') {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "invalid url string");
+ }
+
+ urllen = (long int)HDstrlen(str);
+
+ purl = (parsed_url_t *)H5MM_malloc(sizeof(parsed_url_t));
+ if (purl == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "can't allocate space for parsed_url_t");
+ }
+ purl->magic = S3COMMS_PARSED_URL_MAGIC;
+ purl->scheme = NULL;
+ purl->host = NULL;
+ purl->port = NULL;
+ purl->path = NULL;
+ purl->query = NULL;
+
+ /***************
+ * READ SCHEME *
+ ***************/
+
+ tmpstr = HDstrchr(curstr, ':');
+ if (tmpstr == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "invalid SCHEME construction: probably not URL");
+ }
+ len = tmpstr - curstr;
+ HDassert( (0 <= len) && (len < urllen) );
+
+ /* check for restrictions
+ */
+ for (i = 0; i < len; i++) {
+ /* scheme = [a-zA-Z+-.]+ (terminated by ":") */
+ if (!isalpha(curstr[i]) &&
+ '+' != curstr[i] &&
+ '-' != curstr[i] &&
+ '.' != curstr[i])
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "invalid SCHEME construction");
+ }
+ }
+ /* copy lowercased scheme to structure
+ */
+ purl->scheme = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
+ if (purl->scheme == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "can't allocate space for SCHEME");
+ }
+ (void)HDstrncpy(purl->scheme, curstr, (size_t)len);
+ purl->scheme[len] = '\0';
+ for ( i = 0; i < len; i++ ) {
+ purl->scheme[i] = (char)tolower(purl->scheme[i]);
+ }
+
+ /* Skip "://" */
+ tmpstr += 3;
+ curstr = tmpstr;
+
+ /*************
+ * READ HOST *
+ *************/
+
+ if (*curstr == '[') {
+ /* IPv6 */
+ while (']' != *tmpstr) {
+ if (tmpstr == 0) { /* end of string reached! */
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "reached end of URL: incomplete IPv6 HOST");
+ }
+ tmpstr++;
+ }
+ tmpstr++;
+ } /* end if (IPv6) */
+ else {
+ while (0 != *tmpstr) {
+ if (':' == *tmpstr ||
+ '/' == *tmpstr ||
+ '?' == *tmpstr)
+ {
+ break;
+ }
+ tmpstr++;
+ }
+ } /* end else (IPv4) */
+ len = tmpstr - curstr;
+ if (len == 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "HOST substring cannot be empty");
+ }
+ else
+ if (len > urllen) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem with length of HOST substring");
+ }
+
+ /* copy host
+ */
+ purl->host = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
+ if (purl->host == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "can't allocate space for HOST");
+ }
+ (void)HDstrncpy(purl->host, curstr, (size_t)len);
+ purl->host[len] = 0;
+
+ /*************
+ * READ PORT *
+ *************/
+
+ if (':' == *tmpstr) {
+ tmpstr += 1; /* advance past ':' */
+ curstr = tmpstr;
+ while ((0 != *tmpstr) && ('/' != *tmpstr) && ('?' != *tmpstr)) {
+ tmpstr++;
+ }
+ len = tmpstr - curstr;
+ if (len == 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "PORT element cannot be empty");
+ }
+ else
+ if (len > urllen) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem with length of PORT substring");
+ }
+ for (i = 0; i < len; i ++) {
+ if (!isdigit(curstr[i])) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "PORT is not a decimal string");
+ }
+ }
+
+ /* copy port
+ */
+ purl->port = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
+ if (purl->port == NULL) { /* cannot malloc */
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "can't allocate space for PORT");
+ }
+ (void)HDstrncpy(purl->port, curstr, (size_t)len);
+ purl->port[len] = 0;
+ } /* end if PORT element */
+
+ /*************
+ * READ PATH *
+ *************/
+
+ if ('/' == *tmpstr) {
+ /* advance past '/' */
+ tmpstr += 1;
+ curstr = tmpstr;
+
+ /* seek end of PATH
+ */
+ while ((0 != *tmpstr) && ('?' != *tmpstr)) {
+ tmpstr++;
+ }
+ len = tmpstr - curstr;
+ if (len > urllen) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem with length of PATH substring");
+ }
+ if (len > 0) {
+ purl->path = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
+ if (purl->path == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "can't allocate space for PATH");
+ } /* cannot malloc path pointer */
+ (void)HDstrncpy(purl->path, curstr, (size_t)len);
+ purl->path[len] = 0;
+ }
+ } /* end if PATH element */
+
+ /**************
+ * READ QUERY *
+ **************/
+
+ if ('?' == *tmpstr) {
+ tmpstr += 1;
+ curstr = tmpstr;
+ while (0 != *tmpstr) {
+ tmpstr++;
+ }
+ len = tmpstr - curstr;
+ if (len == 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "QUERY cannot be empty");
+ } else if (len > urllen) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem with length of QUERY substring");
+ }
+ purl->query = (char *)H5MM_malloc(sizeof(char) * (size_t)(len + 1));
+ if (purl->query == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_CANTALLOC, FAIL,
+ "can't allocate space for QUERY");
+ } /* cannot malloc path pointer */
+ (void)HDstrncpy(purl->query, curstr, (size_t)len);
+ purl->query[len] = 0;
+ } /* end if QUERY exists */
+
+ *_purl = purl;
+ ret_value = SUCCEED;
+
+done:
+ if (ret_value == FAIL) {
+ H5FD_s3comms_free_purl(purl);
+ }
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD_s3comms_parse_url() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_percent_encode_char()
+ *
+ * Purpose:
+ *
+ * "Percent-encode" utf-8 character `c`, e.g.,
+ * '$' -> "%24"
+ * '¢' -> "%C2%A2"
+ *
+ * `c` cannot be null.
+ *
+ * Does not (currently) accept multi-byte characters...
+ * limit to (?) u+00ff, well below upper bound for two-byte utf-8 encoding
+ * (u+0080..u+07ff).
+ *
+ * Writes output to `repr`.
+ * `repr` cannot be null.
+ * Assumes adequate space i `repr`...
+ * >>> char[4] or [7] for most characters,
+ * >>> [13] as theoretical maximum.
+ *
+ * Representation `repr` is null-terminated.
+ *
+ * Stores length of representation (without null terminator) at pointer
+ * `repr_len`.
+ *
+ * Return : SUCCEED/FAIL
+ *
+ * - SUCCESS: `SUCCEED`
+ * - percent-encoded representation written to `repr`
+ * - 'repr' is null-terminated
+ * - FAILURE: `FAIL`
+ * - `c` or `repr` was NULL
+ *
+ * Programmer: Jacob Smith
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_percent_encode_char(
+ char *repr,
+ const unsigned char c,
+ size_t *repr_len)
+{
+ unsigned int i = 0;
+ int chars_written = 0;
+ herr_t ret_value = SUCCEED;
+#if S3COMMS_DEBUG
+ unsigned char s[2] = {c, 0};
+ unsigned char hex[3] = {0, 0, 0};
+#endif
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_percent_encode_char.\n");
+#endif
+
+ if (repr == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no destination `repr`.\n")
+ }
+
+#if S3COMMS_DEBUG
+ H5FD_s3comms_bytes_to_hex((char *)hex, s, 1, FALSE);
+ HDfprintf(stdout, " CHAR: \'%s\'\n", s);
+ HDfprintf(stdout, " CHAR-HEX: \"%s\"\n", hex);
+#endif
+
+ if (c <= (unsigned char)0x7f) {
+ /* character represented in a single "byte"
+ * and single percent-code
+ */
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, " SINGLE-BYTE\n");
+#endif
+ *repr_len = 3;
+ chars_written = HDsnprintf(repr, 4, "%%%02X", c);
+ if (chars_written < 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot write char %c",
+ c);
+ }
+ } /* end if single-byte unicode char */
+ else {
+ /* multi-byte, multi-percent representation
+ */
+ unsigned int acc = 0; /* byte accumulator */
+ unsigned int k = 0; /* uint character representation */
+ unsigned int stack_size = 0;
+ unsigned char stack[4] = {0, 0, 0, 0};
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, " MULTI-BYTE\n");
+#endif
+ stack_size = 0;
+ k = (unsigned int)c;
+ *repr_len = 0;
+ do {
+ /* push number onto stack in six-bit slices
+ */
+ acc = k;
+ acc >>= 6; /* cull least */
+ acc <<= 6; /* six bits */
+ stack[stack_size++] = (unsigned char)(k - acc);
+ k = acc >> 6;
+ } while (k > 0);
+
+ /* `stack` now has two to four six-bit 'numbers' to be put into
+ * UTF-8 byte fields.
+ */
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, " STACK:\n {\n");
+ for (i = 0; i < stack_size; i++) {
+ H5FD_s3comms_bytes_to_hex(
+ (char *)hex,
+ (&stack[i]),
+ 1,
+ FALSE);
+ hex[2] = 0;
+ HDfprintf(stdout, " %s,\n", hex);
+ }
+ HDfprintf(stdout, " }\n");
+#endif
+
+ /****************
+ * leading byte *
+ ****************/
+
+ /* prepend 11[1[1]]0 to first byte */
+ /* 110xxxxx, 1110xxxx, or 11110xxx */
+ acc = 0xC0; /* 0x11000000 */
+ acc += (stack_size > 2) ? 0x20 : 0; /* 0x00100000 */
+ acc += (stack_size > 3) ? 0x10 : 0; /* 0x00010000 */
+ stack_size--;
+ chars_written = HDsnprintf(
+ repr,
+ 4,
+ "%%%02X",
+ (unsigned char)(acc + stack[stack_size]));
+ if (chars_written < 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot write char %c",
+ c);
+ }
+ *repr_len += 3;
+
+ /************************
+ * continuation byte(s) *
+ ************************/
+
+ /* 10xxxxxx */
+ for (i = 0; i < stack_size; i++) {
+ chars_written = HDsnprintf(
+ &repr[i*3 + 3],
+ 4,
+ "%%%02X",
+ (unsigned char)(0x80 + stack[stack_size - 1 - i]));
+ if (chars_written < 0) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "cannot write char %c",
+ c);
+ }
+ *repr_len += 3;
+ } /* end for each continuation byte */
+ } /* end else (multi-byte) */
+
+ *(repr + *repr_len) = '\0';
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* H5FD_s3comms_percent_encode_char */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_signing_key()
+ *
+ * Purpose:
+ *
+ * Create AWS4 "Signing Key" from secret key, AWS region, and timestamp.
+ *
+ * Sequentially runs HMAC_SHA256 on strings in specified order,
+ * generating re-usable checksum (according to documentation, valid for
+ * 7 days from time given).
+ *
+ * `secret` is `access key id` for targeted service/bucket/resource.
+ *
+ * `iso8601now` must conform to format, yyyyMMDD'T'hhmmss'Z'
+ * e.g. "19690720T201740Z".
+ *
+ * `region` should be one of AWS service region names, e.g. "us-east-1".
+ *
+ * Hard-coded "service" algorithm requirement to "s3".
+ *
+ * Inputs must be null-terminated strings.
+ *
+ * Writes to `md` the raw byte data, length of `SHA256_DIGEST_LENGTH`.
+ * Programmer must ensure that `md` is appropriately allocated.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - raw byte data of signing key written to `md`
+ * - FAILURE: `FAIL`
+ * - if any input arguments was NULL
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-13
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_signing_key(
+ unsigned char *md,
+ const char *secret,
+ const char *region,
+ const char *iso8601now)
+{
+ char *AWS4_secret = NULL;
+ size_t AWS4_secret_len = 0;
+ unsigned char datekey[SHA256_DIGEST_LENGTH];
+ unsigned char dateregionkey[SHA256_DIGEST_LENGTH];
+ unsigned char dateregionservicekey[SHA256_DIGEST_LENGTH];
+ int ret = 0; /* return value of HDsnprintf */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_signing_key.\n");
+#endif
+
+ if (md == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "Destination `md` cannot be NULL.\n")
+ }
+ if (secret == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "`secret` cannot be NULL.\n")
+ }
+ if (region == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "`region` cannot be NULL.\n")
+ }
+ if (iso8601now == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "`iso8601now` cannot be NULL.\n")
+ }
+
+ AWS4_secret_len = 4 + HDstrlen(secret) + 1;
+ AWS4_secret = (char*)H5MM_malloc(sizeof(char *) * AWS4_secret_len);
+ if (AWS4_secret == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "Could not allocate space.\n")
+ }
+
+ /* prepend "AWS4" to start of the secret key
+ */
+ ret = HDsnprintf(AWS4_secret, AWS4_secret_len,"%s%s", "AWS4", secret);
+ if ((size_t)ret != (AWS4_secret_len - 1)) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem writing AWS4+secret `%s`",
+ secret);
+ }
+
+ /* hash_func, key, len(key), msg, len(msg), digest_dest, digest_len_dest
+ * we know digest length, so ignore via NULL
+ */
+ HMAC(EVP_sha256(),
+ (const unsigned char *)AWS4_secret,
+ (int)HDstrlen(AWS4_secret),
+ (const unsigned char*)iso8601now,
+ 8, /* 8 --> length of 8 --> "yyyyMMDD" */
+ datekey,
+ NULL);
+ HMAC(EVP_sha256(),
+ (const unsigned char *)datekey,
+ SHA256_DIGEST_LENGTH,
+ (const unsigned char *)region,
+ HDstrlen(region),
+ dateregionkey,
+ NULL);
+ HMAC(EVP_sha256(),
+ (const unsigned char *)dateregionkey,
+ SHA256_DIGEST_LENGTH,
+ (const unsigned char *)"s3",
+ 2,
+ dateregionservicekey,
+ NULL);
+ HMAC(EVP_sha256(),
+ (const unsigned char *)dateregionservicekey,
+ SHA256_DIGEST_LENGTH,
+ (const unsigned char *)"aws4_request",
+ 12,
+ md,
+ NULL);
+
+done:
+ H5MM_xfree(AWS4_secret);
+
+ FUNC_LEAVE_NOAPI(ret_value);
+} /* end H5FD_s3comms_signing_key() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_tostringtosign()
+ *
+ * Purpose:
+ *
+ * Get AWS "String to Sign" from Canonical Request, timestamp,
+ * and AWS "region".
+ *
+ * Common between single request and "chunked upload",
+ * conforms to:
+ * "AWS4-HMAC-SHA256\n" +
+ * <ISO8601 date format> + "\n" + // yyyyMMDD'T'hhmmss'Z'
+ * <yyyyMMDD> + "/" + <AWS Region> + "/s3/aws4-request\n" +
+ * hex(SHA256(<CANONICAL-REQUEST>))
+ *
+ * Inputs `creq` (canonical request string), `now` (ISO8601 format),
+ * and `region` (s3 region designator string) must all be
+ * null-terminated strings.
+ *
+ * Result is written to `dest` with null-terminator.
+ * It is left to programmer to ensure `dest` has adequate space.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - "string to sign" written to `dest` and null-terminated
+ * - FAILURE: `FAIL`
+ * - if any of the inputs are NULL
+ * - if an error is encountered while computing checksum
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-??
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_tostringtosign(
+ char *dest,
+ const char *req,
+ const char *now,
+ const char *region)
+{
+ unsigned char checksum[SHA256_DIGEST_LENGTH * 2 + 1];
+ size_t d = 0;
+ char day[9];
+ char hexsum[SHA256_DIGEST_LENGTH * 2 + 1];
+ size_t i = 0;
+ int ret = 0; /* HDsnprintf return value */
+ herr_t ret_value = SUCCEED;
+ char tmp[128];
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_tostringtosign.\n");
+#endif
+
+ if (dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "destination buffer cannot be null.\n")
+ }
+ if (req == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "canonical request cannot be null.\n")
+ }
+ if (now == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "Timestring cannot be NULL.\n")
+ }
+ if (region == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "Region cannot be NULL.\n")
+ }
+
+ for (i = 0; i < 128; i++) {
+ tmp[i] = '\0';
+ }
+ for (i = 0; i < SHA256_DIGEST_LENGTH * 2 + 1; i++) {
+ checksum[i] = '\0';
+ hexsum[i] = '\0';
+ }
+ HDstrncpy(day, now, 8);
+ day[8] = '\0';
+ ret = HDsnprintf(tmp, 127, "%s/%s/s3/aws4_request", day, region);
+ if (ret <= 0 || ret >= 127) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "problem adding day and region to string")
+ }
+
+ HDmemcpy((dest + d), "AWS4-HMAC-SHA256\n", 17);
+ d = 17;
+
+ HDmemcpy((dest+d), now, HDstrlen(now));
+ d += HDstrlen(now);
+ dest[d++] = '\n';
+
+ HDmemcpy((dest + d), tmp, HDstrlen(tmp));
+ d += HDstrlen(tmp);
+ dest[d++] = '\n';
+
+ SHA256((const unsigned char *)req,
+ HDstrlen(req),
+ checksum);
+
+ if (H5FD_s3comms_bytes_to_hex(
+ hexsum,
+ (const unsigned char *)checksum,
+ SHA256_DIGEST_LENGTH,
+ true)
+ == FAIL)
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "could not create hex string");
+ }
+
+ for (i = 0; i < SHA256_DIGEST_LENGTH * 2; i++) {
+ dest[d++] = hexsum[i];
+ }
+
+ dest[d] = '\0';
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5ros3_tostringtosign() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_trim()
+ *
+ * Purpose:
+ *
+ * Remove all whitespace characters from start and end of a string `s`
+ * of length `s_len`, writing trimmed string copy to `dest`.
+ * Stores number of characters remaining at `n_written`.
+ *
+ * Destination for trimmed copy `dest` cannot be null.
+ * `dest` must have adequate space allocated for trimmed copy.
+ * If inadequate space, behavior is undefined, possibly resulting
+ * in segfault or overwrite of other data.
+ *
+ * If `s` is NULL or all whitespace, `dest` is untouched and `n_written`
+ * is set to 0.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - `dest == NULL`
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-18
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_trim(char *dest,
+ char *s,
+ size_t s_len,
+ size_t *n_written)
+{
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "called H5FD_s3comms_trim.\n");
+#endif
+
+ if (dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "destination cannot be null.")
+ }
+ if (s == NULL) {
+ s_len = 0;
+ }
+
+ if (s_len > 0) {
+ /* Find first non-whitespace character from start;
+ * reduce total length per character.
+ */
+ while ((s_len > 0) &&
+ isspace((unsigned char)s[0]) && s_len > 0)
+ {
+ s++;
+ s_len--;
+ }
+
+ /* Find first non-whitespace character from tail;
+ * reduce length per-character.
+ * If length is 0 already, there is no non-whitespace character.
+ */
+ if (s_len > 0) {
+ do {
+ s_len--;
+ } while( isspace((unsigned char)s[s_len]) );
+ s_len++;
+
+ /* write output into dest
+ */
+ HDmemcpy(dest, s, s_len);
+ }
+ }
+
+ *n_written = s_len;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5FD_s3comms_trim() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: H5FD_s3comms_uriencode()
+ *
+ * Purpose:
+ *
+ * URIencode (percent-encode) every byte except "[a-zA-Z0-9]-._~".
+ *
+ * For each character in source string `_s` from `s[0]` to `s[s_len-1]`,
+ * writes to `dest` either the raw character or its percent-encoded
+ * equivalent.
+ *
+ * See `H5FD_s3comms_bytes_to_hex` for information on percent-encoding.
+ *
+ * Space (' ') character encoded as "%20" (not "+")
+ *
+ * Forward-slash ('/') encoded as "%2F" only when `encode_slash == true`.
+ *
+ * Records number of characters written at `n_written`.
+ *
+ * Assumes that `dest` has been allocated with enough space.
+ *
+ * Neither `dest` nor `s` can be NULL.
+ *
+ * `s_len == 0` will have no effect.
+ *
+ * Return:
+ *
+ * - SUCCESS: `SUCCEED`
+ * - FAILURE: `FAIL`
+ * - source strings `s` or destination `dest` are NULL
+ * - error while attempting to percent-encode a character
+ *
+ * Programmer: Jacob Smith
+ * 2017-07-??
+ *
+ *----------------------------------------------------------------------------
+ */
+herr_t
+H5FD_s3comms_uriencode(
+ char *dest,
+ const char *s,
+ size_t s_len,
+ hbool_t encode_slash,
+ size_t *n_written)
+{
+ char c = 0;
+ size_t dest_off = 0;
+ char hex_buffer[13];
+ size_t hex_off = 0;
+ size_t hex_len = 0;
+ herr_t ret_value = SUCCEED;
+ size_t s_off = 0;
+
+ FUNC_ENTER_NOAPI_NOINIT
+
+#if S3COMMS_DEBUG
+ HDfprintf(stdout, "H5FD_s3comms_uriencode called.\n");
+#endif
+
+ if (s == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "source string cannot be NULL");
+ }
+ if (dest == NULL) {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "destination cannot be NULL");
+ }
+
+ /* Write characters to destination, converting to percent-encoded
+ * "hex-utf-8" strings if necessary.
+ * e.g., '$' -> "%24"
+ */
+ for (s_off = 0; s_off < s_len; s_off++) {
+ c = s[s_off];
+ if (isalnum(c) ||
+ c == '.' ||
+ c == '-' ||
+ c == '_' ||
+ c == '~' ||
+ (c == '/' && encode_slash == FALSE))
+ {
+ dest[dest_off++] = c;
+ }
+ else {
+ hex_off = 0;
+ if (H5FD_s3comms_percent_encode_char(
+ hex_buffer,
+ (const unsigned char)c,
+ &hex_len)
+ == FAIL)
+ {
+ hex_buffer[0] = c;
+ hex_buffer[1] = 0;
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "unable to percent-encode character \'%s\' "
+ "at %d in \"%s\"", hex_buffer, (int)s_off, s);
+ }
+
+ for (hex_off = 0; hex_off < hex_len; hex_off++) {
+ dest[dest_off++] = hex_buffer[hex_off];
+ }
+ } /* end else (not a regular character) */
+ } /* end for each character */
+
+ if (dest_off < s_len)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL,
+ "buffer overflow");
+
+ *n_written = dest_off;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5FD_s3comms_uriencode */
+
+#endif /* H5_HAVE_ROS3_VFD */
+
diff --git a/src/H5FDs3comms.h b/src/H5FDs3comms.h
new file mode 100644
index 0000000..c9d6c9d
--- /dev/null
+++ b/src/H5FDs3comms.h
@@ -0,0 +1,610 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*****************************************************************************
+ * Read-Only S3 Virtual File Driver (VFD)
+ *
+ * This is the header for the S3 Communications module
+ *
+ * ***NOT A FILE DRIVER***
+ *
+ * Purpose:
+ *
+ * - Provide structures and functions related to communicating with
+ * Amazon S3 (Simple Storage Service).
+ * - Abstract away the REST API (HTTP,
+ * networked communications) behind a series of uniform function calls.
+ * - Handle AWS4 authentication, if appropriate.
+ * - Fail predictably in event of errors.
+ * - Eventually, support more S3 operations, such as creating, writing to,
+ * and removing Objects remotely.
+ *
+ * translates:
+ * `read(some_file, bytes_offset, bytes_length, &dest_buffer);`
+ * to:
+ * ```
+ * GET myfile HTTP/1.1
+ * Host: somewhere.me
+ * Range: bytes=4096-5115
+ * ```
+ * and places received bytes from HTTP response...
+ * ```
+ * HTTP/1.1 206 Partial-Content
+ * Content-Range: 4096-5115/63239
+ *
+ * <bytes>
+ * ```
+ * ...in destination buffer.
+ *
+ * TODO: put documentation in a consistent place and point to it from here.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-30
+ *
+ *****************************************************************************/
+
+#ifndef H5FDs3comms_H
+#define H5FDs3comms_H
+
+
+#include "H5private.h" /* Generic Functions */
+
+#ifdef H5_HAVE_ROS3_VFD
+
+/* Necessary S3 headers */
+#include <curl/curl.h>
+#include <openssl/evp.h>
+#include <openssl/hmac.h>
+#include <openssl/sha.h>
+
+/*****************
+ * PUBLIC MACROS *
+ *****************/
+
+/* hexadecimal string of pre-computed sha256 checksum of the empty string
+ * hex(sha256sum(""))
+ */
+#define EMPTY_SHA256 \
+"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+/* string length (plus null terminator)
+ * example ISO8601-format string: "20170713T145903Z" (YYYYmmdd'T'HHMMSS'_')
+ */
+#define ISO8601_SIZE 17
+
+/* string length (plus null terminator)
+ * example RFC7231-format string: "Fri, 30 Jun 2017 20:41:55 GMT"
+ */
+#define RFC7231_SIZE 30
+
+/*---------------------------------------------------------------------------
+ *
+ * Macro: ISO8601NOW()
+ *
+ * Purpose:
+ *
+ * write "YYYYmmdd'T'HHMMSS'Z'" (less single-quotes) to dest
+ * e.g., "20170630T204155Z"
+ *
+ * wrapper for strftime()
+ *
+ * It is left to the programmer to check return value of
+ * ISO8601NOW (should equal ISO8601_SIZE - 1).
+ *
+ *---------------------------------------------------------------------------
+ */
+#define ISO8601NOW(dest, now_gm) \
+strftime((dest), ISO8601_SIZE, "%Y%m%dT%H%M%SZ", (now_gm))
+
+/*---------------------------------------------------------------------------
+ *
+ * Macro: RFC7231NOW()
+ *
+ * Purpose:
+ *
+ * write "Day, dd Mmm YYYY HH:MM:SS GMT" to dest
+ * e.g., "Fri, 30 Jun 2017 20:41:55 GMT"
+ *
+ * wrapper for strftime()
+ *
+ * It is left to the programmer to check return value of
+ * RFC7231NOW (should equal RFC7231_SIZE - 1).
+ *
+ *---------------------------------------------------------------------------
+ */
+#define RFC7231NOW(dest, now_gm) \
+strftime((dest), RFC7231_SIZE, "%a, %d %b %Y %H:%M:%S GMT", (now_gm))
+
+
+/* Reasonable maximum length of a credential string.
+ * Provided for error-checking S3COMMS_FORMAT_CREDENTIAL (below).
+ * 17 <- "////aws4_request\0"
+ * 2 < "s3" (service)
+ * 8 <- "YYYYmmdd" (date)
+ * 128 <- (access_id)
+ * 155 :: sum
+ */
+#define S3COMMS_MAX_CREDENTIAL_SIZE 155
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Macro: H5FD_S3COMMS_FORMAT_CREDENTIAL()
+ *
+ * Purpose:
+ *
+ * Format "S3 Credential" string from inputs, for AWS4.
+ *
+ * Wrapper for HDsnprintf().
+ *
+ * _HAS NO ERROR-CHECKING FACILITIES_
+ * It is left to programmer to ensure that return value confers success.
+ * e.g.,
+ * ```
+ * assert( S3COMMS_MAX_CREDENTIAL_SIZE >=
+ * S3COMMS_FORMAT_CREDENTIAL(...) );
+ * ```
+ *
+ * "<access-id>/<date>/<aws-region>/<aws-service>/aws4_request"
+ * assuming that `dest` has adequate space.
+ *
+ * ALL inputs must be null-terminated strings.
+ *
+ * `access` should be the user's access key ID.
+ * `date` must be of format "YYYYmmdd".
+ * `region` should be relevant AWS region, i.e. "us-east-1".
+ * `service` should be "s3".
+ *
+ *---------------------------------------------------------------------------
+ */
+#define S3COMMS_FORMAT_CREDENTIAL(dest, access, iso8601_date, region, service) \
+HDsnprintf((dest), S3COMMS_MAX_CREDENTIAL_SIZE, \
+ "%s/%s/%s/%s/aws4_request", \
+ (access), (iso8601_date), (region), (service))
+
+/*********************
+ * PUBLIC STRUCTURES *
+ *********************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Structure: hrb_node_t
+ *
+ * HTTP Header Field Node
+ *
+ *
+ *
+ * Maintain a ordered (linked) list of HTTP Header fields.
+ *
+ * Provides efficient access and manipulation of a logical sequence of
+ * HTTP header fields, of particular use when composing an
+ * "S3 Canonical Request" for authentication.
+ *
+ * - The creation of a Canoncial Request involves:
+ * - convert field names to lower case
+ * - sort by this lower-case name
+ * - convert ": " name-value separator in HTTP string to ":"
+ * - get sorted lowercase names without field or separator
+ *
+ * As HTTP headers allow headers in any order (excepting the case of multiple
+ * headers with the same name), the list ordering can be optimized for Canonical
+ * Request creation, suggesting alphabtical order. For more expedient insertion
+ * and removal of elements in the list, linked list seems preferable to a
+ * dynamically-expanding array. The usually-smaller number of entries (5 or
+ * fewer) makes performance overhead of traversing the list trivial.
+ *
+ * The above requirements of creating at Canonical Request suggests a reasonable
+ * trade-off of speed for space with the option to compute elements as needed
+ * or to have the various elements prepared and stored in the structure
+ * (e.g. name, value, lowername, concatenated name:value)
+ * The structure currently is implemented to pre-compute.
+ *
+ * At all times, the "first" node of the list should be the least,
+ * alphabetically. For all nodes, the `next` node should be either NULL or
+ * of greater alphabetical value.
+ *
+ * Each node contains its own header field information, plus a pointer to the
+ * next node.
+ *
+ * It is not allowed to have multiple nodes with the same _lowercase_ `name`s
+ * in the same list
+ * (i.e., name is case-insensitive for access and modification.)
+ *
+ * All data (`name`, `value`, `lowername`, and `cat`) are null-terminated
+ * strings allocated specifically for their node.
+ *
+ *
+ *
+ * `magic` (unsigned long)
+ *
+ * "unique" idenfier number for the structure type
+ *
+ * `name` (char *)
+ *
+ * Case-meaningful name of the HTTP field.
+ * Given case is how it is supplied to networking code.
+ * e.g., "Range"
+ *
+ * `lowername` (char *)
+ *
+ * Lowercase copy of name.
+ * e.g., "range"
+ *
+ * `value` (char *)
+ *
+ * Case-meaningful value of HTTP field.
+ * e.g., "bytes=0-9"
+ *
+ * `cat` (char *)
+ *
+ * Concatenated, null-terminated string of HTTP header line,
+ * as the field would appear in an HTTP request.
+ * e.g., "Range: bytes=0-9"
+ *
+ * `next` (hrb_node_t *)
+ *
+ * Pointers to next node in the list, or NULL sentinel as end of list.
+ * Next node must have a greater `lowername` as determined by strcmp().
+ *
+ *----------------------------------------------------------------------------
+ */
+typedef struct hrb_node_t {
+ unsigned long magic;
+ char *name;
+ char *value;
+ char *cat;
+ char *lowername;
+ struct hrb_node_t *next;
+} hrb_node_t;
+#define S3COMMS_HRB_NODE_MAGIC 0x7F5757UL
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Structure: hrb_t
+ *
+ * HTTP Request Buffer structure
+ *
+ *
+ *
+ * Logically represent an HTTP request
+ *
+ * GET /myplace/myfile.h5 HTTP/1.1
+ * Host: over.rainbow.oz
+ * Date: Fri, 01 Dec 2017 12:35:04 CST
+ *
+ * <body>
+ *
+ * ...with fast, efficient access to and modification of primary and field
+ * elements.
+ *
+ * Structure for building HTTP requests while hiding much of the string
+ * processing required "under the hood."
+ *
+ * Information about the request target -- the first line -- and the body text,
+ * if any, are managed directly with this structure. All header fields, e.g.,
+ * "Host" and "Date" above, are created with a linked list of `hrb_node_t` and
+ * included in the request by a pointer to the head of the list.
+ *
+ *
+ *
+ * `magic` (unsigned long)
+ *
+ * "Magic" number confirming that this is an hrb_t structure and
+ * what operations are valid for it.
+ *
+ * Must be S3COMMS_HRB_MAGIC to be valid.
+ *
+ * `body` (char *) :
+ *
+ * Pointer to start of HTTP body.
+ *
+ * Can be NULL, in which case it is treated as the empty string, "".
+ *
+ * `body_len` (size_t) :
+ *
+ * Number of bytes (characters) in `body`. 0 if empty or NULL `body`.
+ *
+ * `first_header` (hrb_node_t *) :
+ *
+ * Pointer to first SORTED header node, if any.
+ * It is left to the programmer to ensure that this node and associated
+ * list is destroyed when done.
+ *
+ * `resource` (char *) :
+ *
+ * Pointer to resource URL string, e.g., "/folder/page.xhtml".
+ *
+ * `verb` (char *) :
+ *
+ * Pointer to HTTP verb string, e.g., "GET".
+ *
+ * `version` (char *) :
+ *
+ * Pointer to HTTP version string, e.g., "HTTP/1.1".
+ *
+ *----------------------------------------------------------------------------
+ */
+typedef struct {
+ unsigned long magic;
+ char *body;
+ size_t body_len;
+ hrb_node_t *first_header;
+ char *resource;
+ char *verb;
+ char *version;
+} hrb_t;
+#define S3COMMS_HRB_MAGIC 0x6DCC84UL
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Structure: parsed_url_t
+ *
+ *
+ * Represent a URL with easily-accessed pointers to logical elements within.
+ * These elements (components) are stored as null-terminated strings (or just
+ * NULLs). These components should be allocated for the structure, making the
+ * data as safe as possible from modification. If a component is NULL, it is
+ * either implicit in or absent from the URL.
+ *
+ * "http://mybucket.s3.amazonaws.com:8080/somefile.h5?param=value&arg=value"
+ * ^--^ ^-----------------------^ ^--^ ^---------^ ^-------------------^
+ * Scheme Host Port Resource Query/-ies
+ *
+ *
+ *
+ * `magic` (unsigned long)
+ *
+ * Structure identification and validation identifier.
+ * Identifies as `parsed_url_t` type.
+ *
+ * `scheme` (char *)
+ *
+ * String representing which protocol is to be expected.
+ * _Must_ be present.
+ * "http", "https", "ftp", e.g.
+ *
+ * `host` (char *)
+ *
+ * String of host, either domain name, IPv4, or IPv6 format.
+ * _Must_ be present.
+ * "over.rainbow.oz", "192.168.0.1", "[0000:0000:0000:0001]"
+ *
+ * `port` (char *)
+ *
+ * String representation of specified port. Must resolve to a valid unsigned
+ * integer.
+ * "9000", "80"
+ *
+ * `path` (char *)
+ *
+ * Path to resource on host. If not specified, assumes root "/".
+ * "lollipop_guild.wav", "characters/witches/white.dat"
+ *
+ * `query` (char *)
+ *
+ * Single string of all query parameters in url (if any).
+ * "arg1=value1&arg2=value2"
+ *
+ *----------------------------------------------------------------------------
+ */
+typedef struct {
+ unsigned long magic;
+ char *scheme; /* required */
+ char *host; /* required */
+ char *port;
+ char *path;
+ char *query;
+} parsed_url_t;
+#define S3COMMS_PARSED_URL_MAGIC 0x21D0DFUL
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Structure: s3r_t
+ *
+ *
+ *
+ * S3 request structure "handle".
+ *
+ * Holds persistent information for Amazon S3 requests.
+ *
+ * Instantiated through `H5FD_s3comms_s3r_open()`, copies data into self.
+ *
+ * Intended to be re-used for operations on a remote object.
+ *
+ * Cleaned up through `H5FD_s3comms_s3r_close()`.
+ *
+ * _DO NOT_ share handle between threads: curl easy handle `curlhandle` has
+ * undefined behavior if called to perform in multiple threads.
+ *
+ *
+ *
+ * `magic` (unsigned long)
+ *
+ * "magic" number identifying this structure as unique type.
+ * MUST equal `S3R_MAGIC` to be valid.
+ *
+ * `curlhandle` (CURL)
+ *
+ * Pointer to the curl_easy handle generated for the request.
+ *
+ * `httpverb` (char *)
+ *
+ * Pointer to NULL-terminated string. HTTP verb,
+ * e.g. "GET", "HEAD", "PUT", etc.
+ *
+ * Default is NULL, resulting in a "GET" request.
+ *
+ * `purl` (parsed_url_t *)
+ *
+ * Pointer to structure holding the elements of URL for file open.
+ *
+ * e.g., "http://bucket.aws.com:8080/myfile.dat?q1=v1&q2=v2"
+ * parsed into...
+ * { scheme: "http"
+ * host: "bucket.aws.com"
+ * port: "8080"
+ * path: "myfile.dat"
+ * query: "q1=v1&q2=v2"
+ * }
+ *
+ * Cannot be NULL.
+ *
+ * `region` (char *)
+ *
+ * Pointer to NULL-terminated string, specifying S3 "region",
+ * e.g., "us-east-1".
+ *
+ * Required to authenticate.
+ *
+ * `secret_id` (char *)
+ *
+ * Pointer to NULL-terminated string for "secret" access id to S3 resource.
+ *
+ * Requred to authenticate.
+ *
+ * `signing_key` (unsigned char *)
+ *
+ * Pointer to `SHA256_DIGEST_LENGTH`-long string for "re-usable" signing
+ * key, generated via
+ * `HMAC-SHA256(HMAC-SHA256(HMAC-SHA256(HMAC-SHA256("AWS4<secret_key>",
+ * "<yyyyMMDD"), "<aws-region>"), "<aws-service>"), "aws4_request")`
+ * which may be re-used for several (up to seven (7)) days from creation?
+ * Computed once upon file open.
+ *
+ * Requred to authenticate.
+ *
+ *----------------------------------------------------------------------------
+ */
+typedef struct {
+ unsigned long magic;
+ CURL *curlhandle;
+ size_t filesize;
+ char *httpverb;
+ parsed_url_t *purl;
+ char *region;
+ char *secret_id;
+ unsigned char *signing_key;
+} s3r_t;
+
+#define S3COMMS_S3R_MAGIC 0x44d8d79
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/*******************************************
+ * DECLARATION OF HTTP FIELD LIST ROUTINES *
+ *******************************************/
+
+H5_DLL herr_t H5FD_s3comms_hrb_node_set(hrb_node_t **L,
+ const char *name,
+ const char *value);
+
+/***********************************************
+ * DECLARATION OF HTTP REQUEST BUFFER ROUTINES *
+ ***********************************************/
+
+H5_DLL herr_t H5FD_s3comms_hrb_destroy(hrb_t **buf);
+
+H5_DLL hrb_t * H5FD_s3comms_hrb_init_request(const char *verb,
+ const char *resource,
+ const char *host);
+
+/*************************************
+ * DECLARATION OF S3REQUEST ROUTINES *
+ *************************************/
+
+H5_DLL herr_t H5FD_s3comms_s3r_close(s3r_t *handle);
+
+H5_DLL size_t H5FD_s3comms_s3r_get_filesize(s3r_t *handle);
+
+H5_DLL s3r_t * H5FD_s3comms_s3r_open(const char url[],
+ const char region[],
+ const char id[],
+ const unsigned char signing_key[]);
+
+H5_DLL herr_t H5FD_s3comms_s3r_read(s3r_t *handle,
+ haddr_t offset,
+ size_t len,
+ void *dest);
+
+/*********************************
+ * DECLARATION OF OTHER ROUTINES *
+ *********************************/
+
+H5_DLL struct tm * gmnow(void);
+
+H5_DLL herr_t H5FD_s3comms_aws_canonical_request(char *canonical_request_dest,
+ int cr_size,
+ char *signed_headers_dest,
+ int sh_size,
+ hrb_t *http_request);
+
+H5_DLL herr_t H5FD_s3comms_bytes_to_hex(char *dest,
+ const unsigned char *msg,
+ size_t msg_len,
+ hbool_t lowercase);
+
+H5_DLL herr_t H5FD_s3comms_free_purl(parsed_url_t *purl);
+
+H5_DLL herr_t H5FD_s3comms_HMAC_SHA256(const unsigned char *key,
+ size_t key_len,
+ const char *msg,
+ size_t msg_len,
+ char *dest);
+
+H5_DLL herr_t H5FD_s3comms_load_aws_profile(const char *name,
+ char *key_id_out,
+ char *secret_access_key_out,
+ char *aws_region_out);
+
+H5_DLL herr_t H5FD_s3comms_nlowercase(char *dest,
+ const char *s,
+ size_t len);
+
+H5_DLL herr_t H5FD_s3comms_parse_url(const char *str,
+ parsed_url_t **purl);
+
+H5_DLL herr_t H5FD_s3comms_percent_encode_char(char *repr,
+ const unsigned char c,
+ size_t *repr_len);
+
+H5_DLL herr_t H5FD_s3comms_signing_key(unsigned char *md,
+ const char *secret,
+ const char *region,
+ const char *iso8601now);
+
+H5_DLL herr_t H5FD_s3comms_tostringtosign(char *dest,
+ const char *req_str,
+ const char *now,
+ const char *region);
+
+H5_DLL herr_t H5FD_s3comms_trim(char *dest,
+ char *s,
+ size_t s_len,
+ size_t *n_written);
+
+H5_DLL herr_t H5FD_s3comms_uriencode(char *dest, const char *s, size_t s_len,
+ hbool_t encode_slash, size_t *n_written);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* H5_HAVE_ROS3_VFD */
+
+#endif /* ifndef H5FDs3comms_H */
+
diff --git a/src/Makefile.am b/src/Makefile.am
index 93a90be..4774ae2 100644
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -53,7 +53,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5Fmount.c H5Fmpi.c H5Fquery.c \
H5Fsfile.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \
H5FD.c H5FDcore.c \
- H5FDdirect.c H5FDfamily.c H5FDint.c H5FDlog.c H5FDmpi.c H5FDmpio.c \
+ H5FDdirect.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c H5FDmpi.c H5FDmpio.c \
H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c \
H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSsection.c H5FSstat.c H5FStest.c \
H5G.c H5Gbtree2.c H5Gcache.c \
@@ -98,9 +98,13 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5Topaque.c \
H5Torder.c \
H5Tpad.c H5Tprecis.c H5Tstrpad.c H5Tvisit.c H5Tvlen.c H5TS.c H5VM.c H5WB.c H5Z.c \
- H5Zdeflate.c H5Zfletcher32.c H5Znbit.c H5Zshuffle.c H5Zszip.c \
- H5Zscaleoffset.c H5Ztrans.c
+ H5Zdeflate.c H5Zfletcher32.c H5Znbit.c H5Zshuffle.c \
+ H5Zscaleoffset.c H5Zszip.c H5Ztrans.c
+# Only compile the read-only S3 VFD if necessary
+if ROS3_VFD_CONDITIONAL
+ libhdf5_la_SOURCES += H5FDros3.c H5FDs3comms.c
+endif
# Public headers
include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5version.h \
@@ -108,8 +112,8 @@ include_HEADERS = hdf5.h H5api_adpt.h H5overflow.h H5pubconf.h H5public.h H5vers
H5Cpublic.h H5Dpublic.h \
H5Epubgen.h H5Epublic.h H5Fpublic.h \
H5FDpublic.h H5FDcore.h H5FDdirect.h \
- H5FDfamily.h H5FDlog.h H5FDmpi.h H5FDmpio.h \
- H5FDmulti.h H5FDsec2.h H5FDstdio.h \
+ H5FDfamily.h H5FDhdfs.h H5FDlog.h H5FDmpi.h H5FDmpio.h \
+ H5FDmulti.h H5FDros3.h H5FDsec2.h H5FDstdio.h \
H5Gpublic.h H5Ipublic.h H5Lpublic.h \
H5MMpublic.h H5Opublic.h H5Ppublic.h \
H5PLextern.h H5PLpublic.h \
diff --git a/src/Makefile.in b/src/Makefile.in
index 97bfbc7..8284041 100644
--- a/src/Makefile.in
+++ b/src/Makefile.in
@@ -163,8 +163,9 @@ am_libhdf5_la_OBJECTS = H5.lo H5checksum.lo H5dbg.lo H5system.lo \
H5Faccum.lo H5Fcwfs.lo H5Fdbg.lo H5Fefc.lo H5Ffake.lo H5Fio.lo \
H5Fmount.lo H5Fmpi.lo H5Fquery.lo H5Fsfile.lo H5Fsuper.lo \
H5Fsuper_cache.lo H5Ftest.lo H5FD.lo H5FDcore.lo H5FDdirect.lo \
- H5FDfamily.lo H5FDint.lo H5FDlog.lo H5FDmpi.lo H5FDmpio.lo \
- H5FDmulti.lo H5FDsec2.lo H5FDspace.lo H5FDstdio.lo H5FL.lo \
+ H5FDfamily.lo H5FDhdfs.lo H5FDint.lo H5FDlog.lo H5FDs3comms.lo \
+ H5FDmpi.lo H5FDmpio.lo H5FDmulti.lo H5FDros3.lo H5FDsec2.lo \
+ H5FDspace.lo H5FDstdio.lo H5FL.lo \
H5FO.lo H5FS.lo H5FScache.lo H5FSdbg.lo H5FSsection.lo \
H5FSstat.lo H5FStest.lo H5G.lo H5Gbtree2.lo H5Gcache.lo \
H5Gcompact.lo H5Gdense.lo H5Gdeprec.lo H5Gent.lo H5Gint.lo \
@@ -530,6 +531,7 @@ H5_CPPFLAGS = @H5_CPPFLAGS@
H5_CXXFLAGS = @H5_CXXFLAGS@
H5_FCFLAGS = @H5_FCFLAGS@
H5_FORTRAN_SHARED = @H5_FORTRAN_SHARED@
+HAVE_LIBHDFS = @HAVE_LIBHDFS@
H5_LDFLAGS = @H5_LDFLAGS@
H5_VERSION = @H5_VERSION@
HADDR_T = @HADDR_T@
@@ -590,6 +592,7 @@ PATH_SEPARATOR = @PATH_SEPARATOR@
PERL = @PERL@
RANLIB = @RANLIB@
ROOT = @ROOT@
+ROS3_VFD = @ROS3_VFD@
RUNPARALLEL = @RUNPARALLEL@
RUNSERIAL = @RUNSERIAL@
R_INTEGER = @R_INTEGER@
@@ -781,8 +784,8 @@ libhdf5_la_SOURCES = H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \
H5Fmount.c H5Fmpi.c H5Fquery.c \
H5Fsfile.c H5Fsuper.c H5Fsuper_cache.c H5Ftest.c \
H5FD.c H5FDcore.c \
- H5FDdirect.c H5FDfamily.c H5FDint.c H5FDlog.c H5FDmpi.c H5FDmpio.c \
- H5FDmulti.c H5FDsec2.c H5FDspace.c H5FDstdio.c \
+ H5FDdirect.c H5FDfamily.c H5FDhdfs.c H5FDint.c H5FDlog.c H5FDmpi.c H5FDmpio.c H5FDs3comms.c \
+ H5FDmulti.c H5FDros3.c H5FDsec2.c H5FDspace.c H5FDstdio.c \
H5FL.c H5FO.c H5FS.c H5FScache.c H5FSdbg.c H5FSsection.c H5FSstat.c H5FStest.c \
H5G.c H5Gbtree2.c H5Gcache.c \
H5Gcompact.c H5Gdense.c H5Gdeprec.c H5Gent.c \
diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in
index 5cb0930..5b65d77 100644
--- a/src/libhdf5.settings.in
+++ b/src/libhdf5.settings.in
@@ -3,37 +3,47 @@
General Information:
-------------------
- HDF5 Version: @H5_VERSION@
- Configured on: @CONFIG_DATE@
- Configured by: @CONFIG_USER@
- Configure mode: @CONFIG_MODE@
- Host system: @host_cpu@-@host_vendor@-@host_os@
- Uname information: @UNAME_INFO@
- Byte sex: @BYTESEX@
- Libraries: @STATIC_SHARED@
- Installation point: @prefix@
+ HDF5 Version: @H5_VERSION@
+ Configured on: @CONFIG_DATE@
+ Configured by: @CONFIG_USER@
+ Configure mode: @CONFIG_MODE@
+ Host system: @host_cpu@-@host_vendor@-@host_os@
+ Uname information: @UNAME_INFO@
+ Byte sex: @BYTESEX@
+ Installation point: @prefix@
Compiling Options:
------------------
- Compilation Mode: @CONFIG_MODE@
+ Build Mode: @BUILD_MODE@
+ Debugging Symbols: @SYMBOLS@
+ Asserts: @ASSERTS@
+ Profiling: @PROFILING@
+ Optimization Level: @OPTIMIZATION@
+
+Linking Options:
+----------------
+ Libraries: @STATIC_SHARED@
+ Statically Linked Executables: @LT_STATIC_EXEC@
+ LDFLAGS: @LDFLAGS@
+ H5_LDFLAGS: @H5_LDFLAGS@
+ AM_LDFLAGS: @AM_LDFLAGS@
+ Extra libraries: @LIBS@
+ Archiver: @AR@
+ AR_FLAGS: @AR_FLAGS@
+ Ranlib: @RANLIB@
+
+Languages:
+----------
+ C: yes
C Compiler: @CC_VERSION@
- CFLAGS: @CFLAGS@
- H5_CFLAGS: @H5_CFLAGS@
- AM_CFLAGS: @AM_CFLAGS@
CPPFLAGS: @CPPFLAGS@
H5_CPPFLAGS: @H5_CPPFLAGS@
AM_CPPFLAGS: @AM_CPPFLAGS@
+ C Flags: @CFLAGS@
+ H5 C Flags: @H5_CFLAGS@
+ AM C Flags: @AM_CFLAGS@
Shared C Library: @enable_shared@
Static C Library: @enable_static@
- Statically Linked Executables: @STATIC_EXEC@
- LDFLAGS: @LDFLAGS@
- H5_LDFLAGS: @H5_LDFLAGS@
- AM_LDFLAGS: @AM_LDFLAGS@
- Extra libraries: @LIBS@
- Archiver: @AR@
- Ranlib: @RANLIB@
- Debugged Packages: @DEBUG_PKG@
- API Tracing: @TRACE_API@
Languages:
----------
@@ -56,19 +66,21 @@ Languages:
Features:
---------
- Parallel HDF5: @PARALLEL@
- High Level library: @HDF5_HL@
- Build HDF5 Tests: @HDF5_TESTS@
- Build HDF5 Tools: @HDF5_TOOLS@
- Threadsafety: @THREADSAFE@
- Default API Mapping: @DEFAULT_API_VERSION@
- With Deprecated Public Symbols: @DEPRECATED_SYMBOLS@
- I/O filters (external): @EXTERNAL_FILTERS@
- MPE: @MPE@
- Direct VFD: @DIRECT_VFD@
- dmalloc: @HAVE_DMALLOC@
-Clear file buffers before write: @CLEARFILEBUF@
- Using memory checker: @USINGMEMCHECKER@
- Function Stack Tracing: @CODESTACK@
- Strict File Format Checks: @STRICT_FORMAT_CHECKS@
- Optimization Instrumentation: @INSTRUMENT@
+ Parallel HDF5: @PARALLEL@
+ High-level library: @HDF5_HL@
+ Build HDF5 Tests: @HDF5_TESTS@
+ Build HDF5 Tools: @HDF5_TOOLS@
+ Threadsafety: @THREADSAFE@
+ Default API mapping: @DEFAULT_API_VERSION@
+ With deprecated public symbols: @DEPRECATED_SYMBOLS@
+ I/O filters (external): @EXTERNAL_FILTERS@
+ MPE: @MPE@
+ Direct VFD: @DIRECT_VFD@
+ (Read-Only) S3 VFD: @ROS3_VFD@
+ (Read-Only) HDFS VFD: @HAVE_LIBHDFS@
+ dmalloc: @HAVE_DMALLOC@
+ Clear file buffers before write: @CLEARFILEBUF@
+ Using memory checker: @USINGMEMCHECKER@
+ Function stack tracing: @CODESTACK@
+ Strict file format checks: @STRICT_FORMAT_CHECKS@
+ Optimization instrumentation: @INSTRUMENT@
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 22cdb90..a67755b 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -207,6 +207,9 @@ set (H5_TESTS
ttsafe # multiple source
getname
vfd
+ ros3
+ s3comms
+ hdfs
ntypes
dangle
dtransform
@@ -250,6 +253,7 @@ endforeach ()
### M U L T I P L E S O U R C E T E S T S ###
##############################################################################
+
######### Special handling for multiple sources #############
#-- Adding test for testhdf5
add_executable (testhdf5 ${testhdf5_SOURCES})
diff --git a/test/Makefile.am b/test/Makefile.am
index fd7f9b5..2e8f85a 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -43,7 +43,7 @@ TEST_PROG=testhdf5 lheap ohdr stab gheap cache cache_api \
dtypes dsets cmpd_dset filter_fail extend external efc objcopy links unlink \
big mtime fillval mount flush1 flush2 app_ref enum \
set_extent ttsafe \
- getname vfd ntypes dangle dtransform reserved cross_read \
+ getname vfd ros3 s3comms hdfs ntypes dangle dtransform reserved cross_read \
freespace mf btree2 fheap file_image unregister
# List programs to be built when testing here. error_test and err_compat are
@@ -157,8 +157,8 @@ CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offse
dtransform.h5 test_filters.h5 get_file_name.h5 tstint[1-2].h5 \
unlink_chunked.h5 btree2.h5 objcopy_src.h5 objcopy_dst.h5 \
objcopy_ext.dat trefer1.h5 trefer2.h5 app_ref.h5 tcheck_version_*.out \
- tcheck_version_*.err efc[0-5].h5 log_vfd_out.log \
- new_multi_file_v16-r.h5 new_multi_file_v16-s.h5 \
+ tcheck_version_*.err efc[0-5].h5 log_vfd_out.log log_ros3_out.log \
+ log_s3comms_out.log new_multi_file_v16-r.h5 new_multi_file_v16-s.h5 \
split_get_file_image_test-m.h5 split_get_file_image_test-r.h5 \
file_image_core_test.h5.copy unregister_filter_1.h5 unregister_filter_2.h5
diff --git a/test/dtypes.c b/test/dtypes.c
index 9747af7..27ad8e7 100644
--- a/test/dtypes.c
+++ b/test/dtypes.c
@@ -23,7 +23,7 @@
#include "H5Iprivate.h" /* For checking that datatype id's don't leak */
/* Number of elements in each test */
-#define NTESTELEM 100000
+#define NTESTELEM 100000
/* For test_compound_8 and test_compound_10 */
#define ARRAY_DIM 4
@@ -32,7 +32,7 @@
* Offset from alinged memory returned by malloc(). This can be used to test
* that type conversions handle non-aligned buffers correctly.
*/
-#define ALIGNMENT 1
+#define ALIGNMENT 1
/*
* Define if you want to test alignment code on a machine that doesn't
@@ -118,22 +118,22 @@ static int num_opaque_conversions_g = 0;
static int opaque_check(int tag_it);
static herr_t convert_opaque(hid_t st, hid_t dt,
H5T_cdata_t *cdata,
- size_t nelmts, size_t buf_stride,
+ size_t nelmts, size_t buf_stride,
size_t bkg_stride, void *_buf,
- void *bkg, hid_t dset_xfer_plid);
+ void *bkg, hid_t dset_xfer_plid);
static int opaque_long(void);
static int opaque_funcs(void);
-
+
/*-------------------------------------------------------------------------
- * Function: reset_hdf5
+ * Function: reset_hdf5
*
- * Purpose: Reset the hdf5 library. This causes statistics to be printed
- * and counters to be reset.
+ * Purpose: Reset the hdf5 library. This causes statistics to be printed
+ * and counters to be reset.
*
- * Return: void
+ * Return: void
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, November 16, 1998
*
* Modifications:
@@ -164,7 +164,7 @@ reset_hdf5(void)
}
-
+
/*-------------------------------------------------------------------------
* Function: test_classes
*
@@ -265,7 +265,7 @@ test_classes(void)
return 1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_copy
*
@@ -286,7 +286,7 @@ static int
test_copy(void)
{
hid_t a_copy;
- herr_t status;
+ herr_t status;
TESTING("H5Tcopy()");
@@ -295,12 +295,12 @@ test_copy(void)
/* We should not be able to close a built-in byte */
H5E_BEGIN_TRY {
- status = H5Tclose (H5T_NATIVE_SCHAR);
+ status = H5Tclose (H5T_NATIVE_SCHAR);
} H5E_END_TRY;
if (status>=0) {
- H5_FAILED();
- HDputs (" Should not be able to close a predefined type!");
- goto error;
+ H5_FAILED();
+ HDputs (" Should not be able to close a predefined type!");
+ goto error;
}
PASSED();
@@ -310,7 +310,7 @@ test_copy(void)
return 1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_detect
*
@@ -487,7 +487,7 @@ error:
return 1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_compound_1
*
@@ -651,19 +651,19 @@ error:
return retval;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_compound_2
+ * Function: test_compound_2
*
- * Purpose: Tests a compound type conversion where the source and
- * destination are the same except for the order of the
- * elements.
+ * Purpose: Tests a compound type conversion where the source and
+ * destination are the same except for the order of the
+ * elements.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, June 17, 1999
*
* Modifications:
@@ -674,19 +674,19 @@ static int
test_compound_2(void)
{
struct st {
- int a, b, c[4], d, e;
+ int a, b, c[4], d, e;
} *s_ptr;
struct dt {
- int e, d, c[4], b, a;
+ int e, d, c[4], b, a;
} *d_ptr;
- const size_t nelmts = NTESTELEM;
- const hsize_t four = 4;
- unsigned char *buf=NULL, *orig=NULL, *bkg=NULL;
- hid_t st=-1, dt=-1;
+ const size_t nelmts = NTESTELEM;
+ const hsize_t four = 4;
+ unsigned char *buf=NULL, *orig=NULL, *bkg=NULL;
+ hid_t st=-1, dt=-1;
hid_t array_dt;
- int64_t nmembs;
- int i;
+ int64_t nmembs;
+ int i;
TESTING("compound element reordering");
@@ -698,15 +698,15 @@ test_compound_2(void)
bkg = (unsigned char*)HDmalloc(nelmts * sizeof(struct dt));
orig = (unsigned char*)HDmalloc(nelmts * sizeof(struct st));
for (i=0; i<(int)nelmts; i++) {
- s_ptr = ((struct st*)((void *)orig)) + i;
- s_ptr->a = i*8+0;
- s_ptr->b = i*8+1;
- s_ptr->c[0] = i*8+2;
- s_ptr->c[1] = i*8+3;
- s_ptr->c[2] = i*8+4;
- s_ptr->c[3] = i*8+5;
- s_ptr->d = i*8+6;
- s_ptr->e = i*8+7;
+ s_ptr = ((struct st*)((void *)orig)) + i;
+ s_ptr->a = i*8+0;
+ s_ptr->b = i*8+1;
+ s_ptr->c[0] = i*8+2;
+ s_ptr->c[1] = i*8+3;
+ s_ptr->c[2] = i*8+4;
+ s_ptr->c[3] = i*8+5;
+ s_ptr->d = i*8+6;
+ s_ptr->e = i*8+7;
}
HDmemcpy(buf, orig, nelmts*sizeof(struct st));
@@ -736,26 +736,26 @@ test_compound_2(void)
/* Compare results */
for (i=0; i<(int)nelmts; i++) {
- s_ptr = ((struct st*)((void *)orig)) + i;
- d_ptr = ((struct dt*)((void *)buf)) + i;
- if (s_ptr->a != d_ptr->a ||
- s_ptr->b != d_ptr->b ||
- s_ptr->c[0] != d_ptr->c[0] ||
- s_ptr->c[1] != d_ptr->c[1] ||
- s_ptr->c[2] != d_ptr->c[2] ||
- s_ptr->c[3] != d_ptr->c[3] ||
- s_ptr->d != d_ptr->d ||
- s_ptr->e != d_ptr->e) {
- H5_FAILED();
- HDprintf(" i=%d\n", i);
- HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n",
- s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2],
- s_ptr->c[3], s_ptr->d, s_ptr->e);
- HDprintf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n",
- d_ptr->a, d_ptr->b, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2],
- d_ptr->c[3], d_ptr->d, d_ptr->e);
- goto error;
- }
+ s_ptr = ((struct st*)((void *)orig)) + i;
+ d_ptr = ((struct dt*)((void *)buf)) + i;
+ if (s_ptr->a != d_ptr->a ||
+ s_ptr->b != d_ptr->b ||
+ s_ptr->c[0] != d_ptr->c[0] ||
+ s_ptr->c[1] != d_ptr->c[1] ||
+ s_ptr->c[2] != d_ptr->c[2] ||
+ s_ptr->c[3] != d_ptr->c[3] ||
+ s_ptr->d != d_ptr->d ||
+ s_ptr->e != d_ptr->e) {
+ H5_FAILED();
+ HDprintf(" i=%d\n", i);
+ HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n",
+ s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2],
+ s_ptr->c[3], s_ptr->d, s_ptr->e);
+ HDprintf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n",
+ d_ptr->a, d_ptr->b, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2],
+ d_ptr->c[3], d_ptr->d, d_ptr->e);
+ goto error;
+ }
}
/* Release resources */
@@ -782,19 +782,19 @@ error:
return 1;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_compound_3
+ * Function: test_compound_3
*
- * Purpose: Tests compound conversions where the source and destination
- * are the same except the destination is missing a couple
- * members which appear in the source.
+ * Purpose: Tests compound conversions where the source and destination
+ * are the same except the destination is missing a couple
+ * members which appear in the source.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, June 17, 1999
*
* Modifications:
@@ -805,19 +805,19 @@ static int
test_compound_3(void)
{
struct st {
- int a, b, c[4], d, e;
+ int a, b, c[4], d, e;
} *s_ptr;
struct dt {
- int a, c[4], e;
+ int a, c[4], e;
} *d_ptr;
- const size_t nelmts = NTESTELEM;
- const hsize_t four = 4;
- unsigned char *buf=NULL, *orig=NULL, *bkg=NULL;
- hid_t st=-1, dt=-1;
+ const size_t nelmts = NTESTELEM;
+ const hsize_t four = 4;
+ unsigned char *buf=NULL, *orig=NULL, *bkg=NULL;
+ hid_t st=-1, dt=-1;
hid_t array_dt;
- int64_t nmembs;
- int i;
+ int64_t nmembs;
+ int i;
TESTING("compound subset conversions");
@@ -866,24 +866,24 @@ test_compound_3(void)
/* Compare results */
for (i=0; i<(int)nelmts; i++) {
- s_ptr = ((struct st*)((void *)orig)) + i;
- d_ptr = ((struct dt*)((void *)buf)) + i;
- if (s_ptr->a != d_ptr->a ||
- s_ptr->c[0] != d_ptr->c[0] ||
- s_ptr->c[1] != d_ptr->c[1] ||
- s_ptr->c[2] != d_ptr->c[2] ||
- s_ptr->c[3] != d_ptr->c[3] ||
- s_ptr->e != d_ptr->e) {
- H5_FAILED();
- HDprintf(" i=%d\n", i);
- HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n",
- s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2],
- s_ptr->c[3], s_ptr->d, s_ptr->e);
- HDprintf(" dst={a=%d, c=[%d,%d,%d,%d], e=%d\n",
- d_ptr->a, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2],
- d_ptr->c[3], d_ptr->e);
- goto error;
- }
+ s_ptr = ((struct st*)((void *)orig)) + i;
+ d_ptr = ((struct dt*)((void *)buf)) + i;
+ if (s_ptr->a != d_ptr->a ||
+ s_ptr->c[0] != d_ptr->c[0] ||
+ s_ptr->c[1] != d_ptr->c[1] ||
+ s_ptr->c[2] != d_ptr->c[2] ||
+ s_ptr->c[3] != d_ptr->c[3] ||
+ s_ptr->e != d_ptr->e) {
+ H5_FAILED();
+ HDprintf(" i=%d\n", i);
+ HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n",
+ s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2],
+ s_ptr->c[3], s_ptr->d, s_ptr->e);
+ HDprintf(" dst={a=%d, c=[%d,%d,%d,%d], e=%d\n",
+ d_ptr->a, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2],
+ d_ptr->c[3], d_ptr->e);
+ goto error;
+ }
}
/* Release resources */
@@ -909,19 +909,19 @@ error:
return 1;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_compound_4
+ * Function: test_compound_4
*
- * Purpose: Tests compound conversions when the destination has the same
- * fields as the source but one or more of the fields are
- * smaller.
+ * Purpose: Tests compound conversions when the destination has the same
+ * fields as the source but one or more of the fields are
+ * smaller.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, June 17, 1999
*
* Modifications:
@@ -933,22 +933,22 @@ test_compound_4(void)
{
struct st {
- int a, b, c[4], d, e;
+ int a, b, c[4], d, e;
} *s_ptr;
struct dt {
- short b;
- int a, c[4];
- short d;
- int e;
+ short b;
+ int a, c[4];
+ short d;
+ int e;
} *d_ptr;
- const size_t nelmts = NTESTELEM;
- const hsize_t four = 4;
- unsigned char *buf=NULL, *orig=NULL, *bkg=NULL;
- hid_t st=-1, dt=-1;
+ const size_t nelmts = NTESTELEM;
+ const hsize_t four = 4;
+ unsigned char *buf=NULL, *orig=NULL, *bkg=NULL;
+ hid_t st=-1, dt=-1;
hid_t array_dt;
- int64_t nmembs;
- int i;
+ int64_t nmembs;
+ int i;
TESTING("compound element shrinking & reordering");
@@ -999,26 +999,26 @@ test_compound_4(void)
/* Compare results */
for (i=0; i<(int)nelmts; i++) {
- s_ptr = ((struct st*)((void *)orig)) + i;
- d_ptr = ((struct dt*)((void *)buf)) + i;
- if (s_ptr->a != d_ptr->a ||
- s_ptr->b != d_ptr->b ||
- s_ptr->c[0] != d_ptr->c[0] ||
- s_ptr->c[1] != d_ptr->c[1] ||
- s_ptr->c[2] != d_ptr->c[2] ||
- s_ptr->c[3] != d_ptr->c[3] ||
- s_ptr->d != d_ptr->d ||
- s_ptr->e != d_ptr->e) {
- H5_FAILED();
- HDprintf(" i=%d\n", i);
- HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n",
- s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2],
- s_ptr->c[3], s_ptr->d, s_ptr->e);
- HDprintf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n",
- d_ptr->a, d_ptr->b, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2],
- d_ptr->c[3], d_ptr->d, d_ptr->e);
- goto error;
- }
+ s_ptr = ((struct st*)((void *)orig)) + i;
+ d_ptr = ((struct dt*)((void *)buf)) + i;
+ if (s_ptr->a != d_ptr->a ||
+ s_ptr->b != d_ptr->b ||
+ s_ptr->c[0] != d_ptr->c[0] ||
+ s_ptr->c[1] != d_ptr->c[1] ||
+ s_ptr->c[2] != d_ptr->c[2] ||
+ s_ptr->c[3] != d_ptr->c[3] ||
+ s_ptr->d != d_ptr->d ||
+ s_ptr->e != d_ptr->e) {
+ H5_FAILED();
+ HDprintf(" i=%d\n", i);
+ HDprintf(" src={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n",
+ s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2],
+ s_ptr->c[3], s_ptr->d, s_ptr->e);
+ HDprintf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d], d=%d, e=%d\n",
+ d_ptr->a, d_ptr->b, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2],
+ d_ptr->c[3], d_ptr->d, d_ptr->e);
+ goto error;
+ }
}
/* Release resources */
@@ -1044,20 +1044,20 @@ error:
return 1;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_compound_5
+ * Function: test_compound_5
*
- * Purpose: Many versions of HDF5 have a bug in the optimized compound
+ * Purpose: Many versions of HDF5 have a bug in the optimized compound
* datatype conversion function, H5T_conv_struct_opt(), which
* is triggered when the top-level type contains a struct
* which must undergo a conversion.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, June 17, 1999
*
* Modifications:
@@ -1149,19 +1149,19 @@ test_compound_5(void)
return retval;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_compound_6
+ * Function: test_compound_6
*
- * Purpose: Tests compound conversions when the destination has the same
- * fields as the source but one or more of the fields are
- * larger.
+ * Purpose: Tests compound conversions when the destination has the same
+ * fields as the source but one or more of the fields are
+ * larger.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Wednesday, December 13, 2000
*
* Modifications:
@@ -1181,11 +1181,11 @@ test_compound_6(void)
long d;
} *d_ptr;
- const size_t nelmts = NTESTELEM;
- unsigned char *buf=NULL, *orig=NULL, *bkg=NULL;
- hid_t st=-1, dt=-1;
- int64_t nmembs;
- int i;
+ const size_t nelmts = NTESTELEM;
+ unsigned char *buf=NULL, *orig=NULL, *bkg=NULL;
+ hid_t st=-1, dt=-1;
+ int64_t nmembs;
+ int i;
TESTING("compound element growing");
@@ -1226,18 +1226,18 @@ test_compound_6(void)
/* Compare results */
for (i=0; i<(int)nelmts; i++) {
- s_ptr = ((struct st*)((void *)orig)) + i;
- d_ptr = ((struct dt*)((void *)buf)) + i;
- if (s_ptr->b != d_ptr->b ||
- s_ptr->d != d_ptr->d) {
- H5_FAILED();
- HDprintf(" i=%d\n", i);
- HDprintf(" src={b=%d, d=%d\n",
+ s_ptr = ((struct st*)((void *)orig)) + i;
+ d_ptr = ((struct dt*)((void *)buf)) + i;
+ if (s_ptr->b != d_ptr->b ||
+ s_ptr->d != d_ptr->d) {
+ H5_FAILED();
+ HDprintf(" i=%d\n", i);
+ HDprintf(" src={b=%d, d=%d\n",
(int)s_ptr->b, (int)s_ptr->d);
- HDprintf(" dst={b=%ld, d=%ld\n",
+ HDprintf(" dst={b=%ld, d=%ld\n",
d_ptr->b, d_ptr->d);
- goto error;
- }
+ goto error;
+ }
}
/* Release resources */
@@ -1264,17 +1264,17 @@ error:
}
/*-------------------------------------------------------------------------
- * Function: test_compound_7
+ * Function: test_compound_7
*
- * Purpose: Tests inserting fields into compound datatypes when the field
+ * Purpose: Tests inserting fields into compound datatypes when the field
* overlaps the end of the compound datatype. Also, tests
* increasing compound type size.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Tuesday, December 18, 2001
*
* Modifications:
@@ -1410,7 +1410,7 @@ error:
return 1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_compound_8
*
@@ -1656,7 +1656,7 @@ test_compound_8(void)
return 1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_compound_9
*
@@ -1676,14 +1676,20 @@ test_compound_8(void)
static int
test_compound_9(void)
{
- typedef struct cmpd_struct {
+ typedef struct cmpd_struct_w {
int i1;
const char* str;
int i2;
- } cmpd_struct;
+ } cmpd_struct_w;
- cmpd_struct wdata = {11, "variable-length string", 22};
- cmpd_struct rdata;
+ typedef struct cmpd_struct_r {
+ int i1;
+ char* str;
+ int i2;
+ } cmpd_struct_r;
+
+ cmpd_struct_w wdata = {11, "variable-length string", 22};
+ cmpd_struct_r rdata;
hid_t file;
hid_t cmpd_tid, str_id, dup_tid;
hid_t space_id;
@@ -1702,13 +1708,13 @@ test_compound_9(void)
} /* end if */
/* Create first compound datatype */
- if((cmpd_tid = H5Tcreate( H5T_COMPOUND, sizeof(struct cmpd_struct))) < 0) {
+ if((cmpd_tid = H5Tcreate( H5T_COMPOUND, sizeof(struct cmpd_struct_w))) < 0) {
H5_FAILED(); AT();
HDprintf("Can't create datatype!\n");
goto error;
} /* end if */
- if(H5Tinsert(cmpd_tid,"i1",HOFFSET(struct cmpd_struct,i1),H5T_NATIVE_INT) < 0) {
+ if(H5Tinsert(cmpd_tid, "i1", HOFFSET(struct cmpd_struct_w, i1), H5T_NATIVE_INT) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'i1'\n");
goto error;
@@ -1721,13 +1727,13 @@ test_compound_9(void)
goto error;
} /* end if */
- if(H5Tinsert(cmpd_tid, "vl_string", HOFFSET(cmpd_struct, str), str_id) < 0) {
+ if(H5Tinsert(cmpd_tid, "vl_string", HOFFSET(cmpd_struct_w, str), str_id) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'i1'\n");
goto error;
} /* end if */
- if(H5Tinsert(cmpd_tid, "i2", HOFFSET(struct cmpd_struct, i2), H5T_NATIVE_INT) < 0) {
+ if(H5Tinsert(cmpd_tid, "i2", HOFFSET(struct cmpd_struct_w, i2), H5T_NATIVE_INT) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'i2'\n");
goto error;
@@ -1878,7 +1884,7 @@ test_compound_9(void)
return 1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_compound_10
*
@@ -2072,7 +2078,7 @@ test_compound_10(void)
return 1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_compound_11
*
@@ -2315,7 +2321,7 @@ error:
return retval;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_compound_12
*
@@ -2357,14 +2363,14 @@ test_compound_12(void)
size+=tmp_size;
if (H5Tset_size(complex_id, size) < 0) goto error;
if (H5Tinsert(complex_id, "real", offset,
- H5T_NATIVE_DOUBLE) < 0) goto error;
+ H5T_NATIVE_DOUBLE) < 0) goto error;
offset = size;
if((tmp_size=H5Tget_size(H5T_NATIVE_DOUBLE))==0) goto error;
size+=tmp_size;
if (H5Tset_size(complex_id, size) < 0) goto error;
if (H5Tinsert(complex_id, "imaginary", offset,
- H5T_NATIVE_DOUBLE) < 0) goto error;
+ H5T_NATIVE_DOUBLE) < 0) goto error;
/* Increase and decrease the size. */
if((tmp_size=H5Tget_size(H5T_NATIVE_DOUBLE))==0) goto error;
@@ -2398,7 +2404,7 @@ test_compound_12(void)
return 1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_compound_13
*
@@ -2500,7 +2506,7 @@ error:
return 1;
} /* end test_compound_13() */
-
+
/*-------------------------------------------------------------------------
* Function: test_compound_14
*
@@ -2523,27 +2529,42 @@ error:
static int
test_compound_14(void)
{
- typedef struct cmpd_struct_1 {
+ typedef struct cmpd_struct_1_w {
char c1;
char c2;
const char* str;
- } cmpd_struct_1;
+ } cmpd_struct_1_w;
- typedef struct cmpd_struct_2 {
+ typedef struct cmpd_struct_1_r {
char c1;
char c2;
char* str;
+ } cmpd_struct_1_r;
+
+ typedef struct cmpd_struct_2_w {
+ char c1;
+ char c2;
+ const char* str;
long l1;
long l2;
long l3;
long l4;
- } cmpd_struct_2;
+ } cmpd_struct_2_w;
- cmpd_struct_1 wdata1 = {'A', 'B', "variable-length string"};
+ typedef struct cmpd_struct_2_r {
+ char c1;
+ char c2;
+ char* str;
+ long l1;
+ long l2;
+ long l3;
+ long l4;
+ } cmpd_struct_2_r;
- cmpd_struct_1 rdata1;
- cmpd_struct_2 wdata2 = {'C', 'D', "another vlen!", 1, 2, -1, 9001};
- cmpd_struct_2 rdata2;
+ cmpd_struct_1_w wdata1 = {'A', 'B', "variable-length string"};
+ cmpd_struct_1_r rdata1;
+ cmpd_struct_2_w wdata2 = {'C', 'D', "another vlen!", 1, 2, -1, 9001};
+ cmpd_struct_2_r rdata2;
hid_t file;
hid_t cmpd_m1_tid, cmpd_f1_tid, cmpd_m2_tid, cmpd_f2_tid, str_id;
hid_t space_id;
@@ -2562,19 +2583,19 @@ test_compound_14(void)
} /* end if */
/* Create memory compound datatype 1 */
- if((cmpd_m1_tid = H5Tcreate( H5T_COMPOUND, sizeof(struct cmpd_struct_1))) < 0) {
+ if((cmpd_m1_tid = H5Tcreate( H5T_COMPOUND, sizeof(struct cmpd_struct_1_w))) < 0) {
H5_FAILED(); AT();
HDprintf("Can't create datatype!\n");
goto error;
} /* end if */
- if(H5Tinsert(cmpd_m1_tid,"c1",HOFFSET(struct cmpd_struct_1,c1),H5T_NATIVE_CHAR) < 0) {
+ if(H5Tinsert(cmpd_m1_tid,"c1",HOFFSET(struct cmpd_struct_1_w, c1), H5T_NATIVE_CHAR) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'c1'\n");
goto error;
} /* end if */
- if(H5Tinsert(cmpd_m1_tid,"c2",HOFFSET(struct cmpd_struct_1,c2),H5T_NATIVE_CHAR) < 0) {
+ if(H5Tinsert(cmpd_m1_tid,"c2",HOFFSET(struct cmpd_struct_1_w, c2), H5T_NATIVE_CHAR) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'c2'\n");
goto error;
@@ -2587,7 +2608,7 @@ test_compound_14(void)
goto error;
} /* end if */
- if(H5Tinsert(cmpd_m1_tid, "vl_string", HOFFSET(cmpd_struct_1, str), str_id) < 0) {
+ if(H5Tinsert(cmpd_m1_tid, "vl_string", HOFFSET(cmpd_struct_1_w, str), str_id) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'vl_string'\n");
goto error;
@@ -2619,49 +2640,49 @@ test_compound_14(void)
} /* end if */
/* Create memory compound datatype 2 */
- if((cmpd_m2_tid = H5Tcreate( H5T_COMPOUND, sizeof(struct cmpd_struct_2))) < 0) {
+ if((cmpd_m2_tid = H5Tcreate( H5T_COMPOUND, sizeof(struct cmpd_struct_2_w))) < 0) {
H5_FAILED(); AT();
HDprintf("Can't create datatype!\n");
goto error;
} /* end if */
- if(H5Tinsert(cmpd_m2_tid,"c1",HOFFSET(struct cmpd_struct_2,c1),H5T_NATIVE_CHAR) < 0) {
+ if(H5Tinsert(cmpd_m2_tid,"c1",HOFFSET(struct cmpd_struct_2_w, c1), H5T_NATIVE_CHAR) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'c1'\n");
goto error;
} /* end if */
- if(H5Tinsert(cmpd_m2_tid,"c2",HOFFSET(struct cmpd_struct_2,c2),H5T_NATIVE_CHAR) < 0) {
+ if(H5Tinsert(cmpd_m2_tid,"c2",HOFFSET(struct cmpd_struct_2_w, c2), H5T_NATIVE_CHAR) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'c2'\n");
goto error;
} /* end if */
- if(H5Tinsert(cmpd_m2_tid, "vl_string", HOFFSET(cmpd_struct_2, str), str_id) < 0) {
+ if(H5Tinsert(cmpd_m2_tid, "vl_string", HOFFSET(cmpd_struct_2_w, str), str_id) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'vl_string'\n");
goto error;
} /* end if */
- if(H5Tinsert(cmpd_m2_tid,"l1",HOFFSET(struct cmpd_struct_2,l1),H5T_NATIVE_LONG) < 0) {
+ if(H5Tinsert(cmpd_m2_tid,"l1",HOFFSET(struct cmpd_struct_2_w, l1), H5T_NATIVE_LONG) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'l1'\n");
goto error;
} /* end if */
- if(H5Tinsert(cmpd_m2_tid,"l2",HOFFSET(struct cmpd_struct_2,l2),H5T_NATIVE_LONG) < 0) {
+ if(H5Tinsert(cmpd_m2_tid,"l2",HOFFSET(struct cmpd_struct_2_w, l2), H5T_NATIVE_LONG) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'l2'\n");
goto error;
} /* end if */
- if(H5Tinsert(cmpd_m2_tid,"l3",HOFFSET(struct cmpd_struct_2,l3),H5T_NATIVE_LONG) < 0) {
+ if(H5Tinsert(cmpd_m2_tid,"l3",HOFFSET(struct cmpd_struct_2_w, l3), H5T_NATIVE_LONG) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'l3'\n");
goto error;
} /* end if */
- if(H5Tinsert(cmpd_m2_tid,"l4",HOFFSET(struct cmpd_struct_2,l4),H5T_NATIVE_LONG) < 0) {
+ if(H5Tinsert(cmpd_m2_tid,"l4",HOFFSET(struct cmpd_struct_2_w, l4), H5T_NATIVE_LONG) < 0) {
H5_FAILED(); AT();
HDprintf("Can't insert field 'l4'\n");
goto error;
@@ -2892,7 +2913,7 @@ test_compound_14(void)
return 1;
} /* end test_compound_14() */
-
+
/*-------------------------------------------------------------------------
* Function: test_compound_15
*
@@ -3051,7 +3072,7 @@ test_compound_15(void)
return 1;
} /* end test_compound_15() */
-
+
/*-------------------------------------------------------------------------
* Function: test_compound_16
*
@@ -3143,7 +3164,7 @@ error:
return 1;
} /* end test_compound_16() */
-
+
/*-------------------------------------------------------------------------
* Function: test_compound_17
*
@@ -3266,7 +3287,7 @@ error:
return 1;
} /* end test_compound_17() */
-
+
/*-------------------------------------------------------------------------
* Function: test_compound_18
*
@@ -3332,7 +3353,7 @@ test_compound_18(void)
FAIL_PUTS_ERROR("created attribute with bad compound datatype")
} /* end if */
- /* Commit the datatype */
+ /* Commit the datatype */
H5E_BEGIN_TRY {
ret = H5Tcommit2(file, "cmpnd", tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
} H5E_END_TRY;
@@ -3392,7 +3413,7 @@ error:
return 1;
} /* end test_compound_18() */
-
+
/*-------------------------------------------------------------------------
* Function: test_query
*
@@ -3638,17 +3659,17 @@ test_query(void)
return 1;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_transient
+ * Function: test_transient
*
- * Purpose: Tests transient datatypes.
+ * Purpose: Tests transient datatypes.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, June 4, 1998
*
* Modifications:
@@ -3658,36 +3679,36 @@ test_query(void)
static int
test_transient (hid_t fapl)
{
- static hsize_t ds_size[2] = {10, 20};
- hid_t file=-1, type=-1, space=-1, dset=-1, t2=-1;
- char filename[1024];
- hid_t ret_id; /* Generic hid_t return value */
- herr_t status;
+ static hsize_t ds_size[2] = {10, 20};
+ hid_t file=-1, type=-1, space=-1, dset=-1, t2=-1;
+ char filename[1024];
+ hid_t ret_id; /* Generic hid_t return value */
+ herr_t status;
TESTING("transient datatypes");
h5_fixname(FILENAME[0], fapl, filename, sizeof filename);
if ((file=H5Fcreate (filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) {
- goto error;
+ goto error;
}
if ((space = H5Screate_simple (2, ds_size, ds_size)) < 0) goto error;
/* Predefined types cannot be modified or closed */
H5E_BEGIN_TRY {
- status = H5Tset_precision(H5T_NATIVE_INT, (size_t)256);
+ status = H5Tset_precision(H5T_NATIVE_INT, (size_t)256);
} H5E_END_TRY;
if (status>=0) {
- H5_FAILED();
- HDputs (" Predefined types should not be modifiable!");
- goto error;
+ H5_FAILED();
+ HDputs (" Predefined types should not be modifiable!");
+ goto error;
}
H5E_BEGIN_TRY {
- status = H5Tclose (H5T_NATIVE_INT);
+ status = H5Tclose (H5T_NATIVE_INT);
} H5E_END_TRY;
if (status>=0) {
- H5_FAILED();
- HDputs (" Predefined types should not be closable!");
- goto error;
+ H5_FAILED();
+ HDputs (" Predefined types should not be closable!");
+ goto error;
}
/* Copying a predefined type results in a modifiable copy */
@@ -3696,29 +3717,29 @@ test_transient (hid_t fapl)
/* It should not be possible to create an attribute for a transient type */
H5E_BEGIN_TRY {
- ret_id = H5Acreate2(type, "attr1", H5T_NATIVE_INT, space, H5P_DEFAULT, H5P_DEFAULT);
+ ret_id = H5Acreate2(type, "attr1", H5T_NATIVE_INT, space, H5P_DEFAULT, H5P_DEFAULT);
} H5E_END_TRY;
if (ret_id>=0) {
- H5_FAILED();
- HDputs (" Attributes should not be allowed for transient types!");
- goto error;
+ H5_FAILED();
+ HDputs (" Attributes should not be allowed for transient types!");
+ goto error;
}
/* Create a dataset from a transient datatype */
if(H5Tclose(type) < 0) goto error;
if((type = H5Tcopy(H5T_NATIVE_INT)) < 0) goto error;
if((dset = H5Dcreate2(file, "dset1", type, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
- goto error;
+ goto error;
/* The type returned from a dataset should not be modifiable */
if((t2 = H5Dget_type(dset)) < 0) goto error;
H5E_BEGIN_TRY {
- status = H5Tset_precision(t2, (size_t)256);
+ status = H5Tset_precision(t2, (size_t)256);
} H5E_END_TRY;
if(status >= 0) {
- H5_FAILED();
- HDputs (" Dataset datatypes should not be modifiable!");
- goto error;
+ H5_FAILED();
+ HDputs (" Dataset datatypes should not be modifiable!");
+ goto error;
}
if(H5Tclose(t2) < 0) goto error;
@@ -3730,12 +3751,12 @@ test_transient (hid_t fapl)
if((dset = H5Dopen2(file, "dset1", H5P_DEFAULT)) < 0) goto error;
if((t2 = H5Dget_type(dset)) < 0) goto error;
H5E_BEGIN_TRY {
- status = H5Tset_precision(t2, (size_t)256);
+ status = H5Tset_precision(t2, (size_t)256);
} H5E_END_TRY;
if(status >= 0) {
- H5_FAILED();
- HDputs (" Dataset datatypes should not be modifiable!");
- goto error;
+ H5_FAILED();
+ HDputs (" Dataset datatypes should not be modifiable!");
+ goto error;
}
if(H5Tclose(t2) < 0) goto error;
@@ -3758,26 +3779,26 @@ test_transient (hid_t fapl)
error:
H5E_BEGIN_TRY {
- H5Tclose (t2);
- H5Tclose (type);
- H5Sclose (space);
- H5Dclose (dset);
- H5Fclose (file);
+ H5Tclose (t2);
+ H5Tclose (type);
+ H5Sclose (space);
+ H5Dclose (dset);
+ H5Fclose (file);
} H5E_END_TRY;
return 1;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_named
+ * Function: test_named
*
- * Purpose: Tests named datatypes.
+ * Purpose: Tests named datatypes.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, June 1, 1998
*
* Modifications:
@@ -3787,29 +3808,29 @@ test_transient (hid_t fapl)
static int
test_named (hid_t fapl)
{
- hid_t file=-1, type=-1, space=-1, dset=-1, t2=-1, t3=-1, attr1=-1;
- herr_t status;
- static hsize_t ds_size[2] = {10, 20};
- size_t i,j;
- unsigned attr_data[10][20];
- char filename[1024];
+ hid_t file=-1, type=-1, space=-1, dset=-1, t2=-1, t3=-1, attr1=-1;
+ herr_t status;
+ static hsize_t ds_size[2] = {10, 20};
+ size_t i,j;
+ unsigned attr_data[10][20];
+ char filename[1024];
TESTING("named datatypes");
h5_fixname(FILENAME[1], fapl, filename, sizeof filename);
if ((file=H5Fcreate (filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) {
- goto error;
+ goto error;
}
if ((space = H5Screate_simple (2, ds_size, ds_size)) < 0) goto error;
/* Predefined types cannot be committed */
H5E_BEGIN_TRY {
- status = H5Tcommit2(file, "test_named_1 (should not exist)", H5T_NATIVE_INT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ status = H5Tcommit2(file, "test_named_1 (should not exist)", H5T_NATIVE_INT, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
} H5E_END_TRY;
if(status >= 0) {
- H5_FAILED();
- HDputs (" Predefined types should not be committable!");
- goto error;
+ H5_FAILED();
+ HDputs (" Predefined types should not be committable!");
+ goto error;
}
/* Copy a predefined datatype and commit the copy */
@@ -3817,34 +3838,34 @@ test_named (hid_t fapl)
if(H5Tcommit2(file, "native-int", type, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) goto error;
if((status = H5Tcommitted(type)) < 0) goto error;
if(0 == status) {
- H5_FAILED();
- HDputs (" H5Tcommitted() returned false!");
- goto error;
+ H5_FAILED();
+ HDputs (" H5Tcommitted() returned false!");
+ goto error;
}
/* We should not be able to modify a type after it has been committed. */
H5E_BEGIN_TRY {
- status = H5Tset_precision (type, (size_t)256);
+ status = H5Tset_precision (type, (size_t)256);
} H5E_END_TRY;
if (status>=0) {
- H5_FAILED();
- HDputs (" Committed type is not constant!");
- goto error;
+ H5_FAILED();
+ HDputs (" Committed type is not constant!");
+ goto error;
}
/* We should not be able to re-commit a committed type */
H5E_BEGIN_TRY {
- status = H5Tcommit2(file, "test_named_2 (should not exist)", type, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ status = H5Tcommit2(file, "test_named_2 (should not exist)", type, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
} H5E_END_TRY;
if(status >= 0) {
- H5_FAILED();
- HDputs (" Committed types should not be recommitted!");
- goto error;
+ H5_FAILED();
+ HDputs (" Committed types should not be recommitted!");
+ goto error;
}
/* It should be possible to define an attribute for the named type */
if((attr1 = H5Acreate2(type, "attr1", H5T_NATIVE_UCHAR, space,
- H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error;
+ H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error;
for(i = 0; i < (size_t)ds_size[0]; i++)
for(j = 0; j < (size_t)ds_size[1]; j++)
attr_data[i][j] = (unsigned)(i * ds_size[1] + j);
@@ -3858,9 +3879,9 @@ test_named (hid_t fapl)
if((t2 = H5Tcopy(type)) < 0) goto error;
if((status = H5Tcommitted(t2)) < 0) goto error;
if(status) {
- H5_FAILED();
- HDputs (" Copying a named type should result in a transient type!");
- goto error;
+ H5_FAILED();
+ HDputs (" Copying a named type should result in a transient type!");
+ goto error;
}
if(H5Tset_precision(t2, (size_t)256) < 0) goto error;
if(H5Tclose(t2) < 0) goto error;
@@ -3873,22 +3894,22 @@ test_named (hid_t fapl)
FAIL_STACK_ERROR
if((status = H5Tcommitted(type)) < 0) goto error;
if(!status) {
- H5_FAILED();
- HDputs (" Opened named types should be named types!");
- goto error;
+ H5_FAILED();
+ HDputs (" Opened named types should be named types!");
+ goto error;
}
/* Create a dataset that uses the named type */
if((dset = H5Dcreate2(file, "dset1", type, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
- goto error;
+ goto error;
/* Get the dataset's datatype and make sure it's a named type */
if((t2 = H5Dget_type(dset)) < 0) goto error;
if((status = H5Tcommitted(t2)) < 0) goto error;
if(!status) {
- H5_FAILED();
- HDputs (" Dataset type should be a named type!");
- goto error;
+ H5_FAILED();
+ HDputs (" Dataset type should be a named type!");
+ goto error;
}
/* Close the dataset, then close its type, then reopen the dataset */
@@ -3900,9 +3921,9 @@ test_named (hid_t fapl)
if((t2 = H5Dget_type(dset)) < 0) goto error;
if((status = H5Tcommitted(t2)) < 0) goto error;
if(!status) {
- H5_FAILED();
- HDputs (" Dataset type should be a named type!");
- goto error;
+ H5_FAILED();
+ HDputs (" Dataset type should be a named type!");
+ goto error;
}
/*
@@ -3919,9 +3940,9 @@ test_named (hid_t fapl)
if((t2 = H5Dget_type(dset)) < 0) goto error;
if((status = H5Tcommitted(t2)) < 0) goto error;
if(!status) {
- H5_FAILED();
- HDputs (" Dataset type should be a named type!");
- goto error;
+ H5_FAILED();
+ HDputs (" Dataset type should be a named type!");
+ goto error;
}
if(H5Tclose(t2) < 0) goto error;
@@ -3940,17 +3961,17 @@ test_named (hid_t fapl)
if((t2 = H5Tcopy(type)) < 0) goto error;
if((status = H5Tcommitted(t2)) < 0) goto error;
if(status) {
- H5_FAILED();
- HDputs (" Copied type should not be a named type!");
- goto error;
+ H5_FAILED();
+ HDputs (" Copied type should not be a named type!");
+ goto error;
}
if((dset = H5Dcreate2(file, "dset3", t2, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto error;
if((t3 = H5Dget_type(dset)) < 0) goto error;
if((status = H5Tcommitted(t3)) < 0) goto error;
if(status) {
- H5_FAILED();
- HDputs (" Datatype from dataset using copied type should not be a named type!");
- goto error;
+ H5_FAILED();
+ HDputs (" Datatype from dataset using copied type should not be a named type!");
+ goto error;
}
if(H5Tclose(t3) < 0) goto error;
if(H5Dclose(dset) < 0) goto error;
@@ -3995,26 +4016,26 @@ test_named (hid_t fapl)
error:
H5E_BEGIN_TRY {
- H5Tclose(t3);
- H5Tclose(t2);
- H5Tclose(type);
- H5Sclose(space);
- H5Dclose(dset);
- H5Fclose(file);
+ H5Tclose(t3);
+ H5Tclose(t2);
+ H5Tclose(type);
+ H5Sclose(space);
+ H5Dclose(dset);
+ H5Fclose(file);
} H5E_END_TRY;
return 1;
}
-
+
/*-------------------------------------------------------------------------
- * Function: mkstr
+ * Function: mkstr
*
- * Purpose: Create a new string datatype
+ * Purpose: Create a new string datatype
*
- * Return: Success: New type
- * Failure: -1
+ * Return: Success: New type
+ * Failure: -1
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, August 10, 1998
*
*-------------------------------------------------------------------------
@@ -4022,7 +4043,7 @@ error:
static hid_t
mkstr(size_t len, H5T_str_t strpad)
{
- hid_t t;
+ hid_t t;
if((t = H5Tcopy(H5T_C_S1)) < 0)
return -1;
@@ -4034,16 +4055,16 @@ mkstr(size_t len, H5T_str_t strpad)
return t;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_str_create
+ * Function: test_str_create
*
- * Purpose: Test string type creation using H5Tcreate
+ * Purpose: Test string type creation using H5Tcreate
*
- * Return: Success: 0
- * Failure: number of errors
+ * Return: Success: 0
+ * Failure: number of errors
*
- * Programmer: Raymond Lu
+ * Programmer: Raymond Lu
* 19 May 2011
*
*-------------------------------------------------------------------------
@@ -4084,10 +4105,10 @@ test_str_create(void)
if(!H5Tequal(vlen_str1, vlen_str2)) goto error;
if((is_vl_str = H5Tis_variable_str(vlen_str1)) < 0) goto error;
- if(!is_vl_str) goto error;
+ if(!is_vl_str) goto error;
if((is_vl_str = H5Tis_variable_str(vlen_str2)) < 0) goto error;
- if(!is_vl_str) goto error;
+ if(!is_vl_str) goto error;
if(H5Tclose(vlen_str1) < 0) goto error;
if(H5Tclose(vlen_str2) < 0) goto error;
@@ -4100,16 +4121,16 @@ error:
return 1;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_conv_str_1
+ * Function: test_conv_str_1
*
- * Purpose: Test string conversions
+ * Purpose: Test string conversions
*
- * Return: Success: 0
- * Failure: number of errors
+ * Return: Success: 0
+ * Failure: number of errors
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, August 10, 1998
*
*-------------------------------------------------------------------------
@@ -4117,8 +4138,8 @@ error:
static int
test_conv_str_1(void)
{
- char *buf = NULL;
- hid_t src_type = -1;
+ char *buf = NULL;
+ hid_t src_type = -1;
hid_t dst_type = -1;
TESTING("string conversions");
@@ -4133,15 +4154,15 @@ test_conv_str_1(void)
HDmemcpy(buf, "abcdefghi\0abcdefghi\0", (size_t)20);
if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcd\0abcd\0abcdefghi\0", (size_t)20)) {
- H5_FAILED();
- HDputs(" Truncated C-string test failed");
- goto error;
+ H5_FAILED();
+ HDputs(" Truncated C-string test failed");
+ goto error;
}
if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcd\0\0\0\0\0\0abcd\0\0\0\0\0\0", (size_t)20)) {
- H5_FAILED();
- HDputs(" Extended C-string test failed");
- goto error;
+ H5_FAILED();
+ HDputs(" Extended C-string test failed");
+ goto error;
}
HDfree(buf);
buf = NULL;
@@ -4157,15 +4178,15 @@ test_conv_str_1(void)
HDmemcpy(buf, "abcdefghijabcdefghij", (size_t)20);
if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcdeabcdeabcdefghij", (size_t)20)) {
- H5_FAILED();
- HDputs(" Truncated C buffer test failed");
- goto error;
+ H5_FAILED();
+ HDputs(" Truncated C buffer test failed");
+ goto error;
}
if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcde\0\0\0\0\0abcde\0\0\0\0\0", (size_t)20)) {
- H5_FAILED();
- HDputs(" Extended C buffer test failed");
- goto error;
+ H5_FAILED();
+ HDputs(" Extended C buffer test failed");
+ goto error;
}
HDfree(buf);
buf = NULL;
@@ -4181,15 +4202,15 @@ test_conv_str_1(void)
HDmemcpy(buf, "abcdefghijabcdefghij", (size_t)20);
if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcdeabcdeabcdefghij", (size_t)20)) {
- H5_FAILED();
- HDputs(" Truncated Fortran-string test failed");
- goto error;
+ H5_FAILED();
+ HDputs(" Truncated Fortran-string test failed");
+ goto error;
}
if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcde abcde ", (size_t)20)) {
- H5_FAILED();
- HDputs(" Extended Fortran-string test failed");
- goto error;
+ H5_FAILED();
+ HDputs(" Extended Fortran-string test failed");
+ goto error;
}
HDfree(buf);
buf = NULL;
@@ -4208,25 +4229,25 @@ test_conv_str_1(void)
HDmemcpy(buf, "abcdefghijabcdefghij", (size_t)20);
if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcdefghijabcdefghij", (size_t)20)) {
- H5_FAILED();
- HDputs(" Non-terminated string test 1");
- goto error;
+ H5_FAILED();
+ HDputs(" Non-terminated string test 1");
+ goto error;
}
H5Tclose(dst_type);
if((dst_type = mkstr((size_t)5, H5T_STR_NULLTERM)) < 0) goto error;
HDmemcpy(buf, "abcdefghijabcdefghij", (size_t)20);
if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcd\0abcd\0abcdefghij", (size_t)20)) {
- H5_FAILED();
- HDputs(" Non-terminated string test 2");
- goto error;
+ H5_FAILED();
+ HDputs(" Non-terminated string test 2");
+ goto error;
}
HDmemcpy(buf, "abcdeabcdexxxxxxxxxx", (size_t)20);
if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcde\0\0\0\0\0abcde\0\0\0\0\0", (size_t)20)) {
- H5_FAILED();
- HDputs(" Non-terminated string test 2");
- goto error;
+ H5_FAILED();
+ HDputs(" Non-terminated string test 2");
+ goto error;
}
HDfree(buf);
buf = NULL;
@@ -4242,30 +4263,30 @@ test_conv_str_1(void)
HDmemcpy(buf, "abcdefghi\0abcdefghi\0", (size_t)20);
if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcdefghi abcdefghi ", (size_t)20)) {
- H5_FAILED();
- HDputs(" C string to Fortran test 1");
- goto error;
+ H5_FAILED();
+ HDputs(" C string to Fortran test 1");
+ goto error;
}
if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcdefghi\0abcdefghi\0", (size_t)20)) {
- H5_FAILED();
- HDputs(" Fortran to C string test 1");
- goto error;
+ H5_FAILED();
+ HDputs(" Fortran to C string test 1");
+ goto error;
}
if (H5Tclose(dst_type) < 0) goto error;
if((dst_type = mkstr((size_t)5, H5T_STR_SPACEPAD)) < 0) goto error;
HDmemcpy(buf, "abcdefgh\0\0abcdefgh\0\0", (size_t)20);
if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcdeabcdeabcdefgh\0\0", (size_t)20)) {
- H5_FAILED();
- HDputs(" C string to Fortran test 2");
- goto error;
+ H5_FAILED();
+ HDputs(" C string to Fortran test 2");
+ goto error;
}
if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcde\0\0\0\0\0abcde\0\0\0\0\0", (size_t)20)) {
- H5_FAILED();
- HDputs(" Fortran to C string test 2");
- goto error;
+ H5_FAILED();
+ HDputs(" Fortran to C string test 2");
+ goto error;
}
if (H5Tclose(src_type) < 0) goto error;
if (H5Tclose(dst_type) < 0) goto error;
@@ -4274,15 +4295,15 @@ test_conv_str_1(void)
HDmemcpy(buf, "abcd\0abcd\0xxxxxxxxxx", (size_t)20);
if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcd abcd ", (size_t)20)) {
- H5_FAILED();
- HDputs(" C string to Fortran test 3");
- goto error;
+ H5_FAILED();
+ HDputs(" C string to Fortran test 3");
+ goto error;
}
if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcd\0abcd\0abcd ", (size_t)20)) {
- H5_FAILED();
- HDputs(" Fortran to C string test 3");
- goto error;
+ H5_FAILED();
+ HDputs(" Fortran to C string test 3");
+ goto error;
}
HDfree(buf);
buf = NULL;
@@ -4298,30 +4319,30 @@ test_conv_str_1(void)
HDmemcpy(buf, "abcdefghijabcdefghij", (size_t)20);
if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcdefghijabcdefghij", (size_t)20)) {
- H5_FAILED();
- HDputs(" C buffer to Fortran test 1");
- goto error;
+ H5_FAILED();
+ HDputs(" C buffer to Fortran test 1");
+ goto error;
}
if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcdefghijabcdefghij", (size_t)20)) {
- H5_FAILED();
- HDputs(" Fortran to C buffer test 1");
- goto error;
+ H5_FAILED();
+ HDputs(" Fortran to C buffer test 1");
+ goto error;
}
if (H5Tclose(dst_type) < 0) goto error;
if((dst_type = mkstr((size_t)5, H5T_STR_SPACEPAD)) < 0) goto error;
HDmemcpy(buf, "abcdefgh\0\0abcdefgh\0\0", (size_t)20);
if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcdeabcdeabcdefgh\0\0", (size_t)20)) {
- H5_FAILED();
- HDputs(" C buffer to Fortran test 2");
- goto error;
+ H5_FAILED();
+ HDputs(" C buffer to Fortran test 2");
+ goto error;
}
if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcde\0\0\0\0\0abcde\0\0\0\0\0", (size_t)20)) {
- H5_FAILED();
- HDputs(" Fortran to C buffer test 2");
- goto error;
+ H5_FAILED();
+ HDputs(" Fortran to C buffer test 2");
+ goto error;
}
if (H5Tclose(src_type) < 0) goto error;
if (H5Tclose(dst_type) < 0) goto error;
@@ -4330,15 +4351,15 @@ test_conv_str_1(void)
HDmemcpy(buf, "abcd\0abcd\0xxxxxxxxxx", (size_t)20);
if (H5Tconvert(src_type, dst_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcd abcd ", (size_t)20)) {
- H5_FAILED();
- HDputs(" C buffer to Fortran test 3");
- goto error;
+ H5_FAILED();
+ HDputs(" C buffer to Fortran test 3");
+ goto error;
}
if (H5Tconvert(dst_type, src_type, (size_t)2, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (HDmemcmp(buf, "abcd\0abcd\0abcd ", (size_t)20)) {
- H5_FAILED();
- HDputs(" Fortran to C buffer test 3");
- goto error;
+ H5_FAILED();
+ HDputs(" Fortran to C buffer test 3");
+ goto error;
}
if(H5Tclose(src_type) < 0) goto error;
if(H5Tclose(dst_type) < 0) goto error;
@@ -4354,7 +4375,7 @@ test_conv_str_1(void)
error:
H5E_BEGIN_TRY {
- H5Tclose(src_type);
+ H5Tclose(src_type);
H5Tclose(dst_type);
} H5E_END_TRY;
@@ -4369,16 +4390,16 @@ error:
return 1;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_conv_str_2
+ * Function: test_conv_str_2
*
- * Purpose: Tests C-to-Fortran and Fortran-to-C string conversion speed.
+ * Purpose: Tests C-to-Fortran and Fortran-to-C string conversion speed.
*
- * Return: Success: 0
- * Failure: number of errors
+ * Return: Success: 0
+ * Failure: number of errors
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Monday, August 10, 1998
*
*-------------------------------------------------------------------------
@@ -4386,12 +4407,12 @@ error:
static int
test_conv_str_2(void)
{
- char *buf = NULL, s[80];
- hid_t c_type = -1;
- hid_t f_type = -1;
- const size_t nelmts = NTESTELEM;
- size_t i, j, nchars;
- int ret_value = 1;
+ char *buf = NULL, s[80];
+ hid_t c_type = -1;
+ hid_t f_type = -1;
+ const size_t nelmts = NTESTELEM;
+ size_t i, j, nchars;
+ int ret_value = 1;
/*
* Initialize types and buffer.
@@ -4400,10 +4421,10 @@ test_conv_str_2(void)
if((f_type = mkstr((size_t)8, H5T_STR_SPACEPAD)) < 0) goto error;
if(NULL == (buf = (char*)HDcalloc(nelmts, (size_t)8))) goto error;
for(i = 0; i < nelmts; i++) {
- nchars = (size_t)(HDrand() % 8);
- for(j = 0; j < nchars; j++)
- buf[i * 8 + j] = (char)('a' + HDrand() % 26);
- while(j < nchars)
+ nchars = (size_t)(HDrand() % 8);
+ for(j = 0; j < nchars; j++)
+ buf[i * 8 + j] = (char)('a' + HDrand() % 26);
+ while(j < nchars)
buf[i * 8 + j++] = '\0';
} /* end for */
@@ -4436,17 +4457,17 @@ error:
return ret_value;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_conv_str_3
+ * Function: test_conv_str_3
*
- * Purpose: Tests some functions that are or aren't supposed to work
+ * Purpose: Tests some functions that are or aren't supposed to work
* for string type.
*
- * Return: Success: 0
- * Failure: number of errors
+ * Return: Success: 0
+ * Failure: number of errors
*
- * Programmer: Raymond Lu
+ * Programmer: Raymond Lu
* Tuesday, April 4, 2006
*
*-------------------------------------------------------------------------
@@ -4454,12 +4475,12 @@ error:
static int
test_conv_str_3(void)
{
- char *buf=NULL;
- hid_t type = -1;
- hid_t super = -1;
- const size_t nelmts = NTESTELEM;
- size_t i, j, nchars;
- int ret_value = 1;
+ char *buf=NULL;
+ hid_t type = -1;
+ hid_t super = -1;
+ const size_t nelmts = NTESTELEM;
+ size_t i, j, nchars;
+ int ret_value = 1;
size_t size;
H5T_pad_t inpad;
H5T_sign_t sign;
@@ -4475,10 +4496,10 @@ test_conv_str_3(void)
if(NULL == (buf = (char*)HDcalloc(nelmts, (size_t)8)))
FAIL_PUTS_ERROR("Allocation failed.");
for(i = 0; i < nelmts; i++) {
- nchars = (size_t)(HDrand() % 8);
- for(j = 0; j < nchars; j++)
- buf[i * 8 + j] = (char)('a' + HDrand() % 26);
- while(j < nchars)
+ nchars = (size_t)(HDrand() % 8);
+ for(j = 0; j < nchars; j++)
+ buf[i * 8 + j] = (char)('a' + HDrand() % 26);
+ while(j < nchars)
buf[i * 8 + j++] = '\0';
} /* end for */
@@ -4541,7 +4562,7 @@ error:
H5Tclose(super);
} H5E_END_TRY;
- if(buf)
+ if(buf)
HDfree(buf);
if(tag)
H5free_memory(tag); /* Technically allocated by API call */
@@ -4554,17 +4575,17 @@ error:
return ret_value; /* Number of errors */
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_conv_enum_1
+ * Function: test_conv_enum_1
*
- * Purpose: Test conversion speed for enum datatypes
+ * Purpose: Test conversion speed for enum datatypes
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Tuesday, January 5, 1999
*
* Modifications:
@@ -4575,11 +4596,11 @@ static int
test_conv_enum_1(void)
{
const size_t nelmts=NTESTELEM;
- int i, val, *buf=NULL;
- hid_t t1 = -1;
- hid_t t2 = -1;
- char s[80];
- int ret_value = 1;
+ int i, val, *buf=NULL;
+ hid_t t1 = -1;
+ hid_t t2 = -1;
+ char s[80];
+ int ret_value = 1;
size_t u;
/* Build the datatypes */
@@ -4587,9 +4608,9 @@ test_conv_enum_1(void)
if((t2 = H5Tenum_create(H5T_NATIVE_INT)) < 0) goto error;
s[1] = '\0';
for(i = 0; i < 26; i++) {
- s[0] = (char)('A' + i);
- H5Tenum_insert(t1, s, &i);
- H5Tenum_insert(t2, s, (val = i * 1000 + i, &val));
+ s[0] = (char)('A' + i);
+ H5Tenum_insert(t1, s, &i);
+ H5Tenum_insert(t2, s, (val = i * 1000 + i, &val));
} /* end for */
/* Initialize the buffer */
@@ -4630,7 +4651,7 @@ error:
return ret_value;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_conv_enum_2
*
@@ -4656,7 +4677,7 @@ test_conv_enum_2(void)
"PURPLE",
"ORANGE",
"WHITE" };
-
+
TESTING("non-native enumeration type conversion");
/* Source enum type */
@@ -4715,17 +4736,17 @@ test_conv_enum_2(void)
return 0;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_conv_bitfield
+ * Function: test_conv_bitfield
*
- * Purpose: Test bitfield conversions.
+ * Purpose: Test bitfield conversions.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, May 20, 1999
*
* Modifications:
@@ -4735,8 +4756,8 @@ test_conv_enum_2(void)
static int
test_conv_bitfield(void)
{
- unsigned char buf[4];
- hid_t st=-1, dt=-1;
+ unsigned char buf[4];
+ hid_t st=-1, dt=-1;
TESTING("bitfield conversions");
@@ -4751,10 +4772,10 @@ test_conv_bitfield(void)
buf[2] = buf[3] = 0x55; /*irrelevant*/
if (H5Tconvert(st, dt, (size_t)1, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (buf[0]!=0xAA || buf[1]!=0xAA || buf[2]!=0 || buf[3]!=0) {
- H5_FAILED();
- HDprintf(" s=0xaaaa, d=0x%02x%02x%02x%02x (test 1)\n",
- buf[3], buf[2], buf[1], buf[0]);
- goto error;
+ H5_FAILED();
+ HDprintf(" s=0xaaaa, d=0x%02x%02x%02x%02x (test 1)\n",
+ buf[3], buf[2], buf[1], buf[0]);
+ goto error;
}
/*
@@ -4770,10 +4791,10 @@ test_conv_bitfield(void)
buf[0] = 0xA8; buf[1] = 0x2A; buf[2] = buf[3] = 0;
if (H5Tconvert(st, dt, (size_t)1, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (buf[0]!=0 || buf[1]!=0xA8 || buf[2]!=0x2A || buf[3]!=0) {
- H5_FAILED();
- HDprintf(" s=0x2AA8 d=0x%02x%02x%02x%02x (test 2)\n",
- buf[3], buf[2], buf[1], buf[0]);
- goto error;
+ H5_FAILED();
+ HDprintf(" s=0x2AA8 d=0x%02x%02x%02x%02x (test 2)\n",
+ buf[3], buf[2], buf[1], buf[0]);
+ goto error;
}
/*
@@ -4784,10 +4805,10 @@ test_conv_bitfield(void)
buf[0] = 0xA8; buf[1] = 0x2A; buf[2] = buf[3] = 0;
if (H5Tconvert(st, dt, (size_t)1, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (buf[0]!=0xff || buf[1]!=0xAB || buf[2]!=0xEA || buf[3]!=0xff) {
- H5_FAILED();
- HDprintf(" s=0x2AA8 d=0x%02x%02x%02x%02x (test 3)\n",
- buf[3], buf[2], buf[1], buf[0]);
- goto error;
+ H5_FAILED();
+ HDprintf(" s=0x2AA8 d=0x%02x%02x%02x%02x (test 3)\n",
+ buf[3], buf[2], buf[1], buf[0]);
+ goto error;
}
H5Tclose(st);
@@ -4812,18 +4833,18 @@ error:
return 1;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_bitfield_funcs
+ * Function: test_bitfield_funcs
*
- * Purpose: Test some datatype functions that are and aren't supposed
+ * Purpose: Test some datatype functions that are and aren't supposed
* work for bitfield type.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Raymond Lu
+ * Programmer: Raymond Lu
* Wednesday, April 5, 2006
*
* Modifications:
@@ -4833,7 +4854,7 @@ error:
static int
test_bitfield_funcs(void)
{
- hid_t type=-1, ntype=-1, super=-1;
+ hid_t type=-1, ntype=-1, super=-1;
size_t size;
char* tag=0;
H5T_pad_t inpad;
@@ -4941,17 +4962,17 @@ error:
return retval;
}
-
+
/*-------------------------------------------------------------------------
- * Function: convert_opaque
+ * Function: convert_opaque
*
- * Purpose: A fake opaque conversion functions
+ * Purpose: A fake opaque conversion functions
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: -1
+ * Failure: -1
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Friday, June 4, 1999
*
* Modifications:
@@ -4960,25 +4981,25 @@ error:
*/
static herr_t
convert_opaque(hid_t H5_ATTR_UNUSED st, hid_t H5_ATTR_UNUSED dt, H5T_cdata_t *cdata,
- size_t H5_ATTR_UNUSED nelmts, size_t H5_ATTR_UNUSED buf_stride,
+ size_t H5_ATTR_UNUSED nelmts, size_t H5_ATTR_UNUSED buf_stride,
size_t H5_ATTR_UNUSED bkg_stride, void H5_ATTR_UNUSED *_buf,
- void H5_ATTR_UNUSED *bkg, hid_t H5_ATTR_UNUSED dset_xfer_plid)
+ void H5_ATTR_UNUSED *bkg, hid_t H5_ATTR_UNUSED dset_xfer_plid)
{
if (H5T_CONV_CONV==cdata->command) num_opaque_conversions_g++;
return 0;
}
-
+
/*-------------------------------------------------------------------------
- * Function: test_opaque
+ * Function: test_opaque
*
- * Purpose: Driver function to test opaque datatypes
+ * Purpose: Driver function to test opaque datatypes
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Raymond Lu
+ * Programmer: Raymond Lu
* June 2, 2004
*
* Modifications:
@@ -5011,17 +5032,17 @@ test_opaque(void)
return num_errors;
}
-
+
/*-------------------------------------------------------------------------
- * Function: opaque_check
+ * Function: opaque_check
*
- * Purpose: Test opaque datatypes
+ * Purpose: Test opaque datatypes
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, May 20, 1999
*
*-------------------------------------------------------------------------
@@ -5030,10 +5051,10 @@ static int
opaque_check(int tag_it)
{
#define OPAQUE_NELMTS 1000
- hid_t st=-1, dt=-1;
- herr_t status;
- char buf[1]; /*not really used*/
- int saved;
+ hid_t st=-1, dt=-1;
+ herr_t status;
+ char buf[1]; /*not really used*/
+ int saved;
saved = num_opaque_conversions_g = 0;
@@ -5049,29 +5070,29 @@ opaque_check(int tag_it)
/* Make sure that we can't convert between the types yet */
H5E_BEGIN_TRY {
- status = H5Tconvert(st, dt, (size_t)OPAQUE_NELMTS, buf, NULL, H5P_DEFAULT);
+ status = H5Tconvert(st, dt, (size_t)OPAQUE_NELMTS, buf, NULL, H5P_DEFAULT);
} H5E_END_TRY;
if (status>=0) {
- H5_FAILED();
- HDprintf(" opaque conversion should have failed but succeeded\n");
- goto error;
+ H5_FAILED();
+ HDprintf(" opaque conversion should have failed but succeeded\n");
+ goto error;
}
/* Register a conversion function */
if (H5Tregister(H5T_PERS_HARD, "o_test", st, dt, convert_opaque) < 0)
- goto error;
+ goto error;
/* Try the conversion again, this time it should work */
if (H5Tconvert(st, dt, (size_t)OPAQUE_NELMTS, buf, NULL, H5P_DEFAULT) < 0) goto error;
if (saved+1 != num_opaque_conversions_g) {
- H5_FAILED();
- HDprintf(" unexpected number of opaque conversions\n");
- goto error;
+ H5_FAILED();
+ HDprintf(" unexpected number of opaque conversions\n");
+ goto error;
}
/* Unregister conversion function */
if (H5Tunregister(H5T_PERS_HARD, "o_test", st, dt, convert_opaque) < 0)
- goto error;
+ goto error;
H5Tclose(st);
H5Tclose(dt);
@@ -5084,16 +5105,16 @@ opaque_check(int tag_it)
return 1;
}
-
+
/*-------------------------------------------------------------------------
- * Function: opaque_long
+ * Function: opaque_long
*
- * Purpose: Test named (committed) opaque datatypes w/very long tags
+ * Purpose: Test named (committed) opaque datatypes w/very long tags
*
- * Return: Success: 0
- * Failure: number of errors
+ * Return: Success: 0
+ * Failure: number of errors
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Tuesday, June 14, 2005
*
*-------------------------------------------------------------------------
@@ -5101,8 +5122,8 @@ opaque_check(int tag_it)
static int
opaque_long(void)
{
- char *long_tag = NULL;
- hid_t dt = -1;
+ char *long_tag = NULL;
+ hid_t dt = -1;
herr_t ret;
/* Build opaque type */
@@ -5115,7 +5136,7 @@ opaque_long(void)
/* Set opaque type's tag */
H5E_BEGIN_TRY {
- ret = H5Tset_tag(dt, long_tag);
+ ret = H5Tset_tag(dt, long_tag);
} H5E_END_TRY;
if(ret != FAIL) TEST_ERROR
@@ -5136,18 +5157,18 @@ error:
return 1;
}
-
+
/*-------------------------------------------------------------------------
- * Function: opaque_funcs
+ * Function: opaque_funcs
*
- * Purpose: Test some type functions that are and aren't supposed to
+ * Purpose: Test some type functions that are and aren't supposed to
* work with opaque type.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Raymond Lu
+ * Programmer: Raymond Lu
* Wednesday, April 5, 2006
*
* Modifications:
@@ -5157,7 +5178,7 @@ error:
static int
opaque_funcs(void)
{
- hid_t type = -1, super=-1;
+ hid_t type = -1, super=-1;
size_t size;
H5T_pad_t inpad;
H5T_cset_t cset;
@@ -5252,7 +5273,7 @@ opaque_funcs(void)
return 1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_encode
*
@@ -5293,7 +5314,7 @@ test_encode(void)
size_t enum_buf_size = 0;
size_t vlstr_buf_size = 0;
unsigned char *cmpd_buf=NULL, *enum_buf=NULL, *vlstr_buf=NULL;
- hid_t ret_id;
+ hid_t ret_id;
herr_t ret;
TESTING("functions of encoding and decoding datatypes");
@@ -5394,7 +5415,7 @@ test_encode(void)
/* Try decoding bogus buffer */
H5E_BEGIN_TRY {
- ret_id = H5Tdecode(cmpd_buf);
+ ret_id = H5Tdecode(cmpd_buf);
} H5E_END_TRY;
if(ret_id!=FAIL) {
H5_FAILED();
@@ -5706,39 +5727,39 @@ test_encode(void)
*/
/* Make sure the reference counts for the decoded datatypes are one. */
- if(H5Iget_ref(decoded_tid1) != 1) {
+ if(H5Iget_ref(decoded_tid1) != 1) {
H5_FAILED();
HDprintf("Decoded datatype has incorrect reference count\n");
goto error;
} /* end if */
- if(H5Iget_ref(decoded_tid2) != 1) {
+ if(H5Iget_ref(decoded_tid2) != 1) {
H5_FAILED();
HDprintf("Decoded datatype has incorrect reference count\n");
goto error;
} /* end if */
- if(H5Iget_ref(decoded_tid3) != 1) {
+ if(H5Iget_ref(decoded_tid3) != 1) {
H5_FAILED();
HDprintf("Decoded datatype has incorrect reference count\n");
goto error;
} /* end if */
- /* Make sure the reference counts for the decoded datatypes can be
+ /* Make sure the reference counts for the decoded datatypes can be
* decremented and the datatypes are closed. */
- if(H5Idec_ref(decoded_tid1) != 0) {
+ if(H5Idec_ref(decoded_tid1) != 0) {
H5_FAILED();
HDprintf("Decoded datatype can't close\n");
goto error;
} /* end if */
- if(H5Idec_ref(decoded_tid2) != 0) {
+ if(H5Idec_ref(decoded_tid2) != 0) {
H5_FAILED();
HDprintf("Decoded datatype can't close\n");
goto error;
} /* end if */
- if(H5Idec_ref(decoded_tid3) != 0) {
+ if(H5Idec_ref(decoded_tid3) != 0) {
H5_FAILED();
HDprintf("Decoded datatype can't close\n");
goto error;
@@ -5746,7 +5767,7 @@ test_encode(void)
/* Make sure the decoded datatypes are already closed. */
H5E_BEGIN_TRY {
- ret = H5Tclose(decoded_tid1);
+ ret = H5Tclose(decoded_tid1);
} H5E_END_TRY;
if(ret!=FAIL) {
H5_FAILED();
@@ -5755,7 +5776,7 @@ test_encode(void)
}
H5E_BEGIN_TRY {
- ret = H5Tclose(decoded_tid2);
+ ret = H5Tclose(decoded_tid2);
} H5E_END_TRY;
if(ret!=FAIL) {
H5_FAILED();
@@ -5764,7 +5785,7 @@ test_encode(void)
}
H5E_BEGIN_TRY {
- ret = H5Tclose(decoded_tid3);
+ ret = H5Tclose(decoded_tid3);
} H5E_END_TRY;
if(ret!=FAIL) {
H5_FAILED();
@@ -5818,7 +5839,7 @@ test_encode(void)
return 1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_latest
*
@@ -5845,7 +5866,7 @@ test_latest(void)
hid_t file = (-1); /* File ID */
hid_t tid1 = (-1), tid2 = (-1); /* Datatype ID */
hid_t fapl = (-1); /* File access property list */
- H5O_info_t oi; /* Stat buffer for committed datatype */
+ H5O_info_t oi; /* Stat buffer for committed datatype */
hsize_t old_dtype_oh_size; /* Size of object header with "old" format */
hsize_t new_dtype_oh_size; /* Size of object header with "new" format */
char filename[1024]; /* Buffer for filename */
@@ -6036,7 +6057,7 @@ conv_except(H5T_conv_except_t except_type, hid_t H5_ATTR_UNUSED src_id, hid_t H5
return(H5T_CONV_UNHANDLED);
}
-
+
/*-------------------------------------------------------------------------
* Function: test_int_float_except
*
@@ -6179,7 +6200,7 @@ error:
#endif /* H5_SIZEOF_INT==4 && H5_SIZEOF_FLOAT==4 */
} /* end test_int_float_except() */
-
+
/*-------------------------------------------------------------------------
* Function: test_set_order
*
@@ -6340,11 +6361,11 @@ error:
return 1;
} /* end test_set_order() */
-
+
/*-------------------------------------------------------------------------
* Function: test_set_order_compound
*
- * Purpose: Tests H5Tset_order/H5Tget_order for complicated compound
+ * Purpose: Tests H5Tset_order/H5Tget_order for complicated compound
* type.
*
* Return: Success: 0
@@ -6376,13 +6397,13 @@ test_set_order_compound(hid_t fapl)
hid_t cmpd = -1, memb_cmpd = -1, memb_array1 = -1, memb_array2 = -1, cmpd_array = -1;
hid_t vl_id = -1;
hsize_t dims[2] = {3, 4}; /* Array dimenstions */
- char filename[1024];
+ char filename[1024];
herr_t ret; /* Generic return value */
TESTING("H5Tset/get_order for compound type");
if((memb_cmpd = H5Tcreate(H5T_COMPOUND, sizeof(atomic_cmpd))) < 0) FAIL_STACK_ERROR
- if(H5Tinsert(memb_cmpd, "i", HOFFSET(atomic_cmpd, i), H5T_NATIVE_INT) < 0) FAIL_STACK_ERROR
+ if(H5Tinsert(memb_cmpd, "i", HOFFSET(atomic_cmpd, i), H5T_NATIVE_INT) < 0) FAIL_STACK_ERROR
if(H5Tinsert(memb_cmpd, "c", HOFFSET(atomic_cmpd, c), H5T_NATIVE_CHAR) < 0) FAIL_STACK_ERROR
if(H5Tinsert(memb_cmpd, "s", HOFFSET(atomic_cmpd, s), H5T_NATIVE_SHORT) < 0) FAIL_STACK_ERROR
if(H5Tinsert(memb_cmpd, "f", HOFFSET(atomic_cmpd, f), H5T_NATIVE_FLOAT) < 0) FAIL_STACK_ERROR
@@ -6402,19 +6423,19 @@ test_set_order_compound(hid_t fapl)
/* Create a compound type using the types above. */
if((cmpd = H5Tcreate(H5T_COMPOUND, sizeof(complex_cmpd))) < 0) FAIL_STACK_ERROR
- if(H5Tinsert(cmpd, "a", HOFFSET(complex_cmpd, a), memb_cmpd) < 0) FAIL_STACK_ERROR
- if(H5Tinsert(cmpd, "vl_type", HOFFSET(complex_cmpd, vl), vl_id) < 0) FAIL_STACK_ERROR
- if(H5Tinsert(cmpd, "b", HOFFSET(complex_cmpd, b), memb_array1) < 0) FAIL_STACK_ERROR
- if(H5Tinsert(cmpd, "d", HOFFSET(complex_cmpd, d), memb_array2) < 0) FAIL_STACK_ERROR
+ if(H5Tinsert(cmpd, "a", HOFFSET(complex_cmpd, a), memb_cmpd) < 0) FAIL_STACK_ERROR
+ if(H5Tinsert(cmpd, "vl_type", HOFFSET(complex_cmpd, vl), vl_id) < 0) FAIL_STACK_ERROR
+ if(H5Tinsert(cmpd, "b", HOFFSET(complex_cmpd, b), memb_array1) < 0) FAIL_STACK_ERROR
+ if(H5Tinsert(cmpd, "d", HOFFSET(complex_cmpd, d), memb_array2) < 0) FAIL_STACK_ERROR
/* The order should be mixed now. */
- if(H5Tget_order(cmpd) != H5T_ORDER_MIXED) FAIL_STACK_ERROR
+ if(H5Tget_order(cmpd) != H5T_ORDER_MIXED) FAIL_STACK_ERROR
/* Create an array of the compound type above */
cmpd_array = H5Tarray_create2(cmpd, 2, dims);
/* The order of the array type should be the same as the compound type */
- if(H5Tget_order(cmpd_array) != H5T_ORDER_MIXED) FAIL_STACK_ERROR
+ if(H5Tget_order(cmpd_array) != H5T_ORDER_MIXED) FAIL_STACK_ERROR
/* Verify that the order can't be 'none'. */
H5E_BEGIN_TRY
@@ -6432,13 +6453,13 @@ test_set_order_compound(hid_t fapl)
if(H5Tset_order(cmpd, H5T_ORDER_BE) < 0) FAIL_STACK_ERROR
/* Verify that the order of the compound type is big-endian */
- if(H5Tget_order(cmpd) != H5T_ORDER_BE) FAIL_STACK_ERROR
+ if(H5Tget_order(cmpd) != H5T_ORDER_BE) FAIL_STACK_ERROR
/* Change the order of the array type to little-endian*/
if(H5Tset_order(cmpd_array, H5T_ORDER_LE) < 0) FAIL_STACK_ERROR
/* Verify that the order of the array type is little-endian */
- if(H5Tget_order(cmpd_array) != H5T_ORDER_LE) FAIL_STACK_ERROR
+ if(H5Tget_order(cmpd_array) != H5T_ORDER_LE) FAIL_STACK_ERROR
/* Create file */
h5_fixname(FILENAME[1], fapl, filename, sizeof filename);
@@ -6473,23 +6494,23 @@ error:
H5Tclose(vl_id);
H5Tclose(cmpd);
H5Tclose(cmpd_array);
- H5Fclose(file);
+ H5Fclose(file);
H5E_END_TRY;
return 1;
} /* end test_set_order_compound() */
-
+
/*-------------------------------------------------------------------------
- * Function: test_named_indirect_reopen
+ * Function: test_named_indirect_reopen
*
- * Purpose: Tests that open named datatypes can be reopened indirectly
+ * Purpose: Tests that open named datatypes can be reopened indirectly
* through H5Dget_type without causing problems.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Neil Fortner
+ * Programmer: Neil Fortner
* Thursday, June 4, 2009
*
* Modifications:
@@ -6499,13 +6520,13 @@ error:
static int
test_named_indirect_reopen(hid_t fapl)
{
- hid_t file=-1, type=-1, reopened_type=-1, strtype=-1, dset=-1, space=-1;
- static hsize_t dims[1] = {3};
+ hid_t file=-1, type=-1, reopened_type=-1, strtype=-1, dset=-1, space=-1;
+ static hsize_t dims[1] = {3};
size_t dt_size;
int enum_value;
const char *tag = "opaque_tag";
char *tag_ret = NULL;
- char filename[1024];
+ char filename[1024];
TESTING("indirectly reopening committed datatypes");
@@ -6669,12 +6690,12 @@ test_named_indirect_reopen(hid_t fapl)
error:
H5E_BEGIN_TRY {
- H5Tclose(type);
- H5Tclose(strtype);
- H5Tclose(reopened_type);
- H5Sclose(space);
- H5Dclose(dset);
- H5Fclose(file);
+ H5Tclose(type);
+ H5Tclose(strtype);
+ H5Tclose(reopened_type);
+ H5Sclose(space);
+ H5Dclose(dset);
+ H5Fclose(file);
} H5E_END_TRY;
if(tag_ret)
H5free_memory(tag_ret);
@@ -6682,18 +6703,18 @@ error:
} /* end test_named_indirect_reopen() */
/*-------------------------------------------------------------------------
- * Function: test_named_indirect_reopen_file
+ * Function: test_named_indirect_reopen_file
*
- * Purpose: Tests that a named compound datatype that refers to a named
+ * Purpose: Tests that a named compound datatype that refers to a named
* string datatype can be reopened indirectly through H5Dget_type,
* and shows the correct H5Tcommitted() state, including after the
* file has been closed and reopened.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: number of errors
+ * Failure: number of errors
*
- * Programmer: Mark Hodson
+ * Programmer: Mark Hodson
* Tuesday, March 17, 2015
*
* Modifications:
@@ -6756,7 +6777,7 @@ test_named_indirect_reopen_file(hid_t fapl)
/* Indirectly reopen string type, verify that they report as committed, and the size doesn't change */
if((reopened_strtype = H5Tget_member_type(reopened_cmptype, 0)) < 0) TEST_ERROR
if(strtype_size != H5Tget_size(reopened_strtype)) TEST_ERROR
- if(H5Tcommitted(reopened_strtype) != 1) TEST_ERROR
+ if(H5Tcommitted(reopened_strtype) != 1) TEST_ERROR
/* Close types and dataset */
if(H5Tclose(reopened_strtype) < 0) TEST_ERROR
@@ -6799,13 +6820,13 @@ test_named_indirect_reopen_file(hid_t fapl)
error:
H5E_BEGIN_TRY {
- H5Tclose(cmptype);
- H5Tclose(strtype);
- H5Tclose(reopened_cmptype);
+ H5Tclose(cmptype);
+ H5Tclose(strtype);
+ H5Tclose(reopened_cmptype);
H5Tclose(reopened_strtype);
- H5Sclose(space);
- H5Dclose(dset);
- H5Fclose(file);
+ H5Sclose(space);
+ H5Dclose(dset);
+ H5Fclose(file);
} H5E_END_TRY;
return 1;
} /* end test_named_indirect_reopen() */
@@ -6910,17 +6931,17 @@ static void create_del_obj_named_test_file(const char *filename, hid_t fapl,
HDassert(status >= 0);
} /* end create_del_obj_named_test_file() */
-
+
/*-------------------------------------------------------------------------
- * Function: test_delete_obj_named
+ * Function: test_delete_obj_named
*
- * Purpose: Tests that delete objects that use named datatypes through
+ * Purpose: Tests that delete objects that use named datatypes through
* different file IDs
*
- * Return: Success: 0
- * Failure: number of errors
+ * Return: Success: 0
+ * Failure: number of errors
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Monday, July 18, 2011
*
*-------------------------------------------------------------------------
@@ -6991,27 +7012,27 @@ test_delete_obj_named(hid_t fapl)
error:
H5E_BEGIN_TRY {
- H5Tclose(attr);
- H5Dclose(dset);
- H5Pclose(fapl2);
- H5Fclose(filea1);
- H5Fclose(filea2);
- H5Fclose(fileb);
+ H5Tclose(attr);
+ H5Dclose(dset);
+ H5Pclose(fapl2);
+ H5Fclose(filea1);
+ H5Fclose(filea2);
+ H5Fclose(fileb);
} H5E_END_TRY;
return 1;
} /* end test_delete_obj_named() */
-
+
/*-------------------------------------------------------------------------
- * Function: test_delete_obj_named_fileid
+ * Function: test_delete_obj_named_fileid
*
- * Purpose: Tests that objects that use named datatypes through
+ * Purpose: Tests that objects that use named datatypes through
* different file IDs get the correct file IDs
*
- * Return: Success: 0
- * Failure: number of errors
+ * Return: Success: 0
+ * Failure: number of errors
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, July 28, 2011
*
*-------------------------------------------------------------------------
@@ -7147,26 +7168,26 @@ test_delete_obj_named_fileid(hid_t fapl)
error:
H5E_BEGIN_TRY {
- H5Tclose(attr);
- H5Dclose(dset);
- H5Pclose(fapl2);
- H5Fclose(filea1);
- H5Fclose(filea2);
- H5Fclose(fileb);
+ H5Tclose(attr);
+ H5Dclose(dset);
+ H5Pclose(fapl2);
+ H5Fclose(filea1);
+ H5Fclose(filea2);
+ H5Fclose(fileb);
} H5E_END_TRY;
return 1;
} /* end test_delete_obj_named_fileid() */
-
+
/*-------------------------------------------------------------------------
- * Function: test_deprec
+ * Function: test_deprec
*
- * Purpose: Tests deprecated API routines for datatypes.
+ * Purpose: Tests deprecated API routines for datatypes.
*
- * Return: Success: 0
- * Failure: number of errors
+ * Return: Success: 0
+ * Failure: number of errors
*
- * Programmer: Quincey Koziol
+ * Programmer: Quincey Koziol
* Thursday, September 27, 2007
*
*-------------------------------------------------------------------------
@@ -7175,17 +7196,17 @@ error:
static int
test_deprec(hid_t fapl)
{
- hid_t file = -1; /* File ID */
- hid_t type = -1; /* Datatype ID */
+ hid_t file = -1; /* File ID */
+ hid_t type = -1; /* Datatype ID */
unsigned rank = 2; /* Rank for array datatype */
hsize_t dims[2] = {3, 3}; /* Dimensions for array datatype */
int perm[2] = {0, 1}; /* Dimensions permutations for array datatype */
hsize_t rdims[2]= {0, 0}; /* Dimensions for querying array datatype */
int rperm[2] = {-2, -2}; /* Dimensions permutations for array datatype */
hbool_t dim_mismatch; /* Whether any dimensions didn't match */
- char filename[1024];
+ char filename[1024];
unsigned u; /* Local index variable */
- herr_t status; /* Generic routine value */
+ herr_t status; /* Generic routine value */
TESTING("deprected API routines for datatypes");
@@ -7213,7 +7234,7 @@ test_deprec(hid_t fapl)
dim_mismatch = TRUE;
} /* end if */
if(dim_mismatch)
- FAIL_PUTS_ERROR(" Dimensions didn't match!")
+ FAIL_PUTS_ERROR(" Dimensions didn't match!")
/* Check the array dimension permutations */
dim_mismatch = FALSE;
@@ -7223,7 +7244,7 @@ test_deprec(hid_t fapl)
dim_mismatch = TRUE;
} /* end if */
if(dim_mismatch)
- FAIL_PUTS_ERROR(" Dimension permutations modified!")
+ FAIL_PUTS_ERROR(" Dimension permutations modified!")
/* Close the datatype */
if(H5Tclose(type) < 0)
@@ -7236,31 +7257,31 @@ test_deprec(hid_t fapl)
/* Predefined types cannot be committed */
H5E_BEGIN_TRY {
- status = H5Tcommit1(file, "test_named_1 (should not exist)", H5T_NATIVE_INT);
+ status = H5Tcommit1(file, "test_named_1 (should not exist)", H5T_NATIVE_INT);
} H5E_END_TRY;
if(status >= 0)
- FAIL_PUTS_ERROR(" Predefined types should not be committable!")
+ FAIL_PUTS_ERROR(" Predefined types should not be committable!")
/* Copy a predefined datatype and commit the copy */
if((type = H5Tcopy(H5T_NATIVE_INT)) < 0) FAIL_STACK_ERROR
if(H5Tcommit1(file, "native-int", type) < 0) FAIL_STACK_ERROR
if((status = H5Tcommitted(type)) < 0) FAIL_STACK_ERROR
if(0 == status)
- FAIL_PUTS_ERROR(" H5Tcommitted() returned false!")
+ FAIL_PUTS_ERROR(" H5Tcommitted() returned false!")
/* We should not be able to modify a type after it has been committed. */
H5E_BEGIN_TRY {
- status = H5Tset_precision(type, (size_t)256);
+ status = H5Tset_precision(type, (size_t)256);
} H5E_END_TRY;
if(status >= 0)
- FAIL_PUTS_ERROR(" Committed type is not constant!")
+ FAIL_PUTS_ERROR(" Committed type is not constant!")
/* We should not be able to re-commit a committed type */
H5E_BEGIN_TRY {
- status = H5Tcommit1(file, "test_named_2 (should not exist)", type);
+ status = H5Tcommit1(file, "test_named_2 (should not exist)", type);
} H5E_END_TRY;
if(status >= 0)
- FAIL_PUTS_ERROR(" Committed types should not be recommitted!")
+ FAIL_PUTS_ERROR(" Committed types should not be recommitted!")
/*
* Close the committed type and reopen it. It should return a named type.
@@ -7269,7 +7290,7 @@ test_deprec(hid_t fapl)
if((type = H5Topen1(file, "native-int")) < 0) FAIL_STACK_ERROR
if((status = H5Tcommitted(type)) < 0) FAIL_STACK_ERROR
if(!status)
- FAIL_PUTS_ERROR(" Opened named types should be named types!")
+ FAIL_PUTS_ERROR(" Opened named types should be named types!")
/* Close */
if(H5Tclose(type) < 0) FAIL_STACK_ERROR
@@ -7299,24 +7320,24 @@ test_deprec(hid_t fapl)
error:
H5E_BEGIN_TRY {
- H5Tclose(type);
- H5Fclose(file);
+ H5Tclose(type);
+ H5Fclose(file);
} H5E_END_TRY;
return 1;
} /* end test_deprec() */
#endif /* H5_NO_DEPRECATED_SYMBOLS */
-
+
/*-------------------------------------------------------------------------
- * Function: test_utf_ascii_conv
+ * Function: test_utf_ascii_conv
*
- * Purpose: Make sure the library doesn't conversion strings between
+ * Purpose: Make sure the library doesn't conversion strings between
* ASCII and UTF8.
*
- * Return: Success: 0
- * Failure: number of errors
+ * Return: Success: 0
+ * Failure: number of errors
*
- * Programmer: Raymond Lu
+ * Programmer: Raymond Lu
* 10 November 2011
*-------------------------------------------------------------------------
*/
@@ -7334,12 +7355,12 @@ test_utf_ascii_conv(void)
char *ascii_r = NULL;
const char *ascii_w = "bar!";
char *utf8_r = NULL;
- char filename[1024];
+ char filename[1024];
char ascii2[4], utf8_2[4];
herr_t status;
TESTING("string conversion between ASCII and UTF");
-
+
/************************************************
* Test VL string conversion from UTF8 to ASCII
************************************************/
@@ -7506,20 +7527,20 @@ test_utf_ascii_conv(void)
error:
H5E_BEGIN_TRY {
- H5Tclose(utf8_vtid);
- H5Tclose(ascii_vtid);
- H5Tclose(utf8_tid);
- H5Tclose(ascii_tid);
- H5Dclose(did);
- H5Sclose(sid);
- H5Fclose(fid);
+ H5Tclose(utf8_vtid);
+ H5Tclose(ascii_vtid);
+ H5Tclose(utf8_tid);
+ H5Tclose(ascii_tid);
+ H5Dclose(did);
+ H5Sclose(sid);
+ H5Fclose(fid);
} H5E_END_TRY;
return 1;
}
-
+
/*-------------------------------------------------------------------------
* Function: main
*
@@ -7539,8 +7560,8 @@ error:
int
main(void)
{
- long nerrors = 0;
- hid_t fapl = -1;
+ long nerrors = 0;
+ hid_t fapl = -1;
/* Set the random # seed */
HDsrandom((unsigned)HDtime(NULL));
@@ -7549,7 +7570,7 @@ main(void)
fapl = h5_fileaccess();
if(ALIGNMENT)
- HDprintf("Testing non-aligned conversions (ALIGNMENT=%d)....\n", ALIGNMENT);
+ HDprintf("Testing non-aligned conversions (ALIGNMENT=%d)....\n", ALIGNMENT);
/* Do the tests */
nerrors += test_classes();
diff --git a/test/extend.c b/test/extend.c
index f8c091b..a31ac0e 100644
--- a/test/extend.c
+++ b/test/extend.c
@@ -219,23 +219,15 @@ error:
/*-------------------------------------------------------------------------
- * Function: main
+ * Function: main
*
- * Purpose: Tests extendible datasets
+ * Purpose: Tests extendible datasets
*
- * Return: Success: exit(0)
- *
- * Failure: exit(non-zero)
+ * Return: EXIT_SUCCESS/EXIT_FAILURE
*
* Programmer: Robb Matzke
* Friday, January 30, 1998
*
- * Modifications:
- * Took main data code out into write_data() routine, to allow
- * different dataset creation property list settings to be tested.
- * Quincey Koziol
- * Tuesday, June 10, 2003
- *
*-------------------------------------------------------------------------
*/
int
@@ -290,16 +282,16 @@ main (void)
if(nerrors) {
HDprintf("***** %d FAILURE%s! *****\n", nerrors, (1 == nerrors) ? "" : "S");
- exit(EXIT_FAILURE);
+ HDexit(EXIT_FAILURE);
} /* end if */
HDprintf("All extend tests passed.\n");
h5_cleanup(FILENAME, fapl);
- return 0;
+ HDexit(EXIT_SUCCESS);
error:
HDprintf("*** One or more extend tests failed ***\n");
- return 1;
-}
+ HDexit(EXIT_FAILURE);
+} /* end main() */
diff --git a/test/filter_fail.c b/test/filter_fail.c
index 2acce01..4be2547 100644
--- a/test/filter_fail.c
+++ b/test/filter_fail.c
@@ -347,17 +347,15 @@ error:
* Purpose: Tests the library's behavior when a mandate filter returns
* failure.
*
- * Return: Success: exit(EXIT_SUCCESS)
- * Failure: exit(EXIT_FAILURE)
+ * Return: EXIT_SUCCESS/EXIT_FAILURE
*
* Programmer: Raymond Lu
* 25 August 2010
*
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
-int main(void)
+int
+main(void)
{
hid_t fapl;
int mdc_nelmts = 0;
@@ -398,7 +396,7 @@ int main(void)
if (nerrors) TEST_ERROR
- return 0;
+ HDexit(EXIT_SUCCESS);
error:
if (nerrors) {
@@ -406,4 +404,4 @@ error:
nerrors, 1==nerrors?"":"S");
HDexit(EXIT_FAILURE);
}
-}
+} /* end main() */
diff --git a/test/flush1.c b/test/flush1.c
index 1e759d0..1098824 100644
--- a/test/flush1.c
+++ b/test/flush1.c
@@ -15,11 +15,11 @@
* Programmer: Robb Matzke <matzke@llnl.gov>
* Friday, October 23, 1998
*
- * Purpose: This is the first half of a two-part test that makes sure
- * that a file can be read after an application crashes as long
- * as the file was flushed first. We simulate a crash by
- * calling _exit(EXIT_SUCCESS) since this doesn't flush HDF5 caches but
- * still exits with success.
+ * Purpose: This is the first half of a two-part test that makes sure
+ * that a file can be read after an application crashes as long
+ * as the file was flushed first. We simulate a crash by
+ * calling _exit(EXIT_SUCCESS) since this doesn't flush HDF5 caches but
+ * still exits with success.
*/
#include "h5test.h"
@@ -30,57 +30,67 @@ const char *FILENAME[] = {
NULL
};
-static double the_data[100][100];
+static double the_data[100][100];
/*-------------------------------------------------------------------------
* Function: create_file
*
* Purpose: Creates files and datasets used in part 1 of the test
*
- * Return: Success: a valid file ID
- * Failure: -1
+ * Return: Success: a valid file ID
+ * Failure: -1
*
- * Programmer: Leon Arber
+ * Programmer: Leon Arber
* Sept. 26, 2006
*
*-------------------------------------------------------------------------
*/
static hid_t
-create_file(char* name, hid_t fapl)
+create_file(const char *filename, hid_t fapl_id)
{
- hid_t file, dcpl, space, dset, groups, grp;
- hsize_t ds_size[2] = {100, 100};
- hsize_t ch_size[2] = {5, 5};
- size_t i, j;
+ hid_t fid = -1; /* file ID */
+ hid_t dcpl, space, dset, groups, grp;
+ hsize_t ds_size[2] = {100, 100};
+ hsize_t ch_size[2] = {5, 5};
+ size_t i, j;
- if((file = H5Fcreate(name, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR
+ if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0)
+ STACK_ERROR
/* Create a chunked dataset */
- if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) FAIL_STACK_ERROR
- if(H5Pset_chunk(dcpl, 2, ch_size) < 0) FAIL_STACK_ERROR
- if((space = H5Screate_simple(2, ds_size, NULL)) < 0) FAIL_STACK_ERROR
- if((dset = H5Dcreate2(file, "dset", H5T_NATIVE_FLOAT, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR
+ if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ FAIL_STACK_ERROR
+ if(H5Pset_chunk(dcpl, 2, ch_size) < 0)
+ FAIL_STACK_ERROR
+ if((space = H5Screate_simple(2, ds_size, NULL)) < 0)
+ FAIL_STACK_ERROR
+ if((dset = H5Dcreate2(fid, "dset", H5T_NATIVE_FLOAT, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ FAIL_STACK_ERROR
/* Write some data */
for(i = 0; i < ds_size[0]; i++)
- /*
- * The extra cast in the following statement is a bug workaround
- * for the Win32 version 5.0 compiler.
- * 1998-11-06 ptl
- */
+ /*
+ * The extra cast in the following statement is a bug workaround
+ * for the Win32 version 5.0 compiler.
+ * 1998-11-06 ptl
+ */
for(j = 0; j < (size_t)ds_size[1]; j++)
- the_data[i][j] = (double)(hssize_t)i/(hssize_t)(j+1);
- if(H5Dwrite(dset, H5T_NATIVE_DOUBLE, space, space, H5P_DEFAULT, the_data) < 0) FAIL_STACK_ERROR
+ the_data[i][j] = (double)(hssize_t)i/(hssize_t)(j+1);
+ if(H5Dwrite(dset, H5T_NATIVE_DOUBLE, space, space, H5P_DEFAULT, the_data) < 0)
+ FAIL_STACK_ERROR
/* Create some groups */
- if((groups = H5Gcreate2(file, "some_groups", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR
+ if((groups = H5Gcreate2(fid, "some_groups", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ FAIL_STACK_ERROR
for(i = 0; i < 100; i++) {
- HDsprintf(name, "grp%02u", (unsigned)i);
- if((grp = H5Gcreate2(groups, name, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR
- if(H5Gclose(grp) < 0) FAIL_STACK_ERROR
+ HDsprintf(filename, "grp%02u", (unsigned)i);
+ if((grp = H5Gcreate2(groups, filename, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ FAIL_STACK_ERROR
+ if(H5Gclose(grp) < 0)
+ FAIL_STACK_ERROR
} /* end for */
- return file;
+ return fid;
error:
HD_exit(1);
@@ -88,15 +98,15 @@ error:
/*-------------------------------------------------------------------------
- * Function: extend_file
+ * Function: extend_file
*
- * Purpose: Add a small dataset to the file.
+ * Purpose: Add a small dataset to the file.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: 1
+ * Failure: 1
*
- * Programmer: Leon Arber
+ * Programmer: Leon Arber
* Oct. 4, 2006
*
* Modifications:
@@ -106,31 +116,34 @@ error:
static hid_t
extend_file(hid_t file)
{
- hid_t dcpl, space, dset;
- hsize_t ds_size[2] = {100, 100};
- hsize_t ch_size[2] = {5, 5};
- size_t i, j;
+ hid_t dcpl, space, dset;
+ hsize_t ds_size[2] = {100, 100};
+ hsize_t ch_size[2] = {5, 5};
+ size_t i, j;
/* Create a chunked dataset */
- if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto error;
- if(H5Pset_chunk(dcpl, 2, ch_size) < 0) goto error;
- if((space = H5Screate_simple(2, ds_size, NULL)) < 0) goto error;
+ if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ goto error;
+ if(H5Pset_chunk(dcpl, 2, ch_size) < 0)
+ goto error;
+ if((space = H5Screate_simple(2, ds_size, NULL)) < 0)
+ goto error;
if((dset = H5Dcreate2(file, "dset2", H5T_NATIVE_FLOAT, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
- goto error;
+ goto error;
/* Write some data */
for (i=0; i<ds_size[0]; i++) {
- /*
- * The extra cast in the following statement is a bug workaround
- * for the Win32 version 5.0 compiler.
- * 1998-11-06 ptl
- */
- for (j=0; j<(size_t)ds_size[1]; j++) {
- the_data[i][j] = (double)(hssize_t)i/(hssize_t)(j+1);
- }
+ /*
+ * The extra cast in the following statement is a bug workaround
+ * for the Win32 version 5.0 compiler.
+ * 1998-11-06 ptl
+ */
+ for (j=0; j<(size_t)ds_size[1]; j++) {
+ the_data[i][j] = (double)(hssize_t)i/(hssize_t)(j+1);
+ }
}
- if (H5Dwrite(dset, H5T_NATIVE_DOUBLE, space, space, H5P_DEFAULT,
- the_data) < 0) goto error;
+ if (H5Dwrite(dset, H5T_NATIVE_DOUBLE, space, space, H5P_DEFAULT, the_data) < 0)
+ goto error;
return file;
@@ -141,22 +154,22 @@ error:
}
/*-------------------------------------------------------------------------
- * Function: main
+ * Function: main
*
- * Purpose: Part 1 of a two-part H5Fflush() test.
+ * Purpose: Part 1 of a two-part H5Fflush() test.
*
- * Return: Success: 0
+ * Return: Success: 0
*
- * Failure: 1
+ * Failure: 1
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Friday, October 23, 1998
*
* Modifications:
- * Leon Arber
- * Sept. 26, 2006, expand test to check for failure if H5Fflush is not called.
- * Oct. 4 2006, expand test to check for partial failure in case file is flushed, but then
- * new datasets are created after the flush.
+ * Leon Arber
+ * Sept. 26, 2006, expand test to check for failure if H5Fflush is not called.
+ * Oct. 4 2006, expand test to check for partial failure in case file is flushed, but then
+ * new datasets are created after the flush.
*
*
*-------------------------------------------------------------------------
@@ -164,40 +177,64 @@ error:
int
main(void)
{
- hid_t file, fapl;
- char name[1024];
+ hid_t fid = -1; /* file ID */
+ hid_t fapl_id = -1; /* file access proplist ID */
+ char filename[1024]; /* filename */
h5_reset();
- fapl = h5_fileaccess();
+ if((fapl_id = h5_fileaccess()) < 0)
+ TEST_ERROR
- TESTING("H5Fflush (part1)");
- /* Create the file */
- h5_fixname(FILENAME[0], fapl, name, sizeof name);
- file = create_file(name, fapl);
- /* Flush and exit without closing the library */
- if (H5Fflush(file, H5F_SCOPE_GLOBAL) < 0) goto error;
+ /*************************************************/
+ /* NOTE: Not closing the file ID is intentional! */
+ /*************************************************/
+
+ /* Create a file and flush */
+ TESTING("H5Fflush (part1 with flush)");
/* Create the file */
- h5_fixname(FILENAME[2], fapl, name, sizeof name);
- file = create_file(name, fapl);
+ h5_fixname(FILENAME[0], fapl_id, filename, sizeof(filename));
+ if((fid = create_file(filename, fapl_id)) < 0)
+ TEST_ERROR
+ if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0)
+ FAIL_STACK_ERROR
+ PASSED();
+
+ /* Create a file which will not be flushed */
+ TESTING("H5Fflush (part1 without flush)");
+ h5_fixname(FILENAME[2], fapl_id, filename, sizeof(filename));
+ if((fid = create_file(filename, fapl_id)) < 0)
+ TEST_ERROR
+ PASSED();
+
/* Flush and exit without closing the library */
- if(H5Fflush(file, H5F_SCOPE_GLOBAL) < 0) goto error;
+ if(H5Fflush(fid, H5F_SCOPE_GLOBAL) < 0) goto error;
/* Add a bit to the file and don't flush the new part */
- extend_file(file);
+ extend_file(fid);
/* Create the other file which will not be flushed */
- h5_fixname(FILENAME[1], fapl, name, sizeof name);
- file = create_file(name, fapl);
+ h5_fixname(FILENAME[1], fapl_id, filename, sizeof(filename));
+ if((fid = create_file(filename, fapl_id)) < 0)
+ TEST_ERROR
+ PASSED();
+ /* Flush console output streams */
+ HDfflush(stdout);
+ HDfflush(stderr);
- PASSED();
- fflush(stdout);
- fflush(stderr);
+ /* DO NOT CLOSE FILE ID! */
+ if(H5Pclose(fapl_id) < 0)
+ STACK_ERROR
- HD_exit(0);
+ /* _exit() is necessary since we want a hard close of the library */
+ HD_exit(EXIT_SUCCESS);
error:
- HD_exit(1);
- return 1;
-}
+ H5E_BEGIN_TRY {
+ H5Pclose(fapl_id);
+ } H5E_END_TRY;
+
+ HDexit(EXIT_FAILURE);
+} /* end main() */
+
diff --git a/test/flush2.c b/test/flush2.c
index ec0ba5b..529fbc0 100644
--- a/test/flush2.c
+++ b/test/flush2.c
@@ -55,9 +55,12 @@ check_dset(hid_t file, const char* name)
size_t i, j;
/* Open the dataset */
- if((dset = H5Dopen2(file, name, H5P_DEFAULT)) < 0) goto error;
- if((space = H5Dget_space(dset)) < 0) goto error;
- if(H5Sget_simple_extent_dims(space, ds_size, NULL) < 0) goto error;
+ if((dset = H5Dopen2(file, name, H5P_DEFAULT)) < 0)
+ goto error;
+ if((space = H5Dget_space(dset)) < 0)
+ goto error;
+ if(H5Sget_simple_extent_dims(space, ds_size, NULL) < 0)
+ goto error;
assert(100 == ds_size[0] && 100 == ds_size[1]);
/* Read some data */
@@ -73,10 +76,8 @@ check_dset(hid_t file, const char* name)
error = fabs(the_data[i][j] - (double)(hssize_t)i / ((hssize_t)j + 1));
if(error > 0.0001F) {
H5_FAILED();
- HDprintf(" dset[%lu][%lu] = %g\n",
- (unsigned long)i, (unsigned long)j, the_data[i][j]);
- HDprintf(" should be %g\n",
- (double)(hssize_t)i/(hssize_t)(j+1));
+ HDprintf(" dset[%lu][%lu] = %g\n", (unsigned long)i, (unsigned long)j, the_data[i][j]);
+ HDprintf(" should be %g\n", (double)(hssize_t)i/(hssize_t)(j+1));
goto error;
} /* end if */
} /* end for */
@@ -110,23 +111,31 @@ check_file(char* filename, hid_t fapl, int flag)
char name[1024];
int i;
- if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) goto error;
- if(check_dset(file, "dset")) goto error;
+ if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0)
+ goto error;
+ if(check_dset(file, "dset"))
+ goto error;
/* Open some groups */
- if((groups = H5Gopen2(file, "some_groups", H5P_DEFAULT)) < 0) goto error;
+ if((groups = H5Gopen2(file, "some_groups", H5P_DEFAULT)) < 0)
+ goto error;
for(i = 0; i < 100; i++) {
HDsprintf(name, "grp%02u", (unsigned)i);
- if((grp = H5Gopen2(groups, name, H5P_DEFAULT)) < 0) goto error;
- if(H5Gclose(grp) < 0) goto error;
+ if((grp = H5Gopen2(groups, name, H5P_DEFAULT)) < 0)
+ goto error;
+ if(H5Gclose(grp) < 0)
+ goto error;
} /* end for */
/* Check to see if that last added dataset in the third file is accessible
* (it shouldn't be...but it might. Flag an error in case it is for now */
- if(flag && check_dset(file, "dset2")) goto error;
+ if(flag && check_dset(file, "dset2"))
+ goto error;
- if(H5Gclose(groups) < 0) goto error;
- if(H5Fclose(file) < 0) goto error;
+ if(H5Gclose(groups) < 0)
+ goto error;
+ if(H5Fclose(file) < 0)
+ goto error;
return 0;
@@ -179,14 +188,13 @@ main(void)
h5_fixname(FILENAME[1], fapl, name, sizeof name);
if(check_file(name, fapl, FALSE))
PASSED()
- else
- {
+ else {
#if defined H5_HAVE_WIN32_API && !defined (hdf5_EXPORTS)
- SKIPPED();
- puts(" DLL will flush the file even when calling _exit, skip this test temporarily");
+ SKIPPED();
+ HDputs(" DLL will flush the file even when calling _exit, skip this test temporarily");
#else
- H5_FAILED()
- goto error;
+ H5_FAILED()
+ goto error;
#endif
}
H5Eset_auto2(H5E_DEFAULT, func, NULL);
@@ -200,14 +208,13 @@ main(void)
h5_fixname(FILENAME[2], fapl, name, sizeof name);
if(check_file(name, fapl, TRUE))
PASSED()
- else
- {
+ else {
#if defined H5_HAVE_WIN32_API && !defined (hdf5_EXPORTS)
- SKIPPED();
- puts(" DLL will flush the file even when calling _exit, skip this test temporarily");
+ SKIPPED();
+ HDputs(" DLL will flush the file even when calling _exit, skip this test temporarily");
#else
- H5_FAILED()
- goto error;
+ H5_FAILED()
+ goto error;
#endif
}
@@ -218,6 +225,6 @@ main(void)
return 0;
error:
- return 1;
-}
+ HDexit(EXIT_FAILURE);
+} /* end main() */
diff --git a/test/h5test.c b/test/h5test.c
index c2e78e4..64b05f4 100644
--- a/test/h5test.c
+++ b/test/h5test.c
@@ -113,8 +113,7 @@ static H5E_auto2_t err_func = NULL;
static herr_t h5_errors(hid_t estack, void *client_data);
static char *h5_fixname_real(const char *base_name, hid_t fapl, const char *suffix,
- char *fullname, size_t size);
-
+ char *fullname, size_t size, hbool_t nest_printf);
/*-------------------------------------------------------------------------
* Function: h5_errors
@@ -477,7 +476,7 @@ h5_test_init(void)
char *
h5_fixname(const char *base_name, hid_t fapl, char *fullname, size_t size)
{
- return (h5_fixname_real(base_name, fapl, ".h5", fullname, size));
+ return (h5_fixname_real(base_name, fapl, ".h5", fullname, size, FALSE));
}
@@ -497,7 +496,7 @@ h5_fixname(const char *base_name, hid_t fapl, char *fullname, size_t size)
char *
h5_fixname_no_suffix(const char *base_name, hid_t fapl, char *fullname, size_t size)
{
- return (h5_fixname_real(base_name, fapl, NULL, fullname, size));
+ return (h5_fixname_real(base_name, fapl, NULL, fullname, size, FALSE));
}
@@ -523,7 +522,7 @@ h5_fixname_no_suffix(const char *base_name, hid_t fapl, char *fullname, size_t s
char *
h5_fixname_printf(const char *base_name, hid_t fapl, char *fullname, size_t size)
{
- return (h5_fixname_real(base_name, fapl, ".h5", fullname, size));
+ return (h5_fixname_real(base_name, fapl, ".h5", fullname, size, TRUE));
}
@@ -551,7 +550,7 @@ h5_fixname_printf(const char *base_name, hid_t fapl, char *fullname, size_t size
*/
static char *
h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix,
- char *fullname, size_t size)
+ char *fullname, size_t size, hbool_t nest_printf)
{
const char *prefix = NULL;
const char *env = NULL; /* HDF5_DRIVER environment variable */
@@ -572,10 +571,30 @@ h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix,
return NULL;
if(suffix) {
- if(H5FD_FAMILY == driver)
- suffix = "%05d.h5";
- else if (H5FD_MULTI == driver)
- suffix = NULL;
+ if(H5FD_FAMILY == driver) {
+ suffix = nest_printf ? "%%05d.h5" : "%05d.h5";
+ }
+ else if (H5FD_MULTI == driver) {
+
+ /* Get the environment variable, if it exists, in case
+ * we are using the split driver since both of those
+ * use the multi VFD under the hood.
+ */
+ env = HDgetenv("HDF5_DRIVER");
+#ifdef HDF5_DRIVER
+ /* Use the environment variable, then the compile-time constant */
+ if(!env)
+ env = HDF5_DRIVER;
+#endif
+ if(env && !HDstrcmp(env, "split")) {
+ /* split VFD */
+ suffix = NULL;
+ }
+ else {
+ /* multi VFD */
+ suffix = NULL;
+ }
+ }
}
}
diff --git a/test/hdfs.c b/test/hdfs.c
new file mode 100644
index 0000000..ab39da6
--- /dev/null
+++ b/test/hdfs.c
@@ -0,0 +1,1767 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Read-Only HDFS Virtual File Driver (VFD)
+ *
+ * Purpose:
+ *
+ * Verify behavior for Read-Only HDFS VFD.
+ *
+ * Demonstrates basic use cases and fapl interaction.
+ *
+ * Programmer: Jacob Smith <jake.smith@hdfgroup.org>
+ * 2018-04-23
+ */
+
+#include "h5test.h" /* testing utilities */
+#include "H5FDhdfs.h" /* this file driver's utilities */
+
+
+#ifdef H5_HAVE_LIBHDFS
+#define HDFS_TEST_DEBUG 0
+#define HDFS_TEST_MAX_BUF_SIZE 256
+#endif /* H5_HAVE_LIBHDFS */
+
+/*****************************************************************************
+ *
+ * FILE-LOCAL TESTING MACROS
+ *
+ * Purpose:
+ *
+ * 1) Upon test failure, goto-jump to single-location teardown in test
+ * function. E.g., `error:` (consistency with HDF corpus) or
+ * `failed:` (reflects purpose).
+ * >>> using "error", in part because `H5E_BEGIN_TRY` expects it.
+ * 2) Increase clarity and reduce overhead found with `TEST_ERROR`.
+ * e.g., "if(somefunction(arg, arg2) < 0) TEST_ERROR:"
+ * requires reading of entire line to know whether this if/call is
+ * part of the test setup, test operation, or a test unto itself.
+ * 3) Provide testing macros with optional user-supplied failure message;
+ * if not supplied (NULL), generate comparison output in the spirit of
+ * test-driven development. E.g., "expected 5 but was -3"
+ * User messages clarify test's purpose in code, encouraging description
+ * without relying on comments.
+ * 4) Configurable expected-actual order in generated comparison strings.
+ * Some prefer `VERIFY(expected, actual)`, others
+ * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch
+ * to satifsy both parties, assuming one paradigm per test file.
+ * (One could #undef and redefine the flag through the file as desired,
+ * but _why_.)
+ *
+ * Provided as courtesy, per consideration for inclusion in the library
+ * proper.
+ *
+ * Macros:
+ *
+ * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order
+ * FAIL_IF() - check condition
+ * FAIL_UNLESS() - check _not_ condition
+ * JSVERIFY() - long-int equality check; prints reason/comparison
+ * JSVERIFY_NOT() - long-int inequality check; prints
+ * JSVERIFY_STR() - string equality check; prints
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *****************************************************************************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * ifdef flag: JSVERIFY_EXP_ACT
+ *
+ * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason])
+ * default, if this is undefined, is (ACTUAL, EXPECTED[, reason])
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_EXP_ACT 1L
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSFAILED_AT()
+ *
+ * Purpose:
+ *
+ * Preface a test failure by printing "*FAILED*" and location to stdout
+ * Similar to `H5_FAILED(); AT();` from h5test.h
+ *
+ * *FAILED* at somefile.c:12 in function_name()...
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSFAILED_AT() { \
+ HDprintf("*FAILED* at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: FAIL_IF()
+ *
+ * Purpose:
+ *
+ * Make tests more accessible and less cluttered than
+ * `if (thing == otherthing()) TEST_ERROR`
+ * paradigm.
+ *
+ * The following lines are roughly equivalent:
+ *
+ * `if (myfunc() < 0) TEST_ERROR;` (as seen elsewhere in HDF tests)
+ * `FAIL_IF(myfunc() < 0)`
+ *
+ * Prints a generic "FAILED AT" line to stdout and jumps to `error`,
+ * similar to `TEST_ERROR` in h5test.h
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *----------------------------------------------------------------------------
+ */
+#define FAIL_IF(condition) \
+if (condition) { \
+ JSFAILED_AT() \
+ goto error; \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: FAIL_UNLESS()
+ *
+ * Purpose:
+ *
+ * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests",
+ * e.g., "a != b".
+ *
+ * Opposite of FAIL_IF; fails if the given condition is _not_ true.
+ *
+ * `FAIL_IF( 5 != my_op() )`
+ * is equivalent to
+ * `FAIL_UNLESS( 5 == my_op() )`
+ * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer.
+ * (see JSVERIFY)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#if 0 /* UNUSED */
+#define FAIL_UNLESS(condition) \
+if (!(condition)) { \
+ JSFAILED_AT() \
+ goto error; \
+}
+#endif /* UNUSED */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSERR_LONG()
+ *
+ * Purpose:
+ *
+ * Print an failure message for long-int arguments.
+ * ERROR-AT printed first.
+ * If `reason` is given, it is printed on own line and newlined after
+ * else, prints "expected/actual" aligned on own lines.
+ *
+ * *FAILED* at myfile.c:488 in somefunc()...
+ * forest must be made of trees.
+ *
+ * or
+ *
+ * *FAILED* at myfile.c:488 in somefunc()...
+ * ! Expected 425
+ * ! Actual 3
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSERR_LONG(expected, actual, reason) { \
+ JSFAILED_AT() \
+ if (reason!= NULL) { \
+ HDprintf("%s\n", (reason)); \
+ } else { \
+ HDprintf(" ! Expected %ld\n ! Actual %ld\n", \
+ (long)(expected), (long)(actual)); \
+ } \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSERR_STR()
+ *
+ * Purpose:
+ *
+ * Print an failure message for string arguments.
+ * ERROR-AT printed first.
+ * If `reason` is given, it is printed on own line and newlined after
+ * else, prints "expected/actual" aligned on own lines.
+ *
+ * *FAILED* at myfile.c:421 in myfunc()...
+ * Blue and Red strings don't match!
+ *
+ * or
+ *
+ * *FAILED* at myfile.c:421 in myfunc()...
+ * !!! Expected:
+ * this is my expected
+ * string
+ * !!! Actual:
+ * not what I expected at all
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSERR_STR(expected, actual, reason) { \
+ JSFAILED_AT() \
+ if ((reason) != NULL) { \
+ HDprintf("%s\n", (reason)); \
+ } else { \
+ HDprintf("!!! Expected:\n%s\n!!!Actual:\n%s\n", \
+ (expected), (actual)); \
+ } \
+}
+
+
+
+#ifdef JSVERIFY_EXP_ACT
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY()
+ *
+ * Purpose:
+ *
+ * Verify that two long integers are equal.
+ * If unequal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY(expected, actual, reason) \
+if ((long)(actual) != (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY_NOT()
+ *
+ * Purpose:
+ *
+ * Verify that two long integers are _not_ equal.
+ * If equal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_NOT(expected, actual, reason) \
+if ((long)(actual) == (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY_NOT */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY_STR()
+ *
+ * Purpose:
+ *
+ * Verify that two strings are equal.
+ * If unequal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_STR(expected, actual, reason) \
+if (strcmp((actual), (expected)) != 0) { \
+ JSERR_STR((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY_STR */
+
+
+#else
+/* JSVERIFY_EXP_ACT not defined
+ *
+ * Repeats macros above, but with actual/expected parameters reversed.
+ */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY()
+ * See: JSVERIFY documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY(actual, expected, reason) \
+if ((long)(actual) != (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY_NOT()
+ * See: JSVERIFY_NOT documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_NOT(actual, expected, reason) \
+if ((long)(actual) == (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY_NOT */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY_STR()
+ * See: JSVERIFY_STR documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_STR(actual, expected, reason) \
+if (strcmp((actual), (expected)) != 0) { \
+ JSERR_STR((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY_STR */
+
+#endif /* ifdef/else JSVERIFY_EXP_ACT */
+
+/********************************
+ * OTHER MACROS AND DEFINITIONS *
+ ********************************/
+
+/* copied from src/hdfs.c
+ */
+#ifdef H5_HAVE_LIBHDFS
+#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1)
+#endif /* H5_HAVE_LIBHDFS */
+
+#define HDFS_NAMENODE_NAME_MAX_SIZE 128
+
+/*******************************
+ * FILE-LOCAL GLOBAL VARIABLES *
+ *******************************/
+
+#ifdef H5_HAVE_LIBHDFS
+static const char filename_missing[] = "/tmp/missing.txt";
+static const char filename_bard[] = "/tmp/t8.shakespeare.txt";
+static const char filename_raven[] = "/tmp/Poe_Raven.txt";
+static const char filename_example_h5[] = "/tmp/t.h5";
+#endif /* H5_HAVE_LIBHDFS */
+
+static H5FD_hdfs_fapl_t default_fa = {
+ 1, /* fa version */
+ "localhost", /* namenode name */
+ 0, /* namenode port */
+ "", /* user name */
+ "", /* kerberos path */
+ 1024, /* buffer size */
+};
+
+/******************
+ * TEST FUNCTIONS *
+ ******************/
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_fapl_config_validation()
+ *
+ * Purpose:
+ *
+ * Test data consistency of fapl configuration.
+ * Tests `H5FD_hdfs_validate_config` indirectly through `H5Pset_fapl_hdfs`.
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2018-04-25
+ *
+ * Changes: None.
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_fapl_config_validation(void)
+{
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct testcase {
+ const char *msg;
+ herr_t expected;
+ H5FD_hdfs_fapl_t config;
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ hid_t fapl_id = -1; /* file access property list ID */
+ H5FD_hdfs_fapl_t config;
+ H5FD_hdfs_fapl_t fa_fetch;
+ herr_t success = SUCCEED;
+ unsigned int i = 0;
+ unsigned int ncases = 6; /* should equal number of cases */
+ struct testcase *case_ptr = NULL; /* dumb work-around for possible */
+ /* dynamic cases creation because */
+ /* of compiler warnings Wlarger-than */
+ struct testcase cases_arr[] = {
+ { "default config fapl",
+ SUCCEED,
+ { 1, /* version */
+ "localhost", /* namenode_name */
+ 0, /* namenode_port number */
+ "some_user", /* user_name */
+ "", /* kerberos_ticket_cache path */
+ -1, /* stream_buffer_size */
+ },
+ },
+ { "invalid version number (2)",
+ FAIL,
+ { 2, /* version */
+ "localhost", /* namenode_name */
+ 0, /* namenode_port number */
+ "some_user", /* user_name */
+ "", /* kerberos_ticket_cache path */
+ -1, /* stream_buffer_size */
+ },
+ },
+ { "invalid version number (0)",
+ FAIL,
+ { 0, /* version */
+ "localhost", /* namenode_name */
+ 0, /* namenode_port number */
+ "some_user", /* user_name */
+ "", /* kerberos_ticket_cache path */
+ -1, /* stream_buffer_size */
+ },
+ },
+ { "nonsense kerberos path still ok?",
+ SUCCEED,
+ { 1, /* version */
+ "localhost", /* namenode_name */
+ 0, /* namenode_port number */
+ "some_user", /* user_name */
+ "pathToSomewhere", /* kerberos_ticket_cache path */
+ -1, /* stream_buffer_size */
+ },
+ },
+ { "namenode port number too high",
+ FAIL,
+ { 1, /* version */
+ "localhost", /* namenode_name */
+ 88000, /* namenode_port number */
+ "some_user", /* user_name */
+ "", /* kerberos_ticket_cache path */
+ -1, /* stream_buffer_size */
+ },
+ },
+ { "negative namenode port number",
+ FAIL,
+ { 1, /* version */
+ "localhost", /* namenode_name */
+ -1, /* namenode_port number */
+ "some_user", /* user_name */
+ "", /* kerberos_ticket_cache path */
+ -1, /* stream_buffer_size */
+ },
+ },
+ };
+
+ TESTING("HDFS fapl configuration validation");
+
+ /*********
+ * TESTS *
+ *********/
+
+ for (i = 0; i < ncases; i++) {
+
+ /*---------------
+ * per-test setup
+ *---------------
+ */
+ case_ptr = &cases_arr[i];
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 ) /* sanity-check */
+
+ /*-----------------------------------
+ * Actually test -- set fapl.
+ * Mute stack trace in failure cases.
+ *-----------------------------------
+ */
+ H5E_BEGIN_TRY {
+ /* `H5FD_hdfs_validate_config(...)` is static/private
+ * to src/hdfs.c and cannot (and should not?) be tested directly?
+ * Instead, validate config through public api.
+ */
+ success = H5Pset_fapl_hdfs(fapl_id, &case_ptr->config);
+ } H5E_END_TRY;
+
+ JSVERIFY( case_ptr->expected, success, case_ptr->msg )
+
+ /* Make sure we can get back what we put in.
+ * Only valid if the fapl configuration does not result in error.
+ */
+ if (success == SUCCEED) {
+ config = case_ptr->config;
+ JSVERIFY( SUCCEED,
+ H5Pget_fapl_hdfs(fapl_id, &fa_fetch),
+ "unable to get fapl" )
+ JSVERIFY( H5FD__CURR_HDFS_FAPL_T_VERSION,
+ fa_fetch.version,
+ "invalid version number" )
+ JSVERIFY( config.version,
+ fa_fetch.version,
+ "version number mismatch" )
+ JSVERIFY( config.namenode_port,
+ fa_fetch.namenode_port,
+ "namenode port mismatch" )
+ JSVERIFY( config.stream_buffer_size,
+ fa_fetch.stream_buffer_size,
+ "streambuffer size mismatch" )
+ JSVERIFY_STR( config.namenode_name,
+ fa_fetch.namenode_name,
+ NULL )
+ JSVERIFY_STR( config.user_name,
+ fa_fetch.user_name,
+ NULL )
+ JSVERIFY_STR( config.kerberos_ticket_cache,
+ fa_fetch.kerberos_ticket_cache,
+ NULL )
+ }
+
+ /*-----------------------------
+ * per-test sanitation/teardown
+ *-----------------------------
+ */
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ } /* for each test case */
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (fapl_id < 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+ return 1;
+
+} /* end test_fapl_config_validation() */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: test_hdfs_fapl()
+ *
+ * Purpose: Tests the file handle interface for the HDFS driver.
+ *
+ * For now, test only fapl & flags. Extend as the
+ * work on the VFD continues.
+ *
+ * Return: Success: 0
+ * Failure: 1
+ *
+ * Programmer: Jacob Smith
+ * 2018-04-25
+ *
+ * Changes: None.
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+test_hdfs_fapl(void)
+{
+ /************************
+ * test-local variables *
+ ************************/
+
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t driver_id = -1; /* ID for this VFD */
+ unsigned long driver_flags = 0; /* VFD feature flags */
+ H5FD_hdfs_fapl_t hdfs_fa_0 = {
+ 1, /* version*/
+ "", /* node name */
+ 9000, /* node port */
+ "", /* username */
+ "", /* kerb cache path */
+ 1024, /* stream buffer size */
+ };
+
+ TESTING("HDFS fapl ");
+
+ /* Set property list and file name for HDFS driver.
+ */
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 )
+
+ FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &hdfs_fa_0) )
+
+ driver_id = H5Pget_driver(fapl_id);
+ FAIL_IF( driver_id < 0 )
+
+ /****************
+ * Check that the VFD feature flags are correct
+ * SPEC MAY CHANGE
+ ******************/
+
+ FAIL_IF( H5FDdriver_query(driver_id, &driver_flags) < 0 )
+
+ JSVERIFY_NOT( 0, (driver_flags & H5FD_FEAT_DATA_SIEVE),
+ "bit(s) in `driver_flags` must align with "
+ "H5FD_FEAT_DATA_SIEVE" )
+
+ JSVERIFY( H5FD_FEAT_DATA_SIEVE, driver_flags,
+ "H5FD_FEAT_DATA_SIEVE should be the only supported flag")
+
+ PASSED();
+ return 0;
+
+error:
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+
+ return 1;
+
+} /* end test_hdfs_fapl() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_vfd_open()
+ *
+ * Purpose:
+ *
+ * Demonstrate/specify VFD-level "Open" failure cases
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2018-06-07
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_vfd_open(void)
+{
+
+#ifndef H5_HAVE_LIBHDFS
+ TESTING("HDFS VFD-level open");
+ SKIPPED();
+ puts(" HDFS VFD is not enabled");
+ fflush(stdout);
+ return 0;
+
+#else
+
+ /*********************
+ * test-local macros *
+ *********************/
+
+/* selectors for which fapl to use in testcase */
+#define FAPL_H5P_DEFAULT -2
+#define FAPL_UNCONFIGURED -3 /* H5P_FILE_ACCESS */
+#define FAPL_HDFS -4
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct test_condition {
+ const char *message;
+ const char *url;
+ unsigned flags;
+ int which_fapl;
+ haddr_t maxaddr;
+ hbool_t might_use_other_driver;
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ struct test_condition failing_conditions[] = {
+ { "default property list (H5P_DEFAULT) is invalid",
+ filename_bard,
+ H5F_ACC_RDONLY,
+ FAPL_H5P_DEFAULT,
+ MAXADDR,
+ TRUE,
+ },
+ { "generic file access property list is invalid",
+ filename_bard,
+ H5F_ACC_RDONLY,
+ FAPL_UNCONFIGURED,
+ MAXADDR,
+ TRUE,
+ },
+ { "filename cannot be null",
+ NULL,
+ H5F_ACC_RDONLY,
+ FAPL_HDFS,
+ MAXADDR,
+ FALSE,
+ },
+ { "filename cannot be empty",
+ "",
+ H5F_ACC_RDONLY,
+ FAPL_HDFS,
+ MAXADDR,
+ FALSE,
+ },
+ { "file at filename must exist",
+ filename_missing,
+ H5F_ACC_RDONLY,
+ FAPL_HDFS,
+ MAXADDR,
+ FALSE,
+ },
+ { "read-write flag not supported",
+ filename_bard,
+ H5F_ACC_RDWR,
+ FAPL_HDFS,
+ MAXADDR,
+ FALSE,
+ },
+ { "truncate flag not supported",
+ filename_bard,
+ H5F_ACC_TRUNC,
+ FAPL_HDFS,
+ MAXADDR,
+ FALSE,
+ },
+ { "create flag not supported",
+ filename_bard,
+ H5F_ACC_CREAT,
+ FAPL_HDFS,
+ MAXADDR,
+ FALSE,
+ },
+ { "EXCL flag not supported",
+ filename_bard,
+ H5F_ACC_EXCL,
+ FAPL_HDFS,
+ MAXADDR,
+ FALSE,
+ },
+ { "maxaddr cannot be 0 (caught in `H5FD_open()`)",
+ filename_bard,
+ H5F_ACC_RDONLY,
+ FAPL_HDFS,
+ 0,
+ FALSE,
+ },
+ };
+ unsigned i = 0;
+ unsigned failing_conditions_count = 10;
+ H5FD_t *fd = NULL;
+ hid_t fapl_hdfs = -1;
+ hid_t fapl_unconfigured = -1;
+
+ TESTING("HDFS VFD-level open");
+
+ fapl_unconfigured = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_unconfigured < 0 )
+
+ fapl_hdfs = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_hdfs < 0 )
+ FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_hdfs, &default_fa) )
+
+ /*********
+ * TESTS *
+ *********/
+
+ /* all the test cases that will _not_ open
+ */
+ for (i = 0; i < failing_conditions_count; i++) {
+ struct test_condition T = failing_conditions[i];
+ hid_t fapl_id = H5P_DEFAULT;
+
+ fd = NULL;
+
+ if (T.which_fapl == FAPL_UNCONFIGURED) {
+ fapl_id = fapl_unconfigured;
+ }
+ else
+ if (T.which_fapl == FAPL_HDFS) {
+ fapl_id = fapl_hdfs;
+ }
+
+#if HDFS_TEST_DEBUG
+ HDfprintf(stderr, "testing: %s\n", T.message);
+#endif /* HDFS_TEST_DEBUG */
+
+ H5E_BEGIN_TRY {
+ fd = H5FDopen(T.url, T.flags, fapl_id, T.maxaddr);
+ } H5E_END_TRY;
+ if (NULL != fd) {
+ if (TRUE == T.might_use_other_driver &&
+ H5FD_HDFS != fd->driver_id)
+ {
+ HDfprintf(stderr, "\n!!!!! WARNING !!!!!\n" \
+ " Successful open of file on local system " \
+ "with non-HDFS VFD.\n");
+ JSVERIFY(SUCCEED, H5FDclose(fd),
+ "unable to close errant open");
+ fd = NULL;
+ }
+ else {
+ JSVERIFY(1, 0, T.message); /* print message and fail */
+ }
+ }
+ }
+
+ FAIL_IF( NULL != fd ) /* sanity check */
+
+#if HDFS_TEST_DEBUG
+ HDfprintf(stderr, "nominal open\n");
+#endif /* HDFS_TEST_DEBUG */
+
+ /* finally, show that a file can be opened
+ */
+ fd = H5FDopen(
+ filename_bard,
+ H5F_ACC_RDONLY,
+ fapl_hdfs,
+ MAXADDR);
+ FAIL_IF( NULL == fd )
+
+ /************
+ * TEARDOWN *
+ ************/
+
+#if HDFS_TEST_DEBUG
+ HDfprintf(stderr, "teardown...\n");
+#endif /* HDFS_TEST_DEBUG */
+
+ FAIL_IF( FAIL == H5FDclose(fd) )
+ fd = NULL;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_hdfs) )
+ fapl_hdfs = -1;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_unconfigured) )
+ fapl_unconfigured = -1;
+
+ PASSED();
+ return 0;
+
+error:
+
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (fd) {
+ (void)H5FDclose(fd);
+ }
+ H5E_BEGIN_TRY {
+ if (fapl_hdfs >= 0) {
+ (void)H5Pclose(fapl_hdfs);
+ }
+ if (fapl_unconfigured >= 0) {
+ (void)H5Pclose(fapl_unconfigured);
+ }
+ } H5E_END_TRY;
+
+ return 1;
+
+#undef FAPL_H5P_DEFAULT
+#undef FAPL_UNCONFIGURED
+#undef FAPL_HDFS
+
+#endif /* H5_HAVE_LIBHDFS */
+
+} /* end test_vfd_open() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_eof_eoa()
+ *
+ * Purpose:
+ *
+ * Demonstrate behavior of get_eof, get_eoa, and set_eoa.
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2018-06-07
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_eof_eoa(void)
+{
+#ifndef H5_HAVE_LIBHDFS
+ TESTING("HDFS eof/eoa gets and sets");
+ SKIPPED();
+ puts(" HDFS VFD is not enabled");
+ fflush(stdout);
+ return 0;
+
+#else
+
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ H5FD_t *fd_shakespeare = NULL;
+ hid_t fapl_id = -1;
+
+ TESTING("HDFS eof/eoa gets and sets");
+
+ /*********
+ * SETUP *
+ *********/
+
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( 0 > fapl_id )
+ FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) )
+
+ fd_shakespeare = H5FDopen(
+ filename_bard,
+ H5F_ACC_RDONLY,
+ fapl_id,
+ HADDR_UNDEF);
+ FAIL_IF( NULL == fd_shakespeare )
+
+ /*********
+ * TESTS *
+ *********/
+
+ /* verify as found
+ */
+ JSVERIFY( 5458199, H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), NULL )
+ JSVERIFY( H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT),
+ H5FDget_eof(fd_shakespeare, H5FD_MEM_DRAW),
+ "mismatch between DEFAULT and RAW memory types" )
+ JSVERIFY( 0,
+ H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT),
+ "EoA should be unset by H5FDopen" )
+
+ /* set EoA below EoF
+ */
+ JSVERIFY( SUCCEED,
+ H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 44442202),
+ "unable to set EoA (lower)" )
+ JSVERIFY( 5458199,
+ H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT),
+ "EoF changed" )
+ JSVERIFY( 44442202,
+ H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT),
+ "EoA unchanged" )
+
+ /* set EoA above EoF
+ */
+ JSVERIFY( SUCCEED,
+ H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 6789012),
+ "unable to set EoA (higher)" )
+ JSVERIFY( 5458199,
+ H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT),
+ "EoF changed" )
+ JSVERIFY( 6789012,
+ H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT),
+ "EoA unchanged" )
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ FAIL_IF( FAIL == H5FDclose(fd_shakespeare) )
+ fd_shakespeare = NULL;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ PASSED();
+ return 0;
+
+error:
+
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (fd_shakespeare != NULL) {
+ (void)H5FDclose(fd_shakespeare);
+ }
+ if (fapl_id >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+
+ return 1;
+
+#endif /* H5_HAVE_LIBHDFS */
+
+} /* end test_eof_eoa() */
+
+
+/*-----------------------------------------------------------------------------
+ *
+ * Function: test_H5FDread_without_eoa_set_fails()
+ *
+ * Purpose:
+ *
+ * Demonstrate a not-obvious constraint by the library, preventing
+ * file read before EoA is set
+ *
+ * Programmer: Jacob Smith
+ * 2018-06-08
+ *
+ *-----------------------------------------------------------------------------
+ */
+static int
+test_H5FDread_without_eoa_set_fails(void)
+{
+#ifndef H5_HAVE_LIBHDFS
+ TESTING("HDFS VFD read-eoa temporal coupling library limitation");
+ SKIPPED();
+ puts(" HDFS VFD is not enabled");
+ fflush(stdout);
+ return 0;
+
+#else
+
+ char buffer[HDFS_TEST_MAX_BUF_SIZE];
+ unsigned int i = 0;
+ H5FD_t *file_shakespeare = NULL;
+ hid_t fapl_id = -1;
+
+ TESTING("HDFS VFD read-eoa temporal coupling library limitation");
+
+ /*********
+ * SETUP *
+ *********/
+
+ /* create HDFS fapl
+ */
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 )
+ FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) )
+
+ file_shakespeare = H5FDopen(
+ filename_bard,
+ H5F_ACC_RDONLY,
+ fapl_id,
+ MAXADDR);
+ FAIL_IF( NULL == file_shakespeare )
+
+ JSVERIFY( 0, H5FDget_eoa(file_shakespeare, H5FD_MEM_DEFAULT),
+ "EoA should remain unset by H5FDopen" )
+
+ /* zero buffer contents */
+ for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) {
+ buffer[i] = 0;
+ }
+
+ /********
+ * TEST *
+ ********/
+
+ H5E_BEGIN_TRY { /* mute stack trace on expected failure */
+ JSVERIFY( FAIL,
+ H5FDread(file_shakespeare,
+ H5FD_MEM_DRAW,
+ H5P_DEFAULT,
+ 1200699,
+ 102,
+ buffer),
+ "cannot read before eoa is set" )
+ } H5E_END_TRY;
+ for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) {
+ JSVERIFY( 0, (unsigned)buffer[i], "buffer was modified by write!" )
+ }
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ FAIL_IF( FAIL == H5FDclose(file_shakespeare) )
+ file_shakespeare = NULL;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ PASSED();
+ return 0;
+
+error:
+
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (file_shakespeare) {
+ (void)H5FDclose(file_shakespeare);
+ }
+ if (fapl_id >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+
+ return 1;
+
+#endif /* H5_HAVE_LIBHDFS */
+
+} /* end test_H5FDread_without_eoa_set_fails() */
+
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_read()
+ *
+ * Purpose:
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2018-06-08
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_read(void)
+{
+#ifndef H5_HAVE_LIBHDFS
+ TESTING("HDFS VFD read/range-gets");
+ SKIPPED();
+ puts(" HDFS VFD is not enabled");
+ fflush(stdout);
+ return 0;
+
+#else
+
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+ struct testcase {
+ const char *message; /* purpose of test case */
+ haddr_t eoa_set; /* set file EOA to this prior to read */
+ size_t addr; /* offset of read in file */
+ size_t len; /* length of read in file */
+ herr_t success; /* expected return value of read function */
+ const char *expected; /* expected contents of buffer; failure ignores */
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+ struct testcase cases[] = {
+ { "successful range-get",
+ 6464,
+ 5691,
+ 32, /* fancy quotes are three bytes each(?) */
+ SUCCEED,
+ "Quoth the Raven “Nevermore.”",
+ },
+ { "read past EOA fails (EOA < EOF < addr)",
+ 3000,
+ 4000,
+ 100,
+ FAIL,
+ NULL,
+ },
+ { "read overlapping EOA fails (EOA < addr < EOF < (addr+len))",
+ 3000,
+ 8000,
+ 100,
+ FAIL,
+ NULL,
+ },
+ { "read past EOA/EOF fails ((EOA==EOF) < addr)",
+ 6464,
+ 7000,
+ 100,
+ FAIL,
+ NULL,
+ },
+ { "read overlapping EOA/EOF fails (addr < (EOA==EOF) < (addr+len))",
+ 6464,
+ 6400,
+ 100,
+ FAIL,
+ NULL,
+ },
+ { "read between EOF and EOA fails (EOF < addr < (addr+len) < EOA)",
+ 8000,
+ 7000,
+ 100,
+ FAIL,
+ NULL,
+ },
+ };
+ unsigned testcase_count = 6;
+ unsigned test_i = 0;
+ struct testcase test;
+ herr_t open_return = FAIL;
+ char buffer[HDFS_TEST_MAX_BUF_SIZE];
+ unsigned int i = 0;
+ H5FD_t *file_raven = NULL;
+ hid_t fapl_id = -1;
+
+ TESTING("HDFS VFD read/range-gets");
+
+ /*********
+ * SETUP *
+ *********/
+
+ /* create HDFS fapl
+ */
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 )
+ FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) )
+
+ /* zero buffer contents */
+ for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) {
+ buffer[i] = 0;
+ }
+
+ /* open file
+ */
+ file_raven = H5FDopen(
+ filename_raven,
+ H5F_ACC_RDONLY,
+ fapl_id,
+ HADDR_UNDEF); /* Demonstrate success with "automatic" value */
+ FAIL_IF( NULL == file_raven )
+
+ JSVERIFY( 6464, H5FDget_eof(file_raven, H5FD_MEM_DEFAULT), NULL )
+
+ /*********
+ * TESTS *
+ *********/
+
+ for (test_i = 0; test_i < testcase_count; test_i++) {
+
+ /* -------------- *
+ * per-test setup *
+ * -------------- */
+
+ test = cases[test_i];
+ open_return = FAIL;
+
+ FAIL_IF( HDFS_TEST_MAX_BUF_SIZE < test.len ) /* buffer too small! */
+
+ FAIL_IF( FAIL ==
+ H5FDset_eoa( file_raven, H5FD_MEM_DEFAULT, test.eoa_set) )
+
+ /* zero buffer contents */
+ for (i = 0; i < HDFS_TEST_MAX_BUF_SIZE; i++) {
+ buffer[i] = 0;
+ }
+
+ /* ------------ *
+ * conduct test *
+ * ------------ */
+
+ H5E_BEGIN_TRY {
+ open_return = H5FDread(
+ file_raven,
+ H5FD_MEM_DRAW,
+ H5P_DEFAULT,
+ test.addr,
+ test.len,
+ buffer);
+ } H5E_END_TRY;
+
+ JSVERIFY( test.success,
+ open_return,
+ test.message )
+
+ if (open_return == SUCCEED) {
+ JSVERIFY_STR( test.expected, buffer, NULL )
+ }
+
+ } /* for each testcase */
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ FAIL_IF( FAIL == H5FDclose(file_raven) )
+ file_raven = NULL;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ PASSED();
+ return 0;
+
+error:
+
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (file_raven != 0) {
+ (void)H5FDclose(file_raven);
+ }
+ if (fapl_id >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+
+ return 1;
+
+#endif /* H5_HAVE_LIBHDFS */
+
+} /* end test_read() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_noops_and_autofails()
+ *
+ * Purpose:
+ *
+ * Demonstrate the unavailable and do-nothing routines unique to
+ * Read-Only VFD.
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-06
+ *
+ * Changes:
+ * + modify from S3VFD codebase to HDFS; Minor changes, mostly.
+ * + Jacob Smith 2018-06-08
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_noops_and_autofails(void)
+{
+#ifndef H5_HAVE_LIBHDFS
+ TESTING("HDFS VFD always-fail and no-op routines");
+ SKIPPED();
+ puts(" HDFS VFD is not enabled");
+ fflush(stdout);
+ return 0;
+
+#else
+
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ hid_t fapl_id = -1;
+ H5FD_t *file = NULL;
+ const char data[36] = "The Force shall be with you, always";
+
+ TESTING("HDFS VFD always-fail and no-op routines");
+
+ /*********
+ * SETUP *
+ *********/
+
+ /* create HDFS fapl
+ */
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 )
+ FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) )
+
+ /* open file
+ */
+ file = H5FDopen(
+ filename_bard,
+ H5F_ACC_RDONLY,
+ fapl_id,
+ HADDR_UNDEF);
+ FAIL_IF( NULL == file )
+
+ /*********
+ * TESTS *
+ *********/
+
+ /* auto-fail calls to write and truncate
+ */
+ H5E_BEGIN_TRY {
+ JSVERIFY( FAIL,
+ H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, 1000, 35, data),
+ "write must fail" )
+ } H5E_END_TRY;
+
+ H5E_BEGIN_TRY {
+ JSVERIFY( FAIL,
+ H5FDtruncate(file, H5P_DEFAULT, FALSE),
+ "truncate must fail" )
+ } H5E_END_TRY;
+
+ H5E_BEGIN_TRY {
+ JSVERIFY( FAIL,
+ H5FDtruncate(file, H5P_DEFAULT, TRUE),
+ "truncate must fail (closing)" )
+ } H5E_END_TRY;
+
+ /* no-op calls to `lock()` and `unlock()`
+ */
+ JSVERIFY( SUCCEED,
+ H5FDlock(file, TRUE),
+ "lock always succeeds; has no effect" )
+ JSVERIFY( SUCCEED,
+ H5FDlock(file, FALSE),
+ NULL )
+ JSVERIFY( SUCCEED,
+ H5FDunlock(file),
+ NULL )
+ /* Lock/unlock with null file or similar error crashes tests.
+ * HDassert in calling heirarchy, `H5FD[un]lock()` and `H5FD_[un]lock()`
+ */
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ FAIL_IF( FAIL == H5FDclose(file) )
+ file = NULL;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ PASSED();
+ return 0;
+
+error:
+
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (fapl_id >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+ if (file != NULL) {
+ (void)H5FDclose(file);
+ }
+
+ return 1;
+
+#endif /* H5_HAVE_LIBHDFS */
+
+} /* end test_noops_and_autofails() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_cmp()
+ *
+ * Purpose:
+ *
+ * Verify "file comparison" behavior.
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-06
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_cmp(void)
+{
+ TESTING("HDFS cmp (comparison)");
+ SKIPPED();
+ HDfprintf(
+ stderr,
+ " TODO: Distinct valid fapls to open the same file.\n");
+
+ return 0;
+
+} /* end test_cmp() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_H5F_integration()
+ *
+ * Purpose:
+ *
+ * Demonstrate H5F (File interface) behavior with files on HDFS.
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-07
+ *
+ * Changes:
+ * + modify from S3VFD codebase to HDFS; Minor changes, mostly.
+ * + Jacob Smith 2018-06-08
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_H5F_integration(void)
+{
+#ifndef H5_HAVE_LIBHDFS
+ TESTING("HDFS file access through HD5F library (H5F API)");
+ SKIPPED();
+ puts(" HDFS VFD is not enabled");
+ fflush(stdout);
+ return 0;
+
+#else
+
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ hid_t file = -1;
+ hid_t fapl_id = -1;
+
+ TESTING("HDFS file access through HD5F library (H5F API)");
+
+ /*********
+ * SETUP *
+ *********/
+
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( 0 > fapl_id )
+ FAIL_IF( FAIL == H5Pset_fapl_hdfs(fapl_id, &default_fa) )
+
+ /*********
+ * TESTS *
+ *********/
+
+ /* Read-Write Open access is not allowed with this file driver.
+ */
+ H5E_BEGIN_TRY {
+ FAIL_IF( 0 <= H5Fopen(
+ filename_example_h5,
+ H5F_ACC_RDWR,
+ fapl_id) )
+ } H5E_END_TRY;
+
+ /* H5Fcreate() is not allowed with this file driver.
+ */
+ H5E_BEGIN_TRY {
+ FAIL_IF( 0 <= H5Fcreate(
+ filename_missing,
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT,
+ fapl_id) )
+ } H5E_END_TRY;
+
+ /* Successful open.
+ */
+ file = H5Fopen(
+ filename_example_h5,
+ H5F_ACC_RDONLY,
+ fapl_id);
+ FAIL_IF( file < 0 )
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ FAIL_IF( FAIL == H5Fclose(file) )
+ file = -1;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * CLEANUP *
+ ***********/
+
+#if HDFS_TEST_DEBUG
+ HDprintf("\nerror!"); fflush(stdout);
+#endif /* HDFS_TEST_DEBUG */
+
+ if (fapl_id >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+ if (file > 0) {
+ (void)H5Fclose(file);
+ }
+
+ return 1;
+
+#endif /* H5_HAVE_LIBHDFS */
+
+} /* test_H5F_integration */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: main
+ *
+ * Purpose: Tests the basic features of Virtual File Drivers
+ *
+ * Return: Success: 0
+ * Failure: 1
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+main(void)
+{
+ int nerrors = 0;
+
+ /******************
+ * commence tests *
+ ******************/
+
+ static char hdfs_namenode_name[HDFS_NAMENODE_NAME_MAX_SIZE] = "";
+ const char *hdfs_namenode_name_env = NULL;
+
+ hdfs_namenode_name_env = HDgetenv("HDFS_TEST_NAMENODE_NAME");
+ if (hdfs_namenode_name_env == NULL || hdfs_namenode_name_env[0] == '\0') {
+ HDstrncpy(hdfs_namenode_name, "localhost", HDFS_NAMENODE_NAME_MAX_SIZE);
+ }
+ else {
+ HDstrncpy( /* TODO: error-check? */
+ default_fa.namenode_name,
+ hdfs_namenode_name_env,
+ HDFS_NAMENODE_NAME_MAX_SIZE);
+ }
+
+ h5_reset();
+
+ HDprintf("Testing hdfs VFD functionality.\n");
+
+ nerrors += test_fapl_config_validation();
+ nerrors += test_hdfs_fapl();
+ nerrors += test_vfd_open();
+ nerrors += test_eof_eoa();
+ nerrors += test_H5FDread_without_eoa_set_fails();
+ nerrors += test_read();
+ nerrors += test_noops_and_autofails();
+ nerrors += test_cmp();
+ nerrors += test_H5F_integration();
+
+ if (nerrors > 0) {
+ HDprintf("***** %d hdfs TEST%s FAILED! *****\n",
+ nerrors,
+ nerrors > 1 ? "S" : "");
+ nerrors = 1;
+ }
+ else {
+ HDprintf("All hdfs tests passed.\n");
+ }
+ return nerrors; /* 0 if no errors, 1 if any errors */
+
+} /* end main() */
+
+
diff --git a/test/hyperslab.c b/test/hyperslab.c
index 9e17a2f..d22a689 100644
--- a/test/hyperslab.c
+++ b/test/hyperslab.c
@@ -1180,13 +1180,11 @@ error:
/*-------------------------------------------------------------------------
* Function: main
*
- * Purpose: Test various hyperslab operations. Give the words
- * `small' and/or `medium' on the command line or only `small'
- * is assumed.
+ * Purpose: Test various hyperslab operations. Give the words
+ * 'small' and/or 'medium' on the command line or only 'small'
+ * is assumed.
*
- * Return: Success: exit(EXIT_SUCCESS)
- *
- * Failure: exit(EXIT_FAILURE)
+ * Return: EXIT_SUCCESS/EXIT_FAILURE
*
* Programmer: Robb Matzke
* Friday, October 10, 1997
@@ -1442,6 +1440,6 @@ main(int argc, char *argv[])
H5close();
#endif /* H5_HAVE_THREADSAFE */
- return 0;
+ HDexit(EXIT_SUCCESS);
}
diff --git a/test/istore.c b/test/istore.c
index 6656abd..ca60b03 100644
--- a/test/istore.c
+++ b/test/istore.c
@@ -579,17 +579,13 @@ error:
/*-------------------------------------------------------------------------
* Function: main
*
- * Purpose: Tests indexed storage stuff.
+ * Purpose: Tests indexed storage
*
- * Return: Success: exit(EXIT_SUCCESS)
- *
- * Failure: exit(EXIT_FAILURE)
+ * Return: EXIT_SUCCESS/EXIT_FAILURE
*
* Programmer: Robb Matzke
* Wednesday, October 15, 1997
*
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
int
@@ -618,9 +614,6 @@ main(int argc, char *argv[])
size_of_test |= TEST_LARGE;
} else {
HDprintf("unrecognized argument: %s\n", argv[i]);
-#if 0
- exit(EXIT_FAILURE);
-#endif
}
}
}
@@ -654,7 +647,7 @@ main(int argc, char *argv[])
h5_fixname(FILENAME[0], fapl, filename, sizeof filename);
if ((file=H5Fcreate(filename, H5F_ACC_TRUNC, fcpl, fapl)) < 0) {
HDprintf("Cannot create file %s; test aborted\n", filename);
- exit(EXIT_FAILURE);
+ HDexit(EXIT_FAILURE);
}
/* Initialize chunk dimensions */
@@ -722,13 +715,13 @@ main(int argc, char *argv[])
if (nerrors) {
HDprintf("***** %d I-STORE TEST%s FAILED! *****\n",
nerrors, 1 == nerrors ? "" : "S");
- exit(EXIT_FAILURE);
+ HDexit(EXIT_FAILURE);
}
HDprintf("All i-store tests passed.\n");
h5_cleanup(FILENAME, fapl);
- return 0;
+ HDexit(EXIT_SUCCESS);
}
diff --git a/test/links.c b/test/links.c
index 89e62ab..6ba9992 100644
--- a/test/links.c
+++ b/test/links.c
@@ -14007,10 +14007,10 @@ main(void)
HDrmdir(TMPDIR);
HDrmdir(TMPDIR2);
- return SUCCEED;
+ HDexit(EXIT_SUCCESS);
error:
HDputs("*** TESTS FAILED ***");
- return 1;
+ HDexit(EXIT_FAILURE);
}
diff --git a/test/links_env.c b/test/links_env.c
index df14675..b07b1e0 100644
--- a/test/links_env.c
+++ b/test/links_env.c
@@ -134,10 +134,9 @@ external_link_env(hid_t fapl, hbool_t new_format)
/*-------------------------------------------------------------------------
* Function: main
*
- * Purpose: Test external link with environment variable HDF5_EXT_PREFIX
+ * Purpose: Test external link with environment variable HDF5_EXT_PREFIX
*
- * Return: Success: exit(EXIT_SUCCESS)
- * Failure: exit(EXIT_FAILURE)
+ * Return: EXIT_SUCCESS/EXIT_FAILURE
*
* Programmer: Vailin Choi; Nov 2010
*
@@ -175,9 +174,10 @@ main(void)
/* clean up tmp_links_env directory created by external link tests */
HDrmdir(TMPDIR);
- return 0;
+ HDexit(EXIT_SUCCESS);
error:
HDputs("*** TESTS FAILED ***");
- return 1;
-}
+ HDexit(EXIT_FAILURE);
+} /* end main() */
+
diff --git a/test/mtime.c b/test/mtime.c
index b50dfeb..2e4b2a4 100644
--- a/test/mtime.c
+++ b/test/mtime.c
@@ -15,10 +15,10 @@
* Programmer: Robb Matzke <matzke@llnl.gov>
* Thursday, July 30, 1998
*
- * Purpose: Determines if the modification time message is working
- * properly. Specifically, the code in H5O_mtime_decode() is
- * very OS-dependent and this test tries to figure out if it's
- * working properly.
+ * Purpose: Determines if the modification time message is working
+ * properly. Specifically, the code in H5O_mtime_decode() is
+ * very OS-dependent and this test tries to figure out if it's
+ * working properly.
*/
#include "h5test.h"
#include "H5srcdir.h"
@@ -33,17 +33,17 @@ const char *FILENAME[] = {
#define TESTFILE2 "tmtimen.h5"
#define MTIME2 1041606478
-
+
/*-------------------------------------------------------------------------
- * Function: main
+ * Function: main
*
- * Purpose: H5O_mtime_decode() test.
+ * Purpose: H5O_mtime_decode() test.
*
- * Return: Success:
+ * Return: Success:
*
- * Failure:
+ * Failure:
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Thursday, July 30, 1998
*
* Modifications:
@@ -58,13 +58,13 @@ const char *FILENAME[] = {
int
main(void)
{
- hid_t fapl, file, space, dset;
- hsize_t size[1] = {2};
- time_t now;
- struct tm *tm;
- H5O_info_t oi1, oi2;
- signed char buf1[32], buf2[32];
- char filename[1024];
+ hid_t fapl, file, space, dset;
+ hsize_t size[1] = {2};
+ time_t now;
+ struct tm *tm;
+ H5O_info_t oi1, oi2;
+ signed char buf1[32], buf2[32];
+ char filename[1024];
h5_reset();
fapl = h5_fileaccess();
@@ -99,18 +99,18 @@ main(void)
/* Compare addresses & times from the two ways of calling H5Oget_info() */
if(oi1.addr != oi2.addr || oi1.ctime != oi2.ctime) {
H5_FAILED();
- puts(" Calling H5Oget_info() with the dataset ID returned");
- puts(" different values than calling it with a file and dataset");
- puts(" name.");
+ HDputs(" Calling H5Oget_info() with the dataset ID returned");
+ HDputs(" different values than calling it with a file and dataset");
+ HDputs(" name.");
goto error;
}
/* Compare times -- they must be within 60 seconds of one another */
if(0 == oi1.ctime) {
SKIPPED();
- puts(" The modification time could not be decoded on this OS.");
- puts(" Modification times will be mantained in the file but");
- puts(" cannot be queried on this system. See H5O_mtime_decode().");
+ HDputs(" The modification time could not be decoded on this OS.");
+ HDputs(" Modification times will be mantained in the file but");
+ HDputs(" cannot be queried on this system. See H5O_mtime_decode().");
return 0;
} else if(HDfabs(HDdifftime(now, oi1.ctime)) > (double)60.0F) {
H5_FAILED();
@@ -130,14 +130,8 @@ main(void)
TESTING("accessing old modification time messages");
{
- char testfile[512]="";
- char *srcdir = HDgetenv("srcdir");
+ const char *testfile = H5_get_srcdir_filename(TESTFILE1); /* Corrected test file name */
- if(srcdir && ((HDstrlen(srcdir) + strlen(TESTFILE1) + 1) < sizeof(testfile))){
- HDstrcpy(testfile, srcdir);
- HDstrcat(testfile, "/");
- }
- HDstrcat(testfile, TESTFILE1);
file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT);
if(file >= 0){
if(H5Oget_info_by_name(file, "/Dataset1", &oi1, H5P_DEFAULT) < 0)
@@ -166,21 +160,15 @@ main(void)
TESTING("accessing new modification time messages");
{
- char testfile[512]="";
- char *srcdir = HDgetenv("srcdir");
+ const char *testfile = H5_get_srcdir_filename(TESTFILE2); /* Corrected test file name */
- if(srcdir && ((HDstrlen(srcdir) + strlen(TESTFILE2) + 1) < sizeof(testfile))){
- HDstrcpy(testfile, srcdir);
- HDstrcat(testfile, "/");
- }
- HDstrcat(testfile, TESTFILE2);
file = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT);
if(file >= 0){
if(H5Oget_info_by_name(file, "/Dataset1", &oi2, H5P_DEFAULT) < 0)
TEST_ERROR;
if(oi2.ctime != MTIME2) {
H5_FAILED();
- puts(" Modification time incorrect.");
+ HDputs(" Modification time incorrect.");
goto error;
}
if(H5Fclose(file) < 0) TEST_ERROR;
@@ -198,7 +186,7 @@ main(void)
if(h5_verify_cached_stabs(FILENAME, fapl) < 0) TEST_ERROR
/* All looks good */
- puts("All modification time tests passed.");
+ HDputs("All modification time tests passed.");
h5_cleanup(FILENAME, fapl);
return 0;
diff --git a/test/objcopy.c b/test/objcopy.c
index db66285..b473e2b 100644
--- a/test/objcopy.c
+++ b/test/objcopy.c
@@ -12,10 +12,10 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * Programmer: Peter X. Cao
- * May 01, 2005
+ * Programmer: Peter X. Cao
+ * May 01, 2005
*
- * Purpose: Test H5Ocopy().
+ * Purpose: Test H5Ocopy().
*/
#include "h5test.h"
diff --git a/test/reserved.c b/test/reserved.c
index e11ae19..7e53d9c 100644
--- a/test/reserved.c
+++ b/test/reserved.c
@@ -22,17 +22,17 @@ const char *FILENAME[] = {
};
/*-------------------------------------------------------------------------
- * Function: rsrv_heap
+ * Function: rsrv_heap
*
- * Purpose: Ensure that heaps reserve file address space.
- * This function does this by creating datasets up to and past
- * the limit of the file, then ensuring that an error (not an
- * assert) was generated and that the file is readable.
+ * Purpose: Ensure that heaps reserve file address space.
+ * This function does this by creating datasets up to and past
+ * the limit of the file, then ensuring that an error (not an
+ * assert) was generated and that the file is readable.
*
- * Return: Success: 0
- * Failure: 1
+ * Return: Success: 0
+ * Failure: 1
*
- * Programmer: James Laird
+ * Programmer: James Laird
* Nat Furrer
* Friday, May 28, 2004
*
@@ -135,17 +135,17 @@ error:
}
/*-------------------------------------------------------------------------
- * Function: rsrv_ohdr
+ * Function: rsrv_ohdr
*
- * Purpose: Ensure that object headers reserve file address space.
- * This function does this by creating attributes of a dataset
- * past the limit of the file, then ensuring that an error (not
- * an assert) was generated and that the file is readable.
+ * Purpose: Ensure that object headers reserve file address space.
+ * This function does this by creating attributes of a dataset
+ * past the limit of the file, then ensuring that an error (not
+ * an assert) was generated and that the file is readable.
*
- * Return: Success: 0
- * Failure: 1
+ * Return: Success: 0
+ * Failure: 1
*
- * Programmer: James Laird
+ * Programmer: James Laird
* Nat Furrer
* Friday, May 28, 2004
*
@@ -257,20 +257,20 @@ error:
}
/*-------------------------------------------------------------------------
- * Function: rsrv_vlen
+ * Function: rsrv_vlen
*
- * Purpose: Ensure that variable length datatypes properly ensure that
+ * Purpose: Ensure that variable length datatypes properly ensure that
* enough file address space exists before writing.
- * This function does this by creating a dataset containing
+ * This function does this by creating a dataset containing
* variable length data past the limit of the file, then
* ensuring that an error (not an assert) was generated and
* that the file is readable.
*
- * Return: Success: 0
- * Failure: 1
+ * Return: Success: 0
+ * Failure: 1
*
- * Programmer: James Laird
- * Nat Furrer
+ * Programmer: James Laird
+ * Nat Furrer
* Thursday, July 1, 2004
*
* Modifications:
@@ -401,15 +401,15 @@ error:
#endif /* BROKEN */
/*-------------------------------------------------------------------------
- * Function: main
+ * Function: main
*
* Purpose:
*
- * Return: Success:
+ * Return: Success:
*
- * Failure:
+ * Failure:
*
- * Programmer: Nat Furrer and James Laird
+ * Programmer: Nat Furrer and James Laird
* Thursday, July 1, 2004
*
* Modifications:
@@ -434,22 +434,22 @@ main(void)
envval = "nomatch";
/* QAK: should be able to use the core driver? */
if (HDstrcmp(envval, "core") && HDstrcmp(envval, "split") && HDstrcmp(envval, "multi") && HDstrcmp(envval, "family")) {
- num_errs+=rsrv_ohdr();
- num_errs+=rsrv_heap();
- num_errs+=rsrv_vlen();
-
- if(num_errs > 0)
- HDprintf("**** %d FAILURE%s! ****\n", num_errs, num_errs==1?"":"S");
- else
- puts("All address space reservation tests passed.");
-
- fapl = h5_fileaccess();
- h5_cleanup(FILENAME, fapl);
- return num_errs;
+ num_errs+=rsrv_ohdr();
+ num_errs+=rsrv_heap();
+ num_errs+=rsrv_vlen();
+
+ if(num_errs > 0)
+ HDprintf("**** %d FAILURE%s! ****\n", num_errs, num_errs==1?"":"S");
+ else
+ HDputs("All address space reservation tests passed.");
+
+ fapl = h5_fileaccess();
+ h5_cleanup(FILENAME, fapl);
+ return num_errs;
}
else
{
- puts("All address space reservation tests skippped - Incompatible with current Virtual File Driver");
+ HDputs("All address space reservation tests skippped - Incompatible with current Virtual File Driver");
}
#endif /* BROKEN */
diff --git a/test/ros3.c b/test/ros3.c
new file mode 100644
index 0000000..73b6ac2
--- /dev/null
+++ b/test/ros3.c
@@ -0,0 +1,1937 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Read-Only S3 Virtual File Driver (VFD)
+ *
+ * Purpose:
+ *
+ * Verify behavior for Read-Only S3 VFD
+ * at the VFL (virtual file layer) level.
+ *
+ * Demonstrates basic use cases and fapl/dxpl interaction.
+ *
+ * Programmer: Jacob Smith <jake.smith@hdfgroup.org>
+ * 2017-10-11
+ */
+
+#include "h5test.h"
+
+#include "H5FDprivate.h" /* Virtual File Driver utilities */
+#include "H5FDros3.h" /* this file driver's utilities */
+#include "H5FDs3comms.h" /* for loading of credentials */
+
+#ifdef H5_HAVE_ROS3_VFD
+
+/* only include the testing macros if needed */
+
+/*****************************************************************************
+ *
+ * FILE-LOCAL TESTING MACROS
+ *
+ * Purpose:
+ *
+ * 1) Upon test failure, goto-jump to single-location teardown in test
+ * function. E.g., `error:` (consistency with HDF corpus) or
+ * `failed:` (reflects purpose).
+ * >>> using "error", in part because `H5E_BEGIN_TRY` expects it.
+ * 2) Increase clarity and reduce overhead found with `TEST_ERROR`.
+ * e.g., "if(somefunction(arg, arg2) < 0) TEST_ERROR:"
+ * requires reading of entire line to know whether this if/call is
+ * part of the test setup, test operation, or a test unto itself.
+ * 3) Provide testing macros with optional user-supplied failure message;
+ * if not supplied (NULL), generate comparison output in the spirit of
+ * test-driven development. E.g., "expected 5 but was -3"
+ * User messages clarify test's purpose in code, encouraging description
+ * without relying on comments.
+ * 4) Configurable expected-actual order in generated comparison strings.
+ * Some prefer `VERIFY(expected, actual)`, others
+ * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch
+ * to satifsy both parties, assuming one paradigm per test file.
+ * (One could #undef and redefine the flag through the file as desired,
+ * but _why_.)
+ *
+ * Provided as courtesy, per consideration for inclusion in the library
+ * proper.
+ *
+ * Macros:
+ *
+ * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order
+ * FAIL_IF() - check condition
+ * FAIL_UNLESS() - check _not_ condition
+ * JSVERIFY() - long-int equality check; prints reason/comparison
+ * JSVERIFY_NOT() - long-int inequality check; prints
+ * JSVERIFY_STR() - string equality check; prints
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *****************************************************************************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * ifdef flag: JSVERIFY_EXP_ACT
+ *
+ * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason])
+ * default, if this is undefined, is (ACTUAL, EXPECTED[, reason])
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_EXP_ACT 1L
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSFAILED_AT()
+ *
+ * Purpose:
+ *
+ * Preface a test failure by printing "*FAILED*" and location to stdout
+ * Similar to `H5_FAILED(); AT();` from h5test.h
+ *
+ * *FAILED* at somefile.c:12 in function_name()...
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSFAILED_AT() { \
+ HDprintf("*FAILED* at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: FAIL_IF()
+ *
+ * Purpose:
+ *
+ * Make tests more accessible and less cluttered than
+ * `if (thing == otherthing()) TEST_ERROR`
+ * paradigm.
+ *
+ * The following lines are roughly equivalent:
+ *
+ * `if (myfunc() < 0) TEST_ERROR;` (as seen elsewhere in HDF tests)
+ * `FAIL_IF(myfunc() < 0)`
+ *
+ * Prints a generic "FAILED AT" line to stdout and jumps to `error`,
+ * similar to `TEST_ERROR` in h5test.h
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *----------------------------------------------------------------------------
+ */
+#define FAIL_IF(condition) \
+if (condition) { \
+ JSFAILED_AT() \
+ goto error; \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: FAIL_UNLESS()
+ *
+ * Purpose:
+ *
+ * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests",
+ * e.g., "a != b".
+ *
+ * Opposite of FAIL_IF; fails if the given condition is _not_ true.
+ *
+ * `FAIL_IF( 5 != my_op() )`
+ * is equivalent to
+ * `FAIL_UNLESS( 5 == my_op() )`
+ * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer.
+ * (see JSVERIFY)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#if 0 /* UNUSED */
+#define FAIL_UNLESS(condition) \
+if (!(condition)) { \
+ JSFAILED_AT() \
+ goto error; \
+}
+#endif
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSERR_LONG()
+ *
+ * Purpose:
+ *
+ * Print an failure message for long-int arguments.
+ * ERROR-AT printed first.
+ * If `reason` is given, it is printed on own line and newlined after
+ * else, prints "expected/actual" aligned on own lines.
+ *
+ * *FAILED* at myfile.c:488 in somefunc()...
+ * forest must be made of trees.
+ *
+ * or
+ *
+ * *FAILED* at myfile.c:488 in somefunc()...
+ * ! Expected 425
+ * ! Actual 3
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSERR_LONG(expected, actual, reason) { \
+ JSFAILED_AT() \
+ if (reason!= NULL) { \
+ HDprintf("%s\n", (reason)); \
+ } else { \
+ HDprintf(" ! Expected %ld\n ! Actual %ld\n", \
+ (long)(expected), (long)(actual)); \
+ } \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSERR_STR()
+ *
+ * Purpose:
+ *
+ * Print an failure message for string arguments.
+ * ERROR-AT printed first.
+ * If `reason` is given, it is printed on own line and newlined after
+ * else, prints "expected/actual" aligned on own lines.
+ *
+ * *FAILED* at myfile.c:421 in myfunc()...
+ * Blue and Red strings don't match!
+ *
+ * or
+ *
+ * *FAILED* at myfile.c:421 in myfunc()...
+ * !!! Expected:
+ * this is my expected
+ * string
+ * !!! Actual:
+ * not what I expected at all
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSERR_STR(expected, actual, reason) { \
+ JSFAILED_AT() \
+ if ((reason) != NULL) { \
+ HDprintf("%s\n", (reason)); \
+ } else { \
+ HDprintf("!!! Expected:\n%s\n!!!Actual:\n%s\n", \
+ (expected), (actual)); \
+ } \
+}
+
+
+
+#ifdef JSVERIFY_EXP_ACT
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY()
+ *
+ * Purpose:
+ *
+ * Verify that two long integers are equal.
+ * If unequal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY(expected, actual, reason) \
+if ((long)(actual) != (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY_NOT()
+ *
+ * Purpose:
+ *
+ * Verify that two long integers are _not_ equal.
+ * If equal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_NOT(expected, actual, reason) \
+if ((long)(actual) == (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY_NOT */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY_STR()
+ *
+ * Purpose:
+ *
+ * Verify that two strings are equal.
+ * If unequal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_STR(expected, actual, reason) \
+if (strcmp((actual), (expected)) != 0) { \
+ JSERR_STR((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY_STR */
+
+
+#else
+/* JSVERIFY_EXP_ACT not defined
+ *
+ * Repeats macros above, but with actual/expected parameters reversed.
+ */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY()
+ * See: JSVERIFY documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY(actual, expected, reason) \
+if ((long)(actual) != (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY_NOT()
+ * See: JSVERIFY_NOT documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_NOT(actual, expected, reason) \
+if ((long)(actual) == (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY_NOT */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY_STR()
+ * See: JSVERIFY_STR documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_STR(actual, expected, reason) \
+if (strcmp((actual), (expected)) != 0) { \
+ JSERR_STR((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY_STR */
+
+#endif /* ifdef/else JSVERIFY_EXP_ACT */
+
+/********************************
+ * OTHER MACROS AND DEFINITIONS *
+ ********************************/
+
+#define MAXADDR (((haddr_t)1<<(8*sizeof(HDoff_t)-1))-1)
+
+#define S3_TEST_PROFILE_NAME "ros3_vfd_test"
+
+#define S3_TEST_MAX_URL_SIZE 256
+
+#define S3_TEST_RESOURCE_TEXT_RESTRICTED "t8.shakespeare.txt"
+#define S3_TEST_RESOURCE_TEXT_PUBLIC "Poe_Raven.txt"
+#define S3_TEST_RESOURCE_H5_PUBLIC "GMODO-SVM01.h5"
+#define S3_TEST_RESOURCE_MISSING "missing.csv"
+
+static char url_text_restricted[S3_TEST_MAX_URL_SIZE] = "";
+static char url_text_public[S3_TEST_MAX_URL_SIZE] = "";
+static char url_h5_public[S3_TEST_MAX_URL_SIZE] = "";
+static char url_missing[S3_TEST_MAX_URL_SIZE] = "";
+static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE] = "";
+static hbool_t s3_test_bucket_defined = FALSE;
+
+/* Global variables for aws test profile.
+ * An attempt is made to read ~/.aws/credentials and ~/.aws/config upon test
+ * startup -- if unable to open either file or cannot load region, id, and key,
+ * tests connecting with S3 will not be run
+ */
+static int s3_test_credentials_loaded = 0;
+static char s3_test_aws_region[16];
+static char s3_test_aws_access_key_id[64];
+static char s3_test_aws_secret_access_key[128];
+
+H5FD_ros3_fapl_t restricted_access_fa = {
+ H5FD_CURR_ROS3_FAPL_T_VERSION, /* fapl version */
+ TRUE, /* authenticate */
+ "", /* aws region */
+ "", /* access key id */
+ ""}; /* secret access key */
+
+H5FD_ros3_fapl_t anonymous_fa = {
+ H5FD_CURR_ROS3_FAPL_T_VERSION,
+ FALSE, "", "", "" };
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_fapl_config_validation()
+ *
+ * Purpose:
+ *
+ * Test data consistency of fapl configuration.
+ * Tests `H5FD_ros3_validate_config` indirectly through `H5Pset_fapl_ros3`.
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_fapl_config_validation(void)
+{
+
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct testcase {
+ const char *msg;
+ herr_t expected;
+ H5FD_ros3_fapl_t config;
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ hid_t fapl_id = -1; /* file access property list ID */
+ H5FD_ros3_fapl_t config;
+ H5FD_ros3_fapl_t fa_fetch;
+ herr_t success = SUCCEED;
+ unsigned int i = 0;
+ unsigned int ncases = 8; /* should equal number of cases */
+ struct testcase *case_ptr = NULL; /* dumb work-around for possible */
+ /* dynamic cases creation because */
+ /* of compiler warnings Wlarger-than */
+ struct testcase cases_arr[] = {
+ { "non-authenticating config allows empties.\n",
+ SUCCEED,
+ { H5FD_CURR_ROS3_FAPL_T_VERSION, /* version */
+ FALSE, /* authenticate */
+ "", /* aws_region */
+ "", /* secret_id */
+ "", /* secret_key */
+ },
+ },
+ { "authenticating config asks for populated strings.\n",
+ FAIL,
+ { H5FD_CURR_ROS3_FAPL_T_VERSION,
+ TRUE,
+ "",
+ "",
+ "",
+ },
+ },
+ { "populated strings; key is the empty string?\n",
+ SUCCEED,
+ { H5FD_CURR_ROS3_FAPL_T_VERSION,
+ TRUE,
+ "region",
+ "me",
+ "",
+ },
+ },
+ { "id cannot be empty.\n",
+ FAIL,
+ { H5FD_CURR_ROS3_FAPL_T_VERSION,
+ TRUE,
+ "",
+ "me",
+ "",
+ },
+ },
+ { "region cannot be empty.\n",
+ FAIL,
+ { H5FD_CURR_ROS3_FAPL_T_VERSION,
+ TRUE,
+ "where",
+ "",
+ "",
+ },
+ },
+ { "all strings populated.\n",
+ SUCCEED,
+ { H5FD_CURR_ROS3_FAPL_T_VERSION,
+ TRUE,
+ "where",
+ "who",
+ "thisIsA GREAT seeeecrit",
+ },
+ },
+ { "incorrect version should fail\n",
+ FAIL,
+ { 12345,
+ FALSE,
+ "",
+ "",
+ "",
+ },
+ },
+ { "non-authenticating config cares not for (de)population"
+ "of strings.\n",
+ SUCCEED,
+ { H5FD_CURR_ROS3_FAPL_T_VERSION,
+ FALSE,
+ "someregion",
+ "someid",
+ "somekey",
+ },
+ },
+ };
+
+ TESTING("ROS3 fapl configuration validation");
+
+ /*********
+ * TESTS *
+ *********/
+
+ if (FALSE == s3_test_bucket_defined) {
+ SKIPPED();
+ puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
+ fflush(stdout);
+ return 0;
+ }
+
+ for (i = 0; i < ncases; i++) {
+
+ /*---------------
+ * per-test setup
+ *---------------
+ */
+ case_ptr = &cases_arr[i];
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 ) /* sanity-check */
+
+ /*-----------------------------------
+ * Actually test.
+ * Mute stack trace in failure cases.
+ *-----------------------------------
+ */
+ H5E_BEGIN_TRY {
+ /* `H5FD_ros3_validate_config(...)` is static/private
+ * to src/ros3.c and cannot (and should not?) be tested directly?
+ * Instead, validate config through public api.
+ */
+ success = H5Pset_fapl_ros3(fapl_id, &case_ptr->config);
+ } H5E_END_TRY;
+
+ JSVERIFY( case_ptr->expected, success, case_ptr->msg )
+
+ /* Make sure we can get back what we put in.
+ * Only valid if the fapl configuration does not result in error.
+ */
+ if (success == SUCCEED) {
+ config = case_ptr->config;
+ JSVERIFY( SUCCEED,
+ H5Pget_fapl_ros3(fapl_id, &fa_fetch),
+ "unable to get fapl" )
+
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION,
+ fa_fetch.version,
+ "invalid version number" )
+ JSVERIFY( config.version,
+ fa_fetch.version,
+ "version number mismatch" )
+ JSVERIFY( config.authenticate,
+ fa_fetch.authenticate,
+ "authentication flag mismatch" )
+ JSVERIFY_STR( config.aws_region,
+ fa_fetch.aws_region,
+ NULL )
+ JSVERIFY_STR( config.secret_id,
+ fa_fetch.secret_id,
+ NULL )
+ JSVERIFY_STR( config.secret_key,
+ fa_fetch.secret_key,
+ NULL )
+ }
+
+ /*-----------------------------
+ * per-test sanitation/teardown
+ *-----------------------------
+ */
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ } /* for each test case */
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (fapl_id < 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+ return 1;
+} /* test_fapl_config_validation */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: test_ros3_fapl()
+ *
+ * Purpose: Tests the file handle interface for the ROS3 driver
+ *
+ * As the ROS3 driver is 1) read only, 2) requires access
+ * to an S3 server, this test is quite
+ * different from the other tests.
+ *
+ * For now, test only fapl & flags. Extend as the
+ * work on the VFD continues.
+ *
+ * Return: Success: 0
+ * Failure: 1
+ *
+ * Programmer: John Mainzer
+ * 7/12/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+test_ros3_fapl(void)
+{
+ /************************
+ * test-local variables *
+ ************************/
+
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t driver_id = -1; /* ID for this VFD */
+ unsigned long driver_flags = 0; /* VFD feature flags */
+ H5FD_ros3_fapl_t ros3_fa_0 = {
+ H5FD_CURR_ROS3_FAPL_T_VERSION, /* version */
+ FALSE, /* authenticate */
+ "", /* aws_region */
+ "", /* secret_id */
+ "plugh", /* secret_key */
+ };
+
+ TESTING("ROS3 fapl ");
+
+ /* Set property list and file name for ROS3 driver.
+ */
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 )
+
+ FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &ros3_fa_0) )
+
+ driver_id = H5Pget_driver(fapl_id);
+ FAIL_IF( driver_id < 0 )
+
+ /****************
+ * Check that the VFD feature flags are correct
+ * SPEC MAY CHANGE
+ ******************/
+
+ FAIL_IF( H5FDdriver_query(driver_id, &driver_flags) < 0 )
+
+ JSVERIFY_NOT( 0, (driver_flags & H5FD_FEAT_DATA_SIEVE),
+ "bit(s) in `driver_flags` must align with "
+ "H5FD_FEAT_DATA_SIEVE" )
+
+ JSVERIFY( H5FD_FEAT_DATA_SIEVE, driver_flags,
+ "H5FD_FEAT_DATA_SIEVE should be the only supported flag")
+
+ PASSED();
+ return 0;
+
+error:
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+
+ return 1;
+
+} /* test_ros3_fapl() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_vfd_open()
+ *
+ * Purpose:
+ *
+ * Demonstrate/specify VFD-level "Open" failure cases
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 1027-11-03
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_vfd_open(void)
+{
+
+ /*********************
+ * test-local macros *
+ *********************/
+
+
+#define FAPL_H5P_DEFAULT -2
+#define FAPL_FILE_ACCESS -3
+#define FAPL_ROS3_ANON -4
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct test_condition {
+ const char *message;
+ const char *url;
+ unsigned flags;
+ int which_fapl;
+ haddr_t maxaddr;
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ struct test_condition tests[] = {
+ { "default property list (H5P_DEFAULT) is invalid",
+ url_text_public,
+ H5F_ACC_RDONLY,
+ FAPL_H5P_DEFAULT,
+ MAXADDR,
+ },
+ { "generic file access property list is invalid",
+ url_text_public,
+ H5F_ACC_RDONLY,
+ FAPL_FILE_ACCESS,
+ MAXADDR,
+ },
+ { "filename cannot be null",
+ NULL,
+ H5F_ACC_RDONLY,
+ FAPL_ROS3_ANON,
+ MAXADDR,
+ },
+ { "filename cannot be empty",
+ "",
+ H5F_ACC_RDONLY,
+ FAPL_ROS3_ANON,
+ MAXADDR,
+ },
+ { "filename must exist",
+ url_missing,
+ H5F_ACC_RDONLY,
+ FAPL_ROS3_ANON,
+ MAXADDR,
+ },
+ { "read-write flag not supported",
+ url_text_public,
+ H5F_ACC_RDWR,
+ FAPL_ROS3_ANON,
+ MAXADDR,
+ },
+ { "truncate flag not supported",
+ url_text_public,
+ H5F_ACC_TRUNC,
+ FAPL_ROS3_ANON,
+ MAXADDR,
+ },
+ { "create flag not supported",
+ url_text_public,
+ H5F_ACC_CREAT,
+ FAPL_ROS3_ANON,
+ MAXADDR,
+ },
+ { "EXCL flag not supported",
+ url_text_public,
+ H5F_ACC_EXCL,
+ FAPL_ROS3_ANON,
+ MAXADDR,
+ },
+ { "maxaddr cannot be 0 (caught in `H5FD_open()`)",
+ url_text_public,
+ H5F_ACC_RDONLY,
+ FAPL_ROS3_ANON,
+ 0,
+ },
+ };
+ H5FD_t *fd = NULL;
+ hbool_t curl_ready = FALSE;
+ hid_t fapl_id = -1;
+ hid_t fapl_file_access = -1;
+ unsigned i = 0;
+ unsigned tests_count = 10;
+
+ TESTING("ROS3 VFD-level open");
+
+ if (FALSE == s3_test_bucket_defined) {
+ SKIPPED();
+ puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
+ fflush(stdout);
+ return 0;
+ }
+
+ FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) )
+ curl_ready = TRUE;
+
+ fapl_file_access = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_file_access < 0 )
+
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 )
+ FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &anonymous_fa) )
+
+ /*********
+ * TESTS *
+ *********/
+
+ /* all the test cases that will _not_ open
+ */
+ for (i = 0; i < tests_count; i++) {
+ struct test_condition T = tests[i];
+ hid_t _fapl_id = H5P_DEFAULT;
+
+ fd = NULL;
+
+ if (T.which_fapl == FAPL_FILE_ACCESS)
+ _fapl_id = fapl_file_access;
+ else if (T.which_fapl == FAPL_ROS3_ANON)
+ _fapl_id = fapl_id;
+
+ H5E_BEGIN_TRY {
+ fd = H5FDopen(T.url, T.flags, _fapl_id, T.maxaddr);
+ } H5E_END_TRY;
+ if (NULL != fd)
+ JSVERIFY(1, 0, T.message); /* wrapper to print message and fail */
+ }
+
+ FAIL_IF( NULL != fd )
+
+ /* finally, show that a file can be opened
+ */
+ fd = H5FDopen(
+ url_text_public,
+ H5F_ACC_RDONLY,
+ fapl_id,
+ MAXADDR);
+ FAIL_IF( NULL == fd )
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ FAIL_IF( FAIL == H5FDclose(fd) )
+ fd = NULL;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_file_access) )
+ fapl_file_access = -1;
+
+ curl_global_cleanup();
+ curl_ready = FALSE;
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (fd) {
+ (void)H5FDclose(fd);
+ }
+ if (fapl_id >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+ if (fapl_file_access >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_file_access);
+ } H5E_END_TRY;
+ }
+ if (curl_ready == TRUE) {
+ curl_global_cleanup();
+ }
+
+ return 1;
+
+#undef FAPL_FILE_ACCESS
+#undef FAPL_H5P_DEFAULT
+#undef FAPL_ROS3_ANON
+
+} /* test_vfd_open */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_eof_eoa()
+ *
+ * Purpose:
+ *
+ * Demonstrate behavior of get_eof, get_eoa, and set_eoa.
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-08
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_eof_eoa(void)
+{
+
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ H5FD_t *fd_shakespeare = NULL;
+ hbool_t curl_ready = FALSE;
+ hid_t fapl_id = -1;
+
+ TESTING("ROS3 eof/eoa gets and sets");
+
+ if (s3_test_credentials_loaded == 0) {
+ SKIPPED();
+ puts(" s3 credentials are not loaded");
+ fflush(stdout);
+ return 0;
+ }
+
+ if (FALSE == s3_test_bucket_defined) {
+ SKIPPED();
+ puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
+ fflush(stdout);
+ return 0;
+ }
+
+ /*********
+ * SETUP *
+ *********/
+
+ FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) )
+ curl_ready = TRUE;
+
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( 0 > fapl_id )
+ FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) )
+
+ fd_shakespeare = H5FDopen(
+ url_text_restricted,
+ H5F_ACC_RDONLY,
+ fapl_id,
+ HADDR_UNDEF);
+ FAIL_IF( NULL == fd_shakespeare )
+
+ /*********
+ * TESTS *
+ *********/
+
+ /* verify as found
+ */
+ JSVERIFY( 5458199, H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT), NULL )
+ JSVERIFY( H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT),
+ H5FDget_eof(fd_shakespeare, H5FD_MEM_DRAW),
+ "mismatch between DEFAULT and RAW memory types" )
+ JSVERIFY( 0,
+ H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT),
+ "EoA should be unset by H5FDopen" )
+
+ /* set EoA below EoF
+ */
+ JSVERIFY( SUCCEED,
+ H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 44442202),
+ "unable to set EoA (lower)" )
+ JSVERIFY( 5458199,
+ H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT),
+ "EoF changed" )
+ JSVERIFY( 44442202,
+ H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT),
+ "EoA unchanged" )
+
+ /* set EoA above EoF
+ */
+ JSVERIFY( SUCCEED,
+ H5FDset_eoa(fd_shakespeare, H5FD_MEM_DEFAULT, 6789012),
+ "unable to set EoA (higher)" )
+ JSVERIFY( 5458199,
+ H5FDget_eof(fd_shakespeare, H5FD_MEM_DEFAULT),
+ "EoF changed" )
+ JSVERIFY( 6789012,
+ H5FDget_eoa(fd_shakespeare, H5FD_MEM_DEFAULT),
+ "EoA unchanged" )
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ FAIL_IF( FAIL == H5FDclose(fd_shakespeare) )
+
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ curl_global_cleanup();
+ curl_ready = FALSE;
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (fd_shakespeare) (void)H5FDclose(fd_shakespeare);
+ if (TRUE == curl_ready) curl_global_cleanup();
+ if (fapl_id >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+
+ return 1;
+
+} /* test_eof_eoa */
+
+
+/*-----------------------------------------------------------------------------
+ *
+ * Function: test_H5FDread_without_eoa_set_fails()
+ *
+ * Purpose:
+ *
+ * Demonstrate a not-obvious constraint by the library, preventing
+ * file read before EoA is set
+ *
+ * Programmer: Jacob Smith
+ * 2018-01-26
+ *
+ *-----------------------------------------------------------------------------
+ */
+static int
+test_H5FDread_without_eoa_set_fails(void)
+{
+ char buffer[256];
+ unsigned int i = 0;
+ H5FD_t *file_shakespeare = NULL;
+ hid_t fapl_id = -1;
+
+ TESTING("ROS3 VFD read-eoa temporal coupling library limitation ");
+
+ if (s3_test_credentials_loaded == 0) {
+ SKIPPED();
+ puts(" s3 credentials are not loaded");
+ fflush(stdout);
+ return 0;
+ }
+
+ if (FALSE == s3_test_bucket_defined) {
+ SKIPPED();
+ puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
+ fflush(stdout);
+ return 0;
+ }
+
+ /*********
+ * SETUP *
+ *********/
+
+ /* create ROS3 fapl
+ */
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 )
+ FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) )
+
+ file_shakespeare = H5FDopen(
+ url_text_restricted,
+ H5F_ACC_RDONLY,
+ fapl_id,
+ MAXADDR);
+ FAIL_IF( NULL == file_shakespeare )
+
+ JSVERIFY( 0, H5FDget_eoa(file_shakespeare, H5FD_MEM_DEFAULT),
+ "EoA should remain unset by H5FDopen" )
+
+ for (i = 0; i < 256; i++)
+ buffer[i] = 0; /* zero buffer contents */
+
+ /********
+ * TEST *
+ ********/
+
+ H5E_BEGIN_TRY { /* mute stack trace on expected failure */
+ JSVERIFY( FAIL,
+ H5FDread(file_shakespeare,
+ H5FD_MEM_DRAW,
+ H5P_DEFAULT,
+ 1200699,
+ 102,
+ buffer),
+ "cannot read before eoa is set" )
+ } H5E_END_TRY;
+ JSVERIFY_STR( "", buffer, "buffer should remain untouched" )
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ FAIL_IF( FAIL == H5FDclose(file_shakespeare) )
+ file_shakespeare = NULL;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (file_shakespeare) { (void)H5FDclose(file_shakespeare); }
+ if (fapl_id >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+
+ return 1;
+
+} /* test_H5FDread_without_eoa_set_fails */
+
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_read()
+ *
+ * Purpose:
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-06
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_read(void)
+{
+
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+ struct testcase {
+ const char *message; /* purpose of test case */
+ haddr_t eoa_set; /* set file EOA to this prior to read */
+ size_t addr; /* offset of read in file */
+ size_t len; /* length of read in file */
+ herr_t success; /* expected return value of read function */
+ const char *expected; /* expected contents of buffer; failure ignores */
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+ struct testcase cases[] = {
+ { "successful range-get",
+ 6464,
+ 5691,
+ 32, /* fancy quotes are three bytes each(?) */
+ SUCCEED,
+ "Quoth the Raven “Nevermore.”",
+ },
+ { "read past EOA fails (EOA < EOF < addr)",
+ 3000,
+ 4000,
+ 100,
+ FAIL,
+ NULL,
+ },
+ { "read overlapping EOA fails (EOA < addr < EOF < (addr+len))",
+ 3000,
+ 8000,
+ 100,
+ FAIL,
+ NULL,
+ },
+ { "read past EOA/EOF fails ((EOA==EOF) < addr)",
+ 6464,
+ 7000,
+ 100,
+ FAIL,
+ NULL,
+ },
+ { "read overlapping EOA/EOF fails (addr < (EOA==EOF) < (addr+len))",
+ 6464,
+ 6400,
+ 100,
+ FAIL,
+ NULL,
+ },
+ { "read between EOF and EOA fails (EOF < addr < (addr+len) < EOA)",
+ 8000,
+ 7000,
+ 100,
+ FAIL,
+ NULL,
+ },
+ };
+ unsigned testcase_count = 6;
+ unsigned test_i = 0;
+ struct testcase test;
+ herr_t open_return = FAIL;
+ char buffer[S3_TEST_MAX_URL_SIZE];
+ unsigned int i = 0;
+ H5FD_t *file_raven = NULL;
+ hid_t fapl_id = -1;
+
+ TESTING("ROS3 VFD read/range-gets");
+
+ if (s3_test_credentials_loaded == 0) {
+ SKIPPED();
+ puts(" s3 credentials are not loaded");
+ fflush(stdout);
+ return 0;
+ }
+
+ if (FALSE == s3_test_bucket_defined) {
+ SKIPPED();
+ puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
+ fflush(stdout);
+ return 0;
+ }
+
+ /*********
+ * SETUP *
+ *********/
+
+ /* create ROS3 fapl
+ */
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 )
+ FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) )
+
+ /* open file
+ */
+ file_raven = H5FDopen( /* will open with "authenticating" fapl */
+ url_text_public, /* TODO: check return state: anon access of restricted says OK? (not NULL) */
+ H5F_ACC_RDONLY,
+ fapl_id,
+ HADDR_UNDEF); /* Demonstrate success with "automatic" value */
+ FAIL_IF( NULL == file_raven )
+
+ JSVERIFY( 6464, H5FDget_eof(file_raven, H5FD_MEM_DEFAULT), NULL )
+
+ /*********
+ * TESTS *
+ *********/
+
+ for (test_i = 0; test_i < testcase_count; test_i++) {
+
+ /* -------------- *
+ * per-test setup *
+ * -------------- */
+
+ test = cases[test_i];
+ open_return = FAIL;
+
+ FAIL_IF( S3_TEST_MAX_URL_SIZE < test.len ) /* buffer too small! */
+
+ FAIL_IF( FAIL ==
+ H5FD_set_eoa( file_raven, H5FD_MEM_DEFAULT, test.eoa_set) )
+
+ for (i = 0; i < S3_TEST_MAX_URL_SIZE; i++) /* zero buffer contents */
+ buffer[i] = 0;
+
+ /* ------------ *
+ * conduct test *
+ * ------------ */
+
+ H5E_BEGIN_TRY {
+ open_return = H5FDread(
+ file_raven,
+ H5FD_MEM_DRAW,
+ H5P_DEFAULT,
+ test.addr,
+ test.len,
+ buffer);
+ } H5E_END_TRY;
+
+ JSVERIFY( test.success,
+ open_return,
+ test.message )
+ if (open_return == SUCCEED)
+ JSVERIFY_STR( test.expected, buffer, NULL )
+
+ } /* for each testcase */
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ FAIL_IF( FAIL == H5FDclose(file_raven) )
+ file_raven = NULL;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (file_raven)
+ (void)H5FDclose(file_raven);
+ if (fapl_id >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+
+ return 1;
+
+} /* test_read */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_noops_and_autofails()
+ *
+ * Purpose:
+ *
+ * Demonstrate the unavailable and do-nothing routines unique to
+ * Read-Only VFD.
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-06
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_noops_and_autofails(void)
+{
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ hbool_t curl_ready = FALSE;
+ hid_t fapl_id = -1;
+ H5FD_t *file = NULL;
+ const char data[36] = "The Force shall be with you, always";
+
+ TESTING("ROS3 VFD always-fail and no-op routines");
+
+
+ if (FALSE == s3_test_bucket_defined) {
+ SKIPPED();
+ puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
+ fflush(stdout);
+ return 0;
+ }
+
+ /*********
+ * SETUP *
+ *********/
+
+ FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) )
+ curl_ready = TRUE;
+
+ /* create ROS3 fapl
+ */
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 )
+ JSVERIFY( SUCCEED, H5Pset_fapl_ros3(fapl_id, &anonymous_fa), NULL )
+
+ /* open file
+ */
+ file = H5FDopen(
+ url_text_public,
+ H5F_ACC_RDONLY,
+ fapl_id,
+ HADDR_UNDEF);
+ FAIL_IF( NULL == file )
+
+ /*********
+ * TESTS *
+ *********/
+
+ /* auto-fail calls to write and truncate
+ */
+ H5E_BEGIN_TRY {
+ JSVERIFY( FAIL,
+ H5FDwrite(file, H5FD_MEM_DRAW, H5P_DEFAULT, 1000, 35, data),
+ "write must fail" )
+ } H5E_END_TRY;
+
+ H5E_BEGIN_TRY {
+ JSVERIFY( FAIL,
+ H5FDtruncate(file, H5P_DEFAULT, FALSE),
+ "truncate must fail" )
+ } H5E_END_TRY;
+
+ H5E_BEGIN_TRY {
+ JSVERIFY( FAIL,
+ H5FDtruncate(file, H5P_DEFAULT, TRUE),
+ "truncate must fail (closing)" )
+ } H5E_END_TRY;
+
+ /* no-op calls to `lock()` and `unlock()`
+ */
+ JSVERIFY( SUCCEED,
+ H5FDlock(file, TRUE),
+ "lock always succeeds; has no effect" )
+ JSVERIFY( SUCCEED,
+ H5FDlock(file, FALSE),
+ NULL )
+ JSVERIFY( SUCCEED,
+ H5FDunlock(file),
+ NULL )
+ /* Lock/unlock with null file or similar error crashes tests.
+ * HDassert in calling heirarchy, `H5FD[un]lock()` and `H5FD_[un]lock()`
+ */
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ FAIL_IF( FAIL == H5FDclose(file) )
+ file = NULL;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ curl_global_cleanup();
+ curl_ready = FALSE;
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (fapl_id >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+ if (file) { (void)H5FDclose(file); }
+ if (curl_ready == TRUE) { curl_global_cleanup(); }
+
+ return 1;
+
+} /* test_noops_and_autofails*/
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_cmp()
+ *
+ * Purpose:
+ *
+ * Verify "file comparison" behavior.
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-06
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_cmp(void)
+{
+
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ H5FD_t *fd_raven = NULL;
+ H5FD_t *fd_shakes = NULL;
+ H5FD_t *fd_raven_2 = NULL;
+ hbool_t curl_ready = FALSE;
+ hid_t fapl_id = -1;
+
+ TESTING("ROS3 cmp (comparison)");
+
+ if (s3_test_credentials_loaded == 0) {
+ SKIPPED();
+ puts(" s3 credentials are not loaded");
+ fflush(stdout);
+ return 0;
+ }
+
+ if (FALSE == s3_test_bucket_defined) {
+ SKIPPED();
+ puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
+ fflush(stdout);
+ return 0;
+ }
+
+ /*********
+ * SETUP *
+ *********/
+
+ FAIL_IF( CURLE_OK != curl_global_init(CURL_GLOBAL_DEFAULT) )
+ curl_ready = TRUE;
+
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( 0 > fapl_id )
+ JSVERIFY( SUCCEED, H5Pset_fapl_ros3(fapl_id, &restricted_access_fa), NULL )
+
+ fd_raven = H5FDopen(
+ url_text_public,
+ H5F_ACC_RDONLY,
+ fapl_id,
+ HADDR_UNDEF);
+ FAIL_IF( NULL == fd_raven )
+
+ fd_shakes = H5FDopen(
+ url_text_restricted,
+ H5F_ACC_RDONLY,
+ fapl_id,
+ HADDR_UNDEF);
+ FAIL_IF( NULL == fd_shakes )
+
+ fd_raven_2 = H5FDopen(
+ url_text_public,
+ H5F_ACC_RDONLY,
+ fapl_id,
+ HADDR_UNDEF);
+ FAIL_IF( NULL == fd_raven_2 )
+
+ /*********
+ * TESTS *
+ *********/
+
+ JSVERIFY( 0, H5FDcmp(fd_raven, fd_raven_2), NULL )
+ JSVERIFY( -1, H5FDcmp(fd_raven, fd_shakes), NULL )
+ JSVERIFY( -1, H5FDcmp(fd_shakes, fd_raven_2), NULL )
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ FAIL_IF( FAIL == H5FDclose(fd_raven) )
+ fd_raven = NULL;
+ FAIL_IF( FAIL == H5FDclose(fd_shakes) )
+ fd_shakes = NULL;
+ FAIL_IF( FAIL == H5FDclose(fd_raven_2) )
+ fd_raven_2 = NULL;
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ curl_global_cleanup();
+ curl_ready = FALSE;
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (fd_raven != NULL) (void)H5FDclose(fd_raven);
+ if (fd_raven_2 != NULL) (void)H5FDclose(fd_raven_2);
+ if (fd_shakes != NULL) (void)H5FDclose(fd_shakes);
+ if (TRUE == curl_ready) curl_global_cleanup();
+ if (fapl_id >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+
+ return 1;
+
+} /* test_cmp */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_H5F_integration()
+ *
+ * Purpose:
+ *
+ * Demonstrate S3 file-open through H5F API.
+ *
+ * Return:
+ *
+ * PASSED : 0
+ * FAILED : 1
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-07
+ *
+ *---------------------------------------------------------------------------
+ */
+static int
+test_H5F_integration(void)
+{
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ hid_t file = -1;
+ hid_t fapl_id = -1;
+
+ TESTING("S3 file access through HD5F library (H5F API)");
+
+ if (s3_test_credentials_loaded == 0) {
+ SKIPPED();
+ puts(" s3 credentials are not loaded");
+ fflush(stdout);
+ return 0;
+ }
+
+ if (FALSE == s3_test_bucket_defined) {
+ SKIPPED();
+ puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
+ fflush(stdout);
+ return 0;
+ }
+
+ /*********
+ * SETUP *
+ *********/
+
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( 0 > fapl_id )
+ FAIL_IF( FAIL == H5Pset_fapl_ros3(fapl_id, &restricted_access_fa) )
+
+ /*********
+ * TESTS *
+ *********/
+
+ /* Read-Write Open access is not allowed with this file driver.
+ */
+ H5E_BEGIN_TRY {
+ FAIL_IF( 0 <= H5Fopen(
+ url_h5_public,
+ H5F_ACC_RDWR,
+ fapl_id) )
+ } H5E_END_TRY;
+
+ /* H5Fcreate() is not allowed with this file driver.
+ */
+ H5E_BEGIN_TRY {
+ FAIL_IF( 0 <= H5Fcreate(
+ url_missing,
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT,
+ fapl_id) )
+ } H5E_END_TRY;
+
+ /* Successful open.
+ */
+ file = H5Fopen(
+ url_h5_public,
+ H5F_ACC_RDONLY,
+ fapl_id);
+ FAIL_IF( file < 0 )
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ FAIL_IF( FAIL == H5Fclose(file) )
+ file = -1;
+
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ fapl_id = -1;
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * CLEANUP *
+ ***********/
+HDprintf("\nerror!"); fflush(stdout);
+
+ if (fapl_id >= 0) {
+ H5E_BEGIN_TRY {
+ (void)H5Pclose(fapl_id);
+ } H5E_END_TRY;
+ }
+ if (file > 0)
+ (void)H5Fclose(file);
+
+ return 1;
+
+} /* test_H5F_integration */
+
+#endif /* H5_HAVE_ROS3_VFD */
+
+
+/*-------------------------------------------------------------------------
+ *
+ * Function: main
+ *
+ * Purpose: Tests the basic features of Virtual File Drivers
+ *
+ * Return: Success: 0
+ * Failure: 1
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+main(void)
+{
+#ifdef H5_HAVE_ROS3_VFD
+ int nerrors = 0;
+ const char *bucket_url_env = NULL;
+
+#endif /* H5_HAVE_ROS3_VFD */
+
+ HDprintf("Testing ros3 VFD functionality.\n");
+
+#ifdef H5_HAVE_ROS3_VFD
+
+ /************************
+ * initialize test urls *
+ ************************/
+
+ bucket_url_env = HDgetenv("HDF5_ROS3_TEST_BUCKET_URL");
+ if (bucket_url_env == NULL || bucket_url_env[0] == '\0') {
+ HDprintf("WARNING: S3 bucket url is not defined in enviornment " \
+ "variable 'HDF5_ROS3_TEST_BUCKET_URL'!\n");
+ } else {
+ HDstrncpy(s3_test_bucket_url, bucket_url_env, S3_TEST_MAX_URL_SIZE);
+ s3_test_bucket_defined = TRUE;
+ }
+
+ if (S3_TEST_MAX_URL_SIZE < HDsnprintf(
+ url_text_restricted,
+ (size_t)S3_TEST_MAX_URL_SIZE,
+ "%s/%s",
+ (const char *)s3_test_bucket_url,
+ (const char *)S3_TEST_RESOURCE_TEXT_RESTRICTED))
+ {
+ HDprintf("* ros3 setup failed (text_restricted) ! *\n");
+ return 1;
+ }
+ if (S3_TEST_MAX_URL_SIZE < HDsnprintf(
+ url_text_public,
+ (size_t)S3_TEST_MAX_URL_SIZE,
+ "%s/%s",
+ (const char *)s3_test_bucket_url,
+ (const char *)S3_TEST_RESOURCE_TEXT_PUBLIC))
+ {
+ HDprintf("* ros3 setup failed (text_public) ! *\n");
+ return 1;
+ }
+ if (S3_TEST_MAX_URL_SIZE < HDsnprintf(
+ url_h5_public,
+ (size_t)S3_TEST_MAX_URL_SIZE,
+ "%s/%s",
+ (const char *)s3_test_bucket_url,
+ (const char *)S3_TEST_RESOURCE_H5_PUBLIC))
+ {
+ HDprintf("* ros3 setup failed (h5_public) ! *\n");
+ return 1;
+ }
+ if (S3_TEST_MAX_URL_SIZE < HDsnprintf(
+ url_missing,
+ S3_TEST_MAX_URL_SIZE,
+ "%s/%s",
+ (const char *)s3_test_bucket_url,
+ (const char *)S3_TEST_RESOURCE_MISSING))
+ {
+ HDprintf("* ros3 setup failed (missing) ! *\n");
+ return 1;
+ }
+
+ /**************************************
+ * load credentials and prepare fapls *
+ **************************************/
+
+ /* "clear" profile data strings */
+ s3_test_aws_access_key_id[0] = '\0';
+ s3_test_aws_secret_access_key[0] = '\0';
+ s3_test_aws_region[0] = '\0';
+
+ /* attempt to load test credentials
+ * if unable, certain tests will be skipped
+ */
+ if (SUCCEED == H5FD_s3comms_load_aws_profile(
+ S3_TEST_PROFILE_NAME,
+ s3_test_aws_access_key_id,
+ s3_test_aws_secret_access_key,
+ s3_test_aws_region))
+ {
+ s3_test_credentials_loaded = 1;
+ HDstrncpy(restricted_access_fa.aws_region,
+ (const char *)s3_test_aws_region,
+ H5FD_ROS3_MAX_REGION_LEN);
+ HDstrncpy(restricted_access_fa.secret_id,
+ (const char *)s3_test_aws_access_key_id,
+ H5FD_ROS3_MAX_SECRET_ID_LEN);
+ HDstrncpy(restricted_access_fa.secret_key,
+ (const char *)s3_test_aws_secret_access_key,
+ H5FD_ROS3_MAX_SECRET_KEY_LEN);
+ }
+
+ /******************
+ * commence tests *
+ ******************/
+
+ h5_reset();
+
+ nerrors += test_fapl_config_validation();
+ nerrors += test_ros3_fapl();
+ nerrors += test_vfd_open();
+ nerrors += test_eof_eoa();
+ nerrors += test_H5FDread_without_eoa_set_fails();
+ nerrors += test_read();
+ nerrors += test_noops_and_autofails();
+ nerrors += test_cmp();
+ nerrors += test_H5F_integration();
+
+ if (nerrors > 0) {
+ HDprintf("***** %d ros3 TEST%s FAILED! *****\n",
+ nerrors,
+ nerrors > 1 ? "S" : "");
+ nerrors = 1;
+ } else {
+ HDprintf("All ros3 tests passed.\n");
+ }
+ return nerrors; /* 0 if no errors, 1 if any errors */
+
+#else
+
+ HDprintf("SKIPPED - read-only S3 VFD not built\n");
+ return EXIT_SUCCESS;
+
+#endif /* H5_HAVE_ROS3_VFD */
+
+} /* main() */
+
diff --git a/test/s3comms.c b/test/s3comms.c
new file mode 100644
index 0000000..9453b75
--- /dev/null
+++ b/test/s3comms.c
@@ -0,0 +1,2730 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Read-Only S3 Virtual File Driver (VFD)
+ *
+ * Purpose: Unit tests for the S3 Communications (s3comms) module.
+ *
+ * Programmer: Jacob Smith <jake.smith@hdfgroup.org>
+ * 2017-10-11
+ */
+
+#include "h5test.h"
+#include "H5FDs3comms.h"
+#include "H5MMprivate.h" /* memory management */
+
+#ifdef H5_HAVE_ROS3_VFD
+
+/*****************************************************************************
+ *
+ * FILE-LOCAL TESTING MACROS
+ *
+ * Purpose:
+ *
+ * 1) Upon test failure, goto-jump to single-location teardown in test
+ * function. E.g., `error:` (consistency with HDF corpus) or
+ * `failed:` (reflects purpose).
+ * >>> using "error", in part because `H5E_BEGIN_TRY` expects it.
+ * 2) Increase clarity and reduce overhead found with `TEST_ERROR`.
+ * e.g., "if(somefunction(arg, arg2) < 0) TEST_ERROR:"
+ * requires reading of entire line to know whether this if/call is
+ * part of the test setup, test operation, or a test unto itself.
+ * 3) Provide testing macros with optional user-supplied failure message;
+ * if not supplied (NULL), generate comparison output in the spirit of
+ * test-driven development. E.g., "expected 5 but was -3"
+ * User messages clarify test's purpose in code, encouraging description
+ * without relying on comments.
+ * 4) Configurable expected-actual order in generated comparison strings.
+ * Some prefer `VERIFY(expected, actual)`, others
+ * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch
+ * to satifsy both parties, assuming one paradigm per test file.
+ * (One could #undef and redefine the flag through the file as desired,
+ * but _why_.)
+ * Provided as courtesy, per consideration for inclusion in the library
+ * proper.
+ *
+ * Macros:
+ *
+ * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order
+ * FAIL_IF() - check condition
+ * FAIL_UNLESS() - check _not_ condition
+ * JSVERIFY() - long-int equality check; prints reason/comparison
+ * JSVERIFY_NOT() - long-int inequality check; prints
+ * JSVERIFY_STR() - string equality check; prints
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *****************************************************************************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * ifdef flag: JSVERIFY_EXP_ACT
+ *
+ * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason])
+ * default, if this is undefined, is (ACTUAL, EXPECTED[, reason])
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_EXP_ACT 1L
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSFAILED_AT()
+ *
+ * Purpose:
+ *
+ * Preface a test failure by printing "*FAILED*" and location to stdout
+ * Similar to `H5_FAILED(); AT();` from h5test.h
+ *
+ * *FAILED* at somefile.c:12 in function_name()...
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSFAILED_AT() { \
+ HDprintf("*FAILED* at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: FAIL_IF()
+ *
+ * Purpose:
+ *
+ * Make tests more accessible and less cluttered than
+ * `if (thing == otherthing()) TEST_ERROR`
+ * paradigm.
+ *
+ * The following lines are roughly equivalent:
+ *
+ * `if (myfunc() < 0) TEST_ERROR;` (as seen elsewhere in HDF tests)
+ * `FAIL_IF(myfunc() < 0)`
+ *
+ * Prints a generic "FAILED AT" line to stdout and jumps to `error`,
+ * similar to `TEST_ERROR` in h5test.h
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *----------------------------------------------------------------------------
+ */
+#define FAIL_IF(condition) \
+if (condition) { \
+ JSFAILED_AT() \
+ goto error; \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: FAIL_UNLESS()
+ *
+ * Purpose:
+ *
+ * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests",
+ * e.g., "a != b".
+ *
+ * Opposite of FAIL_IF; fails if the given condition is _not_ true.
+ *
+ * `FAIL_IF( 5 != my_op() )`
+ * is equivalent to
+ * `FAIL_UNLESS( 5 == my_op() )`
+ * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer.
+ * (see JSVERIFY)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define FAIL_UNLESS(condition) \
+if (!(condition)) { \
+ JSFAILED_AT() \
+ goto error; \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSERR_LONG()
+ *
+ * Purpose:
+ *
+ * Print an failure message for long-int arguments.
+ * ERROR-AT printed first.
+ * If `reason` is given, it is printed on own line and newlined after
+ * else, prints "expected/actual" aligned on own lines.
+ *
+ * *FAILED* at myfile.c:488 in somefunc()...
+ * forest must be made of trees.
+ *
+ * or
+ *
+ * *FAILED* at myfile.c:488 in somefunc()...
+ * ! Expected 425
+ * ! Actual 3
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSERR_LONG(expected, actual, reason) { \
+ JSFAILED_AT() \
+ if (reason!= NULL) { \
+ HDprintf("%s\n", (reason)); \
+ } else { \
+ HDprintf(" ! Expected %ld\n ! Actual %ld\n", \
+ (long)(expected), (long)(actual)); \
+ } \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSERR_STR()
+ *
+ * Purpose:
+ *
+ * Print an failure message for string arguments.
+ * ERROR-AT printed first.
+ * If `reason` is given, it is printed on own line and newlined after
+ * else, prints "expected/actual" aligned on own lines.
+ *
+ * *FAILED* at myfile.c:421 in myfunc()...
+ * Blue and Red strings don't match!
+ *
+ * or
+ *
+ * *FAILED* at myfile.c:421 in myfunc()...
+ * !!! Expected:
+ * this is my expected
+ * string
+ * !!! Actual:
+ * not what I expected at all
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSERR_STR(expected, actual, reason) { \
+ JSFAILED_AT() \
+ if ((reason) != NULL) { \
+ HDprintf("%s\n", (reason)); \
+ } else { \
+ HDprintf("!!! Expected:\n%s\n!!!Actual:\n%s\n", \
+ (expected), (actual)); \
+ } \
+}
+
+#ifdef JSVERIFY_EXP_ACT
+/* VERIFY rountines with paramter order (<expected>, <actual> [, <msg> ])
+ */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY()
+ *
+ * Purpose:
+ *
+ * Verify that two long integers are equal.
+ * If unequal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY(expected, actual, reason) \
+if ((long)(actual) != (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY */
+
+#if 0 /* UNUSED */
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY_NOT()
+ *
+ * Purpose:
+ *
+ * Verify that two long integers are _not_ equal.
+ * If equal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_NOT(expected, actual, reason) \
+if ((long)(actual) == (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY_NOT */
+#endif /* JSVERIFY_NOT unused */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY_STR()
+ *
+ * Purpose:
+ *
+ * Verify that two strings are equal.
+ * If unequal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_STR(expected, actual, reason) \
+if (strcmp((actual), (expected)) != 0) { \
+ JSERR_STR((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY_STR */
+
+
+#else
+/* JSVERIFY_EXP_ACT not defined
+ *
+ * Repeats macros above, but with actual/expected parameters reversed.
+ */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY()
+ * See: JSVERIFY documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY(actual, expected, reason) \
+if ((long)(actual) != (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY */
+
+#if 0 /* UNUSED */
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY_NOT()
+ * See: JSVERIFY_NOT documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_NOT(actual, expected, reason) \
+if ((long)(actual) == (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY_NOT */
+#endif /* JSVERIFY_NOT unused */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY_STR()
+ * See: JSVERIFY_STR documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_STR(actual, expected, reason) \
+if (strcmp((actual), (expected)) != 0) { \
+ JSERR_STR((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY_STR */
+
+#endif /* ifdef/else JSVERIFY_EXP_ACT */
+
+
+#define S3_TEST_PROFILE_NAME "ros3_vfd_test"
+
+#define S3_TEST_RESOURCE_TEXT_RESTRICTED "t8.shakespeare.txt"
+#define S3_TEST_RESOURCE_TEXT_PUBLIC "Poe_Raven.txt"
+#define S3_TEST_RESOURCE_MISSING "missing.csv"
+
+#define S3_TEST_RUN_TIMEOUT 0 /* run tests that might hang */
+#define S3_TEST_MAX_URL_SIZE 256 /* char array size */
+
+/* Global variables for aws test profile.
+ * An attempt is made to read ~/.aws/credentials and ~/.aws/config upon test
+ * startup -- if unable to open either file or cannot load region, id, and key,
+ * tests connecting with S3 will not be run
+ */
+static int s3_test_credentials_loaded = 0;
+static char s3_test_aws_region[16] = "";
+static char s3_test_aws_access_key_id[64] = "";
+static char s3_test_aws_secret_access_key[128] = "";
+static char s3_test_bucket_url[S3_TEST_MAX_URL_SIZE] = "";
+static hbool_t s3_test_bucket_defined = FALSE;
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_macro_format_credential()
+ *
+ * Purpose:
+ *
+ * Demonstrate that the macro `S3COMMS_FORMAT_CREDENTIAL`
+ * performs as expected.
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-19
+ *
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+test_macro_format_credential(void)
+{
+ /************************
+ * test-local variables *
+ ************************/
+
+ char dest[256];
+ const char access[] = "AKIAIOSFODNN7EXAMPLE";
+ const char date[] = "20130524";
+ const char region[] = "us-east-1";
+ const char service[] = "s3";
+ const char expected[] =
+ "AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request";
+
+ TESTING("test_macro_format_credential");
+
+ FAIL_IF( S3COMMS_MAX_CREDENTIAL_SIZE <
+ S3COMMS_FORMAT_CREDENTIAL(dest, access, date, region, service) )
+
+ JSVERIFY_STR( expected, dest, NULL )
+
+ PASSED();
+ return 0;
+
+error:
+ return -1;
+
+} /* end test_macro_format_credential() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_aws_canonical_request()
+ *
+ * Purpose:
+ *
+ * Demonstrate the construction of a Canoncial Request (and Signed Headers)
+ *
+ * Elided / not yet implemented:
+ * Query strings
+ * request "body"
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-04
+ *
+ *---------------------------------------------------------------------------
+ */
+static herr_t
+test_aws_canonical_request(void)
+{
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct header {
+ const char *name;
+ const char *value;
+ };
+
+ struct testcase {
+ const char *exp_request;
+ const char *exp_headers;
+ const char *verb;
+ const char *resource;
+ unsigned int listsize;
+ struct header list[5];
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ struct testcase cases[] = {
+ { "GET\n/some/path.file\n\nhost:somebucket.someserver.somedomain\nrange:bytes=150-244\n\nhost;range\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "host;range",
+ "GET",
+ "/some/path.file",
+ 2,
+ { {"Range", "bytes=150-244"},
+ {"Host", "somebucket.someserver.somedomain"},
+ },
+ },
+ { "HEAD\n/bucketpath/myfile.dat\n\nhost:place.domain\nx-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nx-amz-date:19411207T150803Z\n\nhost;x-amz-content-sha256;x-amz-date\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "host;x-amz-content-sha256;x-amz-date",
+ "HEAD",
+ "/bucketpath/myfile.dat",
+ 3,
+ { {"x-amz-content-sha256", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"},
+ {"host", "place.domain"},
+ {"x-amz-date", "19411207T150803Z"},
+ }
+ },
+ { "PUT\n/\n\n\n\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ "",
+ "PUT",
+ "/",
+ 0,
+ {{"",""},}, /* unused; satisfies compiler */
+ },
+ }; /* struct testcase cases[] */
+ struct testcase *C = NULL;
+ char cr_dest[512]; /* canonical request */
+ hrb_t *hrb = NULL; /* http request buffer object */
+ unsigned int i = 0; /* looping/indexing */
+ unsigned int j = 0; /* looping/indexing */
+ hrb_node_t *node = NULL; /* http headers list pointer */
+ unsigned int n_cases = 3;
+ char sh_dest[64]; /* signed headers */
+
+ TESTING("test_aws_canonical_request");
+
+ for (i = 0; i < n_cases; i++) {
+ /* pre-test bookkeeping
+ */
+ C = &cases[i];
+ for (j = 0; j < 256; j++) { cr_dest[j] = 0; } /* zero request buffer */
+ for (j = 0; j < 64; j++) { sh_dest[j] = 0; } /* zero headers buffer */
+
+ /* create HTTP request object with given verb, resource/path
+ */
+ hrb = H5FD_s3comms_hrb_init_request(C->verb,
+ C->resource,
+ "HTTP/1.1");
+ HDassert(hrb->body == NULL);
+
+ /* Create headers list from test case input
+ */
+ for (j = 0; j < C->listsize; j++) {
+ FAIL_IF( FAIL ==
+ H5FD_s3comms_hrb_node_set(
+ &node,
+ C->list[j].name,
+ C->list[j].value));
+ }
+
+ hrb->first_header = node;
+
+ /* test
+ */
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_aws_canonical_request(
+ cr_dest,
+ 512,
+ sh_dest,
+ 64,
+ hrb),
+ " unable to compose canonical request" )
+ JSVERIFY_STR( C->exp_headers, sh_dest, NULL )
+ JSVERIFY_STR( C->exp_request, cr_dest, NULL )
+
+ /* tear-down
+ */
+ while (node != NULL) {
+ FAIL_IF( FAIL ==
+ H5FD_s3comms_hrb_node_set(&node, node->name, NULL));
+ }
+ HDassert(NULL == node);
+ FAIL_IF( FAIL == H5FD_s3comms_hrb_destroy(&hrb));
+ HDassert(NULL == hrb);
+
+ } /* for each test case */
+
+ /***************
+ * ERROR CASES *
+ ***************/
+
+ /* malformed hrb and/or node-list
+ */
+ JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request(
+ cr_dest,
+ 20,
+ sh_dest,
+ 20,
+ NULL),
+ "http request object cannot be null" )
+
+ hrb = H5FD_s3comms_hrb_init_request("GET", "/", "HTTP/1.1");
+ JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request(
+ NULL,
+ 20,
+ sh_dest,
+ 20,
+ hrb),
+ "canonical request destination cannot be NULL" )
+
+ JSVERIFY( FAIL, H5FD_s3comms_aws_canonical_request(
+ cr_dest,
+ 20,
+ NULL,
+ 20,
+ hrb),
+ "signed headers destination cannot be null" )
+
+ FAIL_IF( FAIL == H5FD_s3comms_hrb_destroy(&hrb) )
+ HDassert( NULL == hrb );
+
+ PASSED();
+ return 0;
+
+error:
+
+ if (node != NULL) {
+ while (node != NULL)
+ (void)H5FD_s3comms_hrb_node_set(&node, node->name, NULL);
+ HDassert( node == NULL );
+ }
+ if (hrb != NULL) {
+ (void)H5FD_s3comms_hrb_destroy(&hrb);
+ }
+
+ return -1;
+
+} /* end test_aws_canonical_request() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_bytes_to_hex
+ *
+ * Purpose:
+ *
+ * Define and verify behavior of `H5FD_s3comms_bytes_to_hex()`.
+ *
+ * Return:
+ *
+ * Success: 0
+ * Failure: -1
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-14
+ *
+ *---------------------------------------------------------------------------
+ */
+static herr_t
+test_bytes_to_hex(void)
+{
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct testcase {
+ const char exp[17]; /* in size * 2 + 1 for null terminator */
+ const unsigned char in[8];
+ size_t size;
+ hbool_t lower;
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ struct testcase cases[] = {
+ { "52F3000C9A",
+ {82,243,0,12,154},
+ 5,
+ FALSE,
+ },
+ { "009a0cf3005200", /* lowercase alphas */
+ {0,154,12,243,0,82,0},
+ 7,
+ TRUE,
+ },
+ { "",
+ {17,63,26,56},
+ 0,
+ FALSE, /* irrelevant */
+ },
+ };
+ int i = 0;
+ int n_cases = 3;
+ char out[17];
+ int out_off = 0;
+
+
+
+ TESTING("bytes-to-hex");
+
+ for (i = 0; i < n_cases; i++) {
+ for (out_off = 0; out_off < 17; out_off++) {
+ out[out_off] = 0;
+ }
+
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_bytes_to_hex(out,
+ cases[i].in,
+ cases[i].size,
+ cases[i].lower),
+ NULL )
+
+ JSVERIFY_STR(cases[i].exp, out, NULL)
+ }
+
+ /* dest cannot be null
+ */
+ JSVERIFY( FAIL,
+ H5FD_s3comms_bytes_to_hex(
+ NULL,
+ (const unsigned char *)"nada",
+ 5,
+ FALSE),
+ "destination cannot be null" )
+
+ PASSED();
+ return 0;
+
+error:
+ return -1;
+
+} /* end test_bytes_to_hex() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_hrb_init_request()
+ *
+ * Purpose:
+ *
+ * Define and verify behavior of `H5FD_s3comms_hrb_init_request()`
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-20
+ *
+ *---------------------------------------------------------------------------
+ */
+static herr_t
+test_hrb_init_request(void)
+{
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct testcase {
+ const char msg[64];
+ const char *verb;
+ const char *resource;
+ const char *exp_res;
+ const char *version;
+ hbool_t ret_null;
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ struct testcase cases[] = {
+ { "get HTTP request just as we provided",
+ "GET",
+ "/path/to/some/file",
+ "/path/to/some/file",
+ "HTTP/1.1",
+ FALSE,
+ },
+ { "null verb substitues to GET",
+ NULL,
+ "/MYPATH/MYFILE.tiff",
+ "/MYPATH/MYFILE.tiff",
+ "HTTP/1.1",
+ FALSE,
+ },
+ { "demonstrate non-GET verb",
+ "HEAD",
+ "/MYPATH/MYFILE.tiff",
+ "/MYPATH/MYFILE.tiff",
+ "HTTP/1.1",
+ FALSE,
+ },
+ { "slash prepended to resource path, if necessary",
+ NULL,
+ "MYPATH/MYFILE.tiff",
+ "/MYPATH/MYFILE.tiff",
+ NULL,
+ FALSE,
+ },
+ { "null resource path causes problem",
+ "GET",
+ NULL,
+ NULL,
+ NULL,
+ TRUE,
+ },
+ };
+ struct testcase *C = NULL;
+ unsigned int i = 0;
+ unsigned int ncases = 5;
+ hrb_t *req = NULL;
+
+ TESTING("hrb_init_request");
+
+ for (i = 0; i < ncases; i++) {
+ C = &cases[i];
+ req = H5FD_s3comms_hrb_init_request(
+ C->verb,
+ C->resource,
+ C->version);
+ if (cases[i].ret_null == TRUE) {
+ FAIL_IF( req != NULL );
+ }
+ else {
+ FAIL_IF( req == NULL );
+ JSVERIFY( S3COMMS_HRB_MAGIC, req->magic, NULL )
+ if (C->verb == NULL) {
+ JSVERIFY_STR( "GET", req->verb, NULL )
+ }
+ else {
+ JSVERIFY_STR( req->verb, C->verb, NULL )
+ }
+ JSVERIFY_STR( "HTTP/1.1", req->version, NULL )
+ JSVERIFY_STR( C->exp_res, req->resource, NULL )
+ FAIL_IF( req->first_header != NULL );
+ FAIL_IF( req->body != NULL );
+ JSVERIFY( 0, req->body_len, NULL )
+ JSVERIFY( SUCCEED, H5FD_s3comms_hrb_destroy(&req),
+ "unable to destroy hrb_t" )
+ FAIL_IF( NULL != req ); /* should annull pointer as well as free */
+ }
+
+ } /* end for each testcase */
+
+ PASSED();
+ return 0;
+
+error:
+ (void)H5FD_s3comms_hrb_destroy(&req);
+
+ return -1;
+
+} /* end test_hrb_init_request() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_hrb_node_set()
+ *
+ * Purpose:
+ *
+ * Test operations on hrb_node_t structure
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-22
+ *
+ *---------------------------------------------------------------------------
+ */
+static herr_t
+test_hrb_node_set(void)
+{
+ /*************************
+ * test-local structures *
+ *************************/
+
+ /* bundle of name/value representing an hrb_node_t
+ */
+ typedef struct node_mock_t {
+ const char *name;
+ const char *value;
+ } node_mock_t;
+
+ /* bundle for a testcase
+ *
+ * `message`
+ * purpose of the testcase
+ *
+ * `delta`
+ * container for name and value strings to pass into node-set function
+ * to to modify the list.
+ *
+ * `returned`
+ * expected return value of node-set function
+ *
+ * `given`
+ * `expected`
+ * string arrays representing the state of the list before and after
+ * modification. The number of strings must be even, with each name
+ * paired to a value. `NULL` terminates the list, with `{NULL}`
+ * representing the empty list.
+ */
+ typedef struct testcase {
+ const char *message;
+ node_mock_t delta;
+ herr_t returned;
+ const char *given[11]; /* name/value pairs in array; NULL sentinel */
+ const char *expected[11];
+ } testcase;
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ testcase cases[] = {
+ { "cannot remove node from null list",
+ { "Host", NULL },
+ FAIL,
+ {NULL},
+ {NULL},
+ },
+ { "cannot create list with NULL field name",
+ { NULL, "somevalue" },
+ FAIL,
+ {NULL},
+ {NULL},
+ },
+ { "create a new list",
+ { "Host", "somevalue" },
+ SUCCEED,
+ {NULL},
+ { "Host", "somevalue",
+ NULL,
+ },
+ },
+ { "insert new node at head list",
+ { "Host", "somevalue" },
+ SUCCEED,
+ { "Range", "bytes=20-40",
+ NULL,
+ },
+ { "Host", "somevalue",
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ },
+ { "append new node at list end",
+ { "x-amz-date", "somevalue" },
+ SUCCEED,
+ { "Range", "bytes=20-40",
+ NULL,
+ },
+ { "Range", "bytes=20-40",
+ "x-amz-date", "somevalue",
+ NULL,
+ },
+ },
+ { "insert new node inside list",
+ { "Intermediary", "somevalue" },
+ SUCCEED,
+ { "Host", "somehost" ,
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ { "Host", "somehost",
+ "Intermediary", "somevalue",
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ },
+ { "modify node",
+ { "Range", "bytes=40-80" },
+ SUCCEED,
+ { "Host", "somehost",
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ { "Host", "somehost",
+ "Range", "bytes=40-80",
+ NULL,
+ },
+ },
+ { "modify node with new case",
+ { "RANGE", "bytes=40-80" },
+ SUCCEED,
+ { "Host", "somehost",
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ { "Host", "somehost",
+ "RANGE", "bytes=40-80",
+ NULL,
+ },
+ },
+ { "cannot add node with no name",
+ { NULL, "bytes=40-80" },
+ FAIL,
+ { "Host", "somehost",
+ NULL,
+ },
+ { "Host", "somehost",
+ NULL,
+ },
+ },
+ { "add node with 'empty' name",
+ { "", "bytes=40-80" },
+ SUCCEED,
+ { "Host", "somehost",
+ NULL,
+ },
+ { "", "bytes=40-80",
+ "Host", "somehost",
+ NULL,
+ },
+ },
+ { "remove node from end of list",
+ { "Host", NULL },
+ SUCCEED,
+ { "Date", "Thr, 25 Jan 2018",
+ "Host", "somehost",
+ NULL,
+ },
+ { "Date", "Thr, 25 Jan 2018",
+ NULL,
+ },
+ },
+ { "remove node from middle of list",
+ { "Host", NULL },
+ SUCCEED,
+ { "Date", "Thr, 25 Jan 2018",
+ "Host", "somehost",
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ { "Date", "Thr, 25 Jan 2018",
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ },
+ { "remove node from start of list",
+ { "Date", NULL },
+ SUCCEED,
+ { "Date", "Thr, 25 Jan 2018",
+ "Host", "somehost",
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ { "Host", "somehost",
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ },
+ { "remove only node in list",
+ { "Date", NULL },
+ SUCCEED,
+ { "Date", "Thr, 25 Jan 2018",
+ NULL,
+ },
+ { NULL,
+ },
+ },
+ { "attempt to remove absent node fails",
+ { "Host", NULL },
+ FAIL,
+ { "Date", "Thr, 25 Jan 2018",
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ { "Date", "Thr, 25 Jan 2018",
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ },
+ { "removal is case-insensitive",
+ { "hOsT", NULL },
+ SUCCEED,
+ { "Date", "Thr, 25 Jan 2018",
+ "Host", "somehost",
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ { "Date", "Thr, 25 Jan 2018",
+ "Range", "bytes=20-40",
+ NULL,
+ },
+ },
+ };
+ unsigned testcases_count = 16;
+ unsigned test_i = 0;
+ hrb_node_t *list = NULL;
+
+ TESTING("hrb_node_t (test_hrb_node_set)");
+
+ for (test_i = 0; test_i < testcases_count; test_i++) {
+ const hrb_node_t *node = NULL;
+ const testcase *test = &(cases[test_i]);
+ unsigned mock_i = 0;
+
+ /*********
+ * SETUP *
+ *********/
+
+ for (mock_i = 0; test->given[mock_i] != NULL; mock_i += 2) {
+ const char *name = test->given[mock_i];
+ const char *valu = test->given[mock_i+1];
+
+ FAIL_IF( SUCCEED !=
+ H5FD_s3comms_hrb_node_set(&list, name, valu) )
+ }
+ /********
+ * TEST *
+ ********/
+
+ /* perform modification on list
+ */
+ JSVERIFY( test->returned,
+ H5FD_s3comms_hrb_node_set(&list,
+ test->delta.name,
+ test->delta.value),
+ test->message )
+
+
+ /* verify resulting list
+ */
+ node = list;
+ mock_i = 0;
+ while (test->expected[mock_i] != NULL && node != NULL) {
+ const char *name = test->expected[mock_i];
+ const char *valu = test->expected[mock_i+1];
+
+ JSVERIFY_STR( name, node->name, NULL )
+ JSVERIFY_STR( valu, node->value, NULL )
+
+ mock_i += 2;
+ node = node->next;
+ }
+ FAIL_IF( test->expected[mock_i] != NULL )
+ FAIL_IF( node != NULL )
+
+ /************
+ * TEARDOWN *
+ ************/
+
+ while (list != NULL) {
+ FAIL_IF( SUCCEED !=
+ H5FD_s3comms_hrb_node_set(&list, list->name, NULL) )
+ }
+ } /* end for each testcase */
+
+ PASSED();
+ return 0;
+
+error:
+ while (list != NULL) {
+ (void)H5FD_s3comms_hrb_node_set(&list, list->name, NULL);
+ }
+
+ return -1;
+
+} /* end test_hrb_node_t() */
+
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_HMAC_SHA256()
+ *
+ * Purpose:
+ *
+ * Define and verify behavior of `H5FD_s3comms_HMAC_SHA256()`
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-19
+ *
+ *---------------------------------------------------------------------------
+ */
+static herr_t
+test_HMAC_SHA256(void)
+{
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct testcase {
+ herr_t ret; /* SUCCEED/FAIL expected from call */
+ const unsigned char key[SHA256_DIGEST_LENGTH];
+ size_t key_len;
+ const char *msg;
+ size_t msg_len;
+ const char *exp; /* not used if ret == FAIL */
+ size_t dest_size; /* if 0, `dest` is not malloc'd */
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ struct testcase cases[] = {
+ { SUCCEED,
+ { 0xdb, 0xb8, 0x93, 0xac, 0xc0, 0x10, 0x96, 0x49,
+ 0x18, 0xf1, 0xfd, 0x43, 0x3a, 0xdd, 0x87, 0xc7,
+ 0x0e, 0x8b, 0x0d, 0xb6, 0xbe, 0x30, 0xc1, 0xfb,
+ 0xea, 0xfe, 0xfa, 0x5e, 0xc6, 0xba, 0x83, 0x78,
+ },
+ SHA256_DIGEST_LENGTH,
+ "AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/aws4_request\n7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972",
+ HDstrlen("AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/aws4_request\n7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972"),
+ "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41",
+ SHA256_DIGEST_LENGTH * 2 + 1, /* +1 for null terminator */
+ },
+ { SUCCEED,
+ {'J','e','f','e'},
+ 4,
+ "what do ya want for nothing?",
+ 28,
+ "5bdcc146bf60754e6a042426089575c75a003f089d2739839dec58b964ec3843",
+ SHA256_DIGEST_LENGTH * 2 + 1,
+ },
+ { FAIL,
+ "DOESN'T MATTER",
+ 14,
+ "ALSO IRRELEVANT",
+ 15,
+ NULL,
+ 0, /* dest -> null, resulting in immediate error */
+ },
+ };
+ char *dest = NULL;
+ int i = 0;
+ int n_cases = 3;
+
+ TESTING("HMAC_SHA256");
+
+ for (i = 0; i < n_cases; i++) {
+ if (cases[i].dest_size == 0) {
+ dest = NULL;
+ } else {
+ dest = (char *)HDmalloc(sizeof(char) * cases[i].dest_size);
+ HDassert(dest != NULL);
+ }
+
+ JSVERIFY( cases[i].ret,
+ H5FD_s3comms_HMAC_SHA256(
+ cases[i].key,
+ cases[i].key_len,
+ cases[i].msg,
+ cases[i].msg_len,
+ dest),
+ cases[i].msg );
+ if (cases[i].ret == SUCCEED) {
+#ifdef VERBOSE
+ if (0 !=
+ strncmp(cases[i].exp,
+ dest,
+ HDstrlen(cases[i].exp)))
+ {
+ /* print out how wrong things are, and then fail
+ */
+ dest = (char *)realloc(dest, cases[i].dest_size + 1);
+ HDassert(dest != NULL);
+ dest[cases[i].dest_size] = 0;
+ HDfprintf(stdout,
+ "ERROR:\n!!! \"%s\"\n != \"%s\"\n",
+ cases[i].exp,
+ dest);
+ TEST_ERROR;
+ }
+#else /* VERBOSE not defined */
+ /* simple pass/fail test
+ */
+ JSVERIFY( 0,
+ strncmp(cases[i].exp, dest, HDstrlen(cases[i].exp)),
+ NULL);
+#endif /* VERBOSE */
+ }
+ free(dest);
+ }
+
+ PASSED();
+ return 0;
+
+error:
+ free(dest);
+ return -1;
+
+} /* end test_HMAC_SHA256() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: test_nlowercase()
+ *
+ * Purpose:
+ *
+ * Define and verify behavior of `H5FD_s3comms_nlowercase()`
+ *
+ * Programmer: Jacob Smith
+ * 2017-19-18
+ *
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+test_nlowercase(void)
+{
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct testcase {
+ const char *in;
+ size_t len;
+ const char *exp;
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ /* any character after in exp on or after exp[len] is undefined.
+ * in this test, kept as the null character for simplicity.
+ */
+ struct testcase cases[] = {
+ { "HALlEluJAh",
+ 6,
+ "hallel",
+ },
+ { "all\0 lower",
+ 10,
+ "all\0 lower",
+ },
+ { "to meeeeeee",
+ 0,
+ "",
+ },
+ };
+ char *dest = NULL;
+ int i = 0;
+ int n_cases = 3;
+
+ TESTING("nlowercase");
+
+ for (i = 0; i < n_cases; i++) {
+ dest = (char *)HDmalloc(sizeof(char) * 16);
+
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_nlowercase(dest,
+ cases[i].in,
+ cases[i].len),
+ cases[i].in )
+ if (cases[i].len > 0) {
+ JSVERIFY( 0, strncmp(dest, cases[i].exp, cases[i].len), NULL )
+ }
+ free(dest);
+ } /* end for each testcase */
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_nlowercase(NULL,
+ cases[0].in,
+ cases[0].len),
+ "null distination should fail" )
+
+ PASSED();
+ return 0;
+
+error:
+ free(dest);
+ return -1;
+
+} /* end test_nlowercase() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_parse_url()
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-??
+ *
+ *---------------------------------------------------------------------------
+ */
+static herr_t
+test_parse_url(void)
+{
+ /*********************
+ * test-local macros *
+ *********************/
+
+ /*************************
+ * test-local structures *
+ *************************/
+
+ typedef struct {
+ const char *scheme;
+ const char *host;
+ const char *port;
+ const char *path;
+ const char *query;
+ } const_purl_t;
+
+ struct testcase {
+ const char *url;
+ herr_t exp_ret; /* expected return; */
+ /* if FAIL, `expected` is unused */
+ const_purl_t expected;
+ const char *msg;
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ parsed_url_t *purl = NULL;
+ unsigned int i = 0;
+ unsigned int ncases = 15;
+ struct testcase cases[] = {
+ { NULL,
+ FAIL,
+ { NULL, NULL, NULL, NULL, NULL },
+ "null url",
+ },
+ { "",
+ FAIL,
+ { NULL, NULL, NULL, NULL, NULL },
+ "empty url",
+ },
+ { "ftp://[1000:4000:0002:2010]",
+ SUCCEED,
+ { "ftp",
+ "[1000:4000:0002:2010]",
+ NULL,
+ NULL,
+ NULL,
+ },
+ "IPv6 ftp and empty path (root)",
+ },
+ { "ftp://[1000:4000:0002:2010]:2040",
+ SUCCEED,
+ { "ftp",
+ "[1000:4000:0002:2010]",
+ "2040",
+ NULL,
+ NULL,
+ },
+ "root IPv6 ftp with port",
+ },
+ { "http://some.domain.org:9000/path/to/resource.txt",
+ SUCCEED,
+ { "http",
+ "some.domain.org",
+ "9000",
+ "path/to/resource.txt",
+ NULL,
+ },
+ "without query",
+ },
+ { "https://domain.me:00/file.txt?some_params unchecked",
+ SUCCEED,
+ { "https",
+ "domain.me",
+ "00",
+ "file.txt",
+ "some_params unchecked",
+ },
+ "with query",
+ },
+ { "ftp://domain.com/",
+ SUCCEED,
+ { "ftp",
+ "domain.com",
+ NULL,
+ NULL,
+ NULL,
+ },
+ "explicit root w/out port",
+ },
+ { "ftp://domain.com:1234/",
+ SUCCEED,
+ { "ftp",
+ "domain.com",
+ "1234",
+ NULL,
+ NULL,
+ },
+ "explicit root with port",
+ },
+ { "ftp://domain.com:1234/file?",
+ FAIL,
+ { NULL, NULL, NULL, NULL, NULL, },
+ "empty query is invalid",
+ },
+ { "ftp://:1234/file",
+ FAIL,
+ { NULL, NULL, NULL, NULL, NULL, },
+ "no host",
+ },
+ { "h&r block",
+ FAIL,
+ { NULL, NULL, NULL, NULL, NULL, },
+ "no scheme (bad URL)",
+ },
+ { "http://domain.com?a=b&d=b",
+ SUCCEED,
+ { "http",
+ "domain.com",
+ NULL,
+ NULL,
+ "a=b&d=b",
+ },
+ "QUERY with implict PATH",
+ },
+ { "http://[5]/path?a=b&d=b",
+ SUCCEED,
+ { "http",
+ "[5]",
+ NULL,
+ "path",
+ "a=b&d=b",
+ },
+ "IPv6 extraction is really dumb",
+ },
+ { "http://[1234:5678:0910:1112]:port/path",
+ FAIL,
+ { NULL, NULL, NULL, NULL, NULL, },
+ "non-decimal PORT (port)",
+ },
+ { "http://mydomain.com:01a3/path",
+ FAIL,
+ { NULL, NULL, NULL, NULL, NULL, },
+ "non-decimal PORT (01a3)",
+ },
+ };
+
+ TESTING("url-parsing functionality");
+
+ /*********
+ * TESTS *
+ *********/
+
+ for (i = 0; i < ncases; i++) {
+ HDassert( purl == NULL );
+
+ JSVERIFY( cases[i].exp_ret,
+ H5FD_s3comms_parse_url(cases[i].url, &purl),
+ cases[i].msg )
+
+ if (cases[i].exp_ret == FAIL) {
+ /* on FAIL, `purl` should be untouched--remains NULL */
+ FAIL_UNLESS( purl == NULL )
+ }
+ else {
+ /* on SUCCEED, `purl` should be set */
+ FAIL_IF( purl == NULL )
+
+ if (cases[i].expected.scheme != NULL) {
+ FAIL_IF( NULL == purl->scheme )
+ JSVERIFY_STR( cases[i].expected.scheme,
+ purl->scheme,
+ cases[i].msg )
+ } else {
+ FAIL_UNLESS( NULL == purl->scheme )
+ }
+
+ if (cases[i].expected.host != NULL) {
+ FAIL_IF( NULL == purl->host )
+ JSVERIFY_STR( cases[i].expected.host,
+ purl->host,
+ cases[i].msg )
+ } else {
+ FAIL_UNLESS( NULL == purl->host )
+ }
+
+ if (cases[i].expected.port != NULL) {
+ FAIL_IF( NULL == purl->port )
+ JSVERIFY_STR( cases[i].expected.port,
+ purl->port,
+ cases[i].msg )
+ } else {
+ FAIL_UNLESS( NULL == purl->port )
+ }
+
+ if (cases[i].expected.path != NULL) {
+ FAIL_IF( NULL == purl->path )
+ JSVERIFY_STR( cases[i].expected.path,
+ purl->path,
+ cases[i].msg )
+ } else {
+ FAIL_UNLESS( NULL == purl->path )
+ }
+
+ if (cases[i].expected.query != NULL) {
+ FAIL_IF( NULL == purl->query )
+ JSVERIFY_STR( cases[i].expected.query,
+ purl->query,
+ cases[i].msg )
+ } else {
+ FAIL_UNLESS( NULL == purl->query )
+ }
+ } /* end if parse-url return SUCCEED/FAIL */
+
+ /* per-test cleanup
+ * well-behaved, even if `purl` is NULL
+ */
+ FAIL_IF( FAIL == H5FD_s3comms_free_purl(purl) )
+ purl = NULL;
+
+ } /* end for each testcase */
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * cleanup *
+ ***********/
+ (void)H5FD_s3comms_free_purl(purl);
+
+ return -1;
+
+} /* end test_parse_url() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_percent_encode_char()
+ *
+ * Purpose:
+ *
+ * Define and verify behavior of `H5FD_s3comms_percent_encode_char()`
+ *
+ * Return:
+ *
+ * Success: 0
+ * Failure: -1
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-14
+ *
+ *---------------------------------------------------------------------------
+ */
+static herr_t
+test_percent_encode_char(void)
+{
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct testcase {
+ const char c;
+ const char *exp;
+ size_t exp_len;
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ struct testcase cases[] = {
+ {'$', "%24", 3}, /* u+0024 dollar sign */
+ {' ', "%20", 3}, /* u+0020 space */
+ {'^', "%5E", 3}, /* u+0094 carat */
+ {'/', "%2F", 3}, /* u+002f solidus (forward slash) */
+ /* {??, "%C5%8C", 6},*/ /* u+014c Latin Capital Letter O with Macron */
+ /* Not included because it is multibyte "wide" character that poses */
+ /* issues both in the underlying function and in being written in */
+ /* this file. */
+ /* {'¢', "%C2%A2", 6}, */ /* u+00a2 cent sign */
+ /* above works, but complains about wide character overflow */
+ /* Elide for now, until it is determined (a) unnecessary or */
+ /* (b) requiring signature change to accommodate wide characters */
+ {'\0', "%00", 3}, /* u+0000 null */
+ };
+ char dest[13];
+ size_t dest_len = 0;
+ int i = 0;
+ int n_cases = 5;
+
+ TESTING("percent encode characters");
+
+ for (i = 0; i < n_cases; i++) {
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_percent_encode_char(
+ dest,
+ (const unsigned char)cases[i].c,
+ &dest_len),
+ NULL )
+ JSVERIFY(cases[i].exp_len, dest_len, NULL )
+ JSVERIFY(0, strncmp(dest, cases[i].exp, dest_len), NULL )
+ JSVERIFY_STR( cases[i].exp, dest, NULL )
+ }
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_percent_encode_char(
+ NULL,
+ (const unsigned char)'^',
+ &dest_len),
+ NULL )
+
+ PASSED();
+ return 0;
+
+error:
+ return -1;
+} /* end test_percent_encode_char() */
+
+
+/*---------------------------------------------------------------------------
+ * Function: test_s3r_open()
+ *
+ * Programmer: Jacob Smith 2018-01-24
+ *
+ * Changes: None
+ *
+ *---------------------------------------------------------------------------
+ */
+static herr_t
+test_s3r_get_filesize(void)
+{
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ char url_raven[S3_TEST_MAX_URL_SIZE];
+ s3r_t *handle = NULL;
+
+ TESTING("s3r_get_filesize");
+
+ /* setup -- compose url to target resource
+ */
+ if (FALSE == s3_test_bucket_defined) {
+ SKIPPED();
+ puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
+ fflush(stdout);
+ return 0;
+ }
+
+ FAIL_IF( S3_TEST_MAX_URL_SIZE <
+ HDsnprintf(url_raven,
+ S3_TEST_MAX_URL_SIZE,
+ "%s/%s",
+ s3_test_bucket_url,
+ S3_TEST_RESOURCE_TEXT_PUBLIC) );
+
+ JSVERIFY( 0, H5FD_s3comms_s3r_get_filesize(NULL),
+ "filesize of the null handle should be 0" )
+
+ handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL);
+ FAIL_IF( handle == NULL )
+
+ JSVERIFY( 6464, H5FD_s3comms_s3r_get_filesize(handle), NULL )
+
+
+ FAIL_IF( SUCCEED != H5FD_s3comms_s3r_close(handle) )
+
+ PASSED();
+ return 0;
+
+error:
+ if (handle != NULL)
+ (void)H5FD_s3comms_s3r_close(handle);
+
+ return -1;
+
+} /* end test_s3r_get_filesize() */
+
+
+/*---------------------------------------------------------------------------
+ * Function: test_s3r_open()
+ *
+ * Programmer: Jacob Smith 2018-01-??
+ *
+ * Changes: None
+ *
+ *---------------------------------------------------------------------------
+ */
+static herr_t
+test_s3r_open(void)
+{
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ char url_missing[S3_TEST_MAX_URL_SIZE];
+ char url_raven[S3_TEST_MAX_URL_SIZE];
+ char url_raven_badport[S3_TEST_MAX_URL_SIZE];
+ char url_shakespeare[S3_TEST_MAX_URL_SIZE];
+ unsigned char signing_key[SHA256_DIGEST_LENGTH];
+ struct tm *now = NULL;
+ char iso8601now[ISO8601_SIZE];
+ s3r_t *handle = NULL;
+ hbool_t curl_ready = FALSE;
+ parsed_url_t *purl = NULL;
+
+ TESTING("s3r_open");
+
+ if (s3_test_credentials_loaded == 0) {
+ SKIPPED();
+ puts(" s3 credentials are not loaded");
+ fflush(stdout);
+ return 0;
+ }
+ if (FALSE == s3_test_bucket_defined) {
+ SKIPPED();
+ puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
+ fflush(stdout);
+ return 0;
+ }
+
+ /******************
+ * PRE-TEST SETUP *
+ ******************/
+
+ FAIL_IF( S3_TEST_MAX_URL_SIZE <
+ HDsnprintf(url_shakespeare,
+ S3_TEST_MAX_URL_SIZE,
+ "%s/%s",
+ s3_test_bucket_url,
+ S3_TEST_RESOURCE_TEXT_RESTRICTED) );
+
+ FAIL_IF( S3_TEST_MAX_URL_SIZE <
+ HDsnprintf(url_missing,
+ S3_TEST_MAX_URL_SIZE,
+ "%s/%s",
+ s3_test_bucket_url,
+ S3_TEST_RESOURCE_MISSING) );
+
+ FAIL_IF( S3_TEST_MAX_URL_SIZE <
+ HDsnprintf(url_raven,
+ S3_TEST_MAX_URL_SIZE,
+ "%s/%s",
+ s3_test_bucket_url,
+ S3_TEST_RESOURCE_TEXT_PUBLIC) );
+
+ /* Set given bucket url with invalid/inactive port number for badport.
+ * Note, this sort of micro-management of parsed_url_t is not advised
+ */
+ FAIL_IF( FAIL == H5FD_s3comms_parse_url(s3_test_bucket_url, &purl) )
+ if (purl->port == NULL) {
+ purl->port = (char *)H5MM_malloc(sizeof(char) * 5);
+ FAIL_IF( purl->port == NULL );
+ FAIL_IF( 5 < HDsnprintf(purl->port, 5, "9000") )
+ } else if (strcmp(purl->port, "9000") != 0) {
+ FAIL_IF( 5 < HDsnprintf(purl->port, 5, "9000") )
+ } else {
+ FAIL_IF( 5 < HDsnprintf(purl->port, 5, "1234") )
+ }
+ FAIL_IF( S3_TEST_MAX_URL_SIZE <
+ HDsnprintf(url_raven_badport,
+ S3_TEST_MAX_URL_SIZE,
+ "%s://%s:%s/%s",
+ purl->scheme,
+ purl->host,
+ purl->port,
+ S3_TEST_RESOURCE_TEXT_PUBLIC) );
+
+ curl_global_init(CURL_GLOBAL_DEFAULT);
+ curl_ready = TRUE;
+
+ now = gmnow();
+ FAIL_IF( now == NULL )
+ FAIL_IF( ISO8601NOW(iso8601now, now) != (ISO8601_SIZE - 1) );
+
+ /* It is desired to have means available to verify that signing_key
+ * was set successfully and to an expected value.
+ */
+ FAIL_IF( FAIL ==
+ H5FD_s3comms_signing_key(
+ signing_key,
+ (const char *)s3_test_aws_secret_access_key,
+ (const char *)s3_test_aws_region,
+ (const char *)iso8601now) );
+
+ /*************************
+ * OPEN NONEXISTENT FILE *
+ *************************/
+
+ /* attempt anonymously
+ */
+ handle = H5FD_s3comms_s3r_open(url_missing, NULL, NULL, NULL);
+ FAIL_IF( handle != NULL );
+
+ /* attempt with authentication
+ */
+ handle = H5FD_s3comms_s3r_open(
+ url_missing,
+ (const char *)s3_test_aws_region,
+ (const char *)s3_test_aws_access_key_id,
+ (const unsigned char *)signing_key);
+ FAIL_IF( handle != NULL );
+
+ /*************************
+ * INACTIVE PORT ON HOST *
+ *************************/
+
+#if S3_TEST_RUN_TIMEOUT
+ HDprintf("Opening on inactive port may hang for a minute; waiting for timeout\n");
+ handle = H5FD_s3comms_s3r_open(url_raven_badport, NULL, NULL, NULL);
+ FAIL_IF( handle != NULL );
+#endif
+
+ /*******************************
+ * INVALID AUTHENTICATION INFO *
+ *******************************/
+
+ /* anonymous access on restricted file
+ */
+ handle = H5FD_s3comms_s3r_open(url_shakespeare, NULL, NULL, NULL);
+ FAIL_IF( handle != NULL );
+
+ /* passed in a bad ID
+ */
+ handle = H5FD_s3comms_s3r_open(
+ url_shakespeare,
+ (const char *)s3_test_aws_region,
+ "I_MADE_UP_MY_ID",
+ (const unsigned char *)signing_key);
+ FAIL_IF( handle != NULL );
+
+ /* using an invalid signing key
+ */
+ handle = H5FD_s3comms_s3r_open(
+ url_shakespeare,
+ (const char *)s3_test_aws_region,
+ (const char *)s3_test_aws_access_key_id,
+ (const unsigned char *)EMPTY_SHA256);
+ FAIL_IF( handle != NULL );
+
+ /*******************************
+ * SUCCESSFUL OPEN (AND CLOSE) *
+ *******************************/
+
+ /* anonymous
+ */
+ handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL);
+ FAIL_IF( handle == NULL );
+ JSVERIFY( 6464, H5FD_s3comms_s3r_get_filesize(handle),
+ "did not get expected filesize" )
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_s3r_close(handle),
+ "unable to close file" )
+ handle = NULL;
+
+ /* using authentication on anonymously-accessible file?
+ */
+ handle = H5FD_s3comms_s3r_open(
+ url_raven,
+ (const char *)s3_test_aws_region,
+ (const char *)s3_test_aws_access_key_id,
+ (const unsigned char *)signing_key);
+ FAIL_IF( handle == NULL );
+ JSVERIFY( 6464, H5FD_s3comms_s3r_get_filesize(handle), NULL )
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_s3r_close(handle),
+ "unable to close file" )
+ handle = NULL;
+
+ /* authenticating
+ */
+ handle = H5FD_s3comms_s3r_open(
+ url_shakespeare,
+ (const char *)s3_test_aws_region,
+ (const char *)s3_test_aws_access_key_id,
+ (const unsigned char *)signing_key);
+ FAIL_IF( handle == NULL );
+ JSVERIFY( 5458199, H5FD_s3comms_s3r_get_filesize(handle), NULL )
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_s3r_close(handle),
+ "unable to close file" )
+ handle = NULL;
+
+
+
+ curl_global_cleanup();
+ curl_ready = FALSE;
+
+ FAIL_IF( FAIL == H5FD_s3comms_free_purl(purl) )
+ purl = NULL;
+
+ PASSED();
+ return 0;
+error:
+ /***********
+ * cleanup *
+ ***********/
+
+ if (handle != NULL)
+ H5FD_s3comms_s3r_close(handle);
+ if (purl != NULL)
+ H5FD_s3comms_free_purl(purl);
+ if (curl_ready == TRUE)
+ curl_global_cleanup();
+
+ return -1;
+
+} /* end test_s3r_open() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_s3r_read()
+ *
+ * Purpose:
+ *
+ * Specify and demonstrate the use and life cycle of an S3 Request handle
+ * `s3r_t`, through its related functions.
+ *
+ * H5FD_s3comms_s3r_open
+ * H5FD_s3comms_s3r_getsize << called by open() _only_
+ * H5FD_s3comms_s3r_read << called by getsize(), multiple times working
+ * H5FD_s3comms_s3r_close
+ *
+ * Shows most basic curl interation.
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-06
+ *
+ *---------------------------------------------------------------------------
+ */
+static herr_t
+test_s3r_read(void)
+{
+
+#define S3COMMS_TEST_BUFFER_SIZE 256
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ char url_raven[S3_TEST_MAX_URL_SIZE];
+ char buffer[S3COMMS_TEST_BUFFER_SIZE];
+ s3r_t *handle = NULL;
+ hbool_t curl_ready = FALSE;
+ unsigned int i = 0;
+
+ TESTING("test_s3r_read");
+
+ /*
+ * initial setup
+ */
+ if (FALSE == s3_test_bucket_defined) {
+ SKIPPED();
+ puts(" environment variable HDF5_ROS3_TEST_BUCKET_URL not defined");
+ fflush(stdout);
+ return 0;
+ }
+
+ curl_global_init(CURL_GLOBAL_DEFAULT);
+ curl_ready = TRUE;
+ FAIL_IF( S3_TEST_MAX_URL_SIZE <
+ HDsnprintf(url_raven,
+ S3_TEST_MAX_URL_SIZE,
+ "%s/%s",
+ s3_test_bucket_url,
+ S3_TEST_RESOURCE_TEXT_PUBLIC) );
+
+ for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++)
+ buffer[i] = '\0';
+
+ /* open file
+ */
+ handle = H5FD_s3comms_s3r_open(url_raven, NULL, NULL, NULL);
+ FAIL_IF( handle == NULL )
+ JSVERIFY( 6464, H5FD_s3comms_s3r_get_filesize(handle), NULL )
+
+ for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++)
+ buffer[i] = '\0';
+
+ /**********************
+ * read start of file *
+ **********************/
+
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_s3r_read(
+ handle,
+ (haddr_t)0,
+ (size_t)118,
+ buffer),
+ NULL )
+ JSVERIFY_STR (
+ "Once upon a midnight dreary, while I pondered, weak and weary,\n" \
+ "Over many a quaint and curious volume of forgotten lore",
+ buffer,
+ NULL )
+
+ for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++)
+ buffer[i] = '\0';
+
+ /************************
+ * read arbitrary range *
+ ************************/
+
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_s3r_read(
+ handle,
+ (haddr_t)2540,
+ (size_t)54,
+ buffer),
+ NULL )
+ JSVERIFY_STR( "the grave and stern decorum of the countenance it wore",
+ buffer,
+ NULL )
+
+ for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++)
+ buffer[i] = '\0';
+
+ /**********************
+ * read one character *
+ **********************/
+
+ JSVERIFY(SUCCEED,
+ H5FD_s3comms_s3r_read(
+ handle,
+ (haddr_t)2540,
+ (size_t)1,
+ buffer),
+ NULL )
+ JSVERIFY_STR( "t", buffer, NULL )
+
+
+ for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++)
+ buffer[i] = '\0';
+
+ /***************
+ * read to EoF *
+ ***************/
+
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_s3r_read(
+ handle,
+ (haddr_t)6370,
+ (size_t)0,
+ buffer),
+ NULL )
+ JSVERIFY( 0,
+ strncmp(buffer,
+ "And my soul from out that shadow that lies floating on the floor\nShall be lifted—nevermore!\n",
+ 94),
+ buffer )
+
+ for (i = 0; i < S3COMMS_TEST_BUFFER_SIZE; i++)
+ buffer[i] = '\0';
+
+ /*****************
+ * read past eof *
+ *****************/
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_s3r_read(
+ handle,
+ (haddr_t)6400,
+ (size_t)100, /* 6400+100 > 6464 */
+ buffer),
+ NULL )
+ JSVERIFY( 0, strcmp("", buffer), NULL )
+
+ /************************
+ * read starts past eof *
+ ************************/
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_s3r_read(
+ handle,
+ (haddr_t)1200699, /* 1200699 > 6464 */
+ (size_t)100,
+ buffer),
+ NULL )
+ JSVERIFY( 0, strcmp("", buffer), NULL )
+
+ /**********************
+ * read starts on eof *
+ **********************/
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_s3r_read(
+ handle,
+ (haddr_t)6464,
+ (size_t)0,
+ buffer),
+ NULL )
+ JSVERIFY( 0, strcmp("", buffer), NULL )
+
+ /*************
+ * TEAR DOWN *
+ *************/
+
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_s3r_close(handle),
+ "unable to close file" )
+ handle = NULL;
+
+ curl_global_cleanup();
+ curl_ready = FALSE;
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * cleanup *
+ ***********/
+
+ if (handle != NULL)
+ H5FD_s3comms_s3r_close(handle);
+
+ if (curl_ready == TRUE)
+ curl_global_cleanup();
+
+ return -1;
+
+#undef S3COMMS_TEST_BUFFER_SIZE
+
+} /* end test_s3r_read() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_signing_key()
+ *
+ * Purpose:
+ *
+ * Define and verify behavior of `H5FD_s3comms_signing_key()`
+ *
+ * More test cases would be a very good idea.
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-18
+ *
+ *---------------------------------------------------------------------------
+ */
+static herr_t
+test_signing_key(void)
+{
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct testcase {
+ const char *region;
+ const char *secret_key;
+ const char *when;
+ unsigned char exp[SHA256_DIGEST_LENGTH];
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ struct testcase cases[] = {
+ { "us-east-1",
+ "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
+ "20130524T000000Z",
+ { 0xdb, 0xb8, 0x93, 0xac, 0xc0, 0x10, 0x96, 0x49,
+ 0x18, 0xf1, 0xfd, 0x43, 0x3a, 0xdd, 0x87, 0xc7,
+ 0x0e, 0x8b, 0x0d, 0xb6, 0xbe, 0x30, 0xc1, 0xfb,
+ 0xea, 0xfe, 0xfa, 0x5e, 0xc6, 0xba, 0x83, 0x78,
+ },
+ },
+ };
+ int i = 0;
+ unsigned char *key = NULL;
+ int ncases = 1;
+
+ TESTING("signing_key");
+
+ for (i = 0; i < ncases; i++) {
+ key = (unsigned char *)HDmalloc(sizeof(unsigned char) * \
+ SHA256_DIGEST_LENGTH);
+ HDassert(key != NULL);
+
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_signing_key(
+ key,
+ cases[i].secret_key,
+ cases[i].region,
+ cases[i].when),
+ NULL )
+
+ JSVERIFY( 0,
+ strncmp((const char *)cases[i].exp,
+ (const char *)key,
+ SHA256_DIGEST_LENGTH),
+ cases[i].exp )
+
+ free(key);
+ key = NULL;
+ }
+
+
+ /***************
+ * ERROR CASES *
+ ***************/
+
+ key = (unsigned char *)HDmalloc(sizeof(unsigned char) * \
+ SHA256_DIGEST_LENGTH);
+ HDassert(key != NULL);
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_signing_key(
+ NULL,
+ cases[0].secret_key,
+ cases[0].region,
+ cases[0].when),
+ "destination cannot be NULL" )
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_signing_key(
+ key,
+ NULL,
+ cases[0].region,
+ cases[0].when),
+ "secret key cannot be NULL" )
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_signing_key(
+ key,
+ cases[0].secret_key,
+ NULL,
+ cases[0].when),
+ "aws region cannot be NULL" )
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_signing_key(
+ key,
+ cases[0].secret_key,
+ cases[0].region,
+ NULL),
+ "time string cannot be NULL" )
+
+ free(key);
+ key = NULL;
+
+ PASSED();
+ return 0;
+
+error:
+ if (key != NULL) {
+ free(key);
+ }
+
+ return -1;
+
+} /* end test_signing_key() */
+
+
+/*---------------------------------------------------------------------------
+ *
+ * Function: test_tostringtosign()
+ *
+ * Purpose:
+ *
+ * Verify that we can get the "string to sign" from a Canonical Request and
+ * related information.
+ *
+ * Demonstrate failure cases.
+ *
+ * Return:
+ *
+ * Success: 0
+ * Failure: -1
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-13
+ *
+ *---------------------------------------------------------------------------
+ */
+static herr_t
+test_tostringtosign(void)
+{
+ /************************
+ * test-local variables *
+ ************************/
+
+ const char canonreq[] = "GET\n/test.txt\n\nhost:examplebucket.s3.amazonaws.com\nrange:bytes=0-9\nx-amz-content-sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855\nx-amz-date:20130524T000000Z\n\nhost;range;x-amz-content-sha256;x-amz-date\ne3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+ const char iso8601now[] = "20130524T000000Z";
+ const char region[] = "us-east-1";
+ char s2s[512];
+
+ TESTING("s3comms tostringtosign");
+
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_tostringtosign(s2s, canonreq, iso8601now, region),
+ "unable to create string to sign" )
+
+ JSVERIFY_STR( "AWS4-HMAC-SHA256\n20130524T000000Z\n20130524/us-east-1/s3/aws4_request\n7344ae5b7ee6c3e7e6b0fe0640412a37625d1fbfff95c48bbb2dc43964946972",
+ s2s, NULL )
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_tostringtosign(s2s, NULL, iso8601now, region),
+ "canonical request string cannot be NULL" )
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_tostringtosign(s2s, canonreq, NULL, region),
+ "time string cannot be NULL" )
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_tostringtosign(s2s, canonreq, iso8601now, NULL),
+ "aws region cannot be NULL" )
+
+ PASSED();
+ return 0;
+
+error :
+ return -1;
+
+} /* end test_tostringtosign() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: test_trim()
+ *
+ * Purpose:
+ *
+ * Define and verify behavior of `H5FD_s3comms_trim()`.
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-14
+ *
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+test_trim(void)
+{
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct testcase {
+ const char *in;
+ size_t in_len;
+ const char *exp;
+ size_t exp_len;
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ struct testcase cases[] = {
+ { "block string",
+ 12,
+ "block string",
+ 12,
+ },
+ { " \n\r \t",
+ 6,
+ "",
+ 0,
+ },
+ { " \twhite b4",
+ 10,
+ "white b4",
+ 8,
+ },
+ { "white after\r\n ",
+ 15,
+ "white after",
+ 11,
+ },
+ { " on\nends\t",
+ 9,
+ "on\nends",
+ 7,
+ },
+ };
+ char dest[32];
+ size_t dest_len = 0;
+ int i = 0;
+ int n_cases = 5;
+ char *str = NULL;
+
+
+
+ TESTING("s3comms trim");
+
+ for (i = 0; i < n_cases; i++) {
+ HDassert(str == NULL);
+ str = (char *)HDmalloc(sizeof(char) * cases[i].in_len);
+ HDassert(str != NULL);
+ HDstrncpy(str, cases[i].in, cases[i].in_len);
+
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_trim(dest, str, cases[i].in_len, &dest_len),
+ NULL )
+ JSVERIFY( cases[i].exp_len, dest_len, cases[i].in )
+ if (dest_len > 0) {
+ JSVERIFY( 0, strncmp(cases[i].exp, dest, dest_len),
+ cases[i].exp )
+ }
+ free(str);
+ str = NULL;
+ } /* end for each testcase */
+
+ JSVERIFY( SUCCEED, H5FD_s3comms_trim(dest, NULL, 3, &dest_len),
+ "should not fail when trimming a null string" );
+ JSVERIFY( 0, dest_len, "trimming NULL string writes 0 characters" )
+
+ HDassert(str == NULL);
+ str = (char *)HDmalloc(sizeof(char *) * 11);
+ HDassert(str != NULL);
+ memcpy(str, "some text ", 11); /* string with null terminator */
+ JSVERIFY( FAIL, H5FD_s3comms_trim(NULL, str, 10, &dest_len),
+ "destination for trim cannot be NULL" );
+ free(str);
+ str = NULL;
+
+ PASSED();
+ return 0;
+
+error:
+ if (str != NULL) {
+ free(str);
+ }
+ return -1;
+
+} /* end test_trim() */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: test_uriencode()
+ *
+ * Purpose:
+ *
+ * Define and verify behavior of `H5FD_s3comms_uriencode()`.
+ *
+ * Programmer: Jacob Smith
+ * 2017-09-14
+ *
+ *----------------------------------------------------------------------------
+ */
+static herr_t
+test_uriencode(void)
+{
+ /*************************
+ * test-local structures *
+ *************************/
+
+ struct testcase {
+ const char *str;
+ size_t s_len;
+ hbool_t encode_slash;
+ const char *expected;
+ };
+
+ /************************
+ * test-local variables *
+ ************************/
+
+ struct testcase cases[] = {
+ { "/path/to/resource.jpg",
+ 21,
+ FALSE,
+ "/path/to/resource.jpg",
+ },
+ { "/path/to/resource.jpg",
+ 21,
+ TRUE,
+ "%2Fpath%2Fto%2Fresource.jpg",
+ },
+ { "string got_spaa ces",
+ 20,
+ TRUE,
+ "string%20got_spaa%20%20ces",
+ },
+ { "sp ac~es/and-sl ash.encoded",
+ 27,
+ TRUE,
+ "sp%20ac~es%2Fand-sl%20ash.encoded",
+ },
+ { "sp ac~es/and-sl ash.unencoded",
+ 29,
+ FALSE,
+ "sp%20ac~es/and-sl%20ash.unencoded",
+ },
+ { "/path/to/resource.txt",
+ 0,
+ FALSE,
+ "",
+
+ }
+ };
+ char *dest = NULL;
+ size_t dest_written = 0;
+ int i = 0;
+ int ncases = 6;
+ size_t str_len = 0;
+
+
+
+ TESTING("s3comms uriencode")
+
+ for (i = 0; i < ncases; i++) {
+ str_len = cases[i].s_len;
+ dest = (char *)HDmalloc(sizeof(char) * str_len * 3 + 1);
+ FAIL_IF( dest == NULL )
+
+ JSVERIFY( SUCCEED,
+ H5FD_s3comms_uriencode(
+ dest,
+ cases[i].str,
+ str_len,
+ cases[i].encode_slash,
+ &dest_written),
+ NULL );
+ JSVERIFY( HDstrlen(cases[i].expected),
+ dest_written,
+ NULL )
+ JSVERIFY( 0,
+ strncmp(dest, cases[i].expected, dest_written),
+ cases[i].expected );
+
+ free(dest);
+ dest = NULL;
+ } /* end for each testcase */
+
+ /***************
+ * ERROR CASES *
+ ***************/
+
+ dest = (char *)HDmalloc(sizeof(char) * 15);
+ HDassert(dest != NULL);
+
+ JSVERIFY( FAIL,
+ H5FD_s3comms_uriencode(NULL, "word$", 5, false, &dest_written),
+ "destination cannot be NULL" );
+ JSVERIFY( FAIL,
+ H5FD_s3comms_uriencode(dest, NULL, 5, false, &dest_written),
+ "source string cannot be NULL" );
+
+ free(dest);
+ dest = NULL;
+
+ PASSED();
+ return 0;
+
+error:
+ if (dest != NULL) {
+ free(dest);
+ }
+ return -1;
+
+} /* end test_uriencode() */
+
+#endif /* H5_HAVE_ROS3_VFD */
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: main()
+ *
+ * Purpose:
+ *
+ * Run unit tests for S3 Communications (s3comms).
+ *
+ * Return:
+ *
+ * Success: 0
+ * Failure: 1
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-12
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+main(void)
+{
+#ifdef H5_HAVE_ROS3_VFD
+ int nerrors = 0;
+ const char *bucket_url_env = NULL;
+
+ h5_reset();
+
+#endif /* H5_HAVE_ROS3_VFD */
+
+ HDprintf("Testing S3Communications functionality.\n");
+
+#ifdef H5_HAVE_ROS3_VFD
+
+ /* "clear" profile data strings */
+ s3_test_aws_access_key_id[0] = '\0';
+ s3_test_aws_secret_access_key[0] = '\0';
+ s3_test_aws_region[0] = '\0';
+ s3_test_bucket_url[0] = '\0';
+
+/* TODO: unit/regression test for H5FD_s3comms_load_aws_profile()
+ * requires a few test files and/or manipulation of default path
+ */
+ /* attempt to load test credentials
+ * if unable, certain tests will be skipped
+ */
+ if (SUCCEED == H5FD_s3comms_load_aws_profile(
+ S3_TEST_PROFILE_NAME,
+ s3_test_aws_access_key_id,
+ s3_test_aws_secret_access_key,
+ s3_test_aws_region))
+ {
+ s3_test_credentials_loaded = 1;
+ }
+
+ bucket_url_env = HDgetenv("HDF5_ROS3_TEST_BUCKET_URL");
+ if (bucket_url_env == NULL || bucket_url_env[0] == '\0') {
+ HDprintf("WARNING: S3 bucket url is not defined in enviornment " \
+ "variable 'HDF5_ROS3_TEST_BUCKET_URL'!\n");
+ }
+ else {
+ HDstrncpy(s3_test_bucket_url, bucket_url_env, S3_TEST_MAX_URL_SIZE);
+ s3_test_bucket_defined = TRUE;
+ }
+
+ /* tests ordered rougly by dependence */
+ nerrors += test_macro_format_credential() < 0 ? 1 : 0;
+ nerrors += test_trim() < 0 ? 1 : 0;
+ nerrors += test_nlowercase() < 0 ? 1 : 0;
+ nerrors += test_uriencode() < 0 ? 1 : 0;
+ nerrors += test_percent_encode_char() < 0 ? 1 : 0;
+ nerrors += test_bytes_to_hex() < 0 ? 1 : 0;
+ nerrors += test_HMAC_SHA256() < 0 ? 1 : 0;
+ nerrors += test_signing_key() < 0 ? 1 : 0;
+ nerrors += test_hrb_node_set() < 0 ? 1 : 0;
+ nerrors += test_hrb_init_request() < 0 ? 1 : 0;
+ nerrors += test_parse_url() < 0 ? 1 : 0;
+ nerrors += test_aws_canonical_request() < 0 ? 1 : 0;
+ nerrors += test_tostringtosign() < 0 ? 1 : 0;
+ nerrors += test_s3r_open() < 0 ? 1 : 0;
+ nerrors += test_s3r_get_filesize() < 0 ? 1 : 0;
+ nerrors += test_s3r_read() < 0 ? 1 : 0;
+
+ if (nerrors) {
+ HDprintf("***** %d S3comms TEST%s FAILED! *****\n",
+ nerrors,
+ nerrors > 1 ? "S" : "");
+ return 1;
+ }
+
+ HDprintf("All S3comms tests passed.\n");
+
+ return 0;
+
+#else
+
+ HDprintf("SKIPPED - read-only S3 VFD not built\n");
+ return EXIT_SUCCESS;
+
+#endif /* H5_HAVE_ROS3_VFD */
+
+} /* end main() */
+
diff --git a/test/testframe.c b/test/testframe.c
index 231bcc9..d16d1bc 100644
--- a/test/testframe.c
+++ b/test/testframe.c
@@ -83,8 +83,7 @@ AddTest(const char *TheName, void (*TheCall) (void), void (*Cleanup) (void), con
HDexit(EXIT_FAILURE);
} /* end if */
if (HDstrlen(TheName) >= MAXTESTNAME) {
- HDprintf("Test name too long, increase MAXTESTNAME(%d).\n",
- MAXTESTNAME);
+ HDprintf("Test name too long, increase MAXTESTNAME(%d).\n", MAXTESTNAME);
HDexit(EXIT_FAILURE);
} /* end if */
@@ -96,7 +95,7 @@ AddTest(const char *TheName, void (*TheCall) (void), void (*Cleanup) (void), con
/* Reallocate array */
if(NULL == (newTest = (TestStruct *)HDrealloc(Test, newAlloc * sizeof(TestStruct)))) {
HDprintf("Out of memory for tests, Index = %u, TestAlloc = %u, newAlloc = %u\n", Index, TestAlloc, newAlloc);
- exit(EXIT_FAILURE);
+ HDexit(EXIT_FAILURE);
} /* end if */
/* Update info */
diff --git a/test/vfd.c b/test/vfd.c
index ee177c4..1c2e656 100644
--- a/test/vfd.c
+++ b/test/vfd.c
@@ -25,8 +25,13 @@
#define FAMILY_SIZE (1*KB)
#define FAMILY_SIZE2 (5*KB)
#define MULTI_SIZE 128
+
#define CORE_INCREMENT (4*KB)
-#define CORE_PAGE_SIZE (1024 * 1024)
+#define CORE_PAGE_SIZE (1024*KB)
+#define CORE_DSET_NAME "core dset"
+#define CORE_DSET_DIM1 1024
+#define CORE_DSET_DIM2 32
+
#define DSET1_NAME "dset1"
#define DSET1_DIM1 1024
#define DSET1_DIM2 32
@@ -53,6 +58,7 @@ const char *FILENAME[] = {
"stdio_file", /*7*/
"windows_file", /*8*/
"new_multi_file_v16",/*9*/
+ "ro_s3_file", /*10*/
NULL
};
@@ -61,7 +67,7 @@ const char *FILENAME[] = {
#define COMPAT_BASENAME "family_v16_"
#define MULTI_COMPAT_BASENAME "multi_file_v16"
-
+
/*-------------------------------------------------------------------------
* Function: test_sec2
*
@@ -78,56 +84,83 @@ const char *FILENAME[] = {
static herr_t
test_sec2(void)
{
- hid_t file = -1; /* file ID */
- hid_t fapl = -1; /* file access property list ID */
- hid_t access_fapl = -1;
- char filename[1024];
- int *fhandle = NULL;
- hsize_t file_size = 0;
+ hid_t fid = -1; /* file ID */
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t fapl_id_out = -1; /* from H5Fget_access_plist */
+ hid_t driver_id = -1; /* ID for this VFD */
+ unsigned long driver_flags = 0; /* VFD feature flags */
+ char filename[1024]; /* filename */
+ void *os_file_handle = NULL; /* OS file handle */
+ hsize_t file_size; /* file size */
TESTING("SEC2 file driver");
/* Set property list and file name for SEC2 driver. */
- if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+ if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
- if(H5Pset_fapl_sec2(fapl) < 0)
+ if(H5Pset_fapl_sec2(fapl_id) < 0)
TEST_ERROR;
- h5_fixname(FILENAME[0], fapl, filename, sizeof filename);
+ h5_fixname(FILENAME[0], fapl_id, filename, sizeof(filename));
- if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
+ /* Check that the VFD feature flags are correct */
+ if ((driver_id = H5Pget_driver(fapl_id)) < 0)
+ TEST_ERROR
+ if (H5FDdriver_query(driver_id, &driver_flags) < 0)
+ TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_METADATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_ACCUMULATE_METADATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_POSIX_COMPAT_HANDLE)) TEST_ERROR
+ /* Check for extra flags not accounted for above */
+ if(driver_flags != (H5FD_FEAT_AGGREGATE_METADATA
+ | H5FD_FEAT_ACCUMULATE_METADATA
+ | H5FD_FEAT_DATA_SIEVE
+ | H5FD_FEAT_AGGREGATE_SMALLDATA
+ | H5FD_FEAT_POSIX_COMPAT_HANDLE))
+ TEST_ERROR
+
+ if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0)
TEST_ERROR;
/* Retrieve the access property list... */
- if((access_fapl = H5Fget_access_plist(file)) < 0)
+ if((fapl_id_out = H5Fget_access_plist(fid)) < 0)
TEST_ERROR;
/* Check that the driver is correct */
- if(H5FD_SEC2 != H5Pget_driver(access_fapl))
+ if(H5FD_SEC2 != H5Pget_driver(fapl_id_out))
TEST_ERROR;
/* ...and close the property list */
- if(H5Pclose(access_fapl) < 0)
+ if(H5Pclose(fapl_id_out) < 0)
TEST_ERROR;
- /* Check file handle API */
- if(H5Fget_vfd_handle(file, H5P_DEFAULT, (void **)&fhandle) < 0)
- TEST_ERROR;
- if(*fhandle < 0)
+ /* Check that we can get an operating-system-specific handle from
+ * the library.
+ */
+ if(H5Fget_vfd_handle(fid, H5P_DEFAULT, &os_file_handle) < 0)
TEST_ERROR;
+ if(os_file_handle == NULL)
+ FAIL_PUTS_ERROR("NULL os-specific vfd/file handle was returned from H5Fget_vfd_handle");
- /* Check file size API */
- if(H5Fget_filesize(file, &file_size) < 0)
- TEST_ERROR;
- /* There is no guarantee the size of metadata in file is constant.
- * Just try to check if it's reasonable. It's 2KB right now.
+ /* There is no garantee the size of metadata in file is constant.
+ * Just try to check if it's reasonable.
+ *
+ * Currently it should be around 2 KB.
*/
+ if(H5Fget_filesize(fid, &file_size) < 0)
+ TEST_ERROR;
if(file_size < 1 * KB || file_size > 4 * KB)
+ FAIL_PUTS_ERROR("suspicious file size obtained from H5Fget_filesize");
+
+ /* Close and delete the file */
+ if(H5Fclose(fid) < 0)
TEST_ERROR;
- h5_delete_test_file(FILENAME[0], fapl);
+ h5_delete_test_file(FILENAME[0], fapl_id);
/* Close the fapl */
- if(H5Pclose(fapl) < 0)
+ if(H5Pclose(fapl_id) < 0)
TEST_ERROR;
PASSED();
@@ -135,209 +168,347 @@ test_sec2(void)
error:
H5E_BEGIN_TRY {
- H5Pclose(fapl);
- H5Fclose(file);
+ H5Pclose(fapl_id);
+ H5Pclose(fapl_id_out);
+ H5Fclose(fid);
} H5E_END_TRY;
return -1;
-}
+} /* end test_sec2() */
-
/*-------------------------------------------------------------------------
- * Function: test_direct
+ * Function: test_core
*
- * Purpose: Tests the file handle interface for DIRECT I/O driver
+ * Purpose: Tests the file handle interface for CORE driver
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Raymond Lu
- * Wednesday, 20 September 2006
+ * Tuesday, Sept 24, 2002
*
*-------------------------------------------------------------------------
*/
static herr_t
-test_direct(void)
+test_core(void)
{
-#ifdef H5_HAVE_DIRECT
- hid_t file=(-1), fapl, access_fapl = -1;
- hid_t dset1=-1, dset2=-1, space1=-1, space2=-1;
- char filename[1024];
- int *fhandle=NULL;
- hsize_t file_size;
- hsize_t dims1[2], dims2[1];
- size_t mbound;
- size_t fbsize;
- size_t cbsize;
- int *points = NULL, *check = NULL, *p1, *p2;
- int wdata2[DSET2_DIM] = {11,12,13,14};
- int rdata2[DSET2_DIM];
- int i, j, n;
-#endif /*H5_HAVE_DIRECT*/
+ hid_t fid = -1; /* file ID */
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t fapl_id_out = -1; /* from H5Fget_access_plist */
+ hid_t driver_id = -1; /* ID for this VFD */
+ unsigned long driver_flags = 0; /* VFD feature flags */
+ hid_t did = -1; /* dataset ID */
+ hid_t sid = -1; /* dataspace ID */
+ char filename[1024]; /* filename */
+ void *os_file_handle = NULL; /* OS file handle */
+ hsize_t file_size; /* file size */
+ size_t increment; /* core VFD increment */
+ hbool_t backing_store; /* use backing store? */
+ hbool_t use_write_tracking; /* write tracking flag */
+ size_t write_tracking_page_size; /* write tracking page size */
+ int *data_w = NULL; /* data written to the dataset */
+ int *data_r = NULL; /* data read from the dataset */
+ int val; /* data value */
+ int *pw = NULL, *pr = NULL; /* pointers for iterating over
+ data arrays (write & read) */
+ hsize_t dims[2]; /* dataspace dimensions */
+ int i, j; /* iterators */
+ htri_t status; /* return value from H5Lexists */
- TESTING("DIRECT I/O file driver");
+ TESTING("CORE file driver");
-#ifndef H5_HAVE_DIRECT
- SKIPPED();
- return 0;
-#else /*H5_HAVE_DIRECT*/
+ /* Get a file access property list and fix up the file name */
+ if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+ TEST_ERROR;
+ h5_fixname(FILENAME[1], fapl_id, filename, sizeof(filename));
- /* Set property list and file name for Direct driver. Set memory alignment boundary
- * and file block size to 512 which is the minimum for Linux 2.6. */
- if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+ /************************************************************************
+ * Check that the backing store flag works by creating a file, close
+ * it, and ensure that the file does not exist.
+ ************************************************************************/
+
+ /* Make sure it's not present at the start of the test */
+ if(HDaccess(filename, F_OK) != -1)
+ if(HDremove(filename) < 0)
+ FAIL_PUTS_ERROR("unable to remove backing store file");
+
+ /* Create and close file w/ backing store off */
+ if(H5Pset_fapl_core(fapl_id, (size_t)CORE_INCREMENT, FALSE) < 0)
TEST_ERROR;
- if(H5Pset_fapl_direct(fapl, MBOUNDARY, FBSIZE, CBSIZE) < 0)
+
+ /* Check that the VFD feature flags are correct.
+ * Note that the H5FDdriver_query() API call does not require a file
+ * so backing-store related flags will not be returned here.
+ */
+ if ((driver_id = H5Pget_driver(fapl_id)) < 0)
+ TEST_ERROR
+ if (H5FDdriver_query(driver_id, &driver_flags) < 0)
+ TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_METADATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_ACCUMULATE_METADATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_ALLOW_FILE_IMAGE)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_CAN_USE_FILE_IMAGE_CALLBACKS)) TEST_ERROR
+ /* Check for extra flags not accounted for above */
+ if(driver_flags != (H5FD_FEAT_AGGREGATE_METADATA
+ | H5FD_FEAT_ACCUMULATE_METADATA
+ | H5FD_FEAT_DATA_SIEVE
+ | H5FD_FEAT_AGGREGATE_SMALLDATA
+ | H5FD_FEAT_ALLOW_FILE_IMAGE
+ | H5FD_FEAT_CAN_USE_FILE_IMAGE_CALLBACKS))
+ TEST_ERROR
+
+ if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0)
TEST_ERROR;
- h5_fixname(FILENAME[5], fapl, filename, sizeof filename);
+ if(H5Fclose(fid) < 0)
+ TEST_ERROR;
+ /* Check for the backing store file */
+ if(HDaccess(filename, F_OK) != -1)
+ FAIL_PUTS_ERROR("file created when backing store set to FALSE");
- /* Verify the file access properties */
- if(H5Pget_fapl_direct(fapl, &mbound, &fbsize, &cbsize) < 0)
+ /************************************************************************
+ * Check basic core VFD operation and properties. This is done with the
+ * backing store on so a file will be created for later use.
+ ************************************************************************/
+
+ /* Turn the backing store on */
+ if(H5Pset_fapl_core(fapl_id, (size_t)CORE_INCREMENT, TRUE) < 0)
TEST_ERROR;
- if(mbound != MBOUNDARY || fbsize != FBSIZE || cbsize != CBSIZE)
+
+ /* Check that write tracking is off by default and that the default
+ * page size is non-zero.
+ */
+ if(H5Pget_core_write_tracking(fapl_id, &use_write_tracking, &write_tracking_page_size) < 0)
TEST_ERROR;
+ if(FALSE != use_write_tracking)
+ FAIL_PUTS_ERROR("write tracking should be off by default");
+ if(0 == write_tracking_page_size)
+ FAIL_PUTS_ERROR("write tracking page size should never be zero");
- if(H5Pset_alignment(fapl, (hsize_t)THRESHOLD, (hsize_t)FBSIZE) < 0)
+ /* Set core VFD properties */
+ if(H5Pset_core_write_tracking(fapl_id, TRUE, CORE_PAGE_SIZE) < 0)
TEST_ERROR;
- H5E_BEGIN_TRY {
- file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
- } H5E_END_TRY;
- if(file<0) {
- H5Pclose (fapl);
- SKIPPED();
- HDprintf(" Probably the file system doesn't support Direct I/O\n");
- return 0;
- }
+ /* Create the file */
+ if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0)
+ TEST_ERROR;
/* Retrieve the access property list... */
- if ((access_fapl = H5Fget_access_plist(file)) < 0)
+ if((fapl_id_out = H5Fget_access_plist(fid)) < 0)
TEST_ERROR;
/* Check that the driver is correct */
- if(H5FD_DIRECT != H5Pget_driver(access_fapl))
+ if(H5FD_CORE != H5Pget_driver(fapl_id_out))
TEST_ERROR;
- /* ...and close the property list */
- if (H5Pclose(access_fapl) < 0)
- TEST_ERROR;
+ /* Get the basic VFD properties from the fapl and ensure that
+ * they are correct.
+ */
+ if(H5Pget_fapl_core(fapl_id_out, &increment, &backing_store) < 0)
+ TEST_ERROR
+ if(increment != (size_t)CORE_INCREMENT)
+ FAIL_PUTS_ERROR("incorrect increment from file fapl");
+ if(backing_store != TRUE)
+ FAIL_PUTS_ERROR("incorrect backing store flag from file fapl");
- /* Check file handle API */
- if(H5Fget_vfd_handle(file, H5P_DEFAULT, (void **)&fhandle) < 0)
- TEST_ERROR;
- if(*fhandle<0)
+ /* Check that the backing store write tracking info was saved */
+ /* TODO: There is a bug where H5Fget_access_plist() does not return
+ * the write tracking properties. Until this bug is fixed, just
+ * test the main fapl_id.
+ */
+ if(H5Pget_core_write_tracking(fapl_id, &use_write_tracking, &write_tracking_page_size) < 0)
TEST_ERROR;
+ if(TRUE != use_write_tracking)
+ FAIL_PUTS_ERROR("write tracking flag incorrect in fapl obtained from H5Fget_access_plist");
+ if(CORE_PAGE_SIZE != write_tracking_page_size)
+ FAIL_PUTS_ERROR("write tracking page size incorrect in fapl obtained from H5Fget_access_plist");
- /* Check file size API */
- if(H5Fget_filesize(file, &file_size) < 0)
+ /* Close the property list */
+ if(H5Pclose(fapl_id_out) < 0)
TEST_ERROR;
- /* There is no guarantee of the number of metadata allocations, but it's
- * 4 currently and the size of the file should be between 3 & 4 file buffer
- * sizes..
+ /* Check that we can get an operating-system-specific handle from
+ * the library.
*/
- if(file_size < (FBSIZE * 3) || file_size >= (FBSIZE * 4))
+ if(H5Fget_vfd_handle(fid, H5P_DEFAULT, &os_file_handle) < 0)
TEST_ERROR;
+ if(os_file_handle == NULL)
+ FAIL_PUTS_ERROR("NULL os-specific vfd/file handle was returned from H5Fget_vfd_handle");
- /* Allocate aligned memory for data set 1. For data set 1, everything is aligned including
- * memory address, size of data, and file address. */
- if(0 != HDposix_memalign(&points, (size_t)FBSIZE, (size_t)(DSET1_DIM1 * DSET1_DIM2 * sizeof(int))))
+ /* There is no garantee the size of metadata in file is constant.
+ * Just try to check if it's reasonable.
+ *
+ * TODO: Needs justification of why is this is a reasonable size.
+ */
+ if(H5Fget_filesize(fid, &file_size) < 0)
TEST_ERROR;
- if(0 != HDposix_memalign(&check, (size_t)FBSIZE, (size_t)(DSET1_DIM1 * DSET1_DIM2 * sizeof(int))))
+ if(file_size < 2 * KB || file_size > 6 * KB)
+ FAIL_PUTS_ERROR("suspicious file size obtained from H5Fget_filesize");
+
+ /* Close the file */
+ if(H5Fclose(fid) < 0)
TEST_ERROR;
- /* Initialize the dset1 */
- p1 = points;
- for(i = n = 0; i < DSET1_DIM1; i++)
- for(j = 0; j < DSET1_DIM2; j++)
- *p1++ = n++;
- /* Create the data space1 */
- dims1[0] = DSET1_DIM1;
- dims1[1] = DSET1_DIM2;
- if((space1 = H5Screate_simple(2, dims1, NULL)) < 0)
+ /************************************************************************
+ * Make changes to the file with the backing store flag OFF to ensure
+ * that they ARE NOT propagated.
+ ************************************************************************/
+
+ /* Open the file with backing store off for read and write.
+ * Changes won't be saved in file.
+ */
+ if(H5Pset_fapl_core(fapl_id, (size_t)CORE_INCREMENT, FALSE) < 0)
+ TEST_ERROR;
+ if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
- /* Create the dset1 */
- if((dset1 = H5Dcreate2(file, DSET1_NAME, H5T_NATIVE_INT, space1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ /* Allocate memory for data set. */
+ if(NULL == (data_w = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int))))
+ FAIL_PUTS_ERROR("unable to allocate memory for input array");
+ if(NULL == (data_r = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int))))
+ FAIL_PUTS_ERROR("unable to allocate memory for output array");
+
+ /* Initialize the buffers */
+ val = 0;
+ pw = data_w;
+ for(i = 0; i < CORE_DSET_DIM1; i++)
+ for(j = 0; j < CORE_DSET_DIM2; j++)
+ *pw++ = val++;
+ HDmemset(data_r, 0, DSET1_DIM1 * DSET1_DIM2 * sizeof(int));
+
+ /* Create the dataspace */
+ dims[0] = CORE_DSET_DIM1;
+ dims[1] = CORE_DSET_DIM2;
+ if((sid = H5Screate_simple(2, dims, NULL)) < 0)
TEST_ERROR;
- /* Write the data to the dset1 */
- if(H5Dwrite(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, points) < 0)
+ /* Create the dataset */
+ if((did = H5Dcreate2(fid, CORE_DSET_NAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
- if(H5Dclose(dset1) < 0)
+ /* Write the data to the dataset */
+ if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_w) < 0)
TEST_ERROR;
- if((dset1 = H5Dopen2(file, DSET1_NAME, H5P_DEFAULT)) < 0)
+ /* Close and reopen the dataset */
+ if(H5Dclose(did) < 0)
+ TEST_ERROR;
+ if((did = H5Dopen2(fid, CORE_DSET_NAME, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Read the data back from dset1 */
- if(H5Dread(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, check) < 0)
+ if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_r) < 0)
TEST_ERROR;
/* Check that the values read are the same as the values written */
- p1 = points;
- p2 = check;
- for(i = 0; i < DSET1_DIM1; i++)
- for(j = 0; j < DSET1_DIM2; j++)
- if(*p1++ != *p2++) {
+ pw = data_w;
+ pr = data_r;
+ for(i = 0; i < CORE_DSET_DIM1; i++)
+ for(j = 0; j < CORE_DSET_DIM2; j++)
+ if(*pr++ != *pw++) {
H5_FAILED();
- HDprintf(" Read different values than written in data set 1.\n");
+ HDprintf(" Read different values than written in data set.\n");
HDprintf(" At index %d,%d\n", i, j);
TEST_ERROR;
- } /* end if */
+ } /* end if */
- /* Create the data space2. For data set 2, memory address and data size are not aligned. */
- dims2[0] = DSET2_DIM;
- if((space2 = H5Screate_simple(1, dims2, NULL)) < 0)
+ /* Close everything except the dataspace ID (needed below)*/
+ if(H5Dclose(did) < 0)
TEST_ERROR;
-
- /* Create the dset2 */
- if((dset2 = H5Dcreate2(file, DSET2_NAME, H5T_NATIVE_INT, space2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ if(H5Fclose(fid) < 0)
TEST_ERROR;
- /* Write the data to the dset1 */
- if(H5Dwrite(dset2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata2) < 0)
+ /* Reopen the file and ensure that the dataset does not exist */
+ if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0)
+ TEST_ERROR;
+ status = H5Lexists(fid, CORE_DSET_NAME, H5P_DEFAULT);
+ if(status < 0)
TEST_ERROR;
+ if(status > 0)
+ FAIL_PUTS_ERROR("core VFD dataset created in file when backing store disabled");
- if(H5Dclose(dset2) < 0)
+ /* Close the file */
+ if(H5Fclose(fid) < 0)
TEST_ERROR;
- if((dset2 = H5Dopen2(file, DSET2_NAME, H5P_DEFAULT)) < 0)
+
+ /************************************************************************
+ * Make changes to the file with the backing store flag ON to ensure
+ * that they ARE propagated.
+ ************************************************************************/
+
+ /* Open the file with backing store on for read and write.
+ * Changes will be saved in file.
+ */
+ if(H5Pset_fapl_core(fapl_id, (size_t)CORE_INCREMENT, TRUE) < 0)
+ TEST_ERROR;
+ if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
- /* Read the data back from dset1 */
- if(H5Dread(dset2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata2) < 0)
+ /* Create the dataset */
+ if((did = H5Dcreate2(fid, CORE_DSET_NAME, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
- /* Check that the values read are the same as the values written */
- for(i = 0; i < DSET2_DIM; i++)
- if(wdata2[i] != rdata2[i]) {
- H5_FAILED();
- HDprintf(" Read different values than written in data set 2.\n");
- HDprintf(" At index %d\n", i);
- TEST_ERROR;
- } /* end if */
+ /* Write the data to the dataset */
+ if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_w) < 0)
+ TEST_ERROR;
- if(H5Sclose(space1) < 0)
+ /* Close everything and reopen */
+ if(H5Dclose(did) < 0)
TEST_ERROR;
- if(H5Dclose(dset1) < 0)
+ if(H5Fclose(fid) < 0)
TEST_ERROR;
- if(H5Sclose(space2) < 0)
+ if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl_id)) < 0)
TEST_ERROR;
- if(H5Dclose(dset2) < 0)
+ if((did = H5Dopen2(fid, CORE_DSET_NAME, H5P_DEFAULT)) < 0)
TEST_ERROR;
- if(H5Fclose(file) < 0)
+
+ /* Read the data back from the dataset */
+ HDmemset(data_r, 0, DSET1_DIM1 * DSET1_DIM2 * sizeof(int));
+ if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_r) < 0)
TEST_ERROR;
- HDassert(points);
- HDfree(points);
- HDassert(check);
- HDfree(check);
- h5_delete_test_file(FILENAME[1], fapl);
+ /* Check that the values read are the same as the values written */
+ pw = data_w;
+ pr = data_r;
+ for(i = 0; i < CORE_DSET_DIM1; i++)
+ for(j = 0; j < CORE_DSET_DIM2; j++)
+ if(*pw++ != *pr++) {
+ H5_FAILED();
+ HDprintf(" Read different values than written in data set.\n");
+ HDprintf(" At index %d,%d\n", i, j);
+ TEST_ERROR;
+ } /* end if */
+
+ /* Check file size API.
+ * There is no garantee the size of metadata in file is constant.
+ * Just try to check if it's reasonable.
+ *
+ * TODO: Needs justification of why is this is a reasonable size.
+ */
+ if(H5Fget_filesize(fid, &file_size) < 0)
+ TEST_ERROR;
+ if(file_size < 64 * KB || file_size > 256 * KB)
+ FAIL_PUTS_ERROR("suspicious file size obtained from H5Fget_filesize");
+
+ /* Close everything */
+ if(H5Sclose(sid) < 0)
+ TEST_ERROR;
+ if(H5Dclose(did) < 0)
+ TEST_ERROR;
+
+ HDfree(data_w);
+ HDfree(data_r);
+
+ /* Close and delete the file */
+ if(H5Fclose(fid) < 0)
+ TEST_ERROR;
+ h5_delete_test_file(FILENAME[1], fapl_id);
/* Close the fapl */
- if(H5Pclose(fapl) < 0)
+ if(H5Pclose(fapl_id) < 0)
TEST_ERROR;
PASSED();
@@ -345,119 +516,122 @@ test_direct(void)
error:
H5E_BEGIN_TRY {
- H5Pclose(fapl);
- H5Sclose(space1);
- H5Dclose(dset1);
- H5Sclose(space2);
- H5Dclose(dset2);
- H5Fclose(file);
+ H5Sclose(sid);
+ H5Dclose(did);
+ H5Pclose(fapl_id_out);
+ H5Pclose(fapl_id);
+ H5Fclose(fid);
} H5E_END_TRY;
- if(points)
- HDfree(points);
- if(check)
- HDfree(check);
+ if(data_w)
+ HDfree(data_w);
+ if(data_r)
+ HDfree(data_r);
return -1;
-#endif /*H5_HAVE_DIRECT*/
-}
+} /* end test_core() */
+
-
/*-------------------------------------------------------------------------
- * Function: test_core
+ * Function: test_direct
*
- * Purpose: Tests the file handle interface for CORE driver
+ * Purpose: Tests the file handle interface for DIRECT I/O driver
*
* Return: Success: 0
* Failure: -1
*
* Programmer: Raymond Lu
- * Tuesday, Sept 24, 2002
+ * Wednesday, 20 September 2006
*
*-------------------------------------------------------------------------
*/
static herr_t
-test_core(void)
+test_direct(void)
{
- hid_t file=(-1), fapl, access_fapl = -1;
+#ifdef H5_HAVE_DIRECT
+ hid_t file=-1, fapl=-1, access_fapl = -1;
+ hid_t dset1=-1, dset2=-1, space1=-1, space2=-1;
char filename[1024];
- void *fhandle=NULL;
+ int *fhandle=NULL;
hsize_t file_size;
- hbool_t use_write_tracking;
- size_t write_tracking_page_size;
- int *points = NULL, *check = NULL, *p1, *p2;
- hid_t dset1=-1, space1=-1;
- hsize_t dims1[2];
+ hsize_t dims1[2], dims2[1];
+ size_t mbound;
+ size_t fbsize;
+ size_t cbsize;
+ int *points = NULL, *check = NULL, *p1 = NULL, *p2 = NULL;
+ int wdata2[DSET2_DIM] = {11,12,13,14};
+ int rdata2[DSET2_DIM];
int i, j, n;
+#endif /*H5_HAVE_DIRECT*/
- TESTING("CORE file driver");
+ TESTING("DIRECT I/O file driver");
- /* Set property list and file name for CORE driver */
+#ifndef H5_HAVE_DIRECT
+ SKIPPED();
+ return 0;
+#else /*H5_HAVE_DIRECT*/
+
+ /* Set property list and file name for Direct driver. Set memory alignment boundary
+ * and file block size to 512 which is the minimum for Linux 2.6. */
if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
TEST_ERROR;
- if(H5Pset_fapl_core(fapl, (size_t)CORE_INCREMENT, TRUE) < 0)
+ if(H5Pset_fapl_direct(fapl, MBOUNDARY, FBSIZE, CBSIZE) < 0)
TEST_ERROR;
- if(H5Pset_core_write_tracking(fapl, TRUE, CORE_PAGE_SIZE) < 0)
+ h5_fixname(FILENAME[5], fapl, filename, sizeof filename);
+
+ /* Verify the file access properties */
+ if(H5Pget_fapl_direct(fapl, &mbound, &fbsize, &cbsize) < 0)
+ TEST_ERROR;
+ if(mbound != MBOUNDARY || fbsize != FBSIZE || cbsize != CBSIZE)
TEST_ERROR;
- h5_fixname(FILENAME[1], fapl, filename, sizeof filename);
- if((file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
+ if(H5Pset_alignment(fapl, (hsize_t)THRESHOLD, (hsize_t)FBSIZE) < 0)
TEST_ERROR;
+ H5E_BEGIN_TRY {
+ file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl);
+ } H5E_END_TRY;
+ if(file<0) {
+ H5Pclose (fapl);
+ SKIPPED();
+ HDprintf(" Probably the file system doesn't support Direct I/O\n");
+ return 0;
+ }
+
/* Retrieve the access property list... */
if ((access_fapl = H5Fget_access_plist(file)) < 0)
TEST_ERROR;
/* Check that the driver is correct */
- if(H5FD_CORE != H5Pget_driver(access_fapl))
- TEST_ERROR;
-
- /* Check that the backing store write tracking info was saved */
- if(H5Pget_core_write_tracking(fapl, &use_write_tracking, &write_tracking_page_size) < 0)
- TEST_ERROR;
- if(TRUE != use_write_tracking)
- TEST_ERROR;
- if(CORE_PAGE_SIZE != write_tracking_page_size)
+ if(H5FD_DIRECT != H5Pget_driver(access_fapl))
TEST_ERROR;
/* ...and close the property list */
if (H5Pclose(access_fapl) < 0)
TEST_ERROR;
- if(H5Fget_vfd_handle(file, H5P_DEFAULT, &fhandle) < 0)
+ /* Check file handle API */
+ if(H5Fget_vfd_handle(file, H5P_DEFAULT, (void **)&fhandle) < 0)
+ TEST_ERROR;
+ if(*fhandle<0)
TEST_ERROR;
- if(fhandle==NULL)
- {
- HDprintf("fhandle==NULL\n");
- TEST_ERROR;
- }
/* Check file size API */
if(H5Fget_filesize(file, &file_size) < 0)
TEST_ERROR;
- /* There is no garantee the size of metadata in file is constant.
- * Just try to check if it's reasonable. Why is this 4KB?
+ /* There is no guarantee of the number of metadata allocations, but it's
+ * 4 currently and the size of the file should be between 3 & 4 file buffer
+ * sizes..
*/
- if(file_size<2*KB || file_size>6*KB)
- TEST_ERROR;
-
- if(H5Fclose(file) < 0)
- TEST_ERROR;
-
-
- /* Open the file with backing store off for read and write.
- * Changes won't be saved in file. */
- if(H5Pset_fapl_core(fapl, (size_t)CORE_INCREMENT, FALSE) < 0)
- TEST_ERROR;
-
- if((file=H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0)
+ if(file_size < (FBSIZE * 3) || file_size >= (FBSIZE * 4))
TEST_ERROR;
- /* Allocate memory for data set. */
- if(NULL == (points = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int))))
+ /* Allocate aligned memory for data set 1. For data set 1, everything is aligned including
+ * memory address, size of data, and file address. */
+ if(0 != HDposix_memalign(&points, (size_t)FBSIZE, (size_t)(DSET1_DIM1 * DSET1_DIM2 * sizeof(int))))
TEST_ERROR;
- if(NULL == (check = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int))))
+ if(0 != HDposix_memalign(&check, (size_t)FBSIZE, (size_t)(DSET1_DIM1 * DSET1_DIM2 * sizeof(int))))
TEST_ERROR;
/* Initialize the dset1 */
@@ -500,74 +674,50 @@ test_core(void)
HDprintf(" Read different values than written in data set 1.\n");
HDprintf(" At index %d,%d\n", i, j);
TEST_ERROR;
- } /* end if */
-
- if(H5Dclose(dset1) < 0)
- TEST_ERROR;
-
- if(H5Fclose(file) < 0)
- TEST_ERROR;
-
- /* Open the file with backing store on for read and write.
- * Changes will be saved in file. */
- if(H5Pset_fapl_core(fapl, (size_t)CORE_INCREMENT, TRUE) < 0)
- TEST_ERROR;
+ } /* end if */
- if((file = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0)
+ /* Create the data space2. For data set 2, memory address and data size are not aligned. */
+ dims2[0] = DSET2_DIM;
+ if((space2 = H5Screate_simple(1, dims2, NULL)) < 0)
TEST_ERROR;
- /* Create the dset1 */
- if((dset1 = H5Dcreate2(file, DSET1_NAME, H5T_NATIVE_INT, space1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ /* Create the dset2 */
+ if((dset2 = H5Dcreate2(file, DSET2_NAME, H5T_NATIVE_INT, space2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Write the data to the dset1 */
- if(H5Dwrite(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, points) < 0)
- TEST_ERROR;
-
- if(H5Dclose(dset1) < 0)
+ if(H5Dwrite(dset2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata2) < 0)
TEST_ERROR;
- if((dset1 = H5Dopen2(file, DSET1_NAME, H5P_DEFAULT)) < 0)
+ if(H5Dclose(dset2) < 0)
TEST_ERROR;
- /* Reallocate memory for reading buffer. */
- HDassert(check);
- HDfree(check);
- if(NULL == (check = (int *)HDmalloc(DSET1_DIM1 * DSET1_DIM2 * sizeof(int))))
+ if((dset2 = H5Dopen2(file, DSET2_NAME, H5P_DEFAULT)) < 0)
TEST_ERROR;
/* Read the data back from dset1 */
- if(H5Dread(dset1, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, check) < 0)
+ if(H5Dread(dset2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rdata2) < 0)
TEST_ERROR;
/* Check that the values read are the same as the values written */
- p1 = points;
- p2 = check;
- for(i = 0; i < DSET1_DIM1; i++)
- for(j = 0; j < DSET1_DIM2; j++)
- if(*p1++ != *p2++) {
- H5_FAILED();
- HDprintf(" Read different values than written in data set 1.\n");
- HDprintf(" At index %d,%d\n", i, j);
- TEST_ERROR;
- } /* end if */
-
- /* Check file size API */
- if(H5Fget_filesize(file, &file_size) < 0)
- TEST_ERROR;
-
- /* There is no garantee the size of metadata in file is constant.
- * Just try to check if it's reasonable. */
- if(file_size<64*KB || file_size>256*KB)
- TEST_ERROR;
+ for(i = 0; i < DSET2_DIM; i++)
+ if(wdata2[i] != rdata2[i]) {
+ H5_FAILED();
+ HDprintf(" Read different values than written in data set 2.\n");
+ HDprintf(" At index %d\n", i);
+ TEST_ERROR;
+ } /* end if */
if(H5Sclose(space1) < 0)
TEST_ERROR;
if(H5Dclose(dset1) < 0)
TEST_ERROR;
- HDassert(points);
+ if(H5Sclose(space2) < 0)
+ TEST_ERROR;
+ if(H5Dclose(dset2) < 0)
+ TEST_ERROR;
+
HDfree(points);
- HDassert(check);
HDfree(check);
/* Close and delete the file */
@@ -585,6 +735,10 @@ test_core(void)
error:
H5E_BEGIN_TRY {
H5Pclose(fapl);
+ H5Sclose(space1);
+ H5Dclose(dset1);
+ H5Sclose(space2);
+ H5Dclose(dset2);
H5Fclose(file);
} H5E_END_TRY;
@@ -594,9 +748,10 @@ error:
HDfree(check);
return -1;
+#endif /*H5_HAVE_DIRECT*/
}
-
+
/*-------------------------------------------------------------------------
* Function: test_family_opens
*
@@ -611,6 +766,14 @@ error:
*
*-------------------------------------------------------------------------
*/
+/* Disable warning for "format not a string literal" here -QAK */
+/*
+ * This pragma only needs to surround the snprintf() calls with
+ * 'first_name' in the code below, but early (4.4.7, at least) gcc only
+ * allows diagnostic pragmas to be toggled outside of functions.
+ */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-nonliteral"
static herr_t
test_family_opens(char *fname, hid_t fa_pl)
{
@@ -667,8 +830,9 @@ test_family_opens(char *fname, hid_t fa_pl)
error:
return -1;
} /* end test_family_opens() */
+#pragma GCC diagnostic pop
+
-
/*-------------------------------------------------------------------------
* Function: test_family
*
@@ -687,6 +851,8 @@ test_family(void)
{
hid_t file=-1, fapl=-1, fapl2=-1, space=-1, dset=-1;
hid_t access_fapl = -1;
+ hid_t driver_id = -1; /* ID for this VFD */
+ unsigned long driver_flags = 0; /* VFD feature flags */
char filename[1024];
char dname[]="dataset";
unsigned int i, j;
@@ -704,6 +870,22 @@ test_family(void)
TEST_ERROR;
h5_fixname(FILENAME[2], fapl, filename, sizeof(filename));
+ /* Check that the VFD feature flags are correct */
+ if ((driver_id = H5Pget_driver(fapl)) < 0)
+ TEST_ERROR
+ if (H5FDdriver_query(driver_id, &driver_flags) < 0)
+ TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_METADATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_ACCUMULATE_METADATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR
+ /* Check for extra flags not accounted for above */
+ if(driver_flags != (H5FD_FEAT_AGGREGATE_METADATA
+ | H5FD_FEAT_ACCUMULATE_METADATA
+ | H5FD_FEAT_DATA_SIEVE
+ | H5FD_FEAT_AGGREGATE_SMALLDATA))
+ TEST_ERROR
+
if((file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
@@ -832,7 +1014,7 @@ error:
return -1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_family_compat
*
@@ -851,6 +1033,14 @@ error:
*
*-------------------------------------------------------------------------
*/
+/* Disable warning for "format not a string literal" here -QAK */
+/*
+ * This pragma only needs to surround the snprintf() calls with
+ * 'newname_individual', etc. in the code below, but early (4.4.7, at least) gcc only
+ * allows diagnostic pragmas to be toggled outside of functions.
+ */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-nonliteral"
static herr_t
test_family_compat(void)
{
@@ -934,26 +1124,34 @@ error:
return -1;
} /* end test_family_compat() */
+#pragma GCC diagnostic pop
+
-
/*-------------------------------------------------------------------------
* Function: test_multi_opens
*
* Purpose: Private function for test_multi() to tests wrong ways of
* reopening multi file.
*
- * Return: Success: 0
- * Failure: -1
+ * Return: SUCCEED/FAIL
*
* Programmer: Raymond Lu
* Thursday, May 19, 2005
*
*-------------------------------------------------------------------------
*/
+/* Disable warning for "format not a string literal" here -QAK */
+/*
+ * This pragma only needs to surround the snprintf() calls with
+ * 'sf_name' in the code below, but early (4.4.7, at least) gcc only
+ * allows diagnostic pragmas to be toggled outside of functions.
+ */
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wformat-nonliteral"
static herr_t
test_multi_opens(char *fname)
{
- hid_t file=-1;
+ hid_t fid = H5I_INVALID_HID;
char super_name[1024]; /*name string "%%s-s.h5"*/
char sf_name[1024]; /*name string "multi_file-s.h5"*/
@@ -962,20 +1160,20 @@ test_multi_opens(char *fname)
HDsnprintf(sf_name, sizeof(sf_name), super_name, fname);
H5E_BEGIN_TRY {
- file = H5Fopen(sf_name, H5F_ACC_RDWR, H5P_DEFAULT);
+ fid = H5Fopen(sf_name, H5F_ACC_RDWR, H5P_DEFAULT);
} H5E_END_TRY;
- return(file >= 0 ? -1 : 0);
-}
+ return(fid >= 0 ? FAIL : SUCCEED);
+} /* end test_multi_opens() */
+#pragma GCC diagnostic pop
+
-
/*-------------------------------------------------------------------------
* Function: test_multi
*
* Purpose: Tests the file handle interface for MUTLI driver
*
- * Return: Success: 0
- * Failure: -1
+ * Return: SUCCEED/FAIL
*
* Programmer: Raymond Lu
* Tuesday, Sept 24, 2002
@@ -988,6 +1186,8 @@ test_multi(void)
hid_t file=-1, fapl=-1, fapl2=-1, dset=-1, space=-1;
hid_t root=-1, attr=-1, aspace=-1, atype=-1;
hid_t access_fapl = -1;
+ hid_t driver_id = -1; /* ID for this VFD */
+ unsigned long driver_flags = 0; /* VFD feature flags */
char filename[1024];
int *fhandle2=NULL, *fhandle=NULL;
hsize_t file_size;
@@ -1044,6 +1244,18 @@ test_multi(void)
TEST_ERROR;
h5_fixname(FILENAME[4], fapl, filename, sizeof filename);
+ /* Check that the VFD feature flags are correct */
+ if ((driver_id = H5Pget_driver(fapl)) < 0)
+ TEST_ERROR
+ if (H5FDdriver_query(driver_id, &driver_flags) < 0)
+ TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR
+ /* Check for extra flags not accounted for above */
+ if(driver_flags != (H5FD_FEAT_DATA_SIEVE
+ | H5FD_FEAT_AGGREGATE_SMALLDATA))
+ TEST_ERROR
+
if((file=H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
@@ -1136,7 +1348,7 @@ test_multi(void)
if((atype = H5Tcopy(H5T_C_S1)) < 0)
TEST_ERROR;
- if(H5Tset_size(atype, strlen(meta) + 1) < 0)
+ if(H5Tset_size(atype, HDstrlen(meta) + 1) < 0)
TEST_ERROR;
if(H5Tset_strpad(atype, H5T_STR_NULLTERM) < 0)
@@ -1171,7 +1383,7 @@ test_multi(void)
PASSED();
- return 0;
+ return SUCCEED;
error:
H5E_BEGIN_TRY {
@@ -1180,18 +1392,19 @@ error:
H5Pclose(fapl);
H5Pclose(fapl2);
H5Fclose(file);
+ H5Aclose(attr);
} H5E_END_TRY;
- return -1;
-}
+ return FAIL;
+} /* end test_multi() */
+
-
/*-------------------------------------------------------------------------
* Function: test_multi_compat
*
* Purpose: Tests the backward compatibility for MULTI driver.
* See if we can open files created with v1.6 library.
* The source file was created by the test/file_handle.c
- * of the v1.6 library. This test verifies the fix for
+ * of the v1.6 library. This test verifies the fix for
* Issue 2598. In v1.6 library, there was EOA for the whole
* MULTI file saved in the super block. We took it out in
* v1.8 library because it's meaningless for the MULTI file.
@@ -1252,8 +1465,9 @@ test_multi_compat(void)
h5_fixname(FILENAME[9], fapl, newname, sizeof newname);
- /* Make copy for the data file in the build directory, to protect the
- * original file in the source directory */
+ /* Make copy for the data file in the build directory, to protect the
+ * original file in the source directory
+ */
HDsprintf(filename_s, "%s-%c.h5", MULTI_COMPAT_BASENAME, 's');
HDsprintf(newname_s, "%s-%c.h5", FILENAME[9], 's');
h5_make_local_copy(filename_s, newname_s);
@@ -1263,7 +1477,8 @@ test_multi_compat(void)
h5_make_local_copy(filename_r, newname_r);
/* Reopen the file for read only. Verify 1.8 library can open file
- * created with 1.6 library. */
+ * created with 1.6 library.
+ */
if((file=H5Fopen(newname, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR;
@@ -1289,7 +1504,7 @@ test_multi_compat(void)
if(H5Fclose(file) < 0)
TEST_ERROR;
- /* Reopen the file for adding another dataset. The new EOA for metadata file
+ /* Reopen the file for adding another dataset. The new EOA for metadata file
* should be written to the file */
if((file=H5Fopen(newname, H5F_ACC_RDWR, fapl)) < 0)
TEST_ERROR;
@@ -1316,7 +1531,7 @@ test_multi_compat(void)
if(H5Fclose(file) < 0)
TEST_ERROR;
- /* Reopen the file for read only again. Verify the library can handle
+ /* Reopen the file for read only again. Verify the library can handle
* the EOA correctly */
if((file=H5Fopen(newname, H5F_ACC_RDONLY, fapl)) < 0)
TEST_ERROR;
@@ -1356,7 +1571,7 @@ error:
return -1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_log
*
@@ -1376,6 +1591,8 @@ test_log(void)
hid_t file = -1;
hid_t fapl = -1;
hid_t access_fapl = -1;
+ hid_t driver_id = -1; /* ID for this VFD */
+ unsigned long driver_flags = 0; /* VFD feature flags */
char filename[1024];
int *fhandle = NULL;
hsize_t file_size = 0;
@@ -1391,6 +1608,24 @@ test_log(void)
TEST_ERROR;
h5_fixname(FILENAME[6], fapl, filename, sizeof filename);
+ /* Check that the VFD feature flags are correct */
+ if ((driver_id = H5Pget_driver(fapl)) < 0)
+ TEST_ERROR
+ if (H5FDdriver_query(driver_id, &driver_flags) < 0)
+ TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_METADATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_ACCUMULATE_METADATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_POSIX_COMPAT_HANDLE)) TEST_ERROR
+ /* Check for extra flags not accounted for above */
+ if(driver_flags != (H5FD_FEAT_AGGREGATE_METADATA
+ | H5FD_FEAT_ACCUMULATE_METADATA
+ | H5FD_FEAT_DATA_SIEVE
+ | H5FD_FEAT_AGGREGATE_SMALLDATA
+ | H5FD_FEAT_POSIX_COMPAT_HANDLE))
+ TEST_ERROR
+
/* Create the test file */
if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
@@ -1443,7 +1678,7 @@ error:
return -1;
}
-
+
/*-------------------------------------------------------------------------
* Function: test_stdio
*
@@ -1463,6 +1698,8 @@ test_stdio(void)
hid_t file = -1;
hid_t fapl = -1;
hid_t access_fapl = -1;
+ hid_t driver_id = -1; /* ID for this VFD */
+ unsigned long driver_flags = 0; /* VFD feature flags */
char filename[1024];
FILE *fhandle = NULL;
hsize_t file_size = 0;
@@ -1476,6 +1713,22 @@ test_stdio(void)
TEST_ERROR;
h5_fixname(FILENAME[7], fapl, filename, sizeof filename);
+ /* Check that the VFD feature flags are correct */
+ if ((driver_id = H5Pget_driver(fapl)) < 0)
+ TEST_ERROR
+ if (H5FDdriver_query(driver_id, &driver_flags) < 0)
+ TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_METADATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_ACCUMULATE_METADATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR
+ /* Check for extra flags not accounted for above */
+ if(driver_flags != (H5FD_FEAT_AGGREGATE_METADATA
+ | H5FD_FEAT_ACCUMULATE_METADATA
+ | H5FD_FEAT_DATA_SIEVE
+ | H5FD_FEAT_AGGREGATE_SMALLDATA))
+ TEST_ERROR
+
if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
@@ -1528,7 +1781,7 @@ error:
}
-
+
/*-------------------------------------------------------------------------
* Function: test_windows
*
@@ -1550,6 +1803,8 @@ test_windows(void)
hid_t file = -1;
hid_t fapl = -1;
hid_t access_fapl = -1;
+ hid_t driver_id = -1; /* ID for this VFD */
+ unsigned long driver_flags = 0; /* VFD feature flags */
char filename[1024];
int *fhandle = NULL;
hsize_t file_size = 0;
@@ -1572,6 +1827,24 @@ test_windows(void)
TEST_ERROR;
h5_fixname(FILENAME[8], fapl, filename, sizeof filename);
+ /* Check that the VFD feature flags are correct */
+ if ((driver_id = H5Pget_driver(fapl)) < 0)
+ TEST_ERROR
+ if (H5FDdriver_query(driver_id, &driver_flags) < 0)
+ TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_METADATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_ACCUMULATE_METADATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_DATA_SIEVE)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_AGGREGATE_SMALLDATA)) TEST_ERROR
+ if(!(driver_flags & H5FD_FEAT_POSIX_COMPAT_HANDLE)) TEST_ERROR
+ /* Check for extra flags not accounted for above */
+ if(driver_flags != (H5FD_FEAT_AGGREGATE_METADATA
+ | H5FD_FEAT_ACCUMULATE_METADATA
+ | H5FD_FEAT_DATA_SIEVE
+ | H5FD_FEAT_AGGREGATE_SMALLDATA
+ | H5FD_FEAT_POSIX_COMPAT_HANDLE))
+ TEST_ERROR
+
if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0)
TEST_ERROR;
@@ -1623,10 +1896,107 @@ error:
return -1;
#endif /* H5_HAVE_WINDOWS */
-}
+} /* end test_windows() */
-
+/*-------------------------------------------------------------------------
+ * Function: test_ros3
+ *
+ * Purpose: Tests the file handle interface for the ROS3 driver
+ *
+ * As the ROS3 driver is 1) read only, 2) requires access
+ * to an S3 server (minio for now), this test is quite
+ * different from the other tests.
+ *
+ * For now, test only fapl & flags. Extend as the
+ * work on the VFD continues.
+ *
+ * Return: Success: 0
+ * Failure: -1
+ *
+ * Programmer: John Mainzer
+ * 7/12/17
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+test_ros3(void)
+{
+#ifdef H5_HAVE_ROS3_VFD
+ hid_t fid = -1; /* file ID */
+ hid_t fapl_id = -1; /* file access property list ID */
+ hid_t fapl_id_out = -1; /* from H5Fget_access_plist */
+ hid_t driver_id = -1; /* ID for this VFD */
+ unsigned long driver_flags = 0; /* VFD feature flags */
+ char filename[1024]; /* filename */
+ void *os_file_handle = NULL; /* OS file handle */
+ hsize_t file_size; /* file size */
+ H5FD_ros3_fapl_t test_ros3_fa;
+ H5FD_ros3_fapl_t ros3_fa_0 =
+ {
+ /* version = */ H5FD_CURR_ROS3_FAPL_T_VERSION,
+ /* authenticate = */ FALSE,
+ /* aws_region = */ "",
+ /* secret_id = */ "",
+ /* secret_key = */ "plugh",
+ };
+#endif /*H5_HAVE_ROS3_VFD */
+
+ TESTING("Read-only S3 file driver");
+
+#ifndef H5_HAVE_ROS3_VFD
+ SKIPPED();
+ return 0;
+#else /* H5_HAVE_ROS3_VFD */
+
+ /* Set property list and file name for ROS3 driver. */
+ if((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+ TEST_ERROR;
+
+ if(H5Pset_fapl_ros3(fapl_id, &ros3_fa_0) < 0)
+ TEST_ERROR;
+
+ /* verify that the ROS3 FAPL entry is set as expected */
+ if(H5Pget_fapl_ros3(fapl_id, &test_ros3_fa) < 0)
+ TEST_ERROR;
+
+ /* need a macro to compare instances of H5FD_ros3_fapl_t */
+ if((test_ros3_fa.version != ros3_fa_0.version) ||
+ (test_ros3_fa.authenticate != ros3_fa_0.authenticate) ||
+ (strcmp(test_ros3_fa.aws_region, ros3_fa_0.aws_region) != 0) ||
+ (strcmp(test_ros3_fa.secret_id, ros3_fa_0.secret_id) != 0) ||
+ (strcmp(test_ros3_fa.secret_key, ros3_fa_0.secret_key) != 0))
+ TEST_ERROR;
+
+ h5_fixname(FILENAME[10], fapl_id, filename, sizeof(filename));
+
+ /* Check that the VFD feature flags are correct */
+ if ((driver_id = H5Pget_driver(fapl_id)) < 0)
+ TEST_ERROR;
+
+ if (H5FDdriver_query(driver_id, &driver_flags) < 0)
+ TEST_ERROR;
+
+ if(!(driver_flags & H5FD_FEAT_DATA_SIEVE))
+ TEST_ERROR
+
+ /* Check for extra flags not accounted for above */
+ if(driver_flags != (H5FD_FEAT_DATA_SIEVE))
+ TEST_ERROR
+
+ PASSED();
+ return 0;
+
+error:
+ H5E_BEGIN_TRY {
+ H5Pclose(fapl_id);
+ H5Pclose(fapl_id_out);
+ H5Fclose(fid);
+ } H5E_END_TRY;
+ return -1;
+#endif /* H5_HAVE_ROS3_VFD */
+} /* end test_ros3() */
+
/*-------------------------------------------------------------------------
* Function: main
*
@@ -1651,22 +2021,24 @@ main(void)
nerrors += test_sec2() < 0 ? 1 : 0;
nerrors += test_core() < 0 ? 1 : 0;
+ nerrors += test_direct() < 0 ? 1 : 0;
nerrors += test_family() < 0 ? 1 : 0;
nerrors += test_family_compat() < 0 ? 1 : 0;
nerrors += test_multi() < 0 ? 1 : 0;
nerrors += test_multi_compat() < 0 ? 1 : 0;
- nerrors += test_direct() < 0 ? 1 : 0;
nerrors += test_log() < 0 ? 1 : 0;
nerrors += test_stdio() < 0 ? 1 : 0;
nerrors += test_windows() < 0 ? 1 : 0;
+ nerrors += test_ros3() < 0 ? 1 : 0;
if(nerrors) {
- HDprintf("***** %d Virtual File Driver TEST%s FAILED! *****\n",
- nerrors, nerrors > 1 ? "S" : "");
- return 1;
- }
+ HDprintf("***** %d Virtual File Driver TEST%s FAILED! *****\n",
+ nerrors, nerrors > 1 ? "S" : "");
+ return 1;
+ } /* end if */
HDprintf("All Virtual File Driver tests passed.\n");
+
return 0;
-}
+} /* end main() */
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
index f088d3b..f19186e 100644
--- a/tools/CMakeLists.txt
+++ b/tools/CMakeLists.txt
@@ -6,6 +6,15 @@ add_subdirectory (lib)
#-- Add the h5diff and test executables
add_subdirectory (h5diff)
+if (BUILD_TESTING)
+# --------------------------------------------------------------------
+# If S3 or HDFS enabled, then we need to test the tools library
+# --------------------------------------------------------------------
+ if (HDF5_ENABLE_ROS3_VFD OR HDF5_ENABLE_HDFS)
+ add_subdirectory (libtest)
+ endif ()
+endif ()
+
#-- Add the h5ls executable
add_subdirectory (h5ls)
diff --git a/tools/h5copy/h5copy.c b/tools/h5copy/h5copy.c
index 1800810..9490cd9 100644
--- a/tools/h5copy/h5copy.c
+++ b/tools/h5copy/h5copy.c
@@ -381,10 +381,10 @@ main (int argc, const char *argv[])
*-------------------------------------------------------------------------*/
if (verbose) {
- printf("Copying file <%s> and object <%s> to file <%s> and object <%s>\n",
+ HDprintf("Copying file <%s> and object <%s> to file <%s> and object <%s>\n",
fname_src, oname_src, fname_dst, oname_dst);
if (flag) {
- printf("Using %s flag\n", str_flag);
+ HDprintf("Using %s flag\n", str_flag);
}
}
@@ -419,7 +419,7 @@ main (int argc, const char *argv[])
/* Display some output if requested */
if(verbose)
- printf("%s: Creating parent groups\n", h5tools_getprogname());
+ HDprintf("%s: Creating parent groups\n", h5tools_getprogname());
} /* end if */
else {
/* error, if parent groups doesn't already exist in destination file */
@@ -490,7 +490,7 @@ main (int argc, const char *argv[])
leave(EXIT_SUCCESS);
done:
- printf("Error in copy...Exiting\n");
+ HDprintf("Error in copy...Exiting\n");
/* free link info path */
if (linkinfo.trg_path)
diff --git a/tools/h5copy/h5copygentest.c b/tools/h5copy/h5copygentest.c
index fdc165e..049f2d5 100644
--- a/tools/h5copy/h5copygentest.c
+++ b/tools/h5copy/h5copygentest.c
@@ -325,7 +325,7 @@ static void gent_nested_vl(hid_t loc_id)
* Function: gent_att_compound_vlstr
*
* Purpose: Generate a dataset and a group.
- * Both has an attribute with a compound datatype consisting
+ * Both has an attribute with a compound datatype consisting
* of a variable length string
*
*-------------------------------------------------------------------------
@@ -336,15 +336,15 @@ static void gent_att_compound_vlstr(hid_t loc_id)
int i;
const char *v;
} s1;
- hsize_t dim[1] = {1}; /* Dimension size */
- hid_t sid = -1; /* Dataspace ID */
- hid_t tid = -1; /* Datatype ID */
- hid_t aid = -1; /* Attribute ID */
- hid_t did = -1; /* Dataset ID */
- hid_t gid = -1; /* Group ID */
- hid_t vl_str_tid = -1; /* Variable length datatype ID */
- hid_t cmpd_tid = -1; /* Compound datatype ID */
- hid_t null_sid = -1; /* Null dataspace ID */
+ hsize_t dim[1] = {1}; /* Dimension size */
+ hid_t sid = -1; /* Dataspace ID */
+ hid_t tid = -1; /* Datatype ID */
+ hid_t aid = -1; /* Attribute ID */
+ hid_t did = -1; /* Dataset ID */
+ hid_t gid = -1; /* Group ID */
+ hid_t vl_str_tid = -1; /* Variable length datatype ID */
+ hid_t cmpd_tid = -1; /* Compound datatype ID */
+ hid_t null_sid = -1; /* Null dataspace ID */
s1 buf; /* Buffer */
buf.i = 9;
diff --git a/tools/h5diff/h5diffgentest.c b/tools/h5diff/h5diffgentest.c
index f59cb47..e6b5103 100644
--- a/tools/h5diff/h5diffgentest.c
+++ b/tools/h5diff/h5diffgentest.c
@@ -106,10 +106,16 @@ size_t H5TOOLS_MALLOCSIZE = (128 * 1024 * 1024);
#define SPACE1_DIM1 0
#define SPACE1_DIM2 0
+/* Error macros */
+#define AT() HDprintf("ERROR at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC);
+#define PROGRAM_ERROR {AT(); goto error;}
+
/* A UD link traversal function. Shouldn't actually be called. */
-static hid_t UD_traverse(H5_ATTR_UNUSED const char * link_name,
+static hid_t
+UD_traverse(H5_ATTR_UNUSED const char * link_name,
H5_ATTR_UNUSED hid_t cur_group, H5_ATTR_UNUSED const void * udata,
- H5_ATTR_UNUSED size_t udata_size, H5_ATTR_UNUSED hid_t lapl_id) {
+ H5_ATTR_UNUSED size_t udata_size, H5_ATTR_UNUSED hid_t lapl_id)
+{
return -1;
}
@@ -270,7 +276,7 @@ int main(void)
/* string dataset and attribute. HDFFV-10028 */
test_objs_strings(DIFF_STRINGS1, DIFF_STRINGS2);
- return 0;
+ return EXIT_SUCCESS;
}
/*-------------------------------------------------------------------------
@@ -4802,9 +4808,9 @@ out:
* types.
* h5diff should show non-comparable output from these common objects.
*-------------------------------------------------------------------------*/
-static void test_objs_nocomparables(const char *fname1, const char *fname2)
+static void
+test_objs_nocomparables(const char *fname1, const char *fname2)
{
- herr_t status = SUCCEED;
hid_t fid1 = -1;
hid_t fid2 = -1;
hid_t topgid1 = -1;
@@ -4821,110 +4827,90 @@ static void test_objs_nocomparables(const char *fname1, const char *fname2)
* Open file(s) to add objects
*------------------------------------------------------------------------*/
/* file1 */
- fid1 = H5Fopen(fname1, H5F_ACC_RDWR, H5P_DEFAULT);
- if (fid1 < 0) {
- HDfprintf(stderr, "Error: %s> H5Fopen failed.\n", fname1);
- status = FAIL;
- goto out;
- }
+ if((fid1 = H5Fopen(fname1, H5F_ACC_RDWR, H5P_DEFAULT)) < 0)
+ PROGRAM_ERROR
/* file2 */
- fid2 = H5Fopen(fname2, H5F_ACC_RDWR, H5P_DEFAULT);
- if (fid2 < 0) {
- HDfprintf(stderr, "Error: %s> H5Fopen failed.\n", fname2);
- status = FAIL;
- goto out;
- }
+ if((fid2 = H5Fopen(fname2, H5F_ACC_RDWR, H5P_DEFAULT)) < 0)
+ PROGRAM_ERROR
/*-----------------------------------------------------------------------
* in file1 : add member objects
*------------------------------------------------------------------------*/
/* parent group */
- topgid1 = H5Gcreate2(fid1, "diffobjtypes", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- if (topgid1 < 0) {
- HDfprintf(stderr, "Error: %s> H5Gcreate2 failed.\n", fname1);
- status = FAIL;
- goto out;
- }
+ if((topgid1 = H5Gcreate2(fid1, "diffobjtypes", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ PROGRAM_ERROR
/* dataset */
- status = write_dset(topgid1, 1, dims, "obj1", H5T_NATIVE_INT, data1);
- if (status == FAIL) {
- HDfprintf(stderr, "Error: %s> write_dset failed\n", fname1);
- goto out;
- }
+ if(write_dset(topgid1, 1, dims, "obj1", H5T_NATIVE_INT, data1) < 0)
+ PROGRAM_ERROR
/* group */
- gid1 = H5Gcreate2(topgid1, "obj2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- if (gid1 < 0) {
- HDfprintf(stderr, "Error: %s> H5Gcreate2 failed.\n", fname1);
- status = FAIL;
- goto out;
- }
+ if((gid1 = H5Gcreate2(topgid1, "obj2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ PROGRAM_ERROR
/* committed type */
- tid1 = H5Tcopy(H5T_NATIVE_INT);
- status = H5Tcommit2(topgid1, "obj3", tid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- if (status < 0) {
- HDfprintf(stderr, "Error: %s> H5Tcommit2 failed.\n", fname1);
- goto out;
- }
+ if((tid1 = H5Tcopy(H5T_NATIVE_INT)) < 0)
+ PROGRAM_ERROR
+ if(H5Tcommit2(topgid1, "obj3", tid1, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0)
+ PROGRAM_ERROR
/*-----------------------------------------------------------------------
* in file2 : add member objects
*------------------------------------------------------------------------*/
/* parent group */
- topgid2 = H5Gcreate2(fid2, "diffobjtypes", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- if (topgid2 < 0) {
- HDfprintf(stderr, "Error: %s> H5Gcreate2 failed.\n", fname2);
- status = FAIL;
- goto out;
- }
+ if((topgid2 = H5Gcreate2(fid2, "diffobjtypes", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ PROGRAM_ERROR
/* group */
- gid2 = H5Gcreate2(topgid2, "obj1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- if (gid2 < 0) {
- HDfprintf(stderr, "Error: %s> H5Gcreate2 failed.\n", fname2);
- status = FAIL;
- goto out;
- }
+ if((gid2 = H5Gcreate2(topgid2, "obj1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ PROGRAM_ERROR
/* committed type */
- tid2 = H5Tcopy(H5T_NATIVE_INT);
- status = H5Tcommit2(topgid2, "obj2", tid2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
- if (status < 0) {
- HDfprintf(stderr, "Error: %s> H5Tcommit2 failed.\n", fname2);
- goto out;
- }
+ if((tid2 = H5Tcopy(H5T_NATIVE_INT)) < 0)
+ PROGRAM_ERROR
+ if(H5Tcommit2(topgid2, "obj2", tid2, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0)
+ PROGRAM_ERROR
/* dataset */
- status = write_dset(topgid2, 1, dims, "obj3", H5T_NATIVE_INT, data2);
- if (status == FAIL) {
- HDfprintf(stderr, "Error: %s> write_dset failed\n", fname2);
- goto out;
- }
+ if(write_dset(topgid2, 1, dims, "obj3", H5T_NATIVE_INT, data2) < 0)
+ PROGRAM_ERROR
-out:
/*-----------------------------------------------------------------------
* Close IDs
*-----------------------------------------------------------------------*/
- if (fid1)
+ if(H5Fclose(fid1) < 0)
+ PROGRAM_ERROR
+ if(H5Fclose(fid2) < 0)
+ PROGRAM_ERROR
+ if(H5Gclose(topgid1) < 0)
+ PROGRAM_ERROR
+ if(H5Gclose(topgid2) < 0)
+ PROGRAM_ERROR
+ if(H5Gclose(gid1) < 0)
+ PROGRAM_ERROR
+ if(H5Gclose(gid2) < 0)
+ PROGRAM_ERROR
+ if(H5Tclose(tid1) < 0)
+ PROGRAM_ERROR
+ if(H5Tclose(tid2) < 0)
+ PROGRAM_ERROR
+
+ return;
+
+error:
+ H5E_BEGIN_TRY {
H5Fclose(fid1);
- if (fid2)
H5Fclose(fid2);
- if (topgid1)
H5Gclose(topgid1);
- if (topgid2)
H5Gclose(topgid2);
- if (gid1)
H5Gclose(gid1);
- if (gid2)
H5Gclose(gid2);
- if (tid1)
H5Tclose(tid1);
- if (tid2)
H5Tclose(tid2);
+ } H5E_END_TRY;
+ return;
}
static hid_t mkstr(int size, H5T_str_t pad)
@@ -7986,29 +7972,33 @@ int write_dset(hid_t loc_id, int rank, hsize_t *dims, const char *name, hid_t ti
hid_t sid = -1;
/* create a space */
- if ((sid = H5Screate_simple(rank, dims, NULL)) < 0)
- goto out;
+ if((sid = H5Screate_simple(rank, dims, NULL)) < 0)
+ PROGRAM_ERROR
/* create the dataset */
- if ((did = H5Dcreate2(loc_id, name, tid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
- goto out;
+ if((did = H5Dcreate2(loc_id, name, tid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ PROGRAM_ERROR
/* write */
- if (buf) {
- if (H5Dwrite(did, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0)
- goto out;
- }
+ if(buf)
+ if(H5Dwrite(did, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0)
+ PROGRAM_ERROR
/* close */
- H5Dclose(did);
- H5Sclose(sid);
+ if(H5Dclose(did) < 0)
+ PROGRAM_ERROR
+ if(H5Sclose(sid) < 0)
+ PROGRAM_ERROR
return SUCCEED;
-out:
+error:
+
+ H5E_BEGIN_TRY {
+ H5Dclose(did);
+ H5Sclose(sid);
+ } H5E_END_TRY;
- H5Dclose(did);
- H5Sclose(sid);
return FAIL;
-}
+} /* end write_dset() */
diff --git a/tools/h5diff/ph5diff_main.c b/tools/h5diff/ph5diff_main.c
index 83240cb..380ab3b 100644
--- a/tools/h5diff/ph5diff_main.c
+++ b/tools/h5diff/ph5diff_main.c
@@ -68,7 +68,7 @@ int main(int argc, const char *argv[])
if(g_nTasks == 1)
{
- printf("Only 1 task available...doing serial diff\n");
+ HDprintf("Only 1 task available...doing serial diff\n");
g_Parallel = 0;
@@ -148,12 +148,12 @@ ph5diff_worker(int nID)
/* Open the files */
if ((file1_id = H5Fopen (filenames[0], H5F_ACC_RDONLY, H5P_DEFAULT)) < 0)
{
- printf ("h5diff Task [%d]: <%s>: unable to open file\n", nID, filenames[0]);
+ HDprintf("h5diff Task [%d]: <%s>: unable to open file\n", nID, filenames[0]);
MPI_Abort(MPI_COMM_WORLD, 0);
}
if ((file2_id = H5Fopen (filenames[1], H5F_ACC_RDONLY, H5P_DEFAULT)) < 0)
{
- printf ("h5diff Task [%d]: <%s>: unable to open file\n", nID, filenames[1]);
+ HDprintf("h5diff Task [%d]: <%s>: unable to open file\n", nID, filenames[1]);
MPI_Abort(MPI_COMM_WORLD, 0);
}
/* enable error reporting */
@@ -170,7 +170,7 @@ ph5diff_worker(int nID)
/* Make certain we've received the filenames and opened the files already */
if(file1_id < 0 || file2_id < 0)
{
- printf("ph5diff_worker: ERROR: work received before/without filenames\n");
+ HDprintf("ph5diff_worker: ERROR: work received before/without filenames\n");
break;
}
@@ -199,7 +199,7 @@ ph5diff_worker(int nID)
char out_data[PRINT_DATA_MAX_SIZE];
int tmp;
- memset(out_data, 0, PRINT_DATA_MAX_SIZE);
+ HDmemset(out_data, 0, PRINT_DATA_MAX_SIZE);
i=0;
rewind(overflow_file);
@@ -210,7 +210,7 @@ ph5diff_worker(int nID)
{
MPI_Send(out_data, PRINT_DATA_MAX_SIZE, MPI_BYTE, 0, MPI_TAG_PRINT_DATA, MPI_COMM_WORLD);
i=0;
- memset(out_data, 0, PRINT_DATA_MAX_SIZE);
+ HDmemset(out_data, 0, PRINT_DATA_MAX_SIZE);
}
}
@@ -221,8 +221,8 @@ ph5diff_worker(int nID)
overflow_file = NULL;
}
- fflush(stdout);
- memset(outBuff, 0, OUTBUFF_SIZE);
+ HDfflush(stdout);
+ HDmemset(outBuff, 0, OUTBUFF_SIZE);
outBuffOffset = 0;
MPI_Send(&diffs, sizeof(diffs), MPI_BYTE, 0, MPI_TAG_TOK_RETURN, MPI_COMM_WORLD);
@@ -238,7 +238,7 @@ ph5diff_worker(int nID)
}
else
{
- printf("ph5diff_worker: ERROR: invalid tag (%d) received\n", Status.MPI_TAG);
+ HDprintf("ph5diff_worker: ERROR: invalid tag (%d) received\n", Status.MPI_TAG);
break;
}
@@ -266,14 +266,14 @@ void print_manager_output(void)
/* If there was something we buffered, let's print it now */
if( (outBuffOffset>0) && g_Parallel)
{
- printf("%s", outBuff);
+ HDprintf("%s", outBuff);
if(overflow_file)
{
int tmp;
rewind(overflow_file);
- while((tmp = getc(overflow_file)) >= 0)
- putchar(tmp);
+ while((tmp = HDgetc(overflow_file)) >= 0)
+ HDputchar(tmp);
fclose(overflow_file);
overflow_file = NULL;
}
@@ -321,6 +321,6 @@ void h5diff_exit(int status)
/* Always exit(0), since MPI implementations do weird stuff when they
* receive a non-zero exit value. - QAK
*/
- exit(0);
+ HDexit(0);
}
diff --git a/tools/h5dump/h5dump.c b/tools/h5dump/h5dump.c
index 9c1dfa3..7c5b3fa 100644
--- a/tools/h5dump/h5dump.c
+++ b/tools/h5dump/h5dump.c
@@ -24,6 +24,27 @@ static int doxml = 0;
static int useschema = 1;
static const char *xml_dtd_uri = NULL;
+#ifdef H5_HAVE_ROS3_VFD
+static H5FD_ros3_fapl_t ros3_fa = {
+ 1, /* version */
+ false, /* authenticate */
+ "", /* aws region */
+ "", /* access key id */
+ "", /* secret access key */
+};
+#endif /* H5_HAVE_ROS3_VFD */
+
+#ifdef H5_HAVE_LIBHDFS
+static H5FD_hdfs_fapl_t hdfs_fa = {
+ 1, /* fapl version */
+ "localhost", /* namenode name */
+ 0, /* namenode port */
+ "", /* kerberos ticket cache */
+ "", /* user name */
+ 2048, /* stream buffer size */
+};
+#endif /* H5_HAVE_LIBHDFS */
+
/* module-scoped variables for XML option */
#define DEFAULT_XSD "http://www.hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd"
#define DEFAULT_DTD "http://www.hdfgroup.org/HDF5/XML/DTD/HDF5-File.dtd"
@@ -186,6 +207,8 @@ static struct long_options l_opts[] = {
{ "no-compact-subset", no_arg, 'C' },
{ "ddl", optional_arg, 'O' },
{ "any_path", require_arg, 'N' },
+ { "s3-cred", require_arg, '$' },
+ { "hdfs-attrs", require_arg, '#' },
{ NULL, 0, '\0' }
};
@@ -239,6 +262,16 @@ usage(const char *prog)
PRINTVALSTREAM(rawoutstream, " -b B, --binary=B Binary file output, of form B\n");
PRINTVALSTREAM(rawoutstream, " -O F, --ddl=F Output ddl text into file F\n");
PRINTVALSTREAM(rawoutstream, " Use blank(empty) filename F to suppress ddl display\n");
+ PRINTVALSTREAM(rawoutstream, " --s3-cred=<cred> Supply S3 authentication information to \"ros3\" vfd.\n");
+ PRINTVALSTREAM(rawoutstream, " <cred> :: \"(<aws-region>,<access-id>,<access-key>)\"\n");
+ PRINTVALSTREAM(rawoutstream, " If absent or <cred> -> \"(,,)\", no authentication.\n");
+ PRINTVALSTREAM(rawoutstream, " Has no effect is filedriver is not `ros3'.\n");
+ PRINTVALSTREAM(rawoutstream, " --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.\n");
+ PRINTVALSTREAM(rawoutstream, " For use with \"--filedriver=hdfs\"\n");
+ PRINTVALSTREAM(rawoutstream, " <attrs> :: (<namenode name>,<namenode port>,\n");
+ PRINTVALSTREAM(rawoutstream, " <kerberos cache path>,<username>,\n");
+ PRINTVALSTREAM(rawoutstream, " <buffer size>)\n");
+ PRINTVALSTREAM(rawoutstream, " Any absent attribute will use a default value.\n");
PRINTVALSTREAM(rawoutstream, "--------------- Object Options ---------------\n");
PRINTVALSTREAM(rawoutstream, " -a P, --attribute=P Print the specified attribute\n");
PRINTVALSTREAM(rawoutstream, " If an attribute name contains a slash (/), escape the\n");
@@ -1268,6 +1301,126 @@ end_collect:
hand = NULL;
h5tools_setstatus(EXIT_SUCCESS);
goto done;
+
+ case '$':
+#ifndef H5_HAVE_ROS3_VFD
+ error_msg("Read-Only S3 VFD not enabled.\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+#else
+ /* s3 credential */
+ {
+ char **s3_cred = NULL;
+ char *s3_cred_string = NULL;
+ const char *ccred[3];
+ unsigned nelems = 0;
+ if ( FAIL ==
+ parse_tuple(opt_arg, ',',
+ &s3_cred_string, &nelems, &s3_cred))
+ {
+ error_msg("unable to parse malformed s3 credentials\n");
+ usage(h5tools_getprogname());
+ free_handler(hand, argc);
+ hand= NULL;
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ if (nelems != 3) {
+ error_msg("s3 credentials expects 3 elements\n");
+ usage(h5tools_getprogname());
+ free_handler(hand, argc);
+ hand= NULL;
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ ccred[0] = (const char *)s3_cred[0];
+ ccred[1] = (const char *)s3_cred[1];
+ ccred[2] = (const char *)s3_cred[2];
+ if (0 == h5tools_populate_ros3_fapl(&ros3_fa, ccred)) {
+ error_msg("Invalid S3 credentials\n");
+ usage(h5tools_getprogname());
+ free_handler(hand, argc);
+ hand= NULL;
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ HDfree(s3_cred);
+ HDfree(s3_cred_string);
+ } /* s3 credential block */
+ break;
+#endif /* H5_HAVE_ROS3_VFD */
+
+ case '#':
+#ifndef H5_HAVE_LIBHDFS
+ error_msg("HDFS VFD is not enabled.\n");
+ goto error;
+#else
+ {
+ /* read hdfs properties tuple and store values in `hdfs_fa`
+ */
+ unsigned nelems = 0;
+ char *props_src = NULL;
+ char **props = NULL;
+ unsigned long k = 0;
+ if (FAIL == parse_tuple(
+ (const char *)opt_arg,
+ ',',
+ &props_src,
+ &nelems,
+ &props))
+ {
+ error_msg("unable to parse hdfs properties tuple\n");
+ goto error;
+ }
+ /* sanity-check tuple count
+ */
+ if (nelems != 5) {
+ h5tools_setstatus(EXIT_FAILURE);
+ goto error;
+ }
+ /* Populate fapl configuration structure with given
+ * properties.
+ * WARNING: No error-checking is done on length of input
+ * strings... Silent overflow is possible, albeit
+ * unlikely.
+ */
+ if (strncmp(props[0], "", 1)) {
+ HDstrncpy(hdfs_fa.namenode_name,
+ (const char *)props[0],
+ HDstrlen(props[0]));
+ }
+ if (strncmp(props[1], "", 1)) {
+ k = strtoul((const char *)props[1], NULL, 0);
+ if (errno == ERANGE) {
+ h5tools_setstatus(EXIT_FAILURE);
+ goto error;
+ }
+ hdfs_fa.namenode_port = (int32_t)k;
+ }
+ if (strncmp(props[2], "", 1)) {
+ HDstrncpy(hdfs_fa.kerberos_ticket_cache,
+ (const char *)props[2],
+ HDstrlen(props[2]));
+ }
+ if (strncmp(props[3], "", 1)) {
+ HDstrncpy(hdfs_fa.user_name,
+ (const char *)props[3],
+ HDstrlen(props[3]));
+ }
+ if (strncmp(props[4], "", 1)) {
+ k = strtoul((const char *)props[4], NULL, 0);
+ if (errno == ERANGE) {
+ h5tools_setstatus(EXIT_FAILURE);
+ goto error;
+ }
+ hdfs_fa.stream_buffer_size = (int32_t)k;
+ }
+ HDfree(props);
+ HDfree(props_src);
+ }
+#endif /* H5_HAVE_LIBHDFS */
+ break;
+
case '?':
default:
usage(h5tools_getprogname());
@@ -1340,6 +1493,7 @@ main(int argc, const char *argv[])
{
hid_t fid = -1;
hid_t gid = -1;
+ hid_t fapl_id = H5P_DEFAULT;
H5E_auto2_t func;
H5E_auto2_t tools_func;
H5O_info_t oi;
@@ -1426,10 +1580,55 @@ main(int argc, const char *argv[])
/* Initialize indexing options */
h5trav_set_index(sort_by, sort_order);
+ if (driver != NULL) {
+ void *conf_fa = NULL;
+
+ if (!strcmp(driver, "ros3")) {
+#ifndef H5_HAVE_ROS3_VFD
+ error_msg("Read-Only S3 VFD not enabled.\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+#else
+ conf_fa = (void *)&ros3_fa;
+#endif /* H5_HAVE_ROS3_VFD */
+ } else if (!HDstrcmp(driver, "hdfs")) {
+#ifndef H5_HAVE_LIBHDFS
+ error_msg("HDFS VFD is not enabled.\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+#else
+ conf_fa = (void *)&hdfs_fa;
+#endif /* H5_HAVE_LIBHDFS */
+ }
+
+ if (conf_fa != NULL) {
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ if (fapl_id < 0) {
+ error_msg("unable to create fapl entry\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ if (0 == h5tools_set_configured_fapl(
+ fapl_id,
+ driver, /* guaranteed "ros3" or "hdfs" */
+ conf_fa)) /* appropriate to driver */
+ {
+ error_msg("unable to set fapl\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ }
+ } /* driver defined */
+
while(opt_ind < argc) {
fname = HDstrdup(argv[opt_ind++]);
- fid = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, driver, NULL, 0);
+ if (fapl_id != H5P_DEFAULT) {
+ fid = H5Fopen(fname, H5F_ACC_RDONLY, fapl_id);
+ }
+ else {
+ fid = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, driver, NULL, 0);
+ }
if (fid < 0) {
error_msg("unable to open file \"%s\"\n", fname);
@@ -1610,6 +1809,11 @@ done:
/* Free tables for objects */
table_list_free();
+ if (fapl_id != H5P_DEFAULT && 0 < H5Pclose(fapl_id)) {
+ error_msg("Can't close fapl entry\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ }
+
if(fid >=0)
if (H5Fclose(fid) < 0)
h5tools_setstatus(EXIT_FAILURE);
@@ -1631,127 +1835,7 @@ done:
H5Eset_auto2(H5E_DEFAULT, func, edata);
leave(h5tools_getstatus());
-}
-
-/*-------------------------------------------------------------------------
- * Function: h5_fileaccess
- *
- * Purpose: Returns a file access template which is the default template
- * but with a file driver set according to the constant or
- * environment variable HDF5_DRIVER
- *
- * Return: Success: A file access property list
- *
- * Failure: -1
- *
- * Programmer: Robb Matzke
- * Thursday, November 19, 1998
- *
- * Modifications:
- *
- *-------------------------------------------------------------------------
- */
-hid_t
-h5_fileaccess(void)
-{
- static const char *multi_letters = "msbrglo";
- const char *val = NULL;
- const char *name;
- char s[1024];
- hid_t fapl = -1;
-
- /* First use the environment variable, then the constant */
- val = HDgetenv("HDF5_DRIVER");
-#ifdef HDF5_DRIVER
- if (!val) val = HDF5_DRIVER;
-#endif
-
- if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) return -1;
- if (!val || !*val) return fapl; /*use default*/
-
- HDstrncpy(s, val, sizeof s);
- s[sizeof(s)-1] = '\0';
- if (NULL==(name=HDstrtok(s, " \t\n\r"))) return fapl;
-
- if (!HDstrcmp(name, "sec2")) {
- /* Unix read() and write() system calls */
- if (H5Pset_fapl_sec2(fapl)<0) return -1;
- }
- else if (!HDstrcmp(name, "stdio")) {
- /* Standard C fread() and fwrite() system calls */
- if (H5Pset_fapl_stdio(fapl)<0) return -1;
- }
- else if (!HDstrcmp(name, "core")) {
- /* In-core temporary file with 1MB increment */
- if (H5Pset_fapl_core(fapl, 1024*1024, FALSE)<0) return -1;
- }
- else if (!HDstrcmp(name, "split")) {
- /* Split meta data and raw data each using default driver */
- if (H5Pset_fapl_split(fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0)
- return -1;
- }
- else if (!HDstrcmp(name, "multi")) {
- /* Multi-file driver, general case of the split driver */
- H5FD_mem_t memb_map[H5FD_MEM_NTYPES];
- hid_t memb_fapl[H5FD_MEM_NTYPES];
- const char *memb_name[H5FD_MEM_NTYPES];
- char sv[H5FD_MEM_NTYPES][1024];
- haddr_t memb_addr[H5FD_MEM_NTYPES];
- H5FD_mem_t mt;
-
- HDmemset(memb_map, 0, sizeof memb_map);
- HDmemset(memb_fapl, 0, sizeof memb_fapl);
- HDmemset(memb_name, 0, sizeof memb_name);
- HDmemset(memb_addr, 0, sizeof memb_addr);
-
- if(HDstrlen(multi_letters)==H5FD_MEM_NTYPES) {
- for (mt=H5FD_MEM_DEFAULT; mt<H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t,mt)) {
- memb_fapl[mt] = H5P_DEFAULT;
- memb_map[mt] = mt;
- HDsprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]);
- memb_name[mt] = sv[mt];
- memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10);
- }
- }
- else {
- error_msg("Bad multi_letters list\n");
- return FAIL;
- }
-
- if (H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name, memb_addr, FALSE) < 0)
- return -1;
- }
- else if (!HDstrcmp(name, "family")) {
- hsize_t fam_size = 100*1024*1024; /*100 MB*/
-
- /* Family of files, each 1MB and using the default driver */
- if ((val=HDstrtok(NULL, " \t\n\r")))
- fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024);
- if (H5Pset_fapl_family(fapl, fam_size, H5P_DEFAULT)<0)
- return -1;
- }
- else if (!HDstrcmp(name, "log")) {
- long log_flags = H5FD_LOG_LOC_IO;
-
- /* Log file access */
- if ((val = HDstrtok(NULL, " \t\n\r")))
- log_flags = HDstrtol(val, NULL, 0);
-
- if (H5Pset_fapl_log(fapl, NULL, (unsigned)log_flags, 0) < 0)
- return -1;
- }
- else if (!HDstrcmp(name, "direct")) {
- /* Substitute Direct I/O driver with sec2 driver temporarily because
- * some output has sec2 driver as the standard. */
- if (H5Pset_fapl_sec2(fapl)<0) return -1;
- }
- else {
- /* Unknown driver */
- return -1;
- }
-
- return fapl;
-}
+} /* main */
/*-------------------------------------------------------------------------
diff --git a/tools/h5dump/h5dump_ddl.c b/tools/h5dump/h5dump_ddl.c
index 46373d2..c324a4a 100644
--- a/tools/h5dump/h5dump_ddl.c
+++ b/tools/h5dump/h5dump_ddl.c
@@ -155,7 +155,7 @@ dump_attr_cb(hid_t oid, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *
return ret;
}
-
+
/*-------------------------------------------------------------------------
* Function: dump_all_cb
*
@@ -181,6 +181,7 @@ static herr_t
dump_all_cb(hid_t group, const char *name, const H5L_info_t *linfo, void H5_ATTR_UNUSED *op_data)
{
hid_t obj;
+ hid_t dapl_id = H5P_DEFAULT; /* dataset access property list ID */
herr_t ret = SUCCEED;
char *obj_path = NULL; /* Full path of object */
h5tools_str_t buffer; /* string into which to render */
@@ -267,7 +268,7 @@ dump_all_cb(hid_t group, const char *name, const H5L_info_t *linfo, void H5_ATTR
break;
case H5O_TYPE_DATASET:
- if((obj = H5Dopen2(group, name, H5P_DEFAULT)) >= 0) {
+ if((obj = H5Dopen2(group, name, dapl_id)) >= 0) {
if(oinfo.rc > 1 || hit_elink) {
obj_t *found_obj; /* Found object */
@@ -304,6 +305,8 @@ dump_all_cb(hid_t group, const char *name, const H5L_info_t *linfo, void H5_ATTR
h5tools_setstatus(EXIT_FAILURE);
ret = FAIL;
+ if (dapl_id != H5P_DEFAULT)
+ H5Pclose(dapl_id);
H5Dclose(obj);
goto done;
}
@@ -341,6 +344,8 @@ dump_all_cb(hid_t group, const char *name, const H5L_info_t *linfo, void H5_ATTR
h5tools_str_append(&buffer, "%s", h5tools_dump_header_format->datasetend);
h5tools_render_element(rawoutstream, outputformat, &ctx, &buffer, &curr_pos, (size_t)outputformat->line_ncols, (hsize_t)0, (hsize_t)0);
+ if (dapl_id != H5P_DEFAULT)
+ H5Pclose(dapl_id);
H5Dclose(obj);
goto done;
}
@@ -350,9 +355,13 @@ dump_all_cb(hid_t group, const char *name, const H5L_info_t *linfo, void H5_ATTR
} /* end if */
dump_function_table->dump_dataset_function(obj, name, NULL);
+ if (dapl_id != H5P_DEFAULT)
+ H5Pclose(dapl_id);
H5Dclose(obj);
}
else {
+ if (dapl_id != H5P_DEFAULT)
+ H5Pclose(dapl_id);
error_msg("unable to dump dataset \"%s\"\n", name);
h5tools_setstatus(EXIT_FAILURE);
ret = FAIL;
@@ -1133,7 +1142,7 @@ dump_data(hid_t obj_id, int obj_data, struct subset_t *sset, int display_index)
h5tools_dump_data(rawoutstream, outputformat, &ctx, obj_id, print_dataset, sset, display_index, display_char);
}
-
+
/*-------------------------------------------------------------------------
* Function: dump_fcpl
*
@@ -1648,10 +1657,11 @@ handle_datasets(hid_t fid, const char *dset, void *data, int pe, const char *dis
{
H5O_info_t oinfo;
hid_t dsetid;
+ hid_t dapl_id = H5P_DEFAULT; /* dataset access property list ID */
struct subset_t *sset = (struct subset_t *)data;
const char *real_name = display_name ? display_name : dset;
- if((dsetid = H5Dopen2(fid, dset, H5P_DEFAULT)) < 0) {
+ if((dsetid = H5Dopen2(fid, dset, dapl_id)) < 0) {
if (pe)
handle_links(fid, dset, data, pe, display_name);
return;
@@ -1775,7 +1785,8 @@ handle_datasets(hid_t fid, const char *dset, void *data, int pe, const char *dis
dump_dataset(dsetid, real_name, sset);
dump_indent -= COL;
}
-
+ if (dapl_id != H5P_DEFAULT)
+ H5Pclose(dapl_id);
if(H5Dclose(dsetid) < 0)
h5tools_setstatus(EXIT_FAILURE);
}
@@ -1883,9 +1894,6 @@ handle_links(hid_t fid, const char *links, void H5_ATTR_UNUSED * data, int H5_AT
break;
case H5L_TYPE_EXTERNAL:
- begin_obj(h5tools_dump_header_format->udlinkbegin, links, h5tools_dump_header_format->udlinkblockbegin);
- PRINTVALSTREAM(rawoutstream, "\n");
- indentation(COL);
begin_obj(h5tools_dump_header_format->extlinkbegin, links, h5tools_dump_header_format->extlinkblockbegin);
PRINTVALSTREAM(rawoutstream, "\n");
if(H5Lget_val(fid, links, buf, linfo.u.val_size, H5P_DEFAULT) >= 0) {
@@ -1894,8 +1902,6 @@ handle_links(hid_t fid, const char *links, void H5_ATTR_UNUSED * data, int H5_AT
if(H5Lunpack_elink_val(buf, linfo.u.val_size, NULL, &elink_file, &elink_path)>=0) {
indentation(COL);
- PRINTSTREAM(rawoutstream, "LINKCLASS %d\n", linfo.type);
- indentation(COL);
PRINTSTREAM(rawoutstream, "TARGETFILE \"%s\"\n", elink_file);
indentation(COL);
PRINTSTREAM(rawoutstream, "TARGETPATH \"%s\"\n", elink_path);
@@ -1919,9 +1925,6 @@ handle_links(hid_t fid, const char *links, void H5_ATTR_UNUSED * data, int H5_AT
begin_obj(h5tools_dump_header_format->udlinkbegin, links, h5tools_dump_header_format->udlinkblockbegin);
PRINTVALSTREAM(rawoutstream, "\n");
indentation(COL);
- begin_obj(h5tools_dump_header_format->udlinkbegin, links, h5tools_dump_header_format->udlinkblockbegin);
- PRINTVALSTREAM(rawoutstream, "\n");
- indentation(COL);
PRINTSTREAM(rawoutstream, "LINKCLASS %d\n", linfo.type);
end_obj(h5tools_dump_header_format->udlinkend, h5tools_dump_header_format->udlinkblockend);
break;
@@ -2007,7 +2010,7 @@ handle_datatypes(hid_t fid, const char *type, void H5_ATTR_UNUSED * data, int pe
}
}
-
+
/*-------------------------------------------------------------------------
* Function: dump_extlink
*
diff --git a/tools/h5dump/h5dump_defines.h b/tools/h5dump/h5dump_defines.h
index b08fbb1..7a9d4c0 100644
--- a/tools/h5dump/h5dump_defines.h
+++ b/tools/h5dump/h5dump_defines.h
@@ -21,25 +21,25 @@
#define COL 3
/* Macros for displaying objects */
-#define begin_obj(obj,name,begin) \
- do { \
- if ((name)) { \
+#define begin_obj(obj,name,begin) \
+ do { \
+ if ((name)) { \
PRINTSTREAM(rawoutstream, "%s \"%s\" %s", (obj), (name), (begin)); \
} \
else { \
- PRINTSTREAM(rawoutstream, "%s %s", (obj), (begin)); \
- } \
+ PRINTSTREAM(rawoutstream, "%s %s", (obj), (begin)); \
+ } \
} while(0);
-#define end_obj(obj,end) \
- do { \
- if(HDstrlen(end)) { \
- PRINTSTREAM(rawoutstream, "%s", end); \
+#define end_obj(obj,end) \
+ do { \
+ if(HDstrlen(end)) { \
+ PRINTSTREAM(rawoutstream, "%s", end); \
if(HDstrlen(obj)) \
- PRINTVALSTREAM(rawoutstream, " "); \
- } \
+ PRINTVALSTREAM(rawoutstream, " "); \
+ } \
if(HDstrlen(obj)) \
- PRINTSTREAM(rawoutstream, "%s", obj); \
+ PRINTSTREAM(rawoutstream, "%s", obj); \
} while(0);
diff --git a/tools/h5dump/h5dump_xml.c b/tools/h5dump/h5dump_xml.c
index 95b81b5..c19685c 100644
--- a/tools/h5dump/h5dump_xml.c
+++ b/tools/h5dump/h5dump_xml.c
@@ -1408,7 +1408,6 @@ xml_print_datatype(hid_t type, unsigned in_group)
/* Print lead-in */
ctx.need_prefix = TRUE;
- h5tools_simple_prefix(rawoutstream, outputformat, &ctx, (hsize_t)0, 0);
/* Render the element */
h5tools_str_reset(&buffer);
@@ -2881,7 +2880,7 @@ xml_print_refs(hid_t did, int source)
goto error;
}
- refbuf = (hobj_ref_t *) buf;
+ refbuf = (hobj_ref_t *)((void *)buf);
/* setup */
HDmemset(&buffer, 0, sizeof(h5tools_str_t));
@@ -3057,7 +3056,7 @@ xml_print_strs(hid_t did, int source)
for (i = 0; i < (hsize_t)ssiz; i++) {
if (is_vlstr) {
- onestring = *(char **) bp;
+ onestring = *(char **)((void *)bp);
if (onestring)
str_size = HDstrlen(onestring);
}
diff --git a/tools/h5dump/h5dumpgentest.c b/tools/h5dump/h5dumpgentest.c
index e6ddbd9..8ba752d 100644
--- a/tools/h5dump/h5dumpgentest.c
+++ b/tools/h5dump/h5dumpgentest.c
@@ -112,6 +112,7 @@
#define FILE83 "tvlenstr_array.h5"
#define FILE84 "tudfilter.h5"
#define FILE85 "tgrpnullspace.h5"
+#define FILE87 "tintsnodata.h5"
/*-------------------------------------------------------------------------
* prototypes
@@ -2683,14 +2684,14 @@ gent_vldatatypes2(void)
for(i = 0; i < SPACE1_DIM1; i++) {
wdata[i].p = (hvl_t *)HDmalloc((i + 1) * sizeof(hvl_t));
if(wdata[i].p == NULL) {
- printf("Cannot allocate memory for VL data! i=%u\n", i);
+ HDprintf("Cannot allocate memory for VL data! i=%u\n", i);
return;
} /* end if */
wdata[i].len = i + 1;
for(t1 = (hvl_t *)wdata[i].p, j = 0; j < (i + 1); j++, t1++) {
t1->p = (unsigned *)HDmalloc((j + 1) * sizeof(unsigned));
if(t1->p == NULL) {
- printf("Cannot allocate memory for VL data! i=%u, j=%u\n",i,j);
+ HDprintf("Cannot allocate memory for VL data! i=%u, j=%u\n",i,j);
return;
} /* end if */
t1->len=j+1;
@@ -3739,7 +3740,7 @@ void gent_multi(void)
memb_map[mt] = mt;
HDsprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]);
memb_name[mt] = sv[mt];
- /*printf("memb_name[%d]=%s, memb_map[%d]=%d; ", mt, memb_name[mt], mt, memb_map[mt]);*/
+ /*HDprintf("memb_name[%d]=%s, memb_map[%d]=%d; ", mt, memb_name[mt], mt, memb_map[mt]);*/
memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10);
}
memb_map[H5FD_MEM_DEFAULT] = H5FD_MEM_SUPER;
@@ -6248,8 +6249,8 @@ static int gent_ldouble(void)
return 0;
- error:
- printf("error !\n");
+error:
+ HDprintf("error !\n");
return -1;
}
@@ -6412,7 +6413,7 @@ gent_bigdims(void)
return;
out:
- printf("Error.....\n");
+ HDprintf("Error.....\n");
H5E_BEGIN_TRY {
H5Pclose(dcpl);
H5Sclose(f_sid);
@@ -6604,7 +6605,7 @@ gent_group_creation_order(void)
return;
out:
- printf("Error.....\n");
+ HDprintf("Error.....\n");
H5E_BEGIN_TRY {
H5Gclose(gid);
H5Pclose(gcpl_id);
@@ -6873,7 +6874,7 @@ gent_attr_creation_order(void)
return;
out:
- printf("Error.....\n");
+ HDprintf("Error.....\n");
H5E_BEGIN_TRY {
H5Gclose(gid);
H5Dclose(did);
@@ -7389,6 +7390,80 @@ gent_attr_intsize(void)
H5Fclose(fid);
}
+static void
+gent_nodata(void)
+{
+ hid_t fid, dataset, space;
+ hsize_t dims[2];
+
+ fid = H5Fcreate(FILE87, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+
+ /* Dataset of 8 bits unsigned int */
+ dims[0] = F66_XDIM; dims[1] = F66_YDIM8;
+ space = H5Screate_simple(2, dims, NULL);
+ dataset = H5Dcreate2(fid, F66_DATASETU08, H5T_STD_U8LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Sclose(space);
+ H5Dclose(dataset);
+
+ /* Dataset of 16 bits unsigned int */
+ dims[0] = F66_XDIM; dims[1] = F66_YDIM16;
+ space = H5Screate_simple(2, dims, NULL);
+ dataset = H5Dcreate2(fid, F66_DATASETU16, H5T_STD_U16LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Sclose(space);
+ H5Dclose(dataset);
+
+ /* Dataset of 32 bits unsigned int */
+ dims[0] = F66_XDIM; dims[1] = F66_YDIM32;
+ space = H5Screate_simple(2, dims, NULL);
+ dataset = H5Dcreate2(fid, F66_DATASETU32, H5T_STD_U32LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Sclose(space);
+ H5Dclose(dataset);
+
+ /* Dataset of 64 bits unsigned int */
+ dims[0] = F66_XDIM; dims[1] = F66_YDIM64;
+ space = H5Screate_simple(2, dims, NULL);
+ dataset = H5Dcreate2(fid, F66_DATASETU64, H5T_STD_U64LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Sclose(space);
+ H5Dclose(dataset);
+
+ /* Dataset of 8 bits signed int */
+ dims[0] = F66_XDIM; dims[1] = F66_YDIM8;
+ space = H5Screate_simple(2, dims, NULL);
+ dataset = H5Dcreate2(fid, F66_DATASETS08, H5T_STD_I8LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Sclose(space);
+ H5Dclose(dataset);
+
+ /* Dataset of 16 bits signed int */
+ dims[0] = F66_XDIM; dims[1] = F66_YDIM16;
+ space = H5Screate_simple(2, dims, NULL);
+ dataset = H5Dcreate2(fid, F66_DATASETS16, H5T_STD_I16LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Sclose(space);
+ H5Dclose(dataset);
+
+ /* Dataset of 32 bits signed int */
+ dims[0] = F66_XDIM; dims[1] = F66_YDIM32;
+ space = H5Screate_simple(2, dims, NULL);
+ dataset = H5Dcreate2(fid, F66_DATASETS32, H5T_STD_I32LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Sclose(space);
+ H5Dclose(dataset);
+
+ /* Dataset of 64 bits signed int */
+ dims[0] = F66_XDIM; dims[1] = F66_YDIM64;
+ space = H5Screate_simple(2, dims, NULL);
+ dataset = H5Dcreate2(fid, F66_DATASETS64, H5T_STD_I64LE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Sclose(space);
+ H5Dclose(dataset);
+
+ /* Double Dummy set for failure tests */
+ dims[0] = F66_XDIM; dims[1] = F66_YDIM8;
+ space = H5Screate_simple(2, dims, NULL);
+ dataset = H5Dcreate2(fid, F66_DUMMYDBL, H5T_IEEE_F64BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
+ H5Sclose(space);
+ H5Dclose(dataset);
+ H5Fclose(fid);
+}
+
+
/*-------------------------------------------------------------------------
* Function: gent_charsets
*
@@ -9294,15 +9369,13 @@ gent_intattrscalars(void)
}
/*-------------------------------------------------------------------------
- * Function: gent_packedbits
+ * Function: gent_intsattrs
*
- * Purpose: Generate a file to be used in the h5dump packed bits tests.
+ * Purpose: Generate a file to be used in the h5dump tests.
* Four datasets of 1, 2, 4 and 8 bytes of unsigned int types are created.
* Four more datasets of 1, 2, 4 and 8 bytes of signed int types are created.
* Fill them with raw data such that no bit will be all zero in a dataset.
* A dummy dataset of double type is created for failure test.
- * Created: Albert Cheng, 2010/5/10.
- * Modified: Allen Byrne, 2011/1/5 Use file to test Signed/Unsigned datatypes
*-------------------------------------------------------------------------
*/
static void
@@ -10051,7 +10124,6 @@ static void gent_compound_complex2(void)
}
H5Tclose(type);
}
-
*/
/* CompoundComplex4D */
/*
@@ -10417,6 +10489,7 @@ int main(void)
gent_intattrscalars();
gent_intsattrs();
gent_bitnopaquefields();
+ gent_nodata();
gent_intsfourdims();
gent_null_space_group();
diff --git a/tools/h5import/h5importtest.c b/tools/h5import/h5importtest.c
index 27995d2..560f1b3 100644
--- a/tools/h5import/h5importtest.c
+++ b/tools/h5import/h5importtest.c
@@ -436,7 +436,7 @@ main(void)
{
char c = bin8w[i];
if ( HDfwrite( &c, sizeof(char), 1, sp) != 1 )
- printf("error writing file\n");
+ HDprintf("error writing file\n");
}
HDfclose(sp);
diff --git a/tools/h5jam/h5jam.c b/tools/h5jam/h5jam.c
index 28a62fd..fae7b9e 100644
--- a/tools/h5jam/h5jam.c
+++ b/tools/h5jam/h5jam.c
@@ -19,10 +19,7 @@
/* Name of tool */
#define PROGRAMNAME "h5jam"
-#define TRUE 1
-#define FALSE 0
-
-hsize_t write_pad (int, hsize_t);
+herr_t write_pad(int ofile, hsize_t old_where, hsize_t *new_where);
hsize_t compute_user_block_size (hsize_t);
hsize_t copy_some_to_file (int, int, hsize_t, hsize_t, ssize_t);
void parse_command_line (int, const char *[]);
@@ -313,7 +310,7 @@ main (int argc, const char *argv[])
h5fsize = (hsize_t)sbuf2.st_size;
if (output_file == NULL) {
- ofid = HDopen (input_file, O_WRONLY, 0);
+ ofid = HDopen(input_file, O_WRONLY, 0);
if (ofid < 0) {
error_msg("unable to open output file \"%s\"\n", output_file);
@@ -323,7 +320,7 @@ main (int argc, const char *argv[])
}
}
else {
- ofid = HDopen (output_file, O_WRONLY | O_CREAT | O_TRUNC, 0644);
+ ofid = HDopen(output_file, O_WRONLY | O_CREAT | O_TRUNC, 0644);
if (ofid < 0) {
error_msg("unable to create output file \"%s\"\n", output_file);
@@ -366,7 +363,13 @@ main (int argc, const char *argv[])
where = copy_some_to_file(ufid, ofid, (hsize_t) 0, startub, (ssize_t) - 1);
/* pad the ub */
- where = write_pad (ofid, where);
+ if(write_pad(ofid, where, &where) < 0) {
+ error_msg("Can't pad file \"%s\"\n", output_file);
+ HDclose (h5fid);
+ HDclose (ufid);
+ HDclose (ofid);
+ leave (EXIT_FAILURE);
+ } /* end if */
if(ub_file)
HDfree(ub_file);
@@ -503,7 +506,7 @@ copy_some_to_file(int infid, int outfid, hsize_t startin, hsize_t startout,
*-------------------------------------------------------------------------
*/
hsize_t
-compute_user_block_size (hsize_t ublock_size)
+compute_user_block_size(hsize_t ublock_size)
{
hsize_t where = 512;
@@ -521,23 +524,30 @@ compute_user_block_size (hsize_t ublock_size)
*
* Returns the size of the padded file.
*/
-hsize_t
-write_pad(int ofile, hsize_t where)
+herr_t
+write_pad(int ofile, hsize_t old_where, hsize_t *new_where)
{
unsigned int i;
char buf[1];
hsize_t psize;
+ if(new_where == NULL)
+ return FAIL;
+
buf[0] = '\0';
- HDlseek(ofile, (off_t) where, SEEK_SET);
+ HDlseek(ofile, (off_t)old_where, SEEK_SET);
- psize = compute_user_block_size (where);
- psize -= where;
+ psize = compute_user_block_size(old_where);
+ psize -= old_where;
for(i = 0; i < psize; i++)
- HDwrite (ofile, buf, 1);
+ if(HDwrite(ofile, buf, 1) < 0)
+ return FAIL;
- return(where + psize); /* the new size of the file. */
-}
+ /* Set the new size of the file. */
+ *new_where = old_where + psize;
+
+ return SUCCEED;
+} /* end write_pad() */
diff --git a/tools/h5jam/h5jamgentest.c b/tools/h5jam/h5jamgentest.c
index e7a1878..c8b0801 100644
--- a/tools/h5jam/h5jamgentest.c
+++ b/tools/h5jam/h5jamgentest.c
@@ -275,10 +275,7 @@ gent_ub(const char * filename, size_t ub_size, size_t ub_fill)
H5Fclose(fid);
/* If a user block is being used, write to it here */
- if(ub_size > 0)
- {
- ssize_t nbytes;
-
+ if(ub_size > 0) {
HDassert(ub_size <= BUF_SIZE);
fd = HDopen(filename, O_RDWR, 0);
@@ -290,8 +287,7 @@ gent_ub(const char * filename, size_t ub_size, size_t ub_fill)
for (u = 0; u < ub_fill; u++)
*bp++ = pattern[u % 10];
- nbytes = HDwrite(fd, buf, ub_size);
- HDassert(nbytes >= 0);
+ HDwrite(fd, buf, ub_size);
HDclose(fd);
}
@@ -304,7 +300,6 @@ create_textfile(const char *name, size_t size)
int fd;
size_t i;
char *bp;
- ssize_t nbytes;
fd = HDcreat(name,0777);
HDassert(fd >= 0);
@@ -316,8 +311,7 @@ create_textfile(const char *name, size_t size)
for(i = 0; i < size; i++)
*bp++ = pattern[i % 10];
- nbytes = HDwrite(fd, buf, size);
- HDassert(nbytes >= 0);
+ HDwrite(fd, buf, size);
HDfree(buf);
diff --git a/tools/h5jam/h5unjam.c b/tools/h5jam/h5unjam.c
index 59e5dae..ffe2aca 100644
--- a/tools/h5jam/h5unjam.c
+++ b/tools/h5jam/h5unjam.c
@@ -19,8 +19,6 @@
/* Name of tool */
#define PROGRAMNAME "h5unjam"
-#define TRUE 1
-#define FALSE 0
#define COPY_BUF_SIZE 1024
hsize_t write_pad( int , hsize_t );
@@ -334,17 +332,19 @@ done:
*-------------------------------------------------------------------------
*/
herr_t
-copy_to_file( FILE *infid, FILE *ofid, ssize_t _where, ssize_t how_much )
+copy_to_file( FILE *infid, FILE *ofid, ssize_t _where, ssize_t show_much )
{
static char buf[COPY_BUF_SIZE];
+ size_t how_much;
off_t where = (off_t)_where;
off_t to;
off_t from;
herr_t ret_value = 0;
/* nothing to copy */
- if(how_much <= 0)
+ if(show_much <= 0)
goto done;
+ how_much = (size_t)show_much;
/* rewind */
HDfseek(infid, 0L, 0);
@@ -379,8 +379,8 @@ copy_to_file( FILE *infid, FILE *ofid, ssize_t _where, ssize_t how_much )
/* Update positions/size */
how_much -= bytes_read;
- from += bytes_read;
- to += bytes_read;
+ from += (off_t)bytes_read;
+ to += (off_t)bytes_read;
/* Write nchars bytes to output file */
bytes_wrote = HDfwrite(buf, (size_t)1, bytes_read, ofid);
diff --git a/tools/h5ls/h5ls.c b/tools/h5ls/h5ls.c
index 78a2cce..fe58610 100644
--- a/tools/h5ls/h5ls.c
+++ b/tools/h5ls/h5ls.c
@@ -157,19 +157,13 @@ static hbool_t print_int_type(h5tools_str_t *buffer, hid_t type, int ind);
static hbool_t print_float_type(h5tools_str_t *buffer, hid_t type, int ind);
static herr_t visit_obj(hid_t file, const char *oname, iter_t *iter);
-
+
/*-------------------------------------------------------------------------
* Function: usage
*
* Purpose: Prints a usage message on stderr and then returns.
*
* Return: void
- *
- * Programmer: Robb Matzke
- * Thursday, July 16, 1998
- *
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
static void
@@ -215,6 +209,15 @@ usage (void)
PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n");
PRINTVALSTREAM(rawoutstream, " --vfd=DRIVER Use the specified virtual file driver\n");
PRINTVALSTREAM(rawoutstream, " -x, --hexdump Show raw data in hexadecimal format\n");
+ PRINTVALSTREAM(rawoutstream, " --s3-cred=C Supply S3 authentication information to \"ros3\" vfd.\n");
+ PRINTVALSTREAM(rawoutstream, " Accepts tuple of \"(<aws-region>,<access-id>,<access-key>)\".\n");
+ PRINTVALSTREAM(rawoutstream, " If absent or C->\"(,,)\", defaults to no-authentication.\n");
+ PRINTVALSTREAM(rawoutstream, " Has no effect if vfd flag not set to \"ros3\".\n");
+ PRINTVALSTREAM(rawoutstream, " --hdfs-attrs=A Supply configuration information to Hadoop VFD.\n");
+ PRINTVALSTREAM(rawoutstream, " Accepts tuple of (<namenode name>,<namenode port>,\n");
+ PRINTVALSTREAM(rawoutstream, " ...<kerberos cache path>,<username>,<buffer size>)\n");
+ PRINTVALSTREAM(rawoutstream, " If absent or A == '(,,,,)', all default values are used.\n");
+ PRINTVALSTREAM(rawoutstream, " Has no effect if vfd flag is not 'hdfs'.\n");
PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, " file/OBJECT\n");
PRINTVALSTREAM(rawoutstream, " Each object consists of an HDF5 file name optionally followed by a\n");
@@ -236,22 +239,14 @@ usage (void)
PRINTVALSTREAM(rawoutstream, " Replaced by --enable-error-stack.\n");
}
-
/*-------------------------------------------------------------------------
* Function: print_string
*
* Purpose: Print a string value by escaping unusual characters. If
- * STREAM is null then we only count how large the output would
- * be.
+ * STREAM is null then we only count how large the output would be.
*
* Return: Number of characters printed.
- *
- * Programmer: Robb Matzke
- * Thursday, November 5, 1998
- *
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
static int
@@ -262,27 +257,33 @@ print_string(h5tools_str_t *buffer, const char *s, hbool_t escape_spaces)
for (/*void*/; s && *s; s++) {
switch (*s) {
case '"':
- if (buffer) h5tools_str_append(buffer, "\\\"");
+ if (buffer)
+ h5tools_str_append(buffer, "\\\"");
nprint += 2;
break;
case '\\':
- if (buffer) h5tools_str_append(buffer, "\\\\");
+ if (buffer)
+ h5tools_str_append(buffer, "\\\\");
nprint += 2;
break;
case '\b':
- if (buffer) h5tools_str_append(buffer, "\\b");
+ if (buffer)
+ h5tools_str_append(buffer, "\\b");
nprint += 2;
break;
case '\f':
- if (buffer) h5tools_str_append(buffer, "\\f");
+ if (buffer)
+ h5tools_str_append(buffer, "\\f");
nprint += 2;
break;
case '\n':
- if (buffer) h5tools_str_append(buffer, "\\n");
+ if (buffer)
+ h5tools_str_append(buffer, "\\n");
nprint += 2;
break;
case '\r':
- if (buffer) h5tools_str_append(buffer, "\\r");
+ if (buffer)
+ h5tools_str_append(buffer, "\\r");
nprint += 2;
break;
case '\t':
@@ -291,21 +292,25 @@ print_string(h5tools_str_t *buffer, const char *s, hbool_t escape_spaces)
break;
case ' ':
if (escape_spaces) {
- if (buffer) h5tools_str_append(buffer, "\\ ");
+ if (buffer)
+ h5tools_str_append(buffer, "\\ ");
nprint += 2;
}
else {
- if (buffer) h5tools_str_append(buffer, " ");
+ if (buffer)
+ h5tools_str_append(buffer, " ");
nprint++;
}
break;
default:
if (isprint((int)*s)) {
- if (buffer) h5tools_str_append(buffer, "%c", *s);
+ if (buffer)
+ h5tools_str_append(buffer, "%c", *s);
nprint++;
}
else {
- if (buffer) h5tools_str_append(buffer, "\\%03o", *((const unsigned char*)s));
+ if (buffer)
+ h5tools_str_append(buffer, "\\%03o", *((const unsigned char*)s));
nprint += 4;
}
break;
@@ -314,21 +319,14 @@ print_string(h5tools_str_t *buffer, const char *s, hbool_t escape_spaces)
return nprint;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_obj_name
*
* Purpose: Print an object name and another string.
*
* Return: Success: TRUE
- *
- * Failure: FALSE, nothing printed
- *
- * Programmer: Quincey Koziol
- * Tuesday, November 6, 2007
- *
- * Modifications:
- *
+ * Failure: FALSE, nothing printed
*-------------------------------------------------------------------------
*/
static int
@@ -339,19 +337,19 @@ print_obj_name(h5tools_str_t *buffer, const iter_t *iter, const char *oname,
const char *name = fullname; /* Pointer to buffer for printing */
int n;
- if(show_file_name_g)
+ if (show_file_name_g)
HDsnprintf(fullname, sizeof(fullname), "%s/%s", iter->fname, oname + iter->name_start);
else
name = oname + iter->name_start;
/* Print the object name, either full name or base name */
- if(fullname_g)
+ if (fullname_g)
n = print_string(buffer, name, TRUE);
else {
const char *last_sep; /* The location of the last group separator */
/* Find the last component of the path name */
- if(NULL == (last_sep = HDstrrchr(name, '/')))
+ if (NULL == (last_sep = HDstrrchr(name, '/')))
last_sep = name;
else {
last_sep++;
@@ -363,123 +361,158 @@ print_obj_name(h5tools_str_t *buffer, const iter_t *iter, const char *oname,
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_native_type
*
* Purpose: Prints the name of a native C data type.
*
* Return: Success: TRUE
- *
- * Failure: FALSE, nothing printed.
- *
- * Programmer: Robb Matzke
- * Thursday, November 5, 1998
- *
- * Modifications:
- * Robb Matzke, 1999-06-11
- * Added the C9x types, but we still prefer to display the types
- * from the C language itself (like `int' vs. `int32_t').
- *
+ * Failure: FALSE, nothing printed.
*-------------------------------------------------------------------------
*/
static hbool_t
-print_native_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind)
+print_native_type(h5tools_str_t *buffer, hid_t type, int ind)
{
- if(!simple_output_g) {
- if (H5Tequal(type, H5T_NATIVE_SCHAR)==TRUE) {
+ if (!simple_output_g) {
+ if (H5Tequal(type, H5T_NATIVE_SCHAR) == TRUE) {
h5tools_str_append(buffer, "native signed char");
- } else if (H5Tequal(type, H5T_NATIVE_UCHAR)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UCHAR) == TRUE) {
h5tools_str_append(buffer, "native unsigned char");
- } else if (H5Tequal(type, H5T_NATIVE_INT)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_INT) == TRUE) {
h5tools_str_append(buffer, "native int");
- } else if (H5Tequal(type, H5T_NATIVE_UINT)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT) == TRUE) {
h5tools_str_append(buffer, "native unsigned int");
- } else if (H5Tequal(type, H5T_NATIVE_SHORT)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_SHORT) == TRUE) {
h5tools_str_append(buffer, "native short");
- } else if (H5Tequal(type, H5T_NATIVE_USHORT)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_USHORT) == TRUE) {
h5tools_str_append(buffer, "native unsigned short");
- } else if (H5Tequal(type, H5T_NATIVE_LONG)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_LONG) == TRUE) {
h5tools_str_append(buffer, "native long");
- } else if (H5Tequal(type, H5T_NATIVE_ULONG)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_ULONG) == TRUE) {
h5tools_str_append(buffer, "native unsigned long");
- } else if (H5Tequal(type, H5T_NATIVE_LLONG)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_LLONG) == TRUE) {
h5tools_str_append(buffer, "native long long");
- } else if (H5Tequal(type, H5T_NATIVE_ULLONG)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_ULLONG) == TRUE) {
h5tools_str_append(buffer, "native unsigned long long");
- } else if (H5Tequal(type, H5T_NATIVE_FLOAT)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_FLOAT) == TRUE) {
h5tools_str_append(buffer, "native float");
- } else if (H5Tequal(type, H5T_NATIVE_DOUBLE)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_DOUBLE) == TRUE) {
h5tools_str_append(buffer, "native double");
+ }
#if H5_SIZEOF_LONG_DOUBLE !=0
- } else if (H5Tequal(type, H5T_NATIVE_LDOUBLE)==TRUE) {
+ else if (H5Tequal(type, H5T_NATIVE_LDOUBLE) == TRUE) {
h5tools_str_append(buffer, "native long double");
+ }
#endif
- } else if (H5Tequal(type, H5T_NATIVE_INT8)==TRUE) {
+ else if (H5Tequal(type, H5T_NATIVE_INT8) == TRUE) {
h5tools_str_append(buffer, "native int8_t");
- } else if (H5Tequal(type, H5T_NATIVE_UINT8)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT8) == TRUE) {
h5tools_str_append(buffer, "native uint8_t");
- } else if (H5Tequal(type, H5T_NATIVE_INT16)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_INT16) == TRUE) {
h5tools_str_append(buffer, "native int16_t");
- } else if (H5Tequal(type, H5T_NATIVE_UINT16)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT16) == TRUE) {
h5tools_str_append(buffer, "native uint16_t");
- } else if (H5Tequal(type, H5T_NATIVE_INT32)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_INT32) == TRUE) {
h5tools_str_append(buffer, "native int32_t");
- } else if (H5Tequal(type, H5T_NATIVE_UINT32)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT32) == TRUE) {
h5tools_str_append(buffer, "native uint32_t");
- } else if (H5Tequal(type, H5T_NATIVE_INT64)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_INT64) == TRUE) {
h5tools_str_append(buffer, "native int64_t");
- } else if (H5Tequal(type, H5T_NATIVE_UINT64)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT64) == TRUE) {
h5tools_str_append(buffer, "native uint64_t");
- } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST8)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_INT_LEAST8) == TRUE) {
h5tools_str_append(buffer, "native int_least8_t");
- } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST8)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST8) == TRUE) {
h5tools_str_append(buffer, "native uint_least8_t");
- } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST16)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_INT_LEAST16) == TRUE) {
h5tools_str_append(buffer, "native int_least16_t");
- } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST16)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST16) == TRUE) {
h5tools_str_append(buffer, "native uint_least16_t");
- } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST32)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_INT_LEAST32) == TRUE) {
h5tools_str_append(buffer, "native int_least32_t");
- } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST32)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST32) == TRUE) {
h5tools_str_append(buffer, "native uint_least32_t");
- } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST64)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_INT_LEAST64) == TRUE) {
h5tools_str_append(buffer, "native int_least64_t");
- } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST64)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST64) == TRUE) {
h5tools_str_append(buffer, "native uint_least64_t");
- } else if (H5Tequal(type, H5T_NATIVE_INT_FAST8)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_INT_FAST8) == TRUE) {
h5tools_str_append(buffer, "native int_fast8_t");
- } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST8)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT_FAST8) == TRUE) {
h5tools_str_append(buffer, "native uint_fast8_t");
- } else if (H5Tequal(type, H5T_NATIVE_INT_FAST16)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_INT_FAST16) == TRUE) {
h5tools_str_append(buffer, "native int_fast16_t");
- } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST16)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT_FAST16) == TRUE) {
h5tools_str_append(buffer, "native uint_fast16_t");
- } else if (H5Tequal(type, H5T_NATIVE_INT_FAST32)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_INT_FAST32) == TRUE) {
h5tools_str_append(buffer, "native int_fast32_t");
- } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST32)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT_FAST32) == TRUE) {
h5tools_str_append(buffer, "native uint_fast32_t");
- } else if (H5Tequal(type, H5T_NATIVE_INT_FAST64)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_INT_FAST64) == TRUE) {
h5tools_str_append(buffer, "native int_fast64_t");
- } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST64)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_UINT_FAST64) == TRUE) {
h5tools_str_append(buffer, "native uint_fast64_t");
- } else if (H5Tequal(type, H5T_NATIVE_B8)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_B8) == TRUE) {
h5tools_str_append(buffer, "native 8-bit field");
- } else if (H5Tequal(type, H5T_NATIVE_B16)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_B16) == TRUE) {
h5tools_str_append(buffer, "native 16-bit field");
- } else if (H5Tequal(type, H5T_NATIVE_B32)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_B32) == TRUE) {
h5tools_str_append(buffer, "native 32-bit field");
- } else if (H5Tequal(type, H5T_NATIVE_B64)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_B64) == TRUE) {
h5tools_str_append(buffer, "native 64-bit field");
- } else if (H5Tequal(type, H5T_NATIVE_HSIZE)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_HSIZE) == TRUE) {
h5tools_str_append(buffer, "native hsize_t");
- } else if (H5Tequal(type, H5T_NATIVE_HSSIZE)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_HSSIZE) == TRUE) {
h5tools_str_append(buffer, "native hssize_t");
- } else if (H5Tequal(type, H5T_NATIVE_HERR)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_HERR) == TRUE) {
h5tools_str_append(buffer, "native herr_t");
- } else if (H5Tequal(type, H5T_NATIVE_HBOOL)==TRUE) {
+ }
+ else if (H5Tequal(type, H5T_NATIVE_HBOOL) == TRUE) {
h5tools_str_append(buffer, "native hbool_t");
- } else {
+ }
+ else {
return print_int_type(buffer, type, ind);
}
} else {
@@ -488,36 +521,29 @@ print_native_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_ieee_type
*
* Purpose: Print the name of an IEEE floating-point data type.
*
* Return: Success: TRUE
- *
- * Failure: FALSE, nothing printed
- *
- * Programmer: Robb Matzke
- * Thursday, November 5, 1998
- *
- * Modifications:
- *
+ * Failure: FALSE, nothing printed
*-------------------------------------------------------------------------
*/
static hbool_t
-print_ieee_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind)
+print_ieee_type(h5tools_str_t *buffer, hid_t type, int ind)
{
- if (H5Tequal(type, H5T_IEEE_F32BE)==TRUE) {
+ if (H5Tequal(type, H5T_IEEE_F32BE) == TRUE) {
h5tools_str_append(buffer, "IEEE 32-bit big-endian float");
}
- else if (H5Tequal(type, H5T_IEEE_F32LE)==TRUE) {
+ else if (H5Tequal(type, H5T_IEEE_F32LE) == TRUE) {
h5tools_str_append(buffer, "IEEE 32-bit little-endian float");
}
- else if (H5Tequal(type, H5T_IEEE_F64BE)==TRUE) {
+ else if (H5Tequal(type, H5T_IEEE_F64BE) == TRUE) {
h5tools_str_append(buffer, "IEEE 64-bit big-endian float");
}
- else if (H5Tequal(type, H5T_IEEE_F64LE)==TRUE) {
+ else if (H5Tequal(type, H5T_IEEE_F64LE) == TRUE) {
h5tools_str_append(buffer, "IEEE 64-bit little-endian float");
}
else {
@@ -526,21 +552,15 @@ print_ieee_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_precision
*
* Purpose: Prints information on the next line about precision and
- * padding if the precision is less than the total data type
- * size.
+ * padding if the precision is less than the total data type
+ * size.
*
* Return: void
- *
- * Programmer: Robb Matzke
- * Thursday, November 5, 1998
- *
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
static void
@@ -555,13 +575,12 @@ print_precision(h5tools_str_t *buffer, hid_t type, int ind)
/* If the precision is less than the total size then show the precision
* and offset on the following line. Also display the padding
* information. */
- if(8 * H5Tget_size(type) != (prec = H5Tget_precision(type))) {
+ if (8 * H5Tget_size(type) != (prec = H5Tget_precision(type))) {
h5tools_str_append(buffer, "\n%*s(%lu bit%s of precision beginning at bit %lu)",
- ind, "", (unsigned long)prec, 1 == prec ? "" : "s",
- (unsigned long)H5Tget_offset(type));
+ ind, "", (unsigned long)prec, 1 == prec ? "" : "s", (unsigned long)H5Tget_offset(type));
H5Tget_pad(type, &plsb, &pmsb);
- if(H5Tget_offset(type) > 0) {
+ if (H5Tget_offset(type) > 0) {
switch(plsb) {
case H5T_PAD_ZERO:
plsb_s = "zero";
@@ -580,7 +599,7 @@ print_precision(h5tools_str_t *buffer, hid_t type, int ind)
break;
}
}
- if((unsigned)H5Tget_offset(type) + prec < 8 * H5Tget_size(type)) {
+ if ((unsigned)H5Tget_offset(type) + prec < 8 * H5Tget_size(type)) {
switch(pmsb) {
case H5T_PAD_ZERO:
pmsb_s = "zero";
@@ -603,61 +622,53 @@ print_precision(h5tools_str_t *buffer, hid_t type, int ind)
h5tools_str_append(buffer, "\n%*s(", ind, "");
if (plsb_s) {
nbits = (unsigned)H5Tget_offset(type);
- h5tools_str_append(buffer, "%lu %s bit%s at bit 0",
- (unsigned long)nbits, plsb_s, 1 == nbits ? "" : "s");
+ h5tools_str_append(buffer, "%lu %s bit%s at bit 0", (unsigned long)nbits, plsb_s, 1 == nbits ? "" : "s");
}
- if (plsb_s && pmsb_s) h5tools_str_append(buffer, ", ");
+ if (plsb_s && pmsb_s)
+ h5tools_str_append(buffer, ", ");
if (pmsb_s) {
nbits = (8 * H5Tget_size(type)) - ((unsigned)H5Tget_offset(type) + prec);
- h5tools_str_append(buffer, "%lu %s bit%s at bit %lu",
- (unsigned long)nbits, pmsb_s, 1 == nbits ? "" : "s",
- (unsigned long)(8 * H5Tget_size(type) - nbits));
+ h5tools_str_append(buffer, "%lu %s bit%s at bit %lu", (unsigned long)nbits, pmsb_s, 1 == nbits ? "" : "s", (unsigned long)(8 * H5Tget_size(type) - nbits));
}
h5tools_str_append(buffer, ")");
}
}
}
-
+
/*-------------------------------------------------------------------------
* Function: print_int_type
*
* Purpose: Print the name of an integer data type. Common information
- * like number of bits, byte order, and sign scheme appear on
- * the first line. Additional information might appear in
- * parentheses on the following lines.
+ * like number of bits, byte order, and sign scheme appear on
+ * the first line. Additional information might appear in
+ * parentheses on the following lines.
*
* Return: Success: TRUE
- *
- * Failure: FALSE, nothing printed
- *
- * Programmer: Robb Matzke
- * Thursday, November 5, 1998
- *
- * Modifications:
- *
+ * Failure: FALSE, nothing printed
*-------------------------------------------------------------------------
*/
static hbool_t
print_int_type(h5tools_str_t *buffer, hid_t type, int ind)
{
- H5T_order_t order; /* byte order value */
- const char *order_s=NULL; /* byte order string */
- H5T_sign_t sign; /* sign scheme value */
- const char *sign_s=NULL; /* sign scheme string */
+ H5T_order_t order; /* byte order value */
+ const char *order_s = NULL; /* byte order string */
+ H5T_sign_t sign; /* sign scheme value */
+ const char *sign_s = NULL; /* sign scheme string */
- if (H5T_INTEGER!=H5Tget_class(type)) return FALSE;
+ if (H5T_INTEGER != H5Tget_class(type))
+ return FALSE;
/* Byte order */
- if (H5Tget_size(type)>1) {
+ if (H5Tget_size(type) > 1) {
order = H5Tget_order(type);
- if (H5T_ORDER_LE==order) {
+ if (H5T_ORDER_LE == order) {
order_s = " little-endian";
}
- else if (H5T_ORDER_BE==order) {
+ else if (H5T_ORDER_BE == order) {
order_s = " big-endian";
}
- else if (H5T_ORDER_VAX==order) {
+ else if (H5T_ORDER_VAX == order) {
order_s = " mixed-endian";
}
else {
@@ -669,11 +680,11 @@ print_int_type(h5tools_str_t *buffer, hid_t type, int ind)
}
/* Sign */
- if ((sign=H5Tget_sign(type))>=0) {
- if (H5T_SGN_NONE==sign) {
+ if ((sign = H5Tget_sign(type)) >= 0) {
+ if (H5T_SGN_NONE == sign) {
sign_s = " unsigned";
}
- else if (H5T_SGN_2==sign) {
+ else if (H5T_SGN_2 == sign) {
sign_s = "";
}
else {
@@ -686,27 +697,19 @@ print_int_type(h5tools_str_t *buffer, hid_t type, int ind)
/* Print size, order, and sign on first line, precision and padding
* information on the subsequent lines */
- h5tools_str_append(buffer, "%lu-bit%s%s integer",
- (unsigned long)(8*H5Tget_size(type)), order_s, sign_s);
+ h5tools_str_append(buffer, "%lu-bit%s%s integer", (unsigned long)(8*H5Tget_size(type)), order_s, sign_s);
print_precision(buffer, type, ind);
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_float_type
*
* Purpose: Print info about a floating point data type.
*
* Return: Success: TRUE
- *
- * Failure: FALSE, nothing printed
- *
- * Programmer: Robb Matzke
- * Thursday, November 5, 1998
- *
- * Modifications:
- *
+ * Failure: FALSE, nothing printed
*-------------------------------------------------------------------------
*/
static hbool_t
@@ -723,18 +726,19 @@ print_float_type(h5tools_str_t *buffer, hid_t type, int ind)
H5T_pad_t pad; /* internal padding value */
const char *pad_s=NULL; /* internal padding string */
- if (H5T_FLOAT!=H5Tget_class(type)) return FALSE;
+ if (H5T_FLOAT != H5Tget_class(type))
+ return FALSE;
/* Byte order */
- if (H5Tget_size(type)>1) {
+ if (H5Tget_size(type) > 1) {
order = H5Tget_order(type);
- if (H5T_ORDER_LE==order) {
+ if (H5T_ORDER_LE == order) {
order_s = " little-endian";
}
- else if (H5T_ORDER_BE==order) {
+ else if (H5T_ORDER_BE == order) {
order_s = " big-endian";
}
- else if (H5T_ORDER_VAX==order) {
+ else if (H5T_ORDER_VAX == order) {
order_s = " mixed-endian";
}
else {
@@ -747,8 +751,7 @@ print_float_type(h5tools_str_t *buffer, hid_t type, int ind)
/* Print size and byte order on first line, precision and padding on
* subsequent lines. */
- h5tools_str_append(buffer, "%lu-bit%s floating-point",
- (unsigned long)(8*H5Tget_size(type)), order_s);
+ h5tools_str_append(buffer, "%lu-bit%s floating-point", (unsigned long)(8*H5Tget_size(type)), order_s);
print_precision(buffer, type, ind);
/* Print sizes, locations, and other information about each field */
@@ -773,15 +776,13 @@ print_float_type(h5tools_str_t *buffer, hid_t type, int ind)
break;
}
h5tools_str_append(buffer, "\n%*s(significant for %lu bit%s at bit %lu%s)", ind, "",
- (unsigned long)msize, 1==msize?"":"s", (unsigned long)mpos,
- norm_s);
+ (unsigned long)msize, 1==msize?"":"s", (unsigned long)mpos, norm_s);
h5tools_str_append(buffer, "\n%*s(exponent for %lu bit%s at bit %lu, bias is 0x%lx)",
- ind, "", (unsigned long)esize, 1==esize?"":"s",
- (unsigned long)epos, (unsigned long)ebias);
+ ind, "", (unsigned long)esize, 1==esize?"":"s", (unsigned long)epos, (unsigned long)ebias);
h5tools_str_append(buffer, "\n%*s(sign bit at %lu)", ind, "", (unsigned long)spos);
/* Display internal padding */
- if (1+esize+msize<H5Tget_precision(type)) {
+ if ((1 + esize + msize) < H5Tget_precision(type)) {
pad = H5Tget_inpad(type);
switch (pad) {
case H5T_PAD_ZERO:
@@ -806,19 +807,14 @@ print_float_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_cmpd_type
*
* Purpose: Print info about a compound data type.
*
* Return: Success: TRUE
- *
- * Failure: FALSE, nothing printed
- *
- * Programmer: Robb Matzke
- * Thursday, November 5, 1998
- *
+ * Failure: FALSE, nothing printed
*-------------------------------------------------------------------------
*/
static hbool_t
@@ -837,14 +833,12 @@ print_cmpd_type(h5tools_str_t *buffer, hid_t type, int ind)
return FALSE;
h5tools_str_append(buffer, "struct {");
- for(i = 0; i < (unsigned)nmembs; i++) {
-
+ for (i = 0; i < (unsigned)nmembs; i++) {
/* Name and offset */
name = H5Tget_member_name(type, i);
h5tools_str_append(buffer, "\n%*s\"", ind+4, "");
n = print_string(buffer, name, FALSE);
- h5tools_str_append(buffer, "\"%*s +%-4lu ", MAX(0, 16-n), "",
- (unsigned long)H5Tget_member_offset(type, i));
+ h5tools_str_append(buffer, "\"%*s +%-4lu ", MAX(0, 16-n), "", (unsigned long)H5Tget_member_offset(type, i));
H5free_memory(name);
/* Member's type */
@@ -853,27 +847,19 @@ print_cmpd_type(h5tools_str_t *buffer, hid_t type, int ind)
H5Tclose(subtype);
}
size = H5Tget_size(type);
- h5tools_str_append(buffer, "\n%*s} %lu byte%s",
- ind, "", (unsigned long)size, 1==size?"":"s");
+ h5tools_str_append(buffer, "\n%*s} %lu byte%s", ind, "", (unsigned long)size, 1==size?"":"s");
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_enum_type
*
* Purpose: Print info about an enumeration data type.
*
* Return: Success: TRUE
- *
- * Failure: FALSE, nothing printed
- *
- * Programmer: Robb Matzke
- * Wednesday, December 23, 1998
- *
- * Modifications:
- *
+ * Failure: FALSE, nothing printed
*-------------------------------------------------------------------------
*/
static hbool_t
@@ -882,9 +868,9 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind)
int nmembs; /* number of members */
hid_t super; /* enum base integer type */
- if(H5T_ENUM != H5Tget_class(type))
+ if (H5T_ENUM != H5Tget_class(type))
return FALSE;
- if((nmembs = H5Tget_nmembers(type)) < 0)
+ if ((nmembs = H5Tget_nmembers(type)) < 0)
return FALSE;
super = H5Tget_super(type);
@@ -892,7 +878,7 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind)
print_type(buffer, super, ind + 4);
h5tools_str_append(buffer, " {");
- if(nmembs > 0) {
+ if (nmembs > 0) {
char **name; /* member names */
unsigned char *value; /* value array */
hid_t native = -1; /* native integer data type */
@@ -904,9 +890,9 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind)
* 1. long long -- the largest native signed integer
* 2. unsigned long long -- the largest native unsigned integer
* 3. raw format */
- if(H5Tget_size(type) <= sizeof(long long)) {
+ if (H5Tget_size(type) <= sizeof(long long)) {
dst_size = sizeof(long long);
- if(H5T_SGN_NONE == H5Tget_sign(type))
+ if (H5T_SGN_NONE == H5Tget_sign(type))
native = H5T_NATIVE_ULLONG;
else
native = H5T_NATIVE_LLONG;
@@ -917,16 +903,16 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind)
/* Get the names and raw values of all members */
name = (char **)HDcalloc((size_t)nmembs, sizeof(char *));
value = (unsigned char *)HDcalloc((size_t)nmembs, MAX(H5Tget_size(type), dst_size));
- for(i = 0; i < (unsigned)nmembs; i++) {
+ for (i = 0; i < (unsigned)nmembs; i++) {
name[i] = H5Tget_member_name(type, i);
H5Tget_member_value(type, i, value + i * H5Tget_size(type));
}
/* Convert values to native data type */
- if(native > 0)
- if(H5Tconvert(super, native, (size_t)nmembs, value, NULL, H5P_DEFAULT) < 0) {
+ if (native > 0)
+ if (H5Tconvert(super, native, (size_t)nmembs, value, NULL, H5P_DEFAULT) < 0) {
/* Release resources */
- for(i = 0; i < (unsigned)nmembs; i++)
+ for (i = 0; i < (unsigned)nmembs; i++)
H5free_memory(name[i]);
HDfree(name);
HDfree(value);
@@ -938,7 +924,7 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind)
/*not implemented yet*/
/* Print members */
- for(i = 0; i < (unsigned)nmembs; i++) {
+ for (i = 0; i < (unsigned)nmembs; i++) {
unsigned char *copy; /* a pointer to value array */
int nchars; /* number of output characters */
@@ -946,14 +932,14 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind)
nchars = print_string(buffer, name[i], TRUE);
h5tools_str_append(buffer, "%*s = ", MAX(0, 16 - nchars), "");
- if(native < 0) {
+ if (native < 0) {
size_t j;
h5tools_str_append(buffer, "0x");
- for(j = 0; j < dst_size; j++)
+ for (j = 0; j < dst_size; j++)
h5tools_str_append(buffer, "%02x", value[i*dst_size+j]);
}
- else if(H5T_SGN_NONE == H5Tget_sign(native)) {
+ else if (H5T_SGN_NONE == H5Tget_sign(native)) {
/*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size"
*strangely, unless use another pointer "copy".*/
copy = value + i * dst_size;
@@ -963,8 +949,7 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind)
/*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size"
*strangely, unless use another pointer "copy".*/
copy = value + i * dst_size;
- h5tools_str_append(buffer, "%"H5_PRINTF_LL_WIDTH"d",
- *((long long*)((void*)copy)));
+ h5tools_str_append(buffer, "%"H5_PRINTF_LL_WIDTH"d", *((long long*)((void*)copy)));
}
}
@@ -984,21 +969,14 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_string_type
*
* Purpose: Print information about a string data type.
*
* Return: Success: TRUE
- *
- * Failure: FALSE, nothing printed
- *
- * Programmer: Robb Matzke
- * Thursday, November 5, 1998
- *
- * Modifications:
- *
+ * Failure: FALSE, nothing printed
*-------------------------------------------------------------------------
*/
static hbool_t
@@ -1009,7 +987,8 @@ print_string_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind)
H5T_cset_t cset;
const char *cset_s=NULL;
- if (H5T_STRING!=H5Tget_class(type)) return FALSE;
+ if (H5T_STRING != H5Tget_class(type))
+ return FALSE;
/* Padding */
pad = H5Tget_strpad(type);
@@ -1085,59 +1064,43 @@ print_string_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_reference_type
*
* Purpose: Prints information about a reference data type.
*
* Return: Success: TRUE
- *
- * Failure: FALSE, nothing printed
- *
- * Programmer: Robb Matzke
- * Thursday, November 5, 1998
- *
- * Modifications:
- * Robb Matzke, 1999-06-04
- * Knows about object and dataset region references.
- *
+ * Failure: FALSE, nothing printed
*-------------------------------------------------------------------------
*/
static hbool_t
print_reference_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind)
{
- if (H5T_REFERENCE!=H5Tget_class(type)) return FALSE;
+ if (H5T_REFERENCE != H5Tget_class(type))
+ return FALSE;
- if (H5Tequal(type, H5T_STD_REF_OBJ)==TRUE) {
+ if (H5Tequal(type, H5T_STD_REF_OBJ) == TRUE) {
h5tools_str_append(buffer, "object reference");
}
- else if (H5Tequal(type, H5T_STD_REF_DSETREG)==TRUE) {
+ else if (H5Tequal(type, H5T_STD_REF_DSETREG) == TRUE) {
h5tools_str_append(buffer, "dataset region reference");
}
else {
- h5tools_str_append(buffer, "%lu-byte unknown reference",
- (unsigned long)H5Tget_size(type));
+ h5tools_str_append(buffer, "%lu-byte unknown reference", (unsigned long)H5Tget_size(type));
}
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_opaque_type
*
* Purpose: Prints information about an opaque data type.
*
* Return: Success: TRUE
- *
- * Failure: FALSE, nothing printed
- *
- * Programmer: Robb Matzke
- * Monday, June 7, 1999
- *
- * Modifications:
- *
+ * Failure: FALSE, nothing printed
*-------------------------------------------------------------------------
*/
static hbool_t
@@ -1146,11 +1109,12 @@ print_opaque_type(h5tools_str_t *buffer, hid_t type, int ind)
char *tag;
size_t size;
- if (H5T_OPAQUE!=H5Tget_class(type)) return FALSE;
+ if (H5T_OPAQUE != H5Tget_class(type))
+ return FALSE;
size = H5Tget_size(type);
h5tools_str_append(buffer, "%lu-byte opaque type", (unsigned long)size);
- if ((tag=H5Tget_tag(type))) {
+ if ((tag = H5Tget_tag(type))) {
h5tools_str_append(buffer, "\n%*s(tag = \"", ind, "");
print_string(buffer, tag, FALSE);
h5tools_str_append(buffer, "\")");
@@ -1159,20 +1123,14 @@ print_opaque_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_vlen_type
*
* Purpose: Print information about a variable-length type
*
* Return: Success: TRUE
- *
* Failure: FALSE
- *
- * Programmer: Robb Matzke
- * Friday, December 1, 2000
- *
- * Modifications:
*-------------------------------------------------------------------------
*/
static hbool_t
@@ -1180,27 +1138,22 @@ print_vlen_type(h5tools_str_t *buffer, hid_t type, int ind)
{
hid_t super;
- if (H5T_VLEN!=H5Tget_class(type)) return FALSE;
+ if (H5T_VLEN != H5Tget_class(type))
+ return FALSE;
- h5tools_str_append(buffer, "variable length of\n%*s", ind+4, "");
+ h5tools_str_append(buffer, "variable length of\n%*s", ind + 4, "");
super = H5Tget_super(type);
- print_type(buffer, super, ind+4);
+ print_type(buffer, super, ind + 4);
H5Tclose(super);
return TRUE;
}
-
+
/*---------------------------------------------------------------------------
* Purpose: Print information about an array type
*
* Return: Success: TRUE
- *
* Failure: FALSE
- *
- * Programmer: Robb Matzke
- * Thursday, January 31, 2002
- *
- * Modifications:
*---------------------------------------------------------------------------
*/
static hbool_t
@@ -1210,7 +1163,7 @@ print_array_type(h5tools_str_t *buffer, hid_t type, int ind)
int ndims, i;
hsize_t *dims=NULL;
- if (H5T_ARRAY!=H5Tget_class(type))
+ if (H5T_ARRAY != H5Tget_class(type))
return FALSE;
ndims = H5Tget_array_ndims(type);
if (ndims) {
@@ -1218,7 +1171,7 @@ print_array_type(h5tools_str_t *buffer, hid_t type, int ind)
H5Tget_array_dims2(type, dims);
/* Print dimensions */
- for (i=0; i<ndims; i++)
+ for (i = 0; i < ndims; i++)
h5tools_str_append(buffer, "%s" HSIZE_T_FORMAT , i?",":"[", dims[i]);
h5tools_str_append(buffer, "]");
@@ -1236,23 +1189,14 @@ print_array_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_bitfield_type
*
* Purpose: Print information about a bitfield type.
*
* Return: Success: TRUE
- *
- * Failure: FALSE, nothing printed
- *
- * Programmer: Pedro Vicente
- * Tuesday, May 20, 2003
- *
- * Modifications:
- * Robb Matzke, LLNL 2003-06-05
- * Generalized Pedro's original if/then/else. Also display
- * precision/offset information.
+ * Failure: FALSE, nothing printed
*-------------------------------------------------------------------------
*/
static hbool_t
@@ -1261,24 +1205,28 @@ print_bitfield_type(h5tools_str_t *buffer, hid_t type, int ind)
H5T_order_t order; /* byte order value */
const char *order_s=NULL; /* byte order string */
- if (H5T_BITFIELD!=H5Tget_class(type)) return FALSE;
+ if (H5T_BITFIELD != H5Tget_class(type))
+ return FALSE;
if (H5Tget_size(type)>1) {
order = H5Tget_order(type);
- if (H5T_ORDER_LE==order) {
+ if (H5T_ORDER_LE == order) {
order_s = " little-endian";
- } else if (H5T_ORDER_BE==order) {
+ }
+ else if (H5T_ORDER_BE == order) {
order_s = " big-endian";
- } else if (H5T_ORDER_VAX==order) {
+ }
+ else if (H5T_ORDER_VAX == order) {
order_s = " mixed-endian";
- } else {
+ }
+ else {
order_s = "unknown-byte-order";
}
- } else {
+ }
+ else {
order_s = "";
}
- h5tools_str_append(buffer, "%lu-bit%s bitfield",
- (unsigned long)(8*H5Tget_size(type)), order_s);
+ h5tools_str_append(buffer, "%lu-bit%s bitfield", (unsigned long)(8*H5Tget_size(type)), order_s);
print_precision(buffer, type, ind);
return TRUE;
}
@@ -1292,16 +1240,9 @@ print_bitfield_type(h5tools_str_t *buffer, hid_t type, int ind)
* there might be line-feeds inside the type definition). The
* first line is assumed to have IND characters before it on
* the same line (printed by the caller).
- *
- * Return: void
- *
- * Programmer: Robb Matzke
- * Thursday, November 5, 1998
- *
- * Modifications:
- * Robb Matzke, 1999-06-11
* Prints the OID of shared data types.
*
+ * Return: void
*-------------------------------------------------------------------------
*/
static void
@@ -1310,24 +1251,23 @@ print_type(h5tools_str_t *buffer, hid_t type, int ind)
H5T_class_t data_class = H5Tget_class(type);
/* Bad data type */
- if (type<0) {
+ if (type < 0) {
h5tools_str_append(buffer,"<ERROR>");
return;
}
/* Shared? If so then print the type's OID */
- if(H5Tcommitted(type)) {
+ if (H5Tcommitted(type)) {
H5O_info_t oi;
- if(H5Oget_info(type, &oi) >= 0)
- h5tools_str_append(buffer,"shared-%lu:"H5_PRINTF_HADDR_FMT" ",
- oi.fileno, oi.addr);
+ if (H5Oget_info(type, &oi) >= 0)
+ h5tools_str_append(buffer,"shared-%lu:"H5_PRINTF_HADDR_FMT" ", oi.fileno, oi.addr);
else
h5tools_str_append(buffer,"shared ");
} /* end if */
/* Print the type */
- if(print_native_type(buffer, type, ind) ||
+ if (print_native_type(buffer, type, ind) ||
print_ieee_type(buffer, type, ind) ||
print_cmpd_type(buffer, type, ind) ||
print_enum_type(buffer, type, ind) ||
@@ -1340,25 +1280,16 @@ print_type(h5tools_str_t *buffer, hid_t type, int ind)
return;
/* Unknown type */
- h5tools_str_append(buffer,"%lu-byte class-%u unknown",
- (unsigned long)H5Tget_size(type), (unsigned)data_class);
+ h5tools_str_append(buffer,"%lu-byte class-%u unknown", (unsigned long)H5Tget_size(type), (unsigned)data_class);
}
-
+
/*-------------------------------------------------------------------------
* Function: dump_dataset_values
*
* Purpose: Prints all values of a dataset.
*
* Return: void
- *
- * Programmer: Robb Matzke
- * Tuesday, July 21, 1998
- *
- * Modifications:
- * Robb Matzke, 1999-09-27
- * Understands the simple_output_g switch which causes data to
- * be displayed in a more machine-readable format.
*-------------------------------------------------------------------------
*/
static void
@@ -1398,7 +1329,8 @@ dump_dataset_values(hid_t dset)
}
outputformat.cmpd_sep = " ";
- if (label_g) outputformat.cmpd_name = "%s=";
+ if (label_g)
+ outputformat.cmpd_name = "%s=";
outputformat.elmt_suf1 = " ";
outputformat.str_locale = ESCAPE_HTML;
@@ -1441,7 +1373,7 @@ dump_dataset_values(hid_t dset)
* command line switch was given. */
outputformat.raw = TRUE;
}
- else if (string_g && 1==size && H5T_INTEGER==H5Tget_class(f_type)) {
+ else if (string_g && 1 == size && H5T_INTEGER == H5Tget_class(f_type)) {
/* Print 1-byte integer data as an ASCI character string instead of
* integers if the `-s' or `--string' command-line option was given. */
outputformat.ascii = TRUE;
@@ -1474,21 +1406,14 @@ dump_dataset_values(hid_t dset)
PRINTVALSTREAM(rawoutstream, "\n");
}
-
+
/*-------------------------------------------------------------------------
* Function: list_attr
*
* Purpose: Prints information about attributes.
*
* Return: Success: 0
- *
- * Failure: -1
- *
- * Programmer: Robb Matzke
- * Friday, June 5, 1998
- *
- * Modifications:
- *
+ * Failure: -1
*-------------------------------------------------------------------------
*/
static herr_t
@@ -1521,7 +1446,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain
print_string(&buffer, attr_name, TRUE);
- if((attr = H5Aopen(obj, attr_name, H5P_DEFAULT))) {
+ if ((attr = H5Aopen(obj, attr_name, H5P_DEFAULT))) {
space = H5Aget_space(attr);
type = H5Aget_type(attr);
@@ -1538,7 +1463,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain
case H5S_SIMPLE:
/* simple dataspace */
h5tools_str_append(&buffer, " {");
- for (i=0; i<ndims; i++) {
+ for (i = 0; i < ndims; i++) {
h5tools_str_append(&buffer, "%s" HSIZE_T_FORMAT, i?", ":"", size[i]);
nelmts *= size[i];
}
@@ -1570,7 +1495,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain
/* Data */
outputformat = *info;
- if(nelmts < 5) {
+ if (nelmts < 5) {
outputformat.idx_fmt = "";
outputformat.line_1st = " Data: ";
outputformat.line_pre = " ";
@@ -1589,10 +1514,9 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain
}
outputformat.line_ncols = (unsigned)width_g;
- if(label_g)
+ if (label_g)
outputformat.cmpd_name = "%s=";
- if(string_g && 1==H5Tget_size(type) &&
- H5T_INTEGER==H5Tget_class(type)) {
+ if (string_g && 1 == H5Tget_size(type) && H5T_INTEGER == H5Tget_class(type)) {
outputformat.ascii = TRUE;
outputformat.elmt_suf1 = "";
outputformat.elmt_suf2 = "";
@@ -1616,13 +1540,13 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain
info = &outputformat;
- if(space_type != H5S_NULL && space_type != H5S_NO_CLASS) {
- if(hexdump_g)
+ if (space_type != H5S_NULL && space_type != H5S_NO_CLASS) {
+ if (hexdump_g)
p_type = H5Tcopy(type);
else
p_type = H5Tget_native_type(type, H5T_DIR_DEFAULT);
- if(p_type >= 0) {
+ if (p_type >= 0) {
/* VL data special information */
unsigned int vl_data = 0; /* contains VL datatypes */
@@ -1661,7 +1585,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain
return 0;
}
-
+
/*-------------------------------------------------------------------------
* Function: dataset_list1
*
@@ -1671,14 +1595,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain
* printed by the caller.
*
* Return: Success: 0
- *
- * Failure: -1
- *
- * Programmer: Robb Matzke
- * Thursday, August 27, 1998
- *
- * Modifications:
- *
+ * Failure: -1
*-------------------------------------------------------------------------
*/
static herr_t
@@ -1706,17 +1623,19 @@ dataset_list1(hid_t dset)
space_type = H5Sget_simple_extent_type(space);
ndims = H5Sget_simple_extent_dims(space, cur_size, max_size);
h5tools_str_append(&buffer, " {");
- for (i=0; i<ndims; i++) {
+ for (i = 0; i < ndims; i++) {
h5tools_str_append(&buffer, "%s"HSIZE_T_FORMAT, i?", ":"", cur_size[i]);
if (max_size[i]==H5S_UNLIMITED) {
h5tools_str_append(&buffer, "/%s", "Inf");
}
- else if (max_size[i]!=cur_size[i] || verbose_g>0) {
+ else if (max_size[i] != cur_size[i] || verbose_g > 0) {
h5tools_str_append(&buffer, "/"HSIZE_T_FORMAT, max_size[i]);
}
}
- if (space_type==H5S_SCALAR) h5tools_str_append(&buffer, "SCALAR");
- else if (space_type==H5S_NULL) h5tools_str_append(&buffer, "NULL");
+ if (space_type == H5S_SCALAR)
+ h5tools_str_append(&buffer, "SCALAR");
+ else if (space_type == H5S_NULL)
+ h5tools_str_append(&buffer, "NULL");
h5tools_str_append(&buffer, "}");
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
H5Sclose (space);
@@ -1726,7 +1645,7 @@ dataset_list1(hid_t dset)
return 0;
}
-
+
/*-------------------------------------------------------------------------
* Function: dataset_list2
*
@@ -1734,14 +1653,7 @@ dataset_list1(hid_t dset)
* information which is general to all objects.
*
* Return: Success: 0
- *
- * Failure: -1
- *
- * Programmer: Robb Matzke
- * Thursday, August 27, 1998
- *
- * Modifications:
- *
+ * Failure: -1
*-------------------------------------------------------------------------
*/
static herr_t
@@ -1777,7 +1689,7 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name)
h5tools_str_reset(&buffer);
- if(verbose_g > 0) {
+ if (verbose_g > 0) {
dcpl = H5Dget_create_plist(dset);
space = H5Dget_space(dset);
type = H5Dget_type(dset);
@@ -1791,7 +1703,7 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name)
ndims = H5Pget_chunk(dcpl, (int)NELMTS(chsize), chsize/*out*/);
h5tools_str_append(&buffer, " %-10s {", "Chunks:");
total = H5Tget_size(type);
- for (i=0; i<ndims; i++) {
+ for (i = 0; i < ndims; i++) {
h5tools_str_append(&buffer, "%s"HSIZE_T_FORMAT, i?", ":"", chsize[i]);
total *= chsize[i];
}
@@ -1802,29 +1714,25 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name)
break;
case H5D_CONTIGUOUS:
/* Print information about external storage */
- if((nf = H5Pget_external_count(dcpl)) > 0) {
- for(i = 0, max_len = 0; i < nf; i++) {
- if(H5Pget_external(dcpl, (unsigned)i, sizeof(f_name), f_name, NULL, NULL) < 0)
+ if ((nf = H5Pget_external_count(dcpl)) > 0) {
+ for (i = 0, max_len = 0; i < nf; i++) {
+ if (H5Pget_external(dcpl, (unsigned)i, sizeof(f_name), f_name, NULL, NULL) < 0)
continue;
n = print_string(NULL, f_name, TRUE);
max_len = MAX(max_len, n);
} /* end for */
- h5tools_str_append(&buffer, " %-10s %d external file%s\n",
- "Extern:", nf, 1==nf?"":"s");
- h5tools_str_append(&buffer, " %4s %10s %10s %10s %s\n",
- "ID", "DSet-Addr", "File-Addr", "Bytes", "File");
- h5tools_str_append(&buffer, " %4s %10s %10s %10s ",
- "----", "----------", "----------", "----------");
- for (i=0; i<max_len; i++) h5tools_str_append(&buffer, "-");
+ h5tools_str_append(&buffer, " %-10s %d external file%s\n", "Extern:", nf, 1==nf?"":"s");
+ h5tools_str_append(&buffer, " %4s %10s %10s %10s %s\n", "ID", "DSet-Addr", "File-Addr", "Bytes", "File");
+ h5tools_str_append(&buffer, " %4s %10s %10s %10s ", "----", "----------", "----------", "----------");
+ for (i = 0; i < max_len; i++)
+ h5tools_str_append(&buffer, "-");
h5tools_str_append(&buffer, "\n");
- for (i=0, total=0; i<nf; i++) {
+ for (i = 0, total=0; i < nf; i++) {
if (H5Pget_external(dcpl, (unsigned)i, sizeof(f_name), f_name, &f_offset, &f_size)<0) {
- h5tools_str_append(&buffer,
- " #%03d %10"H5_PRINTF_LL_WIDTH"u %10s %10s ***ERROR*** %s\n",
- i, total, "", "",
- i+1<nf?"Following addresses are incorrect":"");
+ h5tools_str_append(&buffer, " #%03d %10"H5_PRINTF_LL_WIDTH"u %10s %10s ***ERROR*** %s\n",
+ i, total, "", "", i+1<nf?"Following addresses are incorrect":"");
}
- else if (H5S_UNLIMITED==f_size) {
+ else if (H5S_UNLIMITED == f_size) {
h5tools_str_append(&buffer, " #%03d %10"H5_PRINTF_LL_WIDTH"u %10"H5_PRINTF_LL_WIDTH"u %10s ",
i, total, (hsize_t)f_offset, "INF");
print_string(&buffer, f_name, TRUE);
@@ -1837,9 +1745,8 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name)
h5tools_str_append(&buffer, "\n");
total += f_size;
}
- h5tools_str_append(&buffer, " %4s %10s %10s %10s ",
- "----", "----------", "----------", "----------");
- for (i=0; i<max_len; i++)
+ h5tools_str_append(&buffer, " %4s %10s %10s %10s ", "----", "----------", "----------", "----------");
+ for (i = 0; i < max_len; i++)
h5tools_str_append(&buffer, "-");
h5tools_str_append(&buffer, "\n");
} /* end if */
@@ -1853,16 +1760,13 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name)
used = H5Dget_storage_size(dset);
tclass = H5Tget_class(type);
h5tools_str_append(&buffer, " %-10s ", "Storage:");
- switch (tclass)
- {
-
+ switch (tclass) {
case H5T_VLEN:
h5tools_str_append(&buffer, "information not available");
break;
case H5T_REFERENCE:
- if ( H5Tequal(type, H5T_STD_REF_DSETREG))
- {
+ if (H5Tequal(type, H5T_STD_REF_DSETREG)) {
h5tools_str_append(&buffer, "information not available");
}
break;
@@ -1879,11 +1783,8 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name)
case H5T_ARRAY:
case H5T_NCLASSES:
default:
- h5tools_str_append(&buffer, HSIZE_T_FORMAT" logical byte%s, "HSIZE_T_FORMAT" allocated byte%s",
- total, 1==total?"":"s",
- used, 1==used?"":"s");
- if (used>0)
- {
+ h5tools_str_append(&buffer, HSIZE_T_FORMAT" logical byte%s, "HSIZE_T_FORMAT" allocated byte%s", total, 1==total?"":"s", used, 1==used?"":"s");
+ if (used>0) {
utilization = ((double)total * (double)100.0f) / (double)used;
h5tools_str_append(&buffer, ", %1.2f%% utilization", utilization);
}
@@ -1892,18 +1793,14 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name)
h5tools_str_append(&buffer, "\n");
/* Print information about raw data filters */
- if((nf = H5Pget_nfilters(dcpl)) > 0) {
- for(i = 0; i < nf; i++) {
+ if ((nf = H5Pget_nfilters(dcpl)) > 0) {
+ for (i = 0; i < nf; i++) {
cd_nelmts = NELMTS(cd_values);
- filt_id = H5Pget_filter2(dcpl, (unsigned)i, &filt_flags, &cd_nelmts,
- cd_values, sizeof(f_name), f_name, NULL);
+ filt_id = H5Pget_filter2(dcpl, (unsigned)i, &filt_flags, &cd_nelmts, cd_values, sizeof(f_name), f_name, NULL);
f_name[sizeof(f_name) - 1] = '\0';
HDsnprintf(s, sizeof(s), "Filter-%d:", i);
- h5tools_str_append(&buffer, " %-10s %s-%u %s {", s,
- (f_name[0] ? f_name : "method"),
- (unsigned)filt_id,
- ((filt_flags & H5Z_FLAG_OPTIONAL) ? "OPT" : ""));
- for(cd_num = 0; cd_num < cd_nelmts; cd_num++)
+ h5tools_str_append(&buffer, " %-10s %s-%u %s {", s, (f_name[0] ? f_name : "method"), (unsigned)filt_id, ((filt_flags & H5Z_FLAG_OPTIONAL) ? "OPT" : ""));
+ for (cd_num = 0; cd_num < cd_nelmts; cd_num++)
h5tools_str_append(&buffer, "%s%u", (cd_num ? ", " : ""), cd_values[cd_num]);
h5tools_str_append(&buffer, "}\n");
} /* end for */
@@ -1916,7 +1813,7 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name)
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
/* Print address information */
- if(address_g)
+ if (address_g)
H5Ddebug(dset);
/* Close stuff */
@@ -1927,13 +1824,13 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name)
h5tools_str_close(&buffer);
- if(data_g)
+ if (data_g)
dump_dataset_values(dset);
return 0;
} /* end dataset_list2() */
-
+
/*-------------------------------------------------------------------------
* Function: datatype_list2
*
@@ -1941,20 +1838,13 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name)
* information which is general to all objects.
*
* Return: Success: 0
- *
- * Failure: -1
- *
- * Programmer: Robb Matzke
- * Thursday, November 5, 1998
- *
- * Modifications:
- *
+ * Failure: -1
*-------------------------------------------------------------------------
*/
static herr_t
datatype_list2(hid_t type, const char H5_ATTR_UNUSED *name)
{
- if (verbose_g>0) {
+ if (verbose_g > 0) {
hsize_t curr_pos = 0; /* total data element position */
h5tools_str_t buffer; /* string into which to render */
h5tools_context_t ctx; /* print context */
@@ -1975,19 +1865,14 @@ datatype_list2(hid_t type, const char H5_ATTR_UNUSED *name)
return 0;
}
-
+
/*-------------------------------------------------------------------------
* Function: list_obj
*
* Purpose: Prints information about an object
*
* Return: Success: 0
- *
- * Failure: -1
- *
- * Programmer: Quincey Koziol
- * Tuesday, November 6, 2007
- *
+ * Failure: -1
*-------------------------------------------------------------------------
*/
static herr_t
@@ -2006,26 +1891,26 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void
h5tools_str_reset(&buffer);
/* Print the link's name, either full name or base name */
- if(!iter->symlink_target)
+ if (!iter->symlink_target)
print_obj_name(&buffer, iter, name, "");
/* Check object information */
- if(oinfo->type < 0 || oinfo->type >= H5O_TYPE_NTYPES) {
+ if (oinfo->type < 0 || oinfo->type >= H5O_TYPE_NTYPES) {
h5tools_str_append(&buffer, "Unknown type(%d)", (int)oinfo->type);
obj_type = H5O_TYPE_UNKNOWN;
}
- if(iter->symlink_target)
+ if (iter->symlink_target)
h5tools_str_append(&buffer, "{");
- if(obj_type >= 0 && dispatch_g[obj_type].name)
+ if (obj_type >= 0 && dispatch_g[obj_type].name)
h5tools_str_append(&buffer, "%s", dispatch_g[obj_type].name);
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
/* Check if we've seen this object before */
- if(first_seen) {
+ if (first_seen) {
h5tools_str_reset(&buffer);
h5tools_str_append(&buffer, ", same as ");
print_string(&buffer, first_seen, TRUE);
- if(!iter->symlink_target) {
+ if (!iter->symlink_target) {
h5tools_str_append(&buffer, "\n");
}
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
@@ -2036,7 +1921,7 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void
/* Open the object. Not all objects can be opened. If this is the case
* then return right away.
*/
- if(obj_type >= 0 && (obj = H5Oopen(iter->fid, name, H5P_DEFAULT)) < 0) {
+ if (obj_type >= 0 && (obj = H5Oopen(iter->fid, name, H5P_DEFAULT)) < 0) {
h5tools_str_reset(&buffer);
h5tools_str_append(&buffer, " *ERROR*\n");
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
@@ -2044,9 +1929,9 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void
} /* end if */
/* List the first line of information for the object. */
- if(obj_type >= 0 && dispatch_g[obj_type].list1)
+ if (obj_type >= 0 && dispatch_g[obj_type].list1)
(dispatch_g[obj_type].list1)(obj);
- if(!iter->symlink_target || (verbose_g > 0)) {
+ if (!iter->symlink_target || (verbose_g > 0)) {
h5tools_str_reset(&buffer);
h5tools_str_append(&buffer, "\n");
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
@@ -2054,13 +1939,13 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void
/* Show detailed information about the object, beginning with information
* which is common to all objects. */
- if(verbose_g > 0) {
+ if (verbose_g > 0) {
size_t buf_size = 0;
char* comment = NULL;
ssize_t cmt_bufsize = -1;
/* Display attributes */
- if(obj_type >= 0)
+ if (obj_type >= 0)
H5Aiterate2(obj, H5_INDEX_NAME, H5_ITER_INC, NULL, list_attr, NULL);
/* Object location & reference count */
@@ -2070,7 +1955,7 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
/* Modification time */
- if(oinfo->mtime > 0) {
+ if (oinfo->mtime > 0) {
char buf[256];
struct tm *tm;
@@ -2078,7 +1963,7 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void
tm = HDgmtime(&(oinfo->mtime));
else
tm = HDlocaltime(&(oinfo->mtime));
- if(tm) {
+ if (tm) {
HDstrftime(buf, sizeof(buf), "%Y-%m-%d %H:%M:%S %Z", tm);
h5tools_str_reset(&buffer);
h5tools_str_append(&buffer, " %-10s %s\n", "Modified:", buf);
@@ -2094,9 +1979,9 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void
* If the call to H5Oget_comment returned an error, skip this block */
if (cmt_bufsize > 0) {
comment = (char *)HDmalloc((size_t)cmt_bufsize + 1); /* new_size including null terminator */
- if(comment) {
+ if (comment) {
cmt_bufsize = H5Oget_comment(obj, comment, (size_t)cmt_bufsize);
- if(cmt_bufsize > 0) {
+ if (cmt_bufsize > 0) {
comment[cmt_bufsize] = 0;
h5tools_str_reset(&buffer);
h5tools_str_append(&buffer, " %-10s \"", "Comment:");
@@ -2110,16 +1995,16 @@ list_obj(const char *name, const H5O_info_t *oinfo, const char *first_seen, void
} /* end if */
/* Detailed list for object */
- if(obj_type >= 0 && dispatch_g[obj_type].list2)
+ if (obj_type >= 0 && dispatch_g[obj_type].list2)
(dispatch_g[obj_type].list2)(obj, name);
/* Close the object. */
- if(obj_type >= 0)
+ if (obj_type >= 0)
H5Oclose(obj);
} /* end else */
done:
- if(iter->symlink_target) {
+ if (iter->symlink_target) {
h5tools_str_reset(&buffer);
h5tools_str_append(&buffer, "}\n");
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
@@ -2131,19 +2016,13 @@ done:
} /* end list_obj() */
-
/*-------------------------------------------------------------------------
* Function: list_lnk
*
* Purpose: Prints information about a link
*
* Return: Success: 0
- *
- * Failure: -1
- *
- * Programmer: Quincey Koziol
- * Thursday, November 8, 2007
- *
+ * Failure: -1
*-------------------------------------------------------------------------
*/
static herr_t
@@ -2190,15 +2069,13 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter)
h5tools_str_append(&buffer, buf);
h5tools_str_append(&buffer, "}");
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
- if(follow_symlink_g)
- {
+ if (follow_symlink_g) {
hbool_t orig_grp_literal = grp_literal_g;
h5tools_str_reset(&buffer);
h5tools_str_append(&buffer, " ");
/* Check if we have already seen this softlink */
- if(symlink_is_visited(iter->symlink_list, linfo->type, NULL, buf))
- {
+ if (symlink_is_visited(iter->symlink_list, linfo->type, NULL, buf)) {
h5tools_str_append(&buffer, "{Already Visited}\n");
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
goto done;
@@ -2206,7 +2083,7 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter)
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
/* Add this link to the list of seen softlinks */
- if(symlink_visit_add(iter->symlink_list, linfo->type, NULL, buf) < 0)
+ if (symlink_visit_add(iter->symlink_list, linfo->type, NULL, buf) < 0)
goto done;
/* Adjust user data to specify that we are operating on the
@@ -2215,11 +2092,10 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter)
/* Prevent recursive listing of soft link target if
* recursive_g is off */
- if(!recursive_g)
+ if (!recursive_g)
grp_literal_g = TRUE;
/* Recurse through the soft link */
- if(visit_obj(iter->fid, name, iter) < 0)
- {
+ if (visit_obj(iter->fid, name, iter) < 0) {
grp_literal_g = orig_grp_literal;
goto done;
}
@@ -2251,7 +2127,7 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter)
else if (no_dangling_link_g && ret == 0)
iter->symlink_list->dangle_link = TRUE;
- if(H5Lunpack_elink_val(buf, linfo->u.val_size, NULL, &filename, &path) < 0)
+ if (H5Lunpack_elink_val(buf, linfo->u.val_size, NULL, &filename, &path) < 0)
goto done;
h5tools_str_append(&buffer, "External Link {");
@@ -2259,21 +2135,19 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter)
h5tools_str_append(&buffer, "/");
if(*path != '/')
h5tools_str_append(&buffer, "/");
- h5tools_str_append(&buffer, path);
+ h5tools_str_append(&buffer, path);
h5tools_str_append(&buffer, "}");
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
/* Recurse through the external link */
/* keep the follow_elink_g for backward compatibility with -E */
- if(follow_link)
- {
+ if (follow_link) {
hbool_t orig_grp_literal = grp_literal_g;
h5tools_str_reset(&buffer);
h5tools_str_append(&buffer, " ");
/* Check if we have already seen this elink */
- if(symlink_is_visited(iter->symlink_list, linfo->type, filename, path))
- {
+ if (symlink_is_visited(iter->symlink_list, linfo->type, filename, path)) {
h5tools_str_append(&buffer, "{Already Visited}\n");
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
goto done;
@@ -2281,8 +2155,7 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter)
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
/* Add this link to the list of seen elinks */
- if(symlink_visit_add(iter->symlink_list, linfo->type, filename, path) < 0)
- {
+ if (symlink_visit_add(iter->symlink_list, linfo->type, filename, path) < 0) {
goto done;
}
@@ -2292,11 +2165,11 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter)
/* Prevent recursive listing of external link target if
* recursive_g is off */
- if(!recursive_g)
+ if (!recursive_g)
grp_literal_g = TRUE;
/* Recurse through the external link */
- if(visit_obj(iter->fid, name, iter) < 0) {
+ if (visit_obj(iter->fid, name, iter) < 0) {
grp_literal_g = orig_grp_literal;
goto done;
}
@@ -2325,7 +2198,7 @@ done:
return 0;
} /* end list_lnk() */
-
+
/*-------------------------------------------------------------------------
* Function: visit_obj
*
@@ -2334,11 +2207,6 @@ done:
* Return:
* Success: 0
* Failure: -1
- *
- * Programmer: Neil Fortner
- * Wednesday, August 21, 2008
- * Mostly copied from main()
- *
*-------------------------------------------------------------------------
*/
static herr_t
@@ -2357,8 +2225,8 @@ visit_obj(hid_t file, const char *oname, iter_t *iter)
h5tools_str_reset(&buffer);
/* Retrieve info for object to list */
- if(H5Oget_info_by_name(file, oname, &oi, H5P_DEFAULT) < 0) {
- if(iter->symlink_target) {
+ if (H5Oget_info_by_name(file, oname, &oi, H5P_DEFAULT) < 0) {
+ if (iter->symlink_target) {
h5tools_str_append(&buffer, "{**NOT FOUND**}\n");
iter->symlink_target = FALSE;
}
@@ -2370,9 +2238,9 @@ visit_obj(hid_t file, const char *oname, iter_t *iter)
} /* end if */
/* Check for group iteration */
- if(H5O_TYPE_GROUP == oi.type && !grp_literal_g) {
+ if (H5O_TYPE_GROUP == oi.type && !grp_literal_g) {
/* Get ID for group */
- if(!iter->symlink_target && (iter->gid = H5Gopen2(file, oname, H5P_DEFAULT)) < 0) {
+ if (!iter->symlink_target && (iter->gid = H5Gopen2(file, oname, H5P_DEFAULT)) < 0) {
h5tools_str_append(&buffer, "%s: unable to open '%s' as group\n", iter->fname, oname);
h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0);
goto done; /* Previously "continue", when this code was in main().
@@ -2388,7 +2256,7 @@ visit_obj(hid_t file, const char *oname, iter_t *iter)
h5trav_visit(file, oname, (hbool_t) (display_root_g || iter->symlink_target), recursive_g, list_obj, list_lnk, iter);
/* Close group */
- if(!iter->symlink_target)
+ if (!iter->symlink_target)
H5Gclose(iter->gid);
} /* end if */
else {
@@ -2405,7 +2273,7 @@ done:
return retval;
}
-
+
/*-------------------------------------------------------------------------
* Function: get_width
*
@@ -2415,14 +2283,7 @@ done:
* were borrowed from the GNU less(1).
*
* Return: Success: Number of columns.
- *
- * Failure: Some default number of columms.
- *
- * Programmer: Robb Matzke
- * Friday, November 6, 1998
- *
- * Modifications:
- *
+ * Failure: Some default number of columms.
*-------------------------------------------------------------------------
*/
static int
@@ -2433,7 +2294,7 @@ get_width(void)
/* Try to get it from the COLUMNS environment variable first since it's
* value is sometimes wrong. */
- if ((s=HDgetenv("COLUMNS")) && *s && isdigit((int)*s))
+ if ((s = HDgetenv("COLUMNS")) && *s && isdigit((int)*s))
width = (int)HDstrtol(s, NULL, 0);
#if defined(H5_HAVE_STRUCT_VIDEOCONFIG) && defined(H5_HAVE__GETVIDEOCONFIG)
@@ -2481,7 +2342,8 @@ get_width(void)
#endif
/* Set to at least 1 */
- if (width<1) width = 1;
+ if (width < 1)
+ width = 1;
return width;
}
@@ -2493,25 +2355,19 @@ get_width(void)
* Return:
* Success: TRUE (1)
* Failure: FALSE (0)
- *
- * Programmer:
- * Jonathan Kim (06/15/2010)
- *
*-------------------------------------------------------------------------*/
static hbool_t
is_valid_args(void)
{
hbool_t ret = TRUE;
- if(recursive_g && grp_literal_g)
- {
+ if (recursive_g && grp_literal_g) {
HDfprintf(rawerrorstream, "Error: 'recursive' option not compatible with 'group info' option!\n\n");
ret = FALSE;
goto out;
}
- if(no_dangling_link_g && !follow_symlink_g)
- {
+ if (no_dangling_link_g && !follow_symlink_g) {
HDfprintf(rawerrorstream, "Error: --no-dangling-links must be used along with --follow-symlinks option!\n\n");
ret = FALSE;
goto out;
@@ -2521,19 +2377,13 @@ out:
return ret;
}
-
+
/*-------------------------------------------------------------------------
* Function: leave
*
* Purpose: Close HDF5 and MPI and call exit()
*
* Return: Does not return
- *
- * Programmer: Quincey Koziol
- * Saturday, January 31, 2004
- *
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
static void
@@ -2544,35 +2394,52 @@ leave(int ret)
HDexit(ret);
}
-
+
/*-------------------------------------------------------------------------
* Function: main
*
* Purpose: Opens a file and lists the specified group
*
* Return: Success: 0
- *
- * Failure: 1
- *
- * Programmer: Robb Matzke
- * Monday, March 23, 1998
- *
- * Modifications:
- *
+ * Failure: 1
*-------------------------------------------------------------------------
*/
int
main(int argc, const char *argv[])
{
- hid_t file = -1;
- char *fname = NULL, *oname = NULL, *x;
+ hid_t file = -1;
+ char *fname = NULL, *oname = NULL, *x;
const char *s = NULL;
- char *rest;
- int argno;
+ char *rest;
+ int argno;
static char root_name[] = "/";
char drivername[50];
const char *preferred_driver = NULL;
- int err_exit = 0;
+ int err_exit = 0;
+ hid_t fapl_id = H5P_DEFAULT;
+
+#ifdef H5_HAVE_ROS3_VFD
+ /* default "anonymous" s3 configuration */
+ H5FD_ros3_fapl_t ros3_fa = {
+ 1, /* fapl version */
+ false, /* authenticate */
+ "", /* aws region */
+ "", /* access key id */
+ "", /* secret access key */
+ };
+#endif /* H5_HVAE_ROS3_VFD */
+
+#ifdef H5_HAVE_LIBHDFS
+ /* "default" HDFS configuration */
+ H5FD_hdfs_fapl_t hdfs_fa = {
+ 1, /* fapl version */
+ "localhost", /* namenode name */
+ 0, /* namenode port */
+ "", /* kerberos ticket cache */
+ "", /* user name */
+ 2048, /* stream buffer size */
+ };
+#endif /* H5_HAVE_LIBHDFS */
h5tools_setprogname(PROGRAMNAME);
h5tools_setstatus(EXIT_SUCCESS);
@@ -2589,90 +2456,258 @@ main(int argc, const char *argv[])
width_g = get_width();
/* Switches come before non-switch arguments */
- for(argno = 1; argno < argc && '-' == argv[argno][0]; argno++) {
- if(!HDstrcmp(argv[argno], "--")) {
+ for (argno = 1; argno < argc && '-' == argv[argno][0]; argno++) {
+ if (!HDstrcmp(argv[argno], "--")) {
/* Last switch */
argno++;
break;
- } else if(!HDstrcmp(argv[argno], "--help")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--help")) {
usage();
leave(EXIT_SUCCESS);
- } else if(!HDstrcmp(argv[argno], "--address")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--address")) {
address_g = TRUE;
- } else if(!HDstrcmp(argv[argno], "--data")) {
+ }
+ else if(!HDstrcmp(argv[argno], "--data")) {
data_g = TRUE;
- } else if(!HDstrcmp(argv[argno], "--enable-error-stack")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--enable-error-stack")) {
show_errors_g = TRUE;
/* deprecated --errors */
- } else if(!HDstrcmp(argv[argno], "--errors")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--errors")) {
show_errors_g = TRUE;
- } else if(!HDstrcmp(argv[argno], "--follow-symlinks")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--follow-symlinks")) {
follow_symlink_g = TRUE;
- } else if(!HDstrcmp(argv[argno], "--no-dangling-links")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--no-dangling-links")) {
no_dangling_link_g = TRUE;
- } else if(!HDstrcmp(argv[argno], "--external")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--external")) {
follow_elink_g = TRUE;
- } else if(!HDstrcmp(argv[argno], "--full")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--full")) {
fullname_g = TRUE;
- } else if(!HDstrcmp(argv[argno], "--group")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--group")) {
grp_literal_g = TRUE;
- } else if(!HDstrcmp(argv[argno], "--label")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--label")) {
label_g = TRUE;
- } else if(!HDstrcmp(argv[argno], "--recursive")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--recursive")) {
recursive_g = TRUE;
fullname_g = TRUE;
- } else if(!HDstrcmp(argv[argno], "--simple")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--simple")) {
simple_output_g = TRUE;
- } else if(!HDstrcmp(argv[argno], "--string")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--string")) {
string_g = TRUE;
- } else if(!HDstrncmp(argv[argno], "--vfd=", (size_t)6)) {
+ }
+ else if (!HDstrncmp(argv[argno], "--vfd=", (size_t)6)) {
preferred_driver = argv[argno]+6;
- } else if(!HDstrncmp(argv[argno], "--width=", (size_t)8)) {
+ }
+ else if (!HDstrncmp(argv[argno], "--width=", (size_t)8)) {
width_g = (int)HDstrtol(argv[argno]+8, &rest, 0);
- if(0 == width_g)
+ if (0 == width_g)
no_line_wrap_g = TRUE;
- else if(width_g < 0 || *rest) {
+ else if (width_g < 0 || *rest) {
usage();
leave(EXIT_FAILURE);
}
- } else if(!HDstrcmp(argv[argno], "--width")) {
- if((argno + 1) >= argc) {
+ }
+ else if (!HDstrcmp(argv[argno], "--width")) {
+ if ((argno + 1) >= argc) {
usage();
leave(EXIT_FAILURE);
- } else {
+ }
+ else {
s = argv[++argno];
}
width_g = (int)HDstrtol(s, &rest, 0);
- if(width_g <= 0 || *rest) {
+ if (width_g <= 0 || *rest) {
usage();
leave(EXIT_FAILURE);
}
- } else if(!HDstrcmp(argv[argno], "--verbose")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--verbose")) {
verbose_g++;
- } else if(!HDstrcmp(argv[argno], "--version")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--version")) {
print_version(h5tools_getprogname());
leave(EXIT_SUCCESS);
- } else if(!HDstrcmp(argv[argno], "--hexdump")) {
+ }
+ else if (!HDstrcmp(argv[argno], "--hexdump")) {
hexdump_g = TRUE;
- } else if(!HDstrncmp(argv[argno], "-w", (size_t)2)) {
- if(argv[argno][2]) {
+ }
+ else if (!HDstrncmp(argv[argno], "-w", (size_t)2)) {
+ if (argv[argno][2]) {
s = argv[argno] + 2;
- } else if((argno + 1) >= argc) {
+ }
+ else if ((argno + 1) >= argc) {
usage();
leave(EXIT_FAILURE);
- } else {
+ }
+ else {
s = argv[++argno];
}
width_g = (int)HDstrtol(s, &rest, 0);
- if(0 == width_g)
+ if(0 == width_g) {
no_line_wrap_g = TRUE;
+ }
else if(width_g < 0 || *rest) {
usage();
leave(EXIT_FAILURE);
}
- } else if('-'!=argv[argno][1]) {
+
+ } else if (!HDstrncmp(argv[argno], "--s3-cred=", (size_t)10)) {
+#ifndef H5_HAVE_ROS3_VFD
+ HDfprintf(rawerrorstream, "Error: Read-Only S3 VFD is not enabled\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+#else
+ unsigned nelems = 0;
+ char *start = NULL;
+ char *s3cred_src = NULL;
+ char **s3cred = NULL;
+ char const *ccred[3];
+ /* try to parse s3 credentials tuple
+ */
+ start = strchr(argv[argno], '=');
+ if (start == NULL) {
+ HDfprintf(rawerrorstream,
+ "Error: Unable to parse null credentials tuple\n"
+ " For anonymous access, omit \"--s3-cred\" and use only \"--vfd=ros3\"\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ start++;
+ if (FAIL == parse_tuple((const char *)start, ',', &s3cred_src, &nelems, &s3cred)) {
+ HDfprintf(rawerrorstream, "Error: Unable to parse S3 credentials\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ /* sanity-check tuple count
+ */
+ if (nelems != 3) {
+ HDfprintf(rawerrorstream, "Error: Invalid S3 credentials\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ ccred[0] = (const char *)s3cred[0];
+ ccred[1] = (const char *)s3cred[1];
+ ccred[2] = (const char *)s3cred[2];
+ if (0 == h5tools_populate_ros3_fapl(&ros3_fa, ccred)) {
+ HDfprintf(rawerrorstream, "Error: Invalid S3 credentials\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ HDfree(s3cred);
+ HDfree(s3cred_src);
+#endif /* H5_HAVE_ROS3_VFD */
+
+ }
+ else if (!HDstrncmp(argv[argno], "--hdfs-attrs=", (size_t)13)) {
+#ifndef H5_HAVE_LIBHDFS
+ PRINTVALSTREAM(rawoutstream, "The HDFS VFD is not enabled.\n");
+ leave(EXIT_FAILURE);
+#else
+ /* Parse received configuration data and set fapl config struct */
+
+ hbool_t _debug = FALSE;
+ unsigned nelems = 0;
+ char const *start = NULL;
+ char *props_src = NULL;
+ char **props = NULL;
+ unsigned long k = 0;
+
+ /* try to parse tuple
+ */
+ if (_debug) {
+ HDfprintf(stderr, "configuring hdfs...\n");
+ }
+ start = argv[argno]+13; /* should never segfault: worst case of */
+ if (*start != '(') { /* null-termintor after '='. */
+
+ if (_debug) {
+ HDfprintf(stderr, " no tuple.\n");
+ }
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ if (FAIL == parse_tuple((const char *)start, ',', &props_src, &nelems, &props)) {
+ HDfprintf(stderr, " unable to parse tuple.\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+
+ /* sanity-check tuple count
+ */
+ if (nelems != 5) {
+ HDfprintf(stderr, " expected 5-ple, got `%d`\n", nelems);
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ if (_debug) {
+ HDfprintf(stderr, " got hdfs-attrs tuple: `(%s,%s,%s,%s,%s)`\n",
+ props[0], props[1], props[2], props[3], props[4]);
+ }
+
+ /* Populate fapl configuration structure with given properties.
+ * WARNING: No error-checking is done on length of input strings...
+ * Silent overflow is possible, albeit unlikely.
+ */
+ if (HDstrncmp(props[0], "", 1)) {
+ if (_debug) {
+ HDfprintf(stderr, " setting namenode name: %s\n", props[0]);
+ }
+ HDstrncpy(hdfs_fa.namenode_name, (const char *)props[0], HDstrlen(props[0]));
+ }
+ if (HDstrncmp(props[1], "", 1)) {
+ k = strtoul((const char *)props[1], NULL, 0);
+ if (errno == ERANGE) {
+ HDfprintf(stderr, " supposed port number wasn't.\n");
+ leave(EXIT_FAILURE);
+ }
+ if (_debug) {
+ HDfprintf(stderr, " setting namenode port: %lu\n", k);
+ }
+ hdfs_fa.namenode_port = (int32_t)k;
+ }
+ if (HDstrncmp(props[2], "", 1)) {
+ if (_debug) {
+ HDfprintf(stderr, " setting kerb cache path: %s\n", props[2]);
+ }
+ HDstrncpy(hdfs_fa.kerberos_ticket_cache, (const char *)props[2], HDstrlen(props[2]));
+ }
+ if (HDstrncmp(props[3], "", 1)) {
+ if (_debug) {
+ HDfprintf(stderr, " setting username: %s\n", props[3]);
+ }
+ HDstrncpy(hdfs_fa.user_name, (const char *)props[3], HDstrlen(props[3]));
+ }
+ if (HDstrncmp(props[4], "", 1)) {
+ k = HDstrtoul((const char *)props[4], NULL, 0);
+ if (errno == ERANGE) {
+ HDfprintf(stderr, " supposed buffersize number wasn't.\n");
+ leave(EXIT_FAILURE);
+ }
+ if (_debug) {
+ HDfprintf(stderr, " setting stream buffer size: %lu\n", k);
+ }
+ hdfs_fa.stream_buffer_size = (int32_t)k;
+ }
+ HDfree(props);
+ HDfree(props_src);
+#endif /* H5_HAVE_LIBHDFS */
+
+ }
+ else if('-'!=argv[argno][1]) {
/* Single-letter switches */
for(s = argv[argno] + 1; *s; s++) {
switch(*s) {
@@ -2742,7 +2777,9 @@ main(int argc, const char *argv[])
leave(EXIT_FAILURE);
} /* end switch */
} /* end for */
- } else {
+ }
+ else {
+ HDfprintf(stderr, "Unknown argument: %s\n", argv[argno]);
usage();
leave(EXIT_FAILURE);
}
@@ -2750,18 +2787,54 @@ main(int argc, const char *argv[])
/* If no arguments remain then print a usage message (instead of doing
* absolutely nothing ;-) */
- if(argno >= argc) {
+ if (argno >= argc) {
usage();
leave(EXIT_FAILURE);
} /* end if */
/* Check for conflicting arguments */
- if (!is_valid_args())
- {
+ if (!is_valid_args()) {
usage();
leave(EXIT_FAILURE);
}
+ if (preferred_driver) {
+ void *conf_fa = NULL;
+
+ if (!HDstrcmp(preferred_driver, "ros3")) {
+#ifndef H5_HAVE_ROS3_VFD
+ HDfprintf(rawerrorstream, "Error: Read-Only S3 VFD not enabled.\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+#else
+ conf_fa = (void *)&ros3_fa;
+#endif /* H5_HAVE_ROS3_VFD */
+
+ }
+ else if (!HDstrcmp(preferred_driver, "hdfs")) {
+#ifndef H5_HAVE_LIBHDFS
+ PRINTVALSTREAM(rawoutstream, "The HDFS VFD is not enabled.\n");
+ leave(EXIT_FAILURE);
+#else
+ conf_fa = (void *)&hdfs_fa;
+#endif /* H5_HAVE_LIBHDFS */
+ }
+
+ if (conf_fa != NULL) {
+ HDassert(fapl_id == H5P_DEFAULT);
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ if (fapl_id < 0) {
+ HDfprintf(rawerrorstream, "Error: Unable to create fapl entry\n\n");
+ leave(EXIT_FAILURE);
+ }
+ if (0 == h5tools_set_configured_fapl(fapl_id, preferred_driver, conf_fa)) {
+ HDfprintf(rawerrorstream, "Error: Unable to set fapl\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ }
+ } /* preferred_driver defined */
+
/* Turn off HDF5's automatic error printing unless you're debugging h5ls */
if(!show_errors_g)
H5Eset_auto2(H5E_DEFAULT, NULL, NULL);
@@ -2790,11 +2863,16 @@ main(int argc, const char *argv[])
oname = NULL;
file = -1;
- while(fname && *fname) {
- file = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, preferred_driver, drivername, sizeof drivername);
+ while (fname && *fname) {
+ if (fapl_id != H5P_DEFAULT) {
+ file = H5Fopen(fname, H5F_ACC_RDONLY, fapl_id);
+ }
+ else {
+ file = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, preferred_driver, drivername, sizeof drivername);
+ }
- if(file >= 0) {
- if(verbose_g)
+ if (file >= 0) {
+ if (verbose_g)
PRINTSTREAM(rawoutstream, "Opened \"%s\" with %s driver.\n", fname, drivername);
break; /*success*/
} /* end if */
@@ -2802,27 +2880,27 @@ main(int argc, const char *argv[])
/* Shorten the file name; lengthen the object name */
x = oname;
oname = HDstrrchr(fname, '/');
- if(x)
+ if (x)
*x = '/';
- if(!oname)
+ if (!oname)
break;
*oname = '\0';
} /* end while */
- if(file < 0) {
+ if (file < 0) {
HDfprintf(rawerrorstream, "%s: unable to open file\n", argv[argno-1]);
HDfree(fname);
err_exit = 1;
continue;
} /* end if */
- if(oname) {
+ if (oname) {
/* Always use absolute paths to avoid confusion, keep track of where
* to begin path name output */
*oname = '/';
iter.base_len = HDstrlen(oname);
iter.base_len -= oname[iter.base_len-1] == '/';
x = oname;
- if(NULL == (oname = HDstrdup(oname))) {
+ if (NULL == (oname = HDstrdup(oname))) {
HDfprintf(rawerrorstream, "memory allocation failed\n");
leave(EXIT_FAILURE);
}
@@ -2831,9 +2909,9 @@ main(int argc, const char *argv[])
* is displayed if it is a link or non-group object */
iter.name_start = 1;
}
- if(!oname || !*oname) {
+ if (!oname || !*oname) {
oname = root_name;
- if(recursive_g)
+ if (recursive_g)
display_root_g = TRUE;
iter.base_len = 0;
iter.name_start = 0;
@@ -2854,9 +2932,9 @@ main(int argc, const char *argv[])
symlink_list.objs = NULL;
/* Check for root group as object name */
- if(HDstrcmp(oname, root_name)) {
+ if (HDstrcmp(oname, root_name)) {
/* Check the type of link given */
- if(H5Lget_info(file, oname, &li, H5P_DEFAULT) < 0) {
+ if (H5Lget_info(file, oname, &li, H5P_DEFAULT) < 0) {
hsize_t curr_pos = 0; /* total data element position */
h5tools_str_t buffer; /* string into which to render */
h5tools_context_t ctx; /* print context */
@@ -2875,8 +2953,8 @@ main(int argc, const char *argv[])
li.type = H5L_TYPE_HARD;
/* Open the object and display it's information */
- if(li.type == H5L_TYPE_HARD) {
- if(visit_obj(file, oname, &iter) < 0)
+ if (li.type == H5L_TYPE_HARD) {
+ if (visit_obj(file, oname, &iter) < 0)
leave(EXIT_FAILURE);
} /* end if(li.type == H5L_TYPE_HARD) */
else {
@@ -2887,11 +2965,10 @@ main(int argc, const char *argv[])
}
H5Fclose(file);
HDfree(fname);
- if(x)
+ if (x)
HDfree(oname);
- for(u=0; u < symlink_list.nused; u++)
- {
+ for (u = 0; u < symlink_list.nused; u++) {
if (symlink_list.objs[u].type == H5L_TYPE_EXTERNAL)
HDfree(symlink_list.objs[u].file);
@@ -2904,6 +2981,13 @@ main(int argc, const char *argv[])
err_exit = 1;
} /* end while */
+ if (fapl_id != H5P_DEFAULT) {
+ if (0 < H5Pclose(fapl_id)) {
+ HDfprintf(rawerrorstream, "Error: Unable to set close fapl entry\n\n");
+ leave(EXIT_FAILURE);
+ }
+ }
+
if (err_exit)
leave(EXIT_FAILURE);
else
diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c
index e8c3409..b6f332a 100644
--- a/tools/h5repack/h5repack.c
+++ b/tools/h5repack/h5repack.c
@@ -481,7 +481,7 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p,
} /*H5T_REFERENCE*/
if (options->verbose)
- printf(FORMAT_OBJ_ATTR, "attr", name);
+ HDprintf(FORMAT_OBJ_ATTR, "attr", name);
/*-------------------------------------------------------------------------
* close
@@ -543,7 +543,7 @@ static int check_options(pack_opt_t *options) {
*/
if (options->verbose && have_request(options) /* only print if requested */) {
if (options->all_layout == 1) {
- printf("All objects to modify layout are...\n");
+ HDprintf("All objects to modify layout are...\n");
switch (options->layout_g) {
case H5D_COMPACT:
strcpy(slayout, "compact");
@@ -561,17 +561,17 @@ static int check_options(pack_opt_t *options) {
strcpy(slayout, "invalid layout\n");
HGOTO_DONE(FAIL);
}
- printf(" Apply %s layout to all", slayout);
+ HDprintf(" Apply %s layout to all", slayout);
if (H5D_CHUNKED == options->layout_g) {
- printf("with dimension [ ");
+ HDprintf("with dimension [ ");
for (j = 0; j < options->chunk_g.rank; j++)
- printf("%d ", (int) options->chunk_g.chunk_lengths[j]);
- printf("]");
+ HDprintf("%d ", (int) options->chunk_g.chunk_lengths[j]);
+ HDprintf("]");
}
- printf("\n");
+ HDprintf("\n");
}
else
- printf("No all objects to modify layout\n");
+ HDprintf("No all objects to modify layout\n");
}/* verbose */
for (i = 0; i < options->op_tbl->nelems; i++) {
@@ -579,16 +579,16 @@ static int check_options(pack_opt_t *options) {
if (options->op_tbl->objs[i].chunk.rank > 0) {
if (options->verbose) {
- printf(" <%s> with chunk size ", name);
+ HDprintf(" <%s> with chunk size ", name);
for (k = 0; k < options->op_tbl->objs[i].chunk.rank; k++)
- printf("%d ", (int) options->op_tbl->objs[i].chunk.chunk_lengths[k]);
- printf("\n");
+ HDprintf("%d ", (int) options->op_tbl->objs[i].chunk.chunk_lengths[k]);
+ HDprintf("\n");
}
has_ck = 1;
}
else if (options->op_tbl->objs[i].chunk.rank == -2) {
if (options->verbose)
- printf(" <%s> %s\n", name, "NONE (contiguous)");
+ HDprintf(" <%s> %s\n", name, "NONE (contiguous)");
has_ck = 1;
}
}
@@ -603,33 +603,33 @@ static int check_options(pack_opt_t *options) {
if (options->verbose && have_request(options) /* only print if requested */) {
if (options->all_filter == 1) {
- printf("All objects to apply filter are...\n");
+ HDprintf("All objects to apply filter are...\n");
for (k = 0; k < options->n_filter_g; k++) {
H5Z_filter_t filtn = options->filter_g[k].filtn;
if (filtn < 0) {
- printf(" Unknown\n");
+ HDprintf(" Unknown\n");
continue;
}
switch (filtn) {
case H5Z_FILTER_NONE:
- printf(" Uncompress all\n");
+ HDprintf(" Uncompress all\n");
break;
case H5Z_FILTER_SHUFFLE:
case H5Z_FILTER_FLETCHER32:
- printf(" All with %s\n", get_sfilter(filtn));
+ HDprintf(" All with %s\n", get_sfilter(filtn));
break;
case H5Z_FILTER_SZIP:
case H5Z_FILTER_DEFLATE:
- printf(" All with %s, parameter %d\n", get_sfilter(filtn), options->filter_g[k].cd_values[0]);
+ HDprintf(" All with %s, parameter %d\n", get_sfilter(filtn), options->filter_g[k].cd_values[0]);
break;
default:
- printf(" User Defined %d\n", filtn);
+ HDprintf(" User Defined %d\n", filtn);
break;
} /* k */
};
}
else
- printf("No all objects to apply filter\n");
+ HDprintf("No all objects to apply filter\n");
} /* verbose */
for (i = 0; i < options->op_tbl->nelems; i++) {
@@ -640,9 +640,9 @@ static int check_options(pack_opt_t *options) {
if (options->verbose) {
if(pack.filter[j].filtn >= 0) {
if(pack.filter[j].filtn > H5Z_FILTER_SCALEOFFSET)
- printf(" <%s> with %s filter %d\n", name, get_sfilter(pack.filter[j].filtn), pack.filter[j].filtn);
+ HDprintf(" <%s> with %s filter %d\n", name, get_sfilter(pack.filter[j].filtn), pack.filter[j].filtn);
else
- printf(" <%s> with %s filter\n", name, get_sfilter(pack.filter[j].filtn));
+ HDprintf(" <%s> with %s filter\n", name, get_sfilter(pack.filter[j].filtn));
}
}
has_cp = 1;
@@ -673,7 +673,7 @@ static int check_options(pack_opt_t *options) {
*/
if (options->ublock_filename != NULL && options->ublock_size == 0) {
if (options->verbose) {
- printf("Warning: user block size missing for file %s. Assigning a default size of 1024...\n", options->ublock_filename);
+ HDprintf("Warning: user block size missing for file %s. Assigning a default size of 1024...\n", options->ublock_filename);
options->ublock_size = 1024;
}
}
@@ -708,7 +708,7 @@ static int check_objects(const char* fname, pack_opt_t *options) {
hid_t did = -1;
hid_t sid = -1;
unsigned int i;
- unsigned int uf;
+ int ifil;
trav_table_t *travt = NULL;
/* nothing to do */
@@ -742,31 +742,31 @@ static int check_objects(const char* fname, pack_opt_t *options) {
*/
if (options->verbose)
- printf("Opening file. Searching %d objects to modify ...\n", travt->nobjs);
+ HDprintf("Opening file. Searching %zu objects to modify ...\n", travt->nobjs);
for (i = 0; i < options->op_tbl->nelems; i++) {
char* name = options->op_tbl->objs[i].path;
if (options->verbose)
- printf(" <%s>", name);
+ HDprintf(" <%s>", name);
/* the input object names are present in the file and are valid */
if (h5trav_getindext(name, travt) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "%s Could not find <%s> in file <%s>. Exiting...\n",
(options->verbose ? "\n" : ""), name, fname);
if (options->verbose)
- printf("...Found\n");
+ HDprintf("...Found\n");
- for (uf = 0; uf < options->op_tbl->objs[i].nfilters; uf++) {
- if (options->op_tbl->objs[i].filter[uf].filtn < 0)
+ for (ifil = 0; ifil < options->op_tbl->objs[i].nfilters; ifil++) {
+ if (options->op_tbl->objs[i].filter[ifil].filtn < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "invalid filter");
/* check for extra filter conditions */
- switch (options->op_tbl->objs[i].filter[uf].filtn) {
+ switch (options->op_tbl->objs[i].filter[ifil].filtn) {
/* chunk size must be smaller than pixels per block */
case H5Z_FILTER_SZIP:
{
int j;
hsize_t csize = 1;
- unsigned ppb = options->op_tbl->objs[i].filter[uf].cd_values[0];
+ unsigned ppb = options->op_tbl->objs[i].filter[ifil].cd_values[0];
hsize_t dims[H5S_MAX_RANK];
int rank;
@@ -794,7 +794,7 @@ static int check_objects(const char* fname, pack_opt_t *options) {
}
if (csize < ppb) {
- printf(" <warning: SZIP settings, chunk size is smaller than pixels per block>\n");
+ HDprintf(" <warning: SZIP settings, chunk size is smaller than pixels per block>\n");
HGOTO_DONE(0);
}
}
@@ -802,7 +802,7 @@ static int check_objects(const char* fname, pack_opt_t *options) {
default:
break;
}
- } /* for uf */
+ } /* for ifil */
} /* for i */
done:
diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h
index 5b7bb0b..211d254 100644
--- a/tools/h5repack/h5repack.h
+++ b/tools/h5repack/h5repack.h
@@ -94,21 +94,21 @@ typedef struct {
/* all the above, ready to go to the hrepack call */
typedef struct {
- pack_opttbl_t *op_tbl; /*table with all -c and -f options */
- int all_layout; /*apply the layout to all objects */
- int all_filter; /*apply the filter to all objects */
+ pack_opttbl_t *op_tbl; /*table with all -c and -f options */
+ int all_layout; /*apply the layout to all objects */
+ int all_filter; /*apply the filter to all objects */
filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */
- int n_filter_g; /*number of global filters */
- chunk_info_t chunk_g; /*global chunk INFO for the ALL case */
- H5D_layout_t layout_g; /*global layout information for the ALL case */
- int verbose; /*verbose mode */
- hsize_t min_comp; /*minimum size to compress, in bytes */
- int use_native; /*use a native type in write */
- int latest; /*pack file with the latest file format */
- int grp_compact; /* Set the maximum number of links to store as header messages in the group */
- int grp_indexed; /* Set the minimum number of links to store in the indexed format */
- int msg_size[8]; /* Minimum size of shared messages: dataspace,
- datatype, fill value, filter pipleline, attribute */
+ int n_filter_g; /*number of global filters */
+ chunk_info_t chunk_g; /*global chunk INFO for the ALL case */
+ H5D_layout_t layout_g; /*global layout information for the ALL case */
+ int verbose; /*verbose mode */
+ hsize_t min_comp; /*minimum size to compress, in bytes */
+ int use_native; /*use a native type in write */
+ int latest; /*pack file with the latest file format */
+ int grp_compact; /* Set the maximum number of links to store as header messages in the group */
+ int grp_indexed; /* Set the minimum number of links to store in the indexed format */
+ int msg_size[8]; /* Minimum size of shared messages: dataspace,
+ datatype, fill value, filter pipleline, attribute */
const char *ublock_filename; /* user block file name */
hsize_t ublock_size; /* user block size */
hsize_t meta_block_size; /* metadata aggregation block size (for H5Pset_meta_block_size) */
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index 6b9b079..02a3c00 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -264,7 +264,7 @@ print_user_block(fnamein, fidin);
*-------------------------------------------------------------------------
*/
if (options->verbose)
- printf("Making new file ...\n");
+ HDprintf("Making new file ...\n");
if ((fidout = H5Fcreate(fnameout, H5F_ACC_TRUNC, fcpl, fapl)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fcreate could not create file <%s>:", fnameout);
@@ -592,9 +592,9 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
*/
if (options->verbose) {
- printf("-----------------------------------------\n");
- printf(" Type Filter (Compression) Name\n");
- printf("-----------------------------------------\n");
+ HDprintf("-----------------------------------------\n");
+ HDprintf(" Type Filter (Compression) Name\n");
+ HDprintf("-----------------------------------------\n");
}
if (travt->objs) {
@@ -613,7 +613,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
*/
case H5TRAV_TYPE_GROUP:
if (options->verbose)
- printf(FORMAT_OBJ, "group", travt->objs[i].name);
+ HDprintf(FORMAT_OBJ, "group", travt->objs[i].name);
/* open input group */
if ((grp_in = H5Gopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0)
@@ -848,7 +848,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
if (dset_out == FAIL) {
H5Epush2(H5tools_ERR_STACK_g, __FILE__, FUNC, __LINE__, H5tools_ERR_CLS_g, H5E_tools_g, H5E_tools_min_id_g, "H5Dcreate2 failed");
if (options->verbose)
- printf(" warning: could not create dataset <%s>. Applying original settings\n", travt->objs[i].name);
+ HDprintf(" warning: could not create dataset <%s>. Applying original settings\n", travt->objs[i].name);
if ((dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_in, H5P_DEFAULT)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dcreate2 failed");
@@ -1011,10 +1011,10 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
(in case there was a filter)
*/
if (has_filter && apply_s == 0)
- printf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n", travt->objs[i].name, (int) options->min_comp);
+ HDprintf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n", travt->objs[i].name, (int) options->min_comp);
if (has_filter && apply_f == 0)
- printf(" <warning: could not apply the filter to %s>\n", travt->objs[i].name);
+ HDprintf(" <warning: could not apply the filter to %s>\n", travt->objs[i].name);
} /* verbose */
/*-------------------------------------------------------------------------
@@ -1095,7 +1095,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed");
if (options->verbose)
- printf(FORMAT_OBJ, "dset", travt->objs[i].name);
+ HDprintf(FORMAT_OBJ, "dset", travt->objs[i].name);
} /* end do we have request for filter/chunking */
break;
@@ -1106,7 +1106,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
*/
case H5TRAV_TYPE_NAMED_DATATYPE:
if (options->verbose)
- printf(FORMAT_OBJ, "type", travt->objs[i].name);
+ HDprintf(FORMAT_OBJ, "type", travt->objs[i].name);
if ((type_in = H5Topen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Topen2 failed");
@@ -1145,13 +1145,13 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
case H5TRAV_TYPE_LINK:
case H5TRAV_TYPE_UDLINK:
if (options->verbose)
- printf(FORMAT_OBJ, "link", travt->objs[i].name);
+ HDprintf(FORMAT_OBJ, "link", travt->objs[i].name);
if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Lcopy failed");
if (options->verbose)
- printf(FORMAT_OBJ, "link", travt->objs[i].name);
+ HDprintf(FORMAT_OBJ, "link", travt->objs[i].name);
break;
default:
@@ -1294,15 +1294,15 @@ print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr)
}/*i*/
if (!pr)
- printf(FORMAT_OBJ, "dset", objname);
+ HDprintf(FORMAT_OBJ, "dset", objname);
else {
- char str[255], temp[28];
+ char str[512], temp[512];
HDstrcpy(str, "dset ");
HDstrcat(str, strfilter);
HDsprintf(temp, " (%.3f:1)", ratio);
HDstrcat(str, temp);
- printf(FORMAT_OBJ, str, objname);
+ HDprintf(FORMAT_OBJ, str, objname);
}
}
@@ -1429,10 +1429,10 @@ print_user_block(const char *filename, hid_t fid)
for (i = 0; i < nread; i++) {
- printf("%c ", rbuf[i]);
+ HDprintf("%c ", rbuf[i]);
}
- printf("\n");
+ HDprintf("\n");
if (nread < 0) {
HGOTO_ERROR(H5E_tools_g, H5E_tools_min_id_g, "nread < 0");
diff --git a/tools/h5repack/h5repack_filters.c b/tools/h5repack/h5repack_filters.c
index 8cc7b92..c42d284 100644
--- a/tools/h5repack/h5repack_filters.c
+++ b/tools/h5repack/h5repack_filters.c
@@ -42,7 +42,8 @@ static int aux_copy_obj(hid_t dcpl_id, /* dataset creation property list */
H5D_layout_t layout;
int rank; /* rank of dataset */
hsize_t chsize[64]; /* chunk size in elements */
- unsigned int i;
+ int i;
+ unsigned u;
/* get information about input filters */
if ((nfilters = H5Pget_nfilters(dcpl_id)) < 0)
@@ -65,8 +66,8 @@ static int aux_copy_obj(hid_t dcpl_id, /* dataset creation property list */
if ((rank = H5Pget_chunk(dcpl_id, NELMTS(chsize), chsize/*out*/)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_chunk failed");
objout->chunk.rank = rank;
- for (i = 0; i < rank; i++)
- objout->chunk.chunk_lengths[i] = chsize[i];
+ for (u = 0; u < (unsigned)rank; u++)
+ objout->chunk.chunk_lengths[u] = chsize[u];
}
done:
@@ -85,8 +86,7 @@ static int aux_find_obj(const char* name, /* object name from traverse list */
pack_info_t *obj /*OUT*/) /* info about object to filter */
{
char *pdest = NULL;
- char *pname = NULL;
- int result;
+ const char *pname = NULL;
unsigned int i;
for (i = 0; i < options->op_tbl->nelems; i++) {
@@ -335,12 +335,14 @@ int apply_filters(const char* name, /* object name from traverse list */
sm_nbytes = msize;
for (i = rank; i > 0; --i) {
- hsize_t size = H5TOOLS_BUFSIZE / sm_nbytes;
+ hsize_t size = 0;
+ if(sm_nbytes == 0)
+ HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "number of bytes per stripmine must be > 0");
+ size = H5TOOLS_BUFSIZE / sm_nbytes;
if (size == 0) /* datum size > H5TOOLS_BUFSIZE */
size = 1;
sm_size[i - 1] = MIN(dims[i - 1], size);
sm_nbytes *= sm_size[i - 1];
- HDassert(sm_nbytes > 0);
}
for (i = 0; i < rank; i++) {
diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c
index 2bd0fbb..9c96e56 100644
--- a/tools/h5repack/h5repack_main.c
+++ b/tools/h5repack/h5repack_main.c
@@ -77,6 +77,7 @@ static void usage(const char *prog) {
PRINTVALSTREAM(rawoutstream, " -v, --verbose Verbose mode, print object information\n");
PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n");
PRINTVALSTREAM(rawoutstream, " -n, --native Use a native HDF5 type when repacking\n");
+ PRINTVALSTREAM(rawoutstream, " --enable-error-stack Prints messages from the HDF5 error stack as they occur\n");
PRINTVALSTREAM(rawoutstream, " -L, --latest Use latest version of file format\n");
PRINTVALSTREAM(rawoutstream, " -c L1, --compact=L1 Maximum number of links in header messages\n");
PRINTVALSTREAM(rawoutstream, " -d L2, --indexed=L2 Minimum number of links in the indexed format\n");
@@ -106,9 +107,6 @@ static void usage(const char *prog) {
PRINTVALSTREAM(rawoutstream, " F - is the shared object header message type, any of <dspace|dtype|fill|\n");
PRINTVALSTREAM(rawoutstream, " pline|attr>. If F is not specified, S applies to all messages\n");
PRINTVALSTREAM(rawoutstream, "\n");
- PRINTVALSTREAM(rawoutstream, " --enable-error-stack Prints messages from the HDF5 error stack as they\n");
- PRINTVALSTREAM(rawoutstream, " occur.\n");
- PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, " FILT - is a string with the format:\n");
PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, " <list of objects>:<name of filter>=<filter parameters>\n");
diff --git a/tools/h5repack/h5repack_refs.c b/tools/h5repack/h5repack_refs.c
index 59b652c..fc9e897 100644
--- a/tools/h5repack/h5repack_refs.c
+++ b/tools/h5repack/h5repack_refs.c
@@ -158,7 +158,7 @@ int do_copy_refobjs(hid_t fidin,
if(nelmts) {
buf = (hobj_ref_t *)HDmalloc((unsigned)(nelmts * msize));
if(buf==NULL) {
- printf("cannot read into memory\n" );
+ HDprintf("cannot read into memory\n" );
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed");
} /* end if */
if(H5Dread(dset_in, mtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0)
@@ -166,7 +166,7 @@ int do_copy_refobjs(hid_t fidin,
refbuf = (hobj_ref_t*) HDcalloc((unsigned)nelmts, msize);
if(refbuf == NULL){
- printf("cannot allocate memory\n" );
+ HDprintf("cannot allocate memory\n" );
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDcalloc failed");
} /* end if */
for(u = 0; u < nelmts; u++) {
@@ -183,8 +183,8 @@ int do_copy_refobjs(hid_t fidin,
if(H5Rcreate(&refbuf[u], fidout, refname, H5R_OBJECT, (hid_t)-1) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Rcreate failed");
if(options->verbose) {
- printf(FORMAT_OBJ,"dset",travt->objs[i].name );
- printf("object <%s> object reference created to <%s>\n",
+ HDprintf(FORMAT_OBJ,"dset",travt->objs[i].name );
+ HDprintf("object <%s> object reference created to <%s>\n",
travt->objs[i].name,
refname);
}
@@ -234,7 +234,7 @@ int do_copy_refobjs(hid_t fidin,
if(nelmts) {
buf = (hdset_reg_ref_t *)HDmalloc((unsigned)(nelmts * msize));
if(buf == NULL) {
- printf("cannot read into memory\n");
+ HDprintf("cannot read into memory\n");
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed");
} /* end if */
if(H5Dread(dset_in, mtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0)
@@ -246,7 +246,7 @@ int do_copy_refobjs(hid_t fidin,
*/
refbuf = (hdset_reg_ref_t *)HDcalloc(sizeof(hdset_reg_ref_t), (size_t)nelmts); /*init to zero */
if(refbuf == NULL) {
- printf("cannot allocate memory\n");
+ HDprintf("cannot allocate memory\n");
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDcalloc failed");
} /* end if */
@@ -271,8 +271,8 @@ int do_copy_refobjs(hid_t fidin,
if(H5Sclose(region_id) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed");
if(options->verbose) {
- printf(FORMAT_OBJ,"dset",travt->objs[i].name );
- printf("object <%s> region reference created to <%s>\n",
+ HDprintf(FORMAT_OBJ,"dset",travt->objs[i].name );
+ HDprintf("object <%s> region reference created to <%s>\n",
travt->objs[i].name,
refname);
}
@@ -592,7 +592,7 @@ static int copy_refs_attr(hid_t loc_in,
if((is_ref || is_ref_array) && (H5R_OBJ_REF_BUF_SIZE==msize)) {
buf = (hobj_ref_t *)HDmalloc((unsigned)(nelmts * msize));
if(buf == NULL) {
- printf("cannot read into memory\n");
+ HDprintf("cannot read into memory\n");
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed");
} /* end if */
if(H5Aread(attr_id, mtype_id, buf) < 0)
@@ -600,12 +600,12 @@ static int copy_refs_attr(hid_t loc_in,
refbuf = (hobj_ref_t *)HDcalloc((unsigned)nelmts, msize);
if(refbuf == NULL) {
- printf("cannot allocate memory\n");
+ HDprintf("cannot allocate memory\n");
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDcalloc failed");
} /* end if */
for(i = 0; i < (unsigned)nelmts; i++) {
- if (update_ref_value(attr_id, H5R_OBJECT, &((hobj_ref_t *)buf)[i], fidout, &((hobj_ref_t *)refbuf)[i], travt)<0)
+ if(update_ref_value(attr_id, H5R_OBJECT, &((hobj_ref_t *)buf)[i], fidout, &((hobj_ref_t *)refbuf)[i], travt) < 0)
continue;
if(options->verbose)
printf("object <%s> reference created to <%s>\n", name, refname);
@@ -616,7 +616,7 @@ static int copy_refs_attr(hid_t loc_in,
buf = (hdset_reg_ref_t *)HDmalloc((unsigned)(nelmts * msize));
if(buf == NULL) {
- printf( "cannot read into memory\n" );
+ HDprintf( "cannot read into memory\n" );
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed");
} /* end if */
if(H5Aread(attr_id, mtype_id, buf) < 0)
@@ -628,12 +628,12 @@ static int copy_refs_attr(hid_t loc_in,
*/
refbuf = (hdset_reg_ref_t *)HDcalloc(sizeof(hdset_reg_ref_t), (size_t)nelmts); /*init to zero */
if(refbuf == NULL) {
- printf( "cannot allocate memory\n" );
+ HDprintf( "cannot allocate memory\n" );
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDcalloc failed");
} /* end if */
for(i = 0; i < (unsigned)nelmts; i++) {
- if (update_ref_value(attr_id, H5R_DATASET_REGION, &((hdset_reg_ref_t *)buf)[i], fidout, &((hdset_reg_ref_t *)refbuf)[i], travt)<0)
+ if(update_ref_value(attr_id, H5R_DATASET_REGION, &((hdset_reg_ref_t *)buf)[i], fidout, &((hdset_reg_ref_t *)refbuf)[i], travt) < 0)
continue;
if(options->verbose)
printf("object <%s> region reference created to <%s>\n", name, refname);
@@ -646,7 +646,7 @@ static int copy_refs_attr(hid_t loc_in,
refbuf = buf; /* reuse the read buffer for write */
if(buf == NULL) {
- printf( "cannot read into memory\n" );
+ HDprintf( "cannot read into memory\n" );
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed");
} /* end if */
@@ -687,7 +687,7 @@ static int copy_refs_attr(hid_t loc_in,
refbuf = buf; /* reuse the read buffer for write */
if(buf == NULL) {
- printf( "cannot read into memory\n" );
+ HDprintf( "cannot read into memory\n" );
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "HDmalloc failed");
} /* end if */
@@ -700,7 +700,7 @@ static int copy_refs_attr(hid_t loc_in,
size_t idx = (i * msize) + H5Tget_member_offset(mtype_id, ref_comp_index[j]);
hobj_ref_t ref_out;
- if (update_ref_value(attr_id, H5R_OBJECT, (hobj_ref_t *)(((char *)buf)+idx), fidout, &ref_out, travt)<0)
+ if (update_ref_value(attr_id, H5R_OBJECT, (hobj_ref_t *)((void *)(((char *)buf)+idx)), fidout, &ref_out, travt) < 0) /* Extra (void *) cast to quiet "cast to create alignment" warning - 2019/07/05, QAK */
continue;
HDmemcpy(((char *)buf)+idx, &ref_out, ref_comp_size[j]);
} /* if */
diff --git a/tools/h5repack/testfiles/h5repack-help.txt b/tools/h5repack/testfiles/h5repack-help.txt
index 337e004..8161157 100644
--- a/tools/h5repack/testfiles/h5repack-help.txt
+++ b/tools/h5repack/testfiles/h5repack-help.txt
@@ -6,6 +6,7 @@ usage: h5repack [OPTIONS] file1 file2
-v, --verbose Verbose mode, print object information
-V, --version Print version number and exit
-n, --native Use a native HDF5 type when repacking
+ --enable-error-stack Prints messages from the HDF5 error stack as they occur
-L, --latest Use latest version of file format
-c L1, --compact=L1 Maximum number of links in header messages
-d L2, --indexed=L2 Minimum number of links in the indexed format
@@ -35,9 +36,6 @@ usage: h5repack [OPTIONS] file1 file2
F - is the shared object header message type, any of <dspace|dtype|fill|
pline|attr>. If F is not specified, S applies to all messages
- --enable-error-stack Prints messages from the HDF5 error stack as they
- occur.
-
FILT - is a string with the format:
<list of objects>:<name of filter>=<filter parameters>
diff --git a/tools/h5stat/h5stat.c b/tools/h5stat/h5stat.c
index 54f252b..e3e19d5 100644
--- a/tools/h5stat/h5stat.c
+++ b/tools/h5stat/h5stat.c
@@ -63,14 +63,14 @@ typedef struct iter_t {
ohdr_info_t group_ohdr_info; /* Object header information for groups */
hsize_t max_attrs; /* Maximum attributes from a group */
- unsigned long *num_small_attrs; /* Size of small attributes tracked */
+ unsigned long *num_small_attrs; /* Size of small attributes tracked */
unsigned attr_nbins; /* Number of bins for attribute counts */
unsigned long *attr_bins; /* Pointer to array of bins for attribute counts */
unsigned max_dset_rank; /* Maximum rank of dataset */
unsigned long dset_rank_count[H5S_MAX_RANK]; /* Number of datasets of each rank */
hsize_t max_dset_dims; /* Maximum dimension size of dataset */
- unsigned long *small_dset_dims; /* Size of dimensions of small datasets tracked */
+ unsigned long *small_dset_dims; /* Size of dimensions of small datasets tracked */
unsigned long dset_layouts[H5D_NLAYOUTS]; /* Type of storage for each dataset */
unsigned long dset_comptype[H5_NFILTERS_IMPL]; /* Number of currently implemented filters */
unsigned long dset_ntypes; /* Number of diff. dataset datatypes found */
@@ -97,6 +97,33 @@ typedef struct iter_t {
} iter_t;
+static const char *drivername = "";
+
+#ifdef H5_HAVE_ROS3_VFD
+/* default "anonymous" s3 configuration
+ */
+static H5FD_ros3_fapl_t ros3_fa = {
+ 1, /* fapl version */
+ false, /* authenticate */
+ "", /* aws region */
+ "", /* access key id */
+ "", /* secret access key */
+};
+#endif /* H5_HAVE_ROS3_VFD */
+
+#ifdef H5_HAVE_LIBHDFS
+/* default HDFS access configuration
+ */
+static H5FD_hdfs_fapl_t hdfs_fa = {
+ 1, /* fapl version */
+ "localhost", /* namenode name */
+ 0, /* namenode port */
+ "", /* kerberos ticket cache */
+ "", /* user name */
+ 2048, /* stream buffer size */
+};
+#endif /* H5_HAVE_LIBHDFS */
+
static int display_all = TRUE;
/* Enable the printing of selected statistics */
@@ -124,7 +151,7 @@ struct handler_t {
char **obj;
};
-static const char *s_opts ="Aa:Ddm:EFfhGgl:STO:V";
+static const char *s_opts ="Aa:Ddm:EFfhGgl:STO:Vw:";
/* e.g. "filemetadata" has to precede "file"; "groupmetadata" has to precede "group" etc. */
static struct long_options l_opts[] = {
{"help", no_arg, 'h'},
@@ -216,6 +243,8 @@ static struct long_options l_opts[] = {
{ "summ", no_arg, 'S' },
{ "sum", no_arg, 'S' },
{ "su", no_arg, 'S' },
+ { "s3-cred", require_arg, 'w' },
+ { "hdfs-attrs", require_arg, 'H' },
{ NULL, 0, '\0' }
};
@@ -227,7 +256,7 @@ leave(int ret)
}
-
+
/*-------------------------------------------------------------------------
* Function: usage
*
@@ -264,9 +293,19 @@ static void usage(const char *prog)
HDfprintf(stdout, " than 0. The default threshold is 10.\n");
HDfprintf(stdout, " -S, --summary Print summary of file space information\n");
HDfprintf(stdout, " --enable-error-stack Prints messages from the HDF5 error stack as they occur\n");
+ HDfprintf(stdout, " --s3-cred=<cred> Access file on S3, using provided credential\n");
+ HDfprintf(stdout, " <cred> :: (region,id,key)\n");
+ HDfprintf(stdout, " If <cred> == \"(,,)\", no authentication is used.\n");
+ HDfprintf(stdout, " --hdfs-attrs=<attrs> Access a file on HDFS with given configuration\n");
+ HDfprintf(stdout, " attributes.\n");
+ HDfprintf(stdout, " <attrs> :: (<namenode name>,<namenode port>,\n");
+ HDfprintf(stdout, " <kerberos cache path>,<username>,\n");
+ HDfprintf(stdout, " <buffer size>)\n");
+ HDfprintf(stdout, " If an attribute is empty, a default value will be\n");
+ HDfprintf(stdout, " used.\n");
}
-
+
/*-------------------------------------------------------------------------
* Function: ceil_log10
*
@@ -293,7 +332,7 @@ ceil_log10(unsigned long x)
return ret;
} /* ceil_log10() */
-
+
/*-------------------------------------------------------------------------
* Function: attribute_stats
*
@@ -326,12 +365,12 @@ attribute_stats(iter_t *iter, const H5O_info_t *oi)
/* Add attribute count to proper bin */
bin = ceil_log10((unsigned long)oi->num_attrs);
if((bin + 1) > iter->attr_nbins) {
- iter->attr_bins = (unsigned long *)HDrealloc(iter->attr_bins, (bin + 1) * sizeof(unsigned long));
+ iter->attr_bins = (unsigned long *)HDrealloc(iter->attr_bins, (bin + 1) * sizeof(unsigned long));
HDassert(iter->attr_bins);
/* Initialize counts for intermediate bins */
while(iter->attr_nbins < bin)
- iter->attr_bins[iter->attr_nbins++] = 0;
+ iter->attr_bins[iter->attr_nbins++] = 0;
iter->attr_nbins++;
/* Initialize count for new bin */
@@ -343,7 +382,7 @@ attribute_stats(iter_t *iter, const H5O_info_t *oi)
return 0;
} /* end attribute_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: group_stats
*
@@ -425,7 +464,7 @@ done:
return ret_value;
} /* end group_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: dataset_stats
*
@@ -478,6 +517,7 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "attribute_stats() failed");
/* Get storage info */
+ /* Failure 0 indistinguishable from no-data-stored 0 */
storage = H5Dget_storage_size(did);
/* Gather layout statistics */
@@ -503,7 +543,8 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi)
if(num_ext) {
iter->nexternal += (unsigned long)num_ext;
iter->dset_external_storage_size += (unsigned long)storage;
- } else
+ }
+ else
iter->dset_storage_size += storage;
/* Gather dataspace statistics */
@@ -616,7 +657,7 @@ done:
return ret_value;
} /* end dataset_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: datatype_stats
*
@@ -648,7 +689,7 @@ done:
return ret_value;
} /* end datatype_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: obj_stats
*
@@ -704,7 +745,7 @@ done:
return ret_value;
} /* end obj_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: lnk_stats
*
@@ -742,7 +783,6 @@ lnk_stats(const char H5_ATTR_UNUSED *path, const H5L_info_t *li, void *_iter)
return 0;
} /* end lnk_stats() */
-
/*-------------------------------------------------------------------------
* Function: hand_free
*
@@ -771,7 +811,7 @@ hand_free(struct handler_t *hand)
} /* end if */
} /* end hand_free() */
-
+
/*-------------------------------------------------------------------------
* Function: parse_command_line
*
@@ -839,7 +879,8 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret)
error_msg("Invalid threshold for small groups\n");
goto error;
}
- } else
+ }
+ else
error_msg("Missing threshold for small groups\n");
break;
@@ -861,7 +902,8 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret)
error_msg("Invalid threshold for small datasets\n");
goto error;
}
- } else
+ }
+ else
error_msg("Missing threshold for small datasets\n");
break;
@@ -883,7 +925,8 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret)
error_msg("Invalid threshold for small # of attributes\n");
goto error;
}
- } else
+ }
+ else
error_msg("Missing threshold for small # of attributes\n");
break;
@@ -918,6 +961,105 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret)
} /* end if */
break;
+ case 'w':
+#ifndef H5_HAVE_ROS3_VFD
+ error_msg("Read-Only S3 VFD not enabled.\n");
+ goto error;
+#else
+ {
+ char *cred_str = NULL;
+ unsigned nelems = 0;
+ char **cred = NULL;
+ char const *ccred[3];
+
+ if (FAIL == parse_tuple((const char *)opt_arg, ',', &cred_str, &nelems, &cred)) {
+ error_msg("Unable to parse s3 credential\n");
+ goto error;
+ }
+ if (nelems != 3) {
+ error_msg("s3 credential must have three elements\n");
+ goto error;
+ }
+ ccred[0] = (const char *)cred[0];
+ ccred[1] = (const char *)cred[1];
+ ccred[2] = (const char *)cred[2];
+ if (0 == h5tools_populate_ros3_fapl(&ros3_fa, ccred)) {
+ error_msg("Unable to set ros3 fapl config\n");
+ goto error;
+ }
+ HDfree(cred);
+ HDfree(cred_str);
+ } /* parse s3-cred block */
+ drivername = "ros3";
+ break;
+#endif /* H5_HAVE_ROS3_VFD */
+
+ case 'H':
+#ifndef H5_HAVE_LIBHDFS
+ error_msg("HDFS VFD is not enabled.\n");
+ goto error;
+#else
+ {
+ unsigned nelems = 0;
+ char *props_src = NULL;
+ char **props = NULL;
+ unsigned long k = 0;
+ if (FAIL == parse_tuple((const char *)opt_arg,
+ ',', &props_src, &nelems, &props)) {
+ error_msg("unable to parse hdfs properties tuple\n");
+ goto error;
+ }
+ /* sanity-check tuple count
+ */
+ if (nelems != 5) {
+ char str[64] = "";
+ HDsprintf(str,
+ "expected 5 elements in hdfs properties tuple "
+ "but found %u\n",
+ nelems);
+ HDfree(props);
+ HDfree(props_src);
+ error_msg(str);
+ goto error;
+ }
+ /* Populate fapl configuration structure with given
+ * properties.
+ * TODO/WARNING: No error-checking is done on length of
+ * input strings... Silent overflow is possible,
+ * albeit unlikely.
+ */
+ if (HDstrncmp(props[0], "", 1)) {
+ HDstrncpy(hdfs_fa.namenode_name,(const char *)props[0], HDstrlen(props[0]));
+ }
+ if (HDstrncmp(props[1], "", 1)) {
+ k = strtoul((const char *)props[1], NULL, 0);
+ if (errno == ERANGE) {
+ error_msg("supposed port number wasn't.\n");
+ goto error;
+ }
+ hdfs_fa.namenode_port = (int32_t)k;
+ }
+ if (HDstrncmp(props[2], "", 1)) {
+ HDstrncpy(hdfs_fa.kerberos_ticket_cache, (const char *)props[2], HDstrlen(props[2]));
+ }
+ if (HDstrncmp(props[3], "", 1)) {
+ HDstrncpy(hdfs_fa.user_name, (const char *)props[3], HDstrlen(props[3]));
+ }
+ if (strncmp(props[4], "", 1)) {
+ k = HDstrtoul((const char *)props[4], NULL, 0);
+ if (errno == ERANGE) {
+ error_msg("supposed buffersize number wasn't.\n");
+ goto error;
+ }
+ hdfs_fa.stream_buffer_size = (int32_t)k;
+ }
+ HDfree(props);
+ HDfree(props_src);
+ drivername = "hdfs";
+ }
+ break;
+#endif /* H5_HAVE_LIBHDFS */
+
default:
usage(h5tools_getprogname());
goto error;
@@ -944,7 +1086,7 @@ error:
return -1;
}
-
+
/*-------------------------------------------------------------------------
* Function: iter_free
*
@@ -1004,7 +1146,7 @@ iter_free(iter_t *iter)
} /* end iter_free() */
-
+
/*-------------------------------------------------------------------------
* Function: print_file_info
*
@@ -1024,19 +1166,19 @@ iter_free(iter_t *iter)
static herr_t
print_file_info(const iter_t *iter)
{
- printf("File information\n");
- printf("\t# of unique groups: %lu\n", iter->uniq_groups);
- printf("\t# of unique datasets: %lu\n", iter->uniq_dsets);
- printf("\t# of unique named datatypes: %lu\n", iter->uniq_dtypes);
- printf("\t# of unique links: %lu\n", iter->uniq_links);
- printf("\t# of unique other: %lu\n", iter->uniq_others);
- printf("\tMax. # of links to object: %lu\n", iter->max_links);
+ HDprintf("File information\n");
+ HDprintf("\t# of unique groups: %lu\n", iter->uniq_groups);
+ HDprintf("\t# of unique datasets: %lu\n", iter->uniq_dsets);
+ HDprintf("\t# of unique named datatypes: %lu\n", iter->uniq_dtypes);
+ HDprintf("\t# of unique links: %lu\n", iter->uniq_links);
+ HDprintf("\t# of unique other: %lu\n", iter->uniq_others);
+ HDprintf("\tMax. # of links to object: %lu\n", iter->max_links);
HDfprintf(stdout, "\tMax. # of objects in group: %Hu\n", iter->max_fanout);
return 0;
} /* print_file_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_file_metadata
*
@@ -1059,15 +1201,12 @@ print_file_metadata(const iter_t *iter)
HDfprintf(stdout, "\tUser block: %Hu\n", iter->ublk_size);
HDfprintf(stdout, "\tObject headers: (total/unused)\n");
- HDfprintf(stdout, "\t\tGroups: %Hu/%Hu\n",
- iter->group_ohdr_info.total_size,
- iter->group_ohdr_info.free_size);
+ HDfprintf(stdout, "\t\tGroups: %Hu/%Hu\n", iter->group_ohdr_info.total_size,
+ iter->group_ohdr_info.free_size);
HDfprintf(stdout, "\t\tDatasets(exclude compact data): %Hu/%Hu\n",
- iter->dset_ohdr_info.total_size,
- iter->dset_ohdr_info.free_size);
- HDfprintf(stdout, "\t\tDatatypes: %Hu/%Hu\n",
- iter->dtype_ohdr_info.total_size,
- iter->dtype_ohdr_info.free_size);
+ iter->dset_ohdr_info.total_size, iter->dset_ohdr_info.free_size);
+ HDfprintf(stdout, "\t\tDatatypes: %Hu/%Hu\n", iter->dtype_ohdr_info.total_size,
+ iter->dtype_ohdr_info.free_size);
HDfprintf(stdout, "\tGroups:\n");
HDfprintf(stdout, "\t\tB-tree/List: %Hu\n", iter->groups_btree_storage_size);
@@ -1091,7 +1230,7 @@ print_file_metadata(const iter_t *iter)
return 0;
} /* print_file_metadata() */
-
+
/*-------------------------------------------------------------------------
* Function: print_group_info
*
@@ -1118,37 +1257,37 @@ print_group_info(const iter_t *iter)
unsigned long total; /* Total count for various statistics */
unsigned u; /* Local index variable */
- printf("Small groups (with 0 to %u links):\n", sgroups_threshold-1);
+ HDprintf("Small groups (with 0 to %u links):\n", sgroups_threshold-1);
total = 0;
for(u = 0; u < (unsigned)sgroups_threshold; u++) {
if(iter->num_small_groups[u] > 0) {
- printf("\t# of groups with %u link(s): %lu\n", u, iter->num_small_groups[u]);
+ HDprintf("\t# of groups with %u link(s): %lu\n", u, iter->num_small_groups[u]);
total += iter->num_small_groups[u];
} /* end if */
} /* end for */
- printf("\tTotal # of small groups: %lu\n", total);
+ HDprintf("\tTotal # of small groups: %lu\n", total);
- printf("Group bins:\n");
+ HDprintf("Group bins:\n");
total = 0;
if((iter->group_nbins > 0) && (iter->group_bins[0] > 0)) {
- printf("\t# of groups with 0 link: %lu\n", iter->group_bins[0]);
+ HDprintf("\t# of groups with 0 link: %lu\n", iter->group_bins[0]);
total = iter->group_bins[0];
} /* end if */
power = 1;
for(u = 1; u < iter->group_nbins; u++) {
if(iter->group_bins[u] > 0) {
- printf("\t# of groups with %lu - %lu links: %lu\n", power, (power * 10) - 1,
+ HDprintf("\t# of groups with %lu - %lu links: %lu\n", power, (power * 10) - 1,
iter->group_bins[u]);
total += iter->group_bins[u];
} /* end if */
power *= 10;
} /* end for */
- printf("\tTotal # of groups: %lu\n", total);
+ HDprintf("\tTotal # of groups: %lu\n", total);
return 0;
} /* print_group_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_group_metadata
*
@@ -1164,7 +1303,7 @@ print_group_info(const iter_t *iter)
static herr_t
print_group_metadata(const iter_t *iter)
{
- printf("File space information for groups' metadata (in bytes):\n");
+ HDprintf("File space information for groups' metadata (in bytes):\n");
HDfprintf(stdout, "\tObject headers (total/unused): %Hu/%Hu\n",
iter->group_ohdr_info.total_size, iter->group_ohdr_info.free_size);
@@ -1175,7 +1314,7 @@ print_group_metadata(const iter_t *iter)
return 0;
} /* print_group_metadata() */
-
+
/*-------------------------------------------------------------------------
* Function: print_dataset_info
*
@@ -1197,72 +1336,72 @@ print_dataset_info(const iter_t *iter)
unsigned u; /* Local index variable */
if(iter->uniq_dsets > 0) {
- printf("Dataset dimension information:\n");
- printf("\tMax. rank of datasets: %u\n", iter->max_dset_rank);
- printf("\tDataset ranks:\n");
+ HDprintf("Dataset dimension information:\n");
+ HDprintf("\tMax. rank of datasets: %u\n", iter->max_dset_rank);
+ HDprintf("\tDataset ranks:\n");
for(u = 0; u < H5S_MAX_RANK; u++)
if(iter->dset_rank_count[u] > 0)
- printf("\t\t# of dataset with rank %u: %lu\n", u, iter->dset_rank_count[u]);
+ HDprintf("\t\t# of dataset with rank %u: %lu\n", u, iter->dset_rank_count[u]);
- printf("1-D Dataset information:\n");
+ HDprintf("1-D Dataset information:\n");
HDfprintf(stdout, "\tMax. dimension size of 1-D datasets: %Hu\n", iter->max_dset_dims);
- printf("\tSmall 1-D datasets (with dimension sizes 0 to %u):\n", sdsets_threshold - 1);
+ HDprintf("\tSmall 1-D datasets (with dimension sizes 0 to %u):\n", sdsets_threshold - 1);
total = 0;
for(u = 0; u < (unsigned)sdsets_threshold; u++) {
if(iter->small_dset_dims[u] > 0) {
- printf("\t\t# of datasets with dimension sizes %u: %lu\n", u,
+ HDprintf("\t\t# of datasets with dimension sizes %u: %lu\n", u,
iter->small_dset_dims[u]);
total += iter->small_dset_dims[u];
} /* end if */
} /* end for */
- printf("\t\tTotal # of small datasets: %lu\n", total);
+ HDprintf("\t\tTotal # of small datasets: %lu\n", total);
/* Protect against no datasets in file */
if(iter->dset_dim_nbins > 0) {
- printf("\t1-D Dataset dimension bins:\n");
+ HDprintf("\t1-D Dataset dimension bins:\n");
total = 0;
if(iter->dset_dim_bins[0] > 0) {
- printf("\t\t# of datasets with dimension size 0: %lu\n", iter->dset_dim_bins[0]);
+ HDprintf("\t\t# of datasets with dimension size 0: %lu\n", iter->dset_dim_bins[0]);
total = iter->dset_dim_bins[0];
} /* end if */
power = 1;
for(u = 1; u < iter->dset_dim_nbins; u++) {
if(iter->dset_dim_bins[u] > 0) {
- printf("\t\t# of datasets with dimension size %lu - %lu: %lu\n", power, (power * 10) - 1,
+ HDprintf("\t\t# of datasets with dimension size %lu - %lu: %lu\n", power, (power * 10) - 1,
iter->dset_dim_bins[u]);
total += iter->dset_dim_bins[u];
} /* end if */
power *= 10;
} /* end for */
- printf("\t\tTotal # of datasets: %lu\n", total);
+ HDprintf("\t\tTotal # of datasets: %lu\n", total);
} /* end if */
- printf("Dataset storage information:\n");
+ HDprintf("Dataset storage information:\n");
HDfprintf(stdout, "\tTotal raw data size: %Hu\n", iter->dset_storage_size);
HDfprintf(stdout, "\tTotal external raw data size: %Hu\n", iter->dset_external_storage_size);
- printf("Dataset layout information:\n");
+ HDprintf("Dataset layout information:\n");
for(u = 0; u < H5D_NLAYOUTS; u++)
- printf("\tDataset layout counts[%s]: %lu\n", (u == H5D_COMPACT ? "COMPACT" :
+ HDprintf("\tDataset layout counts[%s]: %lu\n", (u == H5D_COMPACT ? "COMPACT" :
(u == H5D_CONTIGUOUS ? "CONTIG" : (u == H5D_CHUNKED ? "CHUNKED" : "ERROR"))), iter->dset_layouts[u]);
- printf("\tNumber of external files : %lu\n", iter->nexternal);
-
- printf("Dataset filters information:\n");
- printf("\tNumber of datasets with:\n");
- printf("\t\tNO filter: %lu\n", iter->dset_comptype[H5Z_FILTER_ERROR+1]);
- printf("\t\tGZIP filter: %lu\n", iter->dset_comptype[H5Z_FILTER_DEFLATE]);
- printf("\t\tSHUFFLE filter: %lu\n", iter->dset_comptype[H5Z_FILTER_SHUFFLE]);
- printf("\t\tFLETCHER32 filter: %lu\n", iter->dset_comptype[H5Z_FILTER_FLETCHER32]);
- printf("\t\tSZIP filter: %lu\n", iter->dset_comptype[H5Z_FILTER_SZIP]);
- printf("\t\tNBIT filter: %lu\n", iter->dset_comptype[H5Z_FILTER_NBIT]);
- printf("\t\tSCALEOFFSET filter: %lu\n", iter->dset_comptype[H5Z_FILTER_SCALEOFFSET]);
- printf("\t\tUSER-DEFINED filter: %lu\n", iter->dset_comptype[H5_NFILTERS_IMPL-1]);
+ HDprintf("\tNumber of external files : %lu\n", iter->nexternal);
+
+ HDprintf("Dataset filters information:\n");
+ HDprintf("\tNumber of datasets with:\n");
+ HDprintf("\t\tNO filter: %lu\n", iter->dset_comptype[H5Z_FILTER_ERROR+1]);
+ HDprintf("\t\tGZIP filter: %lu\n", iter->dset_comptype[H5Z_FILTER_DEFLATE]);
+ HDprintf("\t\tSHUFFLE filter: %lu\n", iter->dset_comptype[H5Z_FILTER_SHUFFLE]);
+ HDprintf("\t\tFLETCHER32 filter: %lu\n", iter->dset_comptype[H5Z_FILTER_FLETCHER32]);
+ HDprintf("\t\tSZIP filter: %lu\n", iter->dset_comptype[H5Z_FILTER_SZIP]);
+ HDprintf("\t\tNBIT filter: %lu\n", iter->dset_comptype[H5Z_FILTER_NBIT]);
+ HDprintf("\t\tSCALEOFFSET filter: %lu\n", iter->dset_comptype[H5Z_FILTER_SCALEOFFSET]);
+ HDprintf("\t\tUSER-DEFINED filter: %lu\n", iter->dset_comptype[H5_NFILTERS_IMPL-1]);
} /* end if */
return 0;
} /* print_dataset_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_dataset_metadata
*
@@ -1279,7 +1418,7 @@ print_dataset_info(const iter_t *iter)
static herr_t
print_dset_metadata(const iter_t *iter)
{
- printf("File space information for datasets' metadata (in bytes):\n");
+ HDprintf("File space information for datasets' metadata (in bytes):\n");
HDfprintf(stdout, "\tObject headers (total/unused): %Hu/%Hu\n",
iter->dset_ohdr_info.total_size, iter->dset_ohdr_info.free_size);
@@ -1291,7 +1430,7 @@ print_dset_metadata(const iter_t *iter)
return 0;
} /* print_dset_metadata() */
-
+
/*-------------------------------------------------------------------------
* Function: print_dset_dtype_meta
*
@@ -1313,26 +1452,26 @@ print_dset_dtype_meta(const iter_t *iter)
unsigned u; /* Local index variable */
if(iter->dset_ntypes) {
- printf("Dataset datatype information:\n");
- printf("\t# of unique datatypes used by datasets: %lu\n", iter->dset_ntypes);
+ HDprintf("Dataset datatype information:\n");
+ HDprintf("\t# of unique datatypes used by datasets: %lu\n", iter->dset_ntypes);
total = 0;
for(u = 0; u < iter->dset_ntypes; u++) {
H5Tencode(iter->dset_type_info[u].tid, NULL, &dtype_size);
- printf("\tDataset datatype #%u:\n", u);
- printf("\t\tCount (total/named) = (%lu/%lu)\n",
+ HDprintf("\tDataset datatype #%u:\n", u);
+ HDprintf("\t\tCount (total/named) = (%lu/%lu)\n",
iter->dset_type_info[u].count, iter->dset_type_info[u].named);
- printf("\t\tSize (desc./elmt) = (%lu/%lu)\n", (unsigned long)dtype_size,
+ HDprintf("\t\tSize (desc./elmt) = (%lu/%lu)\n", (unsigned long)dtype_size,
(unsigned long)H5Tget_size(iter->dset_type_info[u].tid));
H5Tclose(iter->dset_type_info[u].tid);
total += iter->dset_type_info[u].count;
} /* end for */
- printf("\tTotal dataset datatype count: %lu\n", total);
+ HDprintf("\tTotal dataset datatype count: %lu\n", total);
} /* end if */
return 0;
} /* print_dset_dtype_meta() */
-
+
/*-------------------------------------------------------------------------
* Function: print_attr_info
*
@@ -1354,34 +1493,34 @@ print_attr_info(const iter_t *iter)
unsigned long total; /* Total count for various statistics */
unsigned u; /* Local index variable */
- printf("Small # of attributes (objects with 1 to %u attributes):\n", sattrs_threshold);
+ HDprintf("Small # of attributes (objects with 1 to %u attributes):\n", sattrs_threshold);
total = 0;
for(u = 1; u <= (unsigned)sattrs_threshold; u++) {
if(iter->num_small_attrs[u] > 0) {
- printf("\t# of objects with %u attributes: %lu\n", u, iter->num_small_attrs[u]);
+ HDprintf("\t# of objects with %u attributes: %lu\n", u, iter->num_small_attrs[u]);
total += iter->num_small_attrs[u];
} /* end if */
} /* end for */
- printf("\tTotal # of objects with small # of attributes: %lu\n", total);
+ HDprintf("\tTotal # of objects with small # of attributes: %lu\n", total);
- printf("Attribute bins:\n");
+ HDprintf("Attribute bins:\n");
total = 0;
power = 1;
for(u = 1; u < iter->attr_nbins; u++) {
if(iter->attr_bins[u] > 0) {
- printf("\t# of objects with %lu - %lu attributes: %lu\n", power, (power * 10) - 1,
+ HDprintf("\t# of objects with %lu - %lu attributes: %lu\n", power, (power * 10) - 1,
iter->attr_bins[u]);
total += iter->attr_bins[u];
} /* end if */
power *= 10;
} /* end for */
- printf("\tTotal # of objects with attributes: %lu\n", total);
- printf("\tMax. # of attributes to objects: %lu\n", (unsigned long)iter->max_attrs);
+ HDprintf("\tTotal # of objects with attributes: %lu\n", total);
+ HDprintf("\tMax. # of attributes to objects: %lu\n", (unsigned long)iter->max_attrs);
return 0;
} /* print_attr_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_storage_summary
*
@@ -1437,7 +1576,7 @@ print_storage_summary(const iter_t *iter)
return 0;
} /* print_storage_summary() */
-
+
/*-------------------------------------------------------------------------
* Function: print_file_statistics
*
@@ -1482,7 +1621,7 @@ print_file_statistics(const iter_t *iter)
if(display_summary) print_storage_summary(iter);
} /* print_file_statistics() */
-
+
/*-------------------------------------------------------------------------
* Function: print_object_statistics
*
@@ -1502,10 +1641,10 @@ print_file_statistics(const iter_t *iter)
static void
print_object_statistics(const char *name)
{
- printf("Object name %s\n", name);
+ HDprintf("Object name %s\n", name);
} /* print_object_statistics() */
-
+
/*-------------------------------------------------------------------------
* Function: print_statistics
*
@@ -1531,7 +1670,7 @@ print_statistics(const char *name, const iter_t *iter)
print_file_statistics(iter);
} /* print_statistics() */
-
+
/*-------------------------------------------------------------------------
* Function: main
*
@@ -1552,6 +1691,7 @@ main(int argc, const char *argv[])
void *edata;
void *tools_edata;
struct handler_t *hand = NULL;
+ hid_t fapl_id = H5P_DEFAULT;
h5tools_setprogname(PROGRAMNAME);
h5tools_setstatus(EXIT_SUCCESS);
@@ -1572,6 +1712,42 @@ main(int argc, const char *argv[])
if(parse_command_line(argc, argv, &hand) < 0)
goto done;
+ /* if drivername is not null, probably need to set the fapl */
+ if (HDstrcmp(drivername, "")) {
+ void *conf_fa = NULL;
+
+ if (!HDstrcmp(drivername, "ros3")) {
+#ifndef H5_HAVE_ROS3_VFD
+ error_msg("Read-Only S3 VFD not enabled.\n\n");
+ goto done;
+#else
+ conf_fa = (void *)&ros3_fa;
+#endif /* H5_HAVE_ROS3_VFD */
+
+ }
+ else if (!HDstrcmp(drivername, "hdfs")) {
+#ifndef H5_HAVE_LIBHDFS
+ error_msg("HDFS VFD not enabled.\n\n");
+ goto done;
+#else
+ conf_fa = (void *)&hdfs_fa;
+#endif /* H5_HAVE_LIBHDFS */
+ }
+
+ if (conf_fa != NULL) {
+ HDassert(fapl_id == H5P_DEFAULT);
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ if (fapl_id < 0) {
+ error_msg("Unable to create fapl entry\n");
+ goto done;
+ }
+ if (1 > h5tools_set_configured_fapl(fapl_id, drivername, conf_fa)) {
+ error_msg("Unable to set fapl\n");
+ goto done;
+ }
+ }
+ } /* drivername set */
+
fname = argv[opt_ind];
if(enable_error_stack > 0) {
@@ -1584,7 +1760,7 @@ main(int argc, const char *argv[])
hid_t fcpl;
H5F_info_t finfo;
- printf("Filename: %s\n", fname);
+ HDprintf("Filename: %s\n", fname);
fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
if(fid < 0) {
@@ -1616,7 +1792,7 @@ main(int argc, const char *argv[])
if(iter.num_small_groups == NULL || iter.num_small_attrs == NULL || iter.small_dset_dims == NULL) {
error_msg("Unable to allocate memory for tracking small groups/datasets/attributes\n");
- h5tools_setstatus(EXIT_FAILURE);
+ h5tools_setstatus(EXIT_FAILURE);
goto done;
}
@@ -1642,7 +1818,8 @@ main(int argc, const char *argv[])
if(h5trav_visit(fid, "/", TRUE, TRUE, obj_stats, lnk_stats, &iter) < 0) {
error_msg("unable to traverse objects/links in file \"%s\"\n", fname);
h5tools_setstatus(EXIT_FAILURE);
- } else
+ }
+ else
print_statistics("/", &iter);
} /* end else */
} /* end if */
@@ -1653,6 +1830,13 @@ done:
/* Free iter structure */
iter_free(&iter);
+ if (fapl_id != H5P_DEFAULT) {
+ if (0 < H5Pclose(fapl_id)) {
+ error_msg("unable to close fapl entry\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ }
+ }
+
if(fid >= 0 && H5Fclose(fid) < 0) {
error_msg("unable to close file \"%s\"\n", fname);
h5tools_setstatus(EXIT_FAILURE);
diff --git a/tools/h5stat/testfiles/h5stat_help1.ddl b/tools/h5stat/testfiles/h5stat_help1.ddl
index ce8bb73..e722f74 100644
--- a/tools/h5stat/testfiles/h5stat_help1.ddl
+++ b/tools/h5stat/testfiles/h5stat_help1.ddl
@@ -22,3 +22,13 @@ Usage: h5stat [OPTIONS] file
than 0. The default threshold is 10.
-S, --summary Print summary of file space information
--enable-error-stack Prints messages from the HDF5 error stack as they occur
+ --s3-cred=<cred> Access file on S3, using provided credential
+ <cred> :: (region,id,key)
+ If <cred> == "(,,)", no authentication is used.
+ --hdfs-attrs=<attrs> Access a file on HDFS with given configuration
+ attributes.
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ If an attribute is empty, a default value will be
+ used.
diff --git a/tools/h5stat/testfiles/h5stat_help2.ddl b/tools/h5stat/testfiles/h5stat_help2.ddl
index ce8bb73..e722f74 100644
--- a/tools/h5stat/testfiles/h5stat_help2.ddl
+++ b/tools/h5stat/testfiles/h5stat_help2.ddl
@@ -22,3 +22,13 @@ Usage: h5stat [OPTIONS] file
than 0. The default threshold is 10.
-S, --summary Print summary of file space information
--enable-error-stack Prints messages from the HDF5 error stack as they occur
+ --s3-cred=<cred> Access file on S3, using provided credential
+ <cred> :: (region,id,key)
+ If <cred> == "(,,)", no authentication is used.
+ --hdfs-attrs=<attrs> Access a file on HDFS with given configuration
+ attributes.
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ If an attribute is empty, a default value will be
+ used.
diff --git a/tools/h5stat/testfiles/h5stat_nofile.ddl b/tools/h5stat/testfiles/h5stat_nofile.ddl
index ce8bb73..e722f74 100644
--- a/tools/h5stat/testfiles/h5stat_nofile.ddl
+++ b/tools/h5stat/testfiles/h5stat_nofile.ddl
@@ -22,3 +22,13 @@ Usage: h5stat [OPTIONS] file
than 0. The default threshold is 10.
-S, --summary Print summary of file space information
--enable-error-stack Prints messages from the HDF5 error stack as they occur
+ --s3-cred=<cred> Access file on S3, using provided credential
+ <cred> :: (region,id,key)
+ If <cred> == "(,,)", no authentication is used.
+ --hdfs-attrs=<attrs> Access a file on HDFS with given configuration
+ attributes.
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ If an attribute is empty, a default value will be
+ used.
diff --git a/tools/lib/h5diff_array.c b/tools/lib/h5diff_array.c
index cbfabad..d178866 100644
--- a/tools/lib/h5diff_array.c
+++ b/tools/lib/h5diff_array.c
@@ -782,12 +782,11 @@ static hsize_t diff_datum(
}
/* check object type */
- if (ret_value >= 0)
- if (obj1_type != obj2_type) {
- parallel_print("Different object types referenced: <%s> and <%s>", obj1, obj2);
- opts->not_cmp = 1;
- HGOTO_DONE (opts->err_stat);
- }
+ if (obj1_type != obj2_type) {
+ parallel_print("Different object types referenced: <%s> and <%s>", obj1, obj2);
+ opts->not_cmp = 1;
+ HGOTO_DONE (opts->err_stat);
+ }
if ((obj1_id = H5Rdereference(container1_id, H5R_OBJECT, _mem1)) < 0) {
opts->err_stat = 1;
diff --git a/tools/lib/h5tools_dump.c b/tools/lib/h5tools_dump.c
index 7ec5da8..e76328c 100644
--- a/tools/lib/h5tools_dump.c
+++ b/tools/lib/h5tools_dump.c
@@ -93,7 +93,7 @@ h5tool_format_t h5tools_dataformat = {
const h5tools_dump_header_t h5tools_standardformat = {
"standardformat", /*name */
-"HDF5", /*fileebgin */
+"HDF5", /*filebegin */
"", /*fileend */
SUPER_BLOCK, /*bootblockbegin */
"", /*bootblockend */
@@ -2257,18 +2257,26 @@ h5tools_print_datatype(FILE *stream, h5tools_str_t *buffer, const h5tool_format_
h5tools_render_element(stream, info, ctx, buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0);
ctx->indent_level++;
{
- char *ttag;
+ char *ttag;
- if(NULL == (ttag = H5Tget_tag(type)))
- H5E_THROW(FAIL, H5E_tools_min_id_g, "H5Tget_tag failed");
+ if(NULL == (ttag = H5Tget_tag(type)))
+ H5E_THROW(FAIL, H5E_tools_min_id_g, "H5Tget_tag failed");
- ctx->need_prefix = TRUE;
+ ctx->need_prefix = TRUE;
+
+ h5tools_str_reset(buffer);
+ h5tools_str_append(buffer, "OPAQUE_TAG \"%s\";", ttag);
+ h5tools_render_element(stream, info, ctx, buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0);
+
+ H5free_memory(ttag);
- h5tools_str_reset(buffer);
- h5tools_str_append(buffer, "OPAQUE_TAG \"%s\";", ttag);
- h5tools_render_element(stream, info, ctx, buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0);
+ if((size = H5Tget_size(type)) <= 0) {
+ ctx->need_prefix = TRUE;
- H5free_memory(ttag);
+ h5tools_str_reset(buffer);
+ h5tools_str_append(buffer, "OPAQUE_SIZE \"%s\";", size);
+ h5tools_render_element(stream, info, ctx, buffer, &curr_pos, (size_t)ncols, (hsize_t)0, (hsize_t)0);
+ }
}
ctx->indent_level--;
diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c
index 37d137c..9b564c0 100644
--- a/tools/lib/h5tools_utils.c
+++ b/tools/lib/h5tools_utils.c
@@ -21,6 +21,10 @@
#include "H5private.h"
#include "h5trav.h"
+#ifdef H5_HAVE_ROS3_VFD
+#include "H5FDros3.h"
+#endif
+
/* global variables */
unsigned h5tools_nCols = 80;
/* ``get_option'' variables */
@@ -322,7 +326,229 @@ get_option(int argc, const char **argv, const char *opts, const struct long_opti
return opt_opt;
}
-
+
+/*****************************************************************************
+ *
+ * Function: parse_tuple()
+ *
+ * Purpose:
+ *
+ * Create array of pointers to strings, identified as elements in a tuple
+ * of arbitrary length separated by provided character.
+ * ("tuple" because "nple" looks strange)
+ *
+ * * Receives pointer to start of tuple sequence string, '('.
+ * * Attempts to separate elements by token-character `sep`.
+ * * If the separator character is preceded by a backslash '\',
+ * the backslash is deleted and the separator is included in the
+ * element string as any other character.
+ * * To end an element with a backslash, escape the backslash, e.g.
+ * "(myelem\\,otherelem) -> {"myelem\", "otherelem"}
+ * * In all other cases, a backslash appearing not as part of "\\" or
+ * "\<sep>" digraph will be included berbatim.
+ * * Last two characters in the string MUST be ")\0".
+ *
+ * * Generates a copy of the input string `start`, (src..")\0"), replacing
+ * separators and close-paren with null charaters.
+ * * This string is allocated at runtime and should be freed when done.
+ * * Generates array of char pointers, and directs start of each element
+ * (each pointer) into this copy.
+ * * Each tuple element points to the start of its string (substring)
+ * and ends with a null terminator.
+ * * This array is allocated at runtime and should be freed when done.
+ * * Reallocates and expands elements array during parsing.
+ * * Initially allocated for 2 (plus one null entry), and grows by
+ * powers of 2.
+ * * The final 'slot' in the element array (elements[nelements], e.g.)
+ * always points to NULL.
+ * * The number of elements found and stored are passed out through pointer
+ * to unsigned, `nelems`.
+ *
+ * Return:
+ *
+ * FAIL If malformed--does not look like a tuple "(...)"
+ * or major error was encountered while parsing.
+ * or
+ * SUCCEED String looks properly formed "(...)" and no major errors.
+ *
+ * Stores number of elements through pointer `nelems`.
+ * Stores list of pointers to char (first char in each element
+ * string) through pointer `ptrs_out`.
+ * NOTE: `ptrs_out[nelems] == NULL` should be true.
+ * NOTE: list is malloc'd by function, and should be freed
+ * when done.
+ * Stores "source string" for element pointers through `cpy_out`.
+ * NOTE: Each element substring is null-terminated.
+ * NOTE: There may be extra characters after the last element
+ * (past its null terminator), but is guaranteed to
+ * be null-terminated.
+ * NOTE: `cpy_out` string is malloc'd by function,
+ * and should be freed when done.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-10
+ *
+ * Changes: None.
+ *
+ *****************************************************************************
+ */
+herr_t
+parse_tuple(const char *start,
+ int sep,
+ char **cpy_out,
+ unsigned *nelems,
+ char ***ptrs_out)
+{
+ char *elem_ptr = NULL;
+ char *dest_ptr = NULL;
+ unsigned elems_count = 0;
+ char **elems = NULL; /* more like *elems[], but complier... */
+ char **elems_re = NULL; /* temporary pointer, for realloc */
+ char *cpy = NULL;
+ herr_t ret_value = SUCCEED;
+ unsigned init_slots = 2;
+
+
+
+ /*****************
+ * SANITY-CHECKS *
+ *****************/
+
+ /* must start with "("
+ */
+ if (start[0] != '(') {
+ ret_value = FAIL;
+ goto done;
+ }
+
+ /* must end with ")"
+ */
+ while (start[elems_count] != '\0') {
+ elems_count++;
+ }
+ if (start[elems_count - 1] != ')') {
+ ret_value = FAIL;
+ goto done;
+ }
+
+ elems_count = 0;
+
+
+
+ /***********
+ * PREPARE *
+ ***********/
+
+ /* create list
+ */
+ elems = (char **)HDmalloc(sizeof(char *) * (init_slots + 1));
+ if (elems == NULL) { ret_value = FAIL; goto done; } /* CANTALLOC */
+
+ /* create destination string
+ */
+ start++; /* advance past opening paren '(' */
+ cpy = (char *)HDmalloc(sizeof(char) * (HDstrlen(start))); /* no +1; less '(' */
+ if (cpy == NULL) { ret_value = FAIL; goto done; } /* CANTALLOC */
+
+ /* set pointers
+ */
+ dest_ptr = cpy; /* start writing copy here */
+ elem_ptr = cpy; /* first element starts here */
+ elems[elems_count++] = elem_ptr; /* set first element pointer into list */
+
+
+
+ /*********
+ * PARSE *
+ *********/
+
+ while (*start != '\0') {
+ /* For each character in the source string...
+ */
+ if (*start == '\\') {
+ /* Possibly an escape digraph.
+ */
+ if ((*(start + 1) == '\\') ||
+ (*(start + 1) == sep) )
+ {
+ /* Valid escape digraph of "\\" or "\<sep>".
+ */
+ start++; /* advance past escape char '\' */
+ *(dest_ptr++) = *(start++); /* Copy subsequent char */
+ /* and advance pointers. */
+ } else {
+ /* Not an accepted escape digraph.
+ * Copy backslash character.
+ */
+ *(dest_ptr++) = *(start++);
+ }
+ } else if (*start == sep) {
+ /* Non-escaped separator.
+ * Terminate elements substring in copy, record element, advance.
+ * Expand elements list if appropriate.
+ */
+ *(dest_ptr++) = 0; /* Null-terminate elem substring in copy */
+ /* and advance pointer. */
+ start++; /* Advance src pointer past separator. */
+ elem_ptr = dest_ptr; /* Element pointer points to start of first */
+ /* character after null sep in copy. */
+ elems[elems_count++] = elem_ptr; /* Set elem pointer in list */
+ /* and increment count. */
+
+ /* Expand elements list, if necessary.
+ */
+ if (elems_count == init_slots) {
+ init_slots *= 2;
+ elems_re = (char **)realloc(elems, sizeof(char *) * \
+ (init_slots + 1));
+ if (elems_re == NULL) {
+ /* CANTREALLOC */
+ ret_value = FAIL;
+ goto done;
+ }
+ elems = elems_re;
+ }
+ } else if (*start == ')' && *(start + 1) == '\0') {
+ /* Found terminal, non-escaped close-paren. Last element.
+ * Write null terminator to copy.
+ * Advance source pointer to gently break from loop.
+ * Requred to prevent ")" from always being added to last element.
+ */
+ start++;
+ } else {
+ /* Copy character into destination. Advance pointers.
+ */
+ *(dest_ptr++) = *(start++);
+ }
+ }
+ *dest_ptr = '\0'; /* Null-terminate destination string. */
+ elems[elems_count] = NULL; /* Null-terminate elements list. */
+
+
+
+ /********************
+ * PASS BACK VALUES *
+ ********************/
+
+ *ptrs_out = elems;
+ *nelems = elems_count;
+ *cpy_out = cpy;
+
+done:
+ if (ret_value == FAIL) {
+ /* CLEANUP */
+ if (cpy) free(cpy);
+ if (elems) free(elems);
+ }
+
+ return ret_value;
+
+} /* parse_tuple */
+
+
+
+
+
/*-------------------------------------------------------------------------
* Function: indentation
*
@@ -841,3 +1067,260 @@ done:
return ret_value;
}
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: h5tools_populate_ros3_fapl()
+ *
+ * Purpose:
+ *
+ * Set the values of a ROS3 fapl configuration object.
+ *
+ * If the values pointer is NULL, sets fapl target `fa` to a default
+ * (valid, current-version, non-authenticating) fapl config.
+ *
+ * If `values` pointer is _not_ NULL, expects `values` to contain at least
+ * three non-null pointers to null-terminated strings, corresponding to:
+ * { aws_region,
+ * secret_id,
+ * secret_key,
+ * }
+ * If all three strings are empty (""), the default fapl will be default.
+ * Both aws_region and secret_id values must be both empty or both
+ * populated. If
+ * Only secret_key is allowed to be empty (the empty string, "").
+ * All values are checked against overflow as defined in the ros3 vfd
+ * header file; if a value overruns the permitted space, FAIL is returned
+ * and the function aborts without resetting the fapl to values initially
+ * present.
+ *
+ * Return:
+ *
+ * 0 (failure) if...
+ * * Read-Only S3 VFD is not enabled.
+ * * NULL fapl pointer: (NULL, {...} )
+ * * Warning: In all cases below, fapl will be set as "default"
+ * before error occurs.
+ * * NULL value strings: (&fa, {NULL?, NULL? NULL?, ...})
+ * * Incomplete fapl info:
+ * * empty region, non-empty id, key either way
+ * * (&fa, {"", "...", "?"})
+ * * empty id, non-empty region, key either way
+ * * (&fa, {"...", "", "?"})
+ * * "non-empty key and either id or region empty
+ * * (&fa, {"", "", "...")
+ * * (&fa, {"", "...", "...")
+ * * (&fa, {"...", "", "...")
+ * * Any string would overflow allowed space in fapl definition.
+ * or
+ * 1 (success)
+ * * Sets components in fapl_t pointer, copying strings as appropriate.
+ * * "Default" fapl (valid version, authenticate->False, empty strings)
+ * * `values` pointer is NULL
+ * * (&fa, NULL)
+ * * first three strings in `values` are empty ("")
+ * * (&fa, {"", "", "", ...}
+ * * Authenticating fapl
+ * * region, id, and optional key provided
+ * * (&fa, {"...", "...", ""})
+ * * (&fa, {"...", "...", "..."})
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-13
+ *
+ *----------------------------------------------------------------------------
+ */
+#ifdef H5_HAVE_ROS3_VFD
+int
+h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa,
+ const char **values)
+{
+ int show_progress = 0; /* set to 1 for debugging */
+ int ret_value = 1; /* 1 for success, 0 for failure */
+ /* e.g.? if (!populate()) { then failed } */
+
+ if (show_progress) {
+ HDprintf("called h5tools_populate_ros3_fapl\n");
+ }
+
+ if (fa == NULL) {
+ if (show_progress) {
+ HDprintf(" ERROR: null pointer to fapl_t\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+
+ if (show_progress) {
+ HDprintf(" preset fapl with default values\n");
+ }
+ fa->version = H5FD_CURR_ROS3_FAPL_T_VERSION;
+ fa->authenticate = FALSE;
+ *(fa->aws_region) = '\0';
+ *(fa->secret_id) = '\0';
+ *(fa->secret_key) = '\0';
+
+ /* sanity-check supplied values
+ */
+ if (values != NULL) {
+ if (values[0] == NULL) {
+ if (show_progress) {
+ HDprintf(" ERROR: aws_region value cannot be NULL\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+ if (values[1] == NULL) {
+ if (show_progress) {
+ HDprintf(" ERROR: secret_id value cannot be NULL\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+ if (values[2] == NULL) {
+ if (show_progress) {
+ HDprintf(" ERROR: secret_key value cannot be NULL\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+
+ /* if region and ID are supplied (key optional), write to fapl...
+ * fail if value would overflow
+ */
+ if (*values[0] != '\0' &&
+ *values[1] != '\0')
+ {
+ if (HDstrlen(values[0]) > H5FD_ROS3_MAX_REGION_LEN) {
+ if (show_progress) {
+ HDprintf(" ERROR: aws_region value too long\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+ HDmemcpy(fa->aws_region, values[0],
+ (HDstrlen(values[0]) + 1));
+ if (show_progress) {
+ HDprintf(" aws_region set\n");
+ }
+
+
+ if (HDstrlen(values[1]) > H5FD_ROS3_MAX_SECRET_ID_LEN) {
+ if (show_progress) {
+ HDprintf(" ERROR: secret_id value too long\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+ HDmemcpy(fa->secret_id,
+ values[1],
+ (HDstrlen(values[1]) + 1));
+ if (show_progress) {
+ HDprintf(" secret_id set\n");
+ }
+
+ if (HDstrlen(values[2]) > H5FD_ROS3_MAX_SECRET_KEY_LEN) {
+ if (show_progress) {
+ HDprintf(" ERROR: secret_key value too long\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+ HDmemcpy(fa->secret_key,
+ values[2],
+ (HDstrlen(values[2]) + 1));
+ if (show_progress) {
+ HDprintf(" secret_key set\n");
+ }
+
+ fa->authenticate = TRUE;
+ if (show_progress) {
+ HDprintf(" set to authenticate\n");
+ }
+
+ } else if (*values[0] != '\0' ||
+ *values[1] != '\0' ||
+ *values[2] != '\0')
+ {
+ if (show_progress) {
+ HDprintf(
+ " ERROR: invalid assortment of empty/non-empty values\n"
+ );
+ }
+ ret_value = 0;
+ goto done;
+ }
+ } /* values != NULL */
+
+done:
+ return ret_value;
+
+} /* h5tools_populate_ros3_fapl */
+#endif /* H5_HAVE_ROS3_VFD */
+
+
+/*-----------------------------------------------------------------------------
+ *
+ * Function: h5tools_set_configured_fapl
+ *
+ * Purpose: prepare fapl_id with the given property list, according to
+ * VFD prototype.
+ *
+ * Return: 0 on failure, 1 on success
+ *
+ * Programmer: Jacob Smith
+ * 2018-05-21
+ *
+ * Changes: None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+int
+h5tools_set_configured_fapl(hid_t fapl_id,
+ const char vfd_name[],
+ void *fapl_t_ptr)
+{
+ int ret_value = 1;
+
+ if (fapl_id < 0) {
+ return 0;
+ }
+
+ if (!strcmp("", vfd_name)) {
+ goto done;
+
+#ifdef H5_HAVE_ROS3_VFD
+ } else if (!strcmp("ros3", vfd_name)) {
+ if ((fapl_id == H5P_DEFAULT) ||
+ (fapl_t_ptr == NULL) ||
+ (FAIL == H5Pset_fapl_ros3(
+ fapl_id,
+ (H5FD_ros3_fapl_t *)fapl_t_ptr)))
+ {
+ ret_value = 0;
+ goto done;
+ }
+#endif /* H5_HAVE_ROS3_VFD */
+
+#ifdef H5_HAVE_LIBHDFS
+ } else if (!strcmp("hdfs", vfd_name)) {
+ if ((fapl_id == H5P_DEFAULT) ||
+ (fapl_t_ptr == NULL) ||
+ (FAIL == H5Pset_fapl_hdfs(
+ fapl_id,
+ (H5FD_hdfs_fapl_t *)fapl_t_ptr)))
+ {
+ ret_value = 0;
+ goto done;
+ }
+#endif /* H5_HAVE_LIBHDFS */
+
+ } else {
+ ret_value = 0; /* unrecognized fapl type "name" */
+ }
+
+done:
+ return ret_value;
+
+} /* h5tools_set_configured_fapl() */
+
diff --git a/tools/lib/h5tools_utils.h b/tools/lib/h5tools_utils.h
index 4c2bf1e..0fa5250 100644
--- a/tools/lib/h5tools_utils.h
+++ b/tools/lib/h5tools_utils.h
@@ -123,6 +123,11 @@ H5TOOLS_DLLVAR unsigned h5tools_nCols; /*max number of columns for
H5TOOLS_DLL void indentation(unsigned);
H5TOOLS_DLL void print_version(const char *progname);
H5TOOLS_DLL void parallel_print(const char* format, ... );
+H5TOOLS_DLL herr_t parse_tuple(const char *start,
+ int sep,
+ char **cpy_out,
+ unsigned *nelems,
+ char ***ptrs_out);
H5TOOLS_DLL void error_msg(const char *fmt, ...);
H5TOOLS_DLL void warn_msg(const char *fmt, ...);
H5TOOLS_DLL void help_ref_msg(FILE *output);
@@ -174,6 +179,14 @@ H5TOOLS_DLL void h5tools_setprogname(const char*progname);
H5TOOLS_DLL int h5tools_getstatus(void);
H5TOOLS_DLL void h5tools_setstatus(int d_status);
H5TOOLS_DLL int h5tools_getenv_update_hyperslab_bufsize(void);
+H5TOOLS_DLL int h5tools_set_configured_fapl(hid_t fapl_id,
+ const char vfd_name[],
+ void *fapl_t_ptr);
+#ifdef H5_HAVE_ROS3_VFD
+H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa,
+ const char **values);
+#endif /* H5_HAVE_ROS3_VFD */
+
#ifdef __cplusplus
}
#endif
diff --git a/tools/libtest/CMakeLists.txt b/tools/libtest/CMakeLists.txt
new file mode 100644
index 0000000..f3d28da
--- /dev/null
+++ b/tools/libtest/CMakeLists.txt
@@ -0,0 +1,18 @@
+cmake_minimum_required (VERSION 3.10)
+project (HDF5_TOOLS_LIBTEST C)
+
+#-----------------------------------------------------------------------------
+# Add the h5tools_utils test executables
+#-----------------------------------------------------------------------------
+add_executable (h5tools_test_utils ${HDF5_TOOLS_LIBTEST_SOURCE_DIR}/h5tools_test_utils.c)
+target_include_directories(h5tools_test_utils PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
+if (NOT ONLY_SHARED_LIBS)
+ TARGET_C_PROPERTIES (h5tools_test_utils STATIC)
+ target_link_libraries (h5tools_test_utils PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} ${HDF5_TEST_LIB_TARGET})
+else ()
+ TARGET_C_PROPERTIES (h5tools_test_utils SHARED)
+ target_link_libraries (h5tools_test_utils PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} ${HDF5_TEST_LIBSH_TARGET})
+endif ()
+set_target_properties (h5tools_test_utils PROPERTIES FOLDER tools)
+
+include (CMakeTests.cmake)
diff --git a/tools/libtest/CMakeTests.cmake b/tools/libtest/CMakeTests.cmake
new file mode 100644
index 0000000..4feee9b
--- /dev/null
+++ b/tools/libtest/CMakeTests.cmake
@@ -0,0 +1,49 @@
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+
+##############################################################################
+##############################################################################
+### T E S T I N G ###
+##############################################################################
+##############################################################################
+
+
+##############################################################################
+##############################################################################
+### T H E T E S T S M A C R O S ###
+##############################################################################
+##############################################################################
+
+ macro (ADD_H5_TEST resultfile resultcode)
+ add_test (
+ NAME H5LIBTEST-${resultfile}-clear-objects
+ COMMAND ${CMAKE_COMMAND}
+ -E remove
+ ${resultfile}.out
+ ${resultfile}.out.err
+ )
+ if (NOT "${last_test}" STREQUAL "")
+ set_tests_properties (H5LIBTEST-${resultfile}-clear-objects PROPERTIES DEPENDS ${last_test})
+ endif ()
+ add_test (NAME H5LIBTEST-${resultfile} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5tools_test_utils> ${ARGN})
+ if (NOT "${resultcode}" STREQUAL "0")
+ set_tests_properties (H5LIBTEST-${resultfile} PROPERTIES WILL_FAIL "true")
+ endif ()
+ set_tests_properties (H5LIBTEST-${resultfile} PROPERTIES DEPENDS H5LIBTEST-${resultfile}-clear-objects)
+ endmacro ()
+
+##############################################################################
+##############################################################################
+### T H E T E S T S ###
+##############################################################################
+##############################################################################
+ ADD_H5_TEST (h5tools_utils-default 0)
diff --git a/tools/libtest/Makefile.am b/tools/libtest/Makefile.am
new file mode 100644
index 0000000..a93e25d
--- /dev/null
+++ b/tools/libtest/Makefile.am
@@ -0,0 +1,34 @@
+#
+# Read-Only S3 Virtual File Driver (VFD)
+# Copyright (c) 2017-2018, The HDF Group.
+#
+# All rights reserved.
+#
+# NOTICE:
+# All information contained herein is, and remains, the property of The HDF
+# Group. The intellectual and technical concepts contained herein are
+# proprietary to The HDF Group. Dissemination of this information or
+# reproduction of this material is strictly forbidden unless prior written
+# permission is obtained from The HDF Group.
+##
+## Makefile.am
+## Run automake to generate a Makefile.in from this file.
+#
+# HDF5 Library Makefile(.in)
+#
+
+include $(top_srcdir)/config/commence.am
+
+# Include src and tools/lib directories
+AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib
+
+# All programs depend on the hdf5 and h5tools libraries
+LDADD=$(LIBH5TOOLS) $(LIBHDF5)
+
+
+# main target
+bin_PROGRAMS=h5tools_test_utils
+# check_PROGRAMS=$(TEST_PROG)
+
+
+include $(top_srcdir)/config/conclude.am
diff --git a/tools/libtest/h5tools_test_utils.c b/tools/libtest/h5tools_test_utils.c
new file mode 100644
index 0000000..7908519
--- /dev/null
+++ b/tools/libtest/h5tools_test_utils.c
@@ -0,0 +1,1265 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Purpose: unit-test functionality of the routines in `tools/lib/h5tools_utils`
+ *
+ * Jacob Smith 2017-11-10
+ */
+
+#include "h5tools_utils.h"
+#include "h5test.h"
+
+#define UTIL_TEST_DEBUG 0
+
+#ifndef __js_test__
+
+#define __js_test__ 1L
+
+/*****************************************************************************
+ *
+ * FILE-LOCAL TESTING MACROS
+ *
+ * Purpose:
+ *
+ * 1. Upon test failure, goto-jump to single-location teardown in test
+ * function. E.g., `error:` (consistency with HDF corpus) or
+ * `failed:` (reflects purpose).
+ * >>> using "error", in part because `H5E_BEGIN_TRY` expects it.
+ * 2. Increase clarity and reduce overhead found with `TEST_ERROR`.
+ * e.g., "if(somefunction(arg, arg2) < 0) TEST_ERROR:"
+ * requires reading of entire line to know whether this if/call is
+ * part of the test setup, test operation, or a test unto itself.
+ * 3. Provide testing macros with optional user-supplied failure message;
+ * if not supplied (NULL), generate comparison output in the spirit of
+ * test-driven development. E.g., "expected 5 but was -3"
+ * User messages clarify test's purpose in code, encouraging description
+ * without relying on comments.
+ * 4. Configurable expected-actual order in generated comparison strings.
+ * Some prefer `VERIFY(expected, actual)`, others
+ * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch
+ * to satifsy both parties, assuming one paradigm per test file.
+ * (One could #undef and redefine the flag through the file as desired,
+ * but _why_.)
+ *
+ * Provided as courtesy, per consideration for inclusion in the library
+ * proper.
+ *
+ * Macros:
+ *
+ * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order
+ * FAIL_IF() - check condition
+ * FAIL_UNLESS() - check _not_ condition
+ * JSVERIFY() - long-int equality check; prints reason/comparison
+ * JSVERIFY_NOT() - long-int inequality check; prints
+ * JSVERIFY_STR() - string equality check; prints
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *****************************************************************************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * ifdef flag: JSVERIFY_EXP_ACT
+ *
+ * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason])
+ * default, if this is undefined, is (ACTUAL, EXPECTED[, reason])
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_EXP_ACT 1L
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSFAILED_AT()
+ *
+ * Purpose:
+ *
+ * Preface a test failure by printing "*FAILED*" and location to stdout
+ * Similar to `H5_FAILED(); AT();` from h5test.h
+ *
+ * *FAILED* at somefile.c:12 in function_name()...
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSFAILED_AT() { \
+ HDprintf("*FAILED* at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: FAIL_IF()
+ *
+ * Purpose:
+ *
+ * Make tests more accessible and less cluttered than
+ * `if (thing == otherthing()) TEST_ERROR`
+ * paradigm.
+ *
+ * The following lines are roughly equivalent:
+ *
+ * `if (myfunc() < 0) TEST_ERROR;` (as seen elsewhere in HDF tests)
+ * `FAIL_IF(myfunc() < 0)`
+ *
+ * Prints a generic "FAILED AT" line to stdout and jumps to `error`,
+ * similar to `TEST_ERROR` in h5test.h
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *----------------------------------------------------------------------------
+ */
+#define FAIL_IF(condition) \
+if (condition) { \
+ JSFAILED_AT() \
+ goto error; \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: FAIL_UNLESS()
+ *
+ * Purpose:
+ *
+ * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests",
+ * e.g., "a != b".
+ *
+ * Opposite of FAIL_IF; fails if the given condition is _not_ true.
+ *
+ * `FAIL_IF( 5 != my_op() )`
+ * is equivalent to
+ * `FAIL_UNLESS( 5 == my_op() )`
+ * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer.
+ * (see JSVERIFY)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define FAIL_UNLESS(condition) \
+if (!(condition)) { \
+ JSFAILED_AT() \
+ goto error; \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSERR_LONG()
+ *
+ * Purpose:
+ *
+ * Print an failure message for long-int arguments.
+ * ERROR-AT printed first.
+ * If `reason` is given, it is printed on own line and newlined after
+ * else, prints "expected/actual" aligned on own lines.
+ *
+ * *FAILED* at myfile.c:488 in somefunc()...
+ * forest must be made of trees.
+ *
+ * or
+ *
+ * *FAILED* at myfile.c:488 in somefunc()...
+ * ! Expected 425
+ * ! Actual 3
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSERR_LONG(expected, actual, reason) { \
+ JSFAILED_AT() \
+ if (reason!= NULL) { \
+ HDprintf("%s\n", (reason)); \
+ } else { \
+ HDprintf(" ! Expected %ld\n ! Actual %ld\n", \
+ (long)(expected), (long)(actual)); \
+ } \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSERR_STR()
+ *
+ * Purpose:
+ *
+ * Print an failure message for string arguments.
+ * ERROR-AT printed first.
+ * If `reason` is given, it is printed on own line and newlined after
+ * else, prints "expected/actual" aligned on own lines.
+ *
+ * *FAILED* at myfile.c:421 in myfunc()...
+ * Blue and Red strings don't match!
+ *
+ * or
+ *
+ * *FAILED* at myfile.c:421 in myfunc()...
+ * !!! Expected:
+ * this is my expected
+ * string
+ * !!! Actual:
+ * not what I expected at all
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSERR_STR(expected, actual, reason) { \
+ JSFAILED_AT() \
+ if ((reason) != NULL) { \
+ HDprintf("%s\n", (reason)); \
+ } else { \
+ HDprintf("!!! Expected:\n%s\n!!!Actual:\n%s\n", \
+ (expected), (actual)); \
+ } \
+}
+
+#ifdef JSVERIFY_EXP_ACT
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY()
+ *
+ * Purpose:
+ *
+ * Verify that two long integers are equal.
+ * If unequal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY(expected, actual, reason) \
+if ((long)(actual) != (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY_NOT()
+ *
+ * Purpose:
+ *
+ * Verify that two long integers are _not_ equal.
+ * If equal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_NOT(expected, actual, reason) \
+if ((long)(actual) == (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY_NOT */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY_STR()
+ *
+ * Purpose:
+ *
+ * Verify that two strings are equal.
+ * If unequal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_STR(expected, actual, reason) \
+if (HDstrcmp((actual), (expected)) != 0) { \
+ JSERR_STR((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY_STR */
+
+
+#else /* JSVERIFY_EXP_ACT not defined */
+ /* Repeats macros above, but with actual/expected parameters reversed. */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY()
+ * See: JSVERIFY documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY(actual, expected, reason) \
+if ((long)(actual) != (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY_NOT()
+ * See: JSVERIFY_NOT documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_NOT(actual, expected, reason) \
+if ((long)(actual) == (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY_NOT */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY_STR()
+ * See: JSVERIFY_STR documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_STR(actual, expected, reason) \
+if (HDstrcmp((actual), (expected)) != 0) { \
+ JSERR_STR((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY_STR */
+
+#endif /* ifdef/else JSVERIFY_EXP_ACT */
+
+#endif /* __js_test__ */
+
+/* if > 0, be very verbose when performing tests */
+#define H5TOOLS_UTILS_TEST_DEBUG 0
+
+/******************/
+/* TEST FUNCTIONS */
+/******************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: test_parse_tuple()
+ *
+ * Purpose:
+ *
+ * Provide unit tests and specification for the `parse_tuple()` function.
+ *
+ * Return:
+ *
+ * 0 Tests passed.
+ * 1 Tests failed.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-11
+ *
+ * Changes: None.
+ *
+ *----------------------------------------------------------------------------
+ */
+static unsigned
+test_parse_tuple(void)
+{
+ /*************************
+ * TEST-LOCAL STRUCTURES *
+ *************************/
+
+ struct testcase {
+ const char *test_msg; /* info about test case */
+ const char *in_str; /* input string */
+ int sep; /* separator "character" */
+ herr_t exp_ret; /* expected SUCCEED / FAIL */
+ unsigned exp_nelems; /* expected number of elements */
+ /* (no more than 7!) */
+ const char *exp_elems[7]; /* list of elements (no more than 7!) */
+ };
+
+ /******************
+ * TEST VARIABLES *
+ ******************/
+
+ struct testcase cases[] = {
+ { "bad start",
+ "words(before)",
+ ';',
+ FAIL,
+ 0,
+ {NULL},
+ },
+ { "tuple not closed",
+ "(not ok",
+ ',',
+ FAIL,
+ 0,
+ {NULL},
+ },
+ { "empty tuple",
+ "()",
+ '-',
+ SUCCEED,
+ 1,
+ {""},
+ },
+ { "no separator",
+ "(stuff keeps on going)",
+ ',',
+ SUCCEED,
+ 1,
+ {"stuff keeps on going"},
+ },
+ { "4-ple, escaped seperator",
+ "(elem0,elem1,el\\,em2,elem3)", /* "el\,em" */
+ ',',
+ SUCCEED,
+ 4,
+ {"elem0", "elem1", "el,em2", "elem3"},
+ },
+ { "5-ple, escaped escaped separator",
+ "(elem0,elem1,el\\\\,em2,elem3)",
+ ',',
+ SUCCEED,
+ 5,
+ {"elem0", "elem1", "el\\", "em2", "elem3"},
+ },
+ { "escaped non-comma separator",
+ "(5-2-7-2\\-6-2)",
+ '-',
+ SUCCEED,
+ 5,
+ {"5","2","7","2-6","2"},
+ },
+ { "embedded close-paren",
+ "(be;fo)re)",
+ ';',
+ SUCCEED,
+ 2,
+ {"be", "fo)re"},
+ },
+ { "embedded non-escaping backslash",
+ "(be;fo\\re)",
+ ';',
+ SUCCEED,
+ 2,
+ {"be", "fo\\re"},
+ },
+ { "double close-paren at end",
+ "(be;fore))",
+ ';',
+ SUCCEED,
+ 2,
+ {"be", "fore)"},
+ },
+ { "empty elements",
+ "(;a1;;a4;)",
+ ';',
+ SUCCEED,
+ 5,
+ {"", "a1", "", "a4", ""},
+ },
+ { "nested tuples with different separators",
+ "((4,e,a);(6,2,a))",
+ ';',
+ SUCCEED,
+ 2,
+ {"(4,e,a)","(6,2,a)"},
+ },
+ { "nested tuples with same separators",
+ "((4,e,a),(6,2,a))",
+ ',',
+ SUCCEED,
+ 6,
+ {"(4","e","a)","(6","2","a)"},
+ },
+ { "real-world use case",
+ "(us-east-2,AKIAIMC3D3XLYXLN5COA,ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+)",
+ ',',
+ SUCCEED,
+ 3,
+ {"us-east-2",
+ "AKIAIMC3D3XLYXLN5COA",
+ "ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+"},
+ }
+ };
+ struct testcase tc;
+ unsigned n_tests = 14;
+ unsigned i = 0;
+ unsigned count = 0;
+ unsigned elem_i = 0;
+ char **parsed = NULL;
+ char *cpy = NULL;
+ herr_t success = TRUE;
+ hbool_t show_progress = FALSE;
+
+
+
+ TESTING("arbitrary-count tuple parsing");
+
+#if H5TOOLS_UTILS_TEST_DEBUG > 0
+ show_progress = TRUE;
+#endif /* H5TOOLS_UTILS_TEST_DEBUG */
+
+ /*********
+ * TESTS *
+ *********/
+
+ for (i = 0; i < n_tests; i++) {
+
+ /* SETUP
+ */
+ HDassert(parsed == NULL);
+ HDassert(cpy == NULL);
+ tc = cases[i];
+ if (show_progress == TRUE) {
+ HDprintf("testing %d: %s...\n", i, tc.test_msg);
+ }
+
+ /* VERIFY
+ */
+ success = parse_tuple(tc.in_str, tc.sep, &cpy, &count, &parsed);
+
+ JSVERIFY( tc.exp_ret, success, "function returned incorrect value" )
+ JSVERIFY( tc.exp_nelems, count, NULL )
+ if (success == SUCCEED) {
+ FAIL_IF( parsed == NULL )
+ for (elem_i = 0; elem_i < count; elem_i++) {
+ JSVERIFY_STR( tc.exp_elems[elem_i], parsed[elem_i], NULL )
+ }
+ /* TEARDOWN */
+ HDassert(parsed != NULL);
+ HDassert(cpy != NULL);
+ HDfree(parsed);
+ parsed = NULL;
+ HDfree(cpy);
+ cpy = NULL;
+ } else {
+ FAIL_IF( parsed != NULL )
+ } /* if parse_tuple() == SUCCEED or no */
+
+ } /* for each testcase */
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (parsed != NULL) HDfree(parsed);
+ if (cpy != NULL) HDfree(cpy);
+
+ return 1;
+
+} /* test_parse_tuple */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: test_populate_ros3_fa()
+ *
+ * Purpose: Verify behavior of `populate_ros3_fa()`
+ *
+ * Return: 0 if test passes
+ * 1 if failure
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-13
+ *
+ * Changes: None
+ *
+ *----------------------------------------------------------------------------
+ */
+static unsigned
+test_populate_ros3_fa(void)
+{
+#ifdef H5_HAVE_ROS3_VFD
+ /*************************
+ * TEST-LOCAL STRUCTURES *
+ *************************/
+
+ /************************
+ * TEST-LOCAL VARIABLES *
+ ************************/
+
+ hbool_t show_progress = FALSE;
+ int bad_version = 0xf87a; /* arbitrarily wrong version number */
+#endif /* H5_HAVE_ROS3_VFD */
+
+ TESTING("programmatic ros3 fapl population");
+
+#ifndef H5_HAVE_ROS3_VFD
+ HDputs(" -SKIP-");
+ HDputs(" Read-Only S3 VFD not enabled");
+ HDfflush(stdout);
+ return 0;
+#else
+#if H5TOOLS_UTILS_TEST_DEBUG > 0
+ show_progress = TRUE;
+#endif /* H5TOOLS_UTILS_TEST_DEBUG */
+
+ HDassert(bad_version != H5FD_CURR_ROS3_FAPL_T_VERSION);
+
+ /*********
+ * TESTS *
+ *********/
+
+ /* NULL fapl config pointer fails
+ */
+ {
+ const char *values[] = {"x", "y", "z"};
+
+ if (show_progress) { HDprintf("NULL fapl pointer\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(NULL, values),
+ "fapl pointer cannot be null" )
+ }
+
+ /* NULL values pointer yields default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"};
+
+ if (show_progress) { HDprintf("NULL values pointer\n"); }
+
+ JSVERIFY( 1, h5tools_populate_ros3_fapl(&fa, NULL),
+ "NULL values pointer yields \"default\" fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* all-empty values
+ * yields default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"};
+ const char *values[] = {"", "", ""};
+
+ if (show_progress) { HDprintf("all empty values\n"); }
+
+ JSVERIFY( 1, h5tools_populate_ros3_fapl(&fa, values),
+ "empty values yields \"default\" fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* successfully set fapl with values
+ * excess value is ignored
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"x", "y", "z", "a"};
+
+ if (show_progress) { HDprintf("successful full set\n"); }
+
+ JSVERIFY( 1, h5tools_populate_ros3_fapl(&fa, values),
+ "four values" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( TRUE, fa.authenticate, NULL )
+ JSVERIFY_STR( "x", fa.aws_region, NULL )
+ JSVERIFY_STR( "y", fa.secret_id, NULL )
+ JSVERIFY_STR( "z", fa.secret_key, NULL )
+ }
+
+ /* NULL region
+ * yeilds default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {NULL, "y", "z", NULL};
+
+ if (show_progress) { HDprintf("NULL region\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* empty region
+ * yeilds default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"", "y", "z", NULL};
+
+ if (show_progress) { HDprintf("empty region; non-empty id, key\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* region overflow
+ * yeilds default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {
+ "somewhere over the rainbow not too high " \
+ "there is another rainbow bounding some darkened sky",
+ "y",
+ "z"};
+
+ if (show_progress) { HDprintf("region overflow\n"); }
+
+ HDassert(HDstrlen(values[0]) > H5FD_ROS3_MAX_REGION_LEN);
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* NULL id
+ * yields default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"x", NULL, "z", NULL};
+
+ if (show_progress) { HDprintf("NULL id\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* empty id (non-empty region, key)
+ * yeilds default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"x", "", "z", NULL};
+
+ if (show_progress) { HDprintf("empty id; non-empty region and key\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* id overflow
+ * partial set: region
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {
+ "x",
+ "Why is it necessary to solve the problem? " \
+ "What benefits will you receive by solving the problem? " \
+ "What is the unknown? " \
+ "What is it you don't yet understand? " \
+ "What is the information you have? " \
+ "What isn't the problem? " \
+ "Is the information insufficient, redundant, or contradictory? " \
+ "Should you draw a diagram or figure of the problem? " \
+ "What are the boundaries of the problem? " \
+ "Can you separate the various parts of the problem?",
+ "z"};
+
+ if (show_progress) { HDprintf("id overflow\n"); }
+
+ HDassert(HDstrlen(values[1]) > H5FD_ROS3_MAX_SECRET_ID_LEN);
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "x", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* NULL key
+ * yields default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"x", "y", NULL, NULL};
+
+ if (show_progress) { HDprintf("NULL key\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* empty key (non-empty region, id)
+ * yeilds authenticating fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"x", "y", "", NULL};
+
+ if (show_progress) { HDprintf("empty key; non-empty region and id\n"); }
+
+ JSVERIFY( 1, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( TRUE, fa.authenticate, NULL )
+ JSVERIFY_STR( "x", fa.aws_region, NULL )
+ JSVERIFY_STR( "y", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* empty key, region (non-empty id)
+ * yeilds default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"", "y", "", NULL};
+
+ if (show_progress) { HDprintf("empty key and region; non-empty id\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* empty key, id (non-empty region)
+ * yeilds default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"x", "", "", NULL};
+
+ if (show_progress) { HDprintf("empty key and id; non-empty region\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* key overflow
+ * partial set: region, id
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {
+ "x",
+ "y",
+ "Why is it necessary to solve the problem? " \
+ "What benefits will you receive by solving the problem? " \
+ "What is the unknown? " \
+ "What is it you don't yet understand? " \
+ "What is the information you have? " \
+ "What isn't the problem? " \
+ "Is the information insufficient, redundant, or contradictory? " \
+ "Should you draw a diagram or figure of the problem? " \
+ "What are the boundaries of the problem? " \
+ "Can you separate the various parts of the problem?"};
+
+ if (show_progress) { HDprintf("key overflow\n"); }
+
+ HDassert(HDstrlen(values[2]) > H5FD_ROS3_MAX_SECRET_KEY_LEN);
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD_CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "x", fa.aws_region, NULL )
+ JSVERIFY_STR( "y", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* use case
+ */
+ {
+ H5FD_ros3_fapl_t fa = {0, 0, "", "", ""};
+ const char *values[] = {
+ "us-east-2",
+ "AKIAIMC3D3XLYXLN5COA",
+ "ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+"
+ };
+ JSVERIFY( 1,
+ h5tools_populate_ros3_fapl(&fa, values),
+ "unable to set use case" )
+ JSVERIFY( 1, fa.version, "version check" )
+ JSVERIFY( 1, fa.authenticate, "should authenticate" )
+ }
+
+ PASSED();
+ return 0;
+
+error :
+ /***********
+ * CLEANUP *
+ ***********/
+
+ return 1;
+
+#endif /* H5_HAVE_ROS3_VFD */
+
+} /* test_populate_ros3_fa */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: test_set_configured_fapl()
+ *
+ * Purpose: Verify `h5tools_set_configured_fapl()` with ROS3 VFD
+ *
+ * Return: 0 if test passes
+ * 1 if failure
+ *
+ * Programmer: Jacob Smith
+ * 2018-07-12
+ *
+ * Changes: None
+ *
+ *----------------------------------------------------------------------------
+ */
+static unsigned
+test_set_configured_fapl(void)
+{
+#define UTIL_TEST_NOFAPL 1
+#define UTIL_TEST_DEFAULT 2
+#define UTIL_TEST_CREATE 3
+
+ /*************************
+ * TEST-LOCAL STRUCTURES *
+ *************************/
+ typedef struct testcase {
+ const char message[88];
+ int expected;
+ int fapl_choice;
+ const char vfdname[12];
+ void *conf_fa;
+ } testcase;
+
+ typedef struct other_fa_t {
+ int a;
+ int b;
+ int c;
+ } other_fa_t;
+
+ /************************
+ * TEST-LOCAL VARIABLES *
+ ************************/
+
+ hid_t fapl_id = -1;
+ other_fa_t wrong_fa = {0x432, 0xf82, 0x9093};
+ H5FD_ros3_fapl_t ros3_anon_fa = {1, FALSE, "", "", ""};
+ H5FD_ros3_fapl_t ros3_auth_fa = {
+ 1, /* fapl version */
+ TRUE, /* authenticate */
+ "us-east-1", /* aws region */
+ "12345677890abcdef", /* simulate access key ID */
+ "oiwnerwe9u0234nJw0-aoj+dsf", /* simulate secret key */
+ };
+ H5FD_hdfs_fapl_t hdfs_fa = {
+ 1, /* fapl version */
+ "", /* namenode name */
+ 0, /* namenode port */
+ "", /* kerberos ticket cache */
+ "", /* user name */
+ 2048, /* stream buffer size */
+ };
+ unsigned n_cases = 7; /* number of common testcases */
+ testcase cases[] = {
+ { "(common) should fail: no fapl id",
+ 0,
+ UTIL_TEST_NOFAPL,
+ "",
+ NULL,
+ },
+ { "(common) should fail: no fapl id (with struct)",
+ 0,
+ UTIL_TEST_NOFAPL,
+ "",
+ &wrong_fa,
+ },
+ { "(common) H5P_DEFAULT with no struct should succeed",
+ 1,
+ UTIL_TEST_DEFAULT,
+ "",
+ NULL,
+ },
+ { "(common) H5P_DEFAULT with (ignored) struct should succeed",
+ 1,
+ UTIL_TEST_DEFAULT,
+ "",
+ &wrong_fa,
+ },
+ { "(common) provided fapl entry should not fail",
+ 1,
+ UTIL_TEST_CREATE,
+ "",
+ NULL,
+ },
+ { "(common) provided fapl entry should not fail; ignores struct",
+ 1,
+ UTIL_TEST_CREATE,
+ "",
+ &wrong_fa,
+ },
+ { "(common) should fail: unrecoginzed vfd name",
+ 0,
+ UTIL_TEST_DEFAULT,
+ "unknown",
+ NULL,
+ },
+
+#ifdef H5_HAVE_ROS3_VFD
+ /* WARNING: add number of ROS3 test cases after array definition
+ */
+ { "(ROS3) should fail: no fapl id, no struct",
+ 0,
+ UTIL_TEST_NOFAPL,
+ "ros3",
+ NULL,
+ },
+ { "(ROS3) should fail: no fapl id",
+ 0,
+ UTIL_TEST_NOFAPL,
+ "ros3",
+ &ros3_anon_fa,
+ },
+ { "(ROS3) should fail: no struct",
+ 0,
+ UTIL_TEST_CREATE,
+ "ros3",
+ NULL,
+ },
+ { "(ROS3) successful set",
+ 1,
+ UTIL_TEST_CREATE,
+ "ros3",
+ &ros3_anon_fa,
+ },
+ { "(ROS3) should fail: attempt to set DEFAULT fapl",
+ 0,
+ UTIL_TEST_DEFAULT,
+ "ros3",
+ &ros3_anon_fa,
+ },
+#endif /* H5_HAVE_ROS3_VFD */
+
+#ifdef H5_HAVE_LIBHDFS
+ /* WARNING: add number of HDFS test cases after array definition
+ */
+ { "(HDFS) should fail: no fapl id, no struct",
+ 0,
+ UTIL_TEST_NOFAPL,
+ "hdfs",
+ NULL,
+ },
+ { "(HDFS) should fail: no fapl id",
+ 0,
+ UTIL_TEST_NOFAPL,
+ "hdfs",
+ &hdfs_fa,
+ },
+ { "(HDFS) should fail: no struct",
+ 0,
+ UTIL_TEST_CREATE,
+ "hdfs",
+ NULL,
+ },
+ { "(HDFS) successful set",
+ 1,
+ UTIL_TEST_CREATE,
+ "hdfs",
+ &hdfs_fa,
+ },
+ { "(HDFS) should fail: attempt to set DEFAULT fapl",
+ 0,
+ UTIL_TEST_DEFAULT,
+ "hdfs",
+ &hdfs_fa,
+ },
+#endif /* H5_HAVE_LIBHDFS */
+
+ }; /* testcases `cases` array */
+
+#ifdef H5_HAVE_ROS3_VFD
+ n_cases += 5;
+#endif /* H5_HAVE_ROS3_VFD */
+
+#ifdef H5_HAVE_LIBHDFS
+ n_cases += 5;
+#endif /* H5_HAVE_LIBHDFS */
+
+ TESTING("programmatic fapl set");
+
+ for (unsigned i = 0; i < n_cases; i++) {
+ int result;
+ testcase C = cases[i];
+
+ fapl_id = -1;
+
+#if UTIL_TEST_DEBUG
+ HDfprintf(stderr, "setup test %d\t%s\n", i, C.message); fflush(stderr);
+#endif /* UTIL_TEST_DEBUG */
+
+ /* per-test setup */
+ if (C.fapl_choice == UTIL_TEST_DEFAULT) {
+ fapl_id = H5P_DEFAULT;
+ } else if (C.fapl_choice == UTIL_TEST_CREATE) {
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 )
+ }
+
+#if UTIL_TEST_DEBUG
+ HDfprintf(stderr, "before test\n"); fflush(stderr);
+#endif /* UTIL_TEST_DEBUG */
+
+ /* test */
+ result = h5tools_set_configured_fapl(
+ fapl_id,
+ C.vfdname,
+ C.conf_fa);
+ JSVERIFY( result, C.expected, C.message )
+
+#if UTIL_TEST_DEBUG
+ HDfprintf(stderr, "after test\n"); fflush(stderr);
+#endif /* UTIL_TEST_DEBUG */
+
+ /* per-test-teardown */
+ if (fapl_id > 0) {
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ }
+ fapl_id = -1;
+
+#if UTIL_TEST_DEBUG
+ HDfprintf(stderr, "after cleanup\n"); fflush(stderr);
+#endif /* UTIL_TEST_DEBUG */
+
+ }
+
+#if UTIL_TEST_DEBUG
+ HDfprintf(stderr, "after loop\n"); fflush(stderr);
+#endif /* UTIL_TEST_DEBUG */
+
+ PASSED();
+ return 0;
+
+error :
+ /***********
+ * CLEANUP *
+ ***********/
+
+#if UTIL_TEST_DEBUG
+ HDfprintf(stderr, "ERROR\n"); fflush(stderr);
+#endif /* UTIL_TEST_DEBUG */
+
+ if (fapl_id > 0) {
+ (void)H5Pclose(fapl_id);
+ }
+
+ return 1;
+
+#undef UTIL_TEST_NOFAPL
+#undef UTIL_TEST_DEFAULT
+#undef UTIL_TEST_CREATE
+} /* test_set_configured_fapl */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: main()
+ *
+ * Purpose: Run all test functions.
+ *
+ * Return: 0 iff all test pass
+ * 1 iff any failures
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-10
+ *
+ * Changes: None.
+ *
+ *----------------------------------------------------------------------------
+ */
+int
+main(void)
+{
+ unsigned nerrors = 0;
+
+#ifdef _H5TEST_
+ h5reset(); /* h5test? */
+#endif /* _H5TEST_ */
+
+ HDfprintf(stdout, "Testing h5tools_utils corpus.\n");
+
+ nerrors += test_parse_tuple();
+ nerrors += test_populate_ros3_fa();
+ nerrors += test_set_configured_fapl();
+
+ if (nerrors > 0) {
+ HDfprintf(stdout, "***** %d h5tools_utils TEST%s FAILED! *****\n",
+ nerrors,
+ nerrors > 1 ? "S" : "");
+ nerrors = 1;
+ } else {
+ HDfprintf(stdout, "All h5tools_utils tests passed\n");
+ }
+
+ return (int)nerrors;
+
+} /* main */
+
+
diff --git a/tools/misc/h5debug.c b/tools/misc/h5debug.c
index 245b5a1..676e050 100644
--- a/tools/misc/h5debug.c
+++ b/tools/misc/h5debug.c
@@ -51,7 +51,7 @@
#define INDENT 3
#define VCOL 50
-
+
/*-------------------------------------------------------------------------
* Function: get_H5B2_class
*
@@ -74,45 +74,45 @@ get_H5B2_class(const uint8_t *sig)
const H5B2_class_t *cls;
switch(subtype) {
- case H5B2_TEST_ID:
- cls = H5B2_TEST;
- break;
+ case H5B2_TEST_ID:
+ cls = H5B2_TEST;
+ break;
- case H5B2_FHEAP_HUGE_INDIR_ID:
- cls = H5HF_HUGE_BT2_INDIR;
- break;
+ case H5B2_FHEAP_HUGE_INDIR_ID:
+ cls = H5HF_HUGE_BT2_INDIR;
+ break;
- case H5B2_FHEAP_HUGE_FILT_INDIR_ID:
- cls = H5HF_HUGE_BT2_FILT_INDIR;
- break;
+ case H5B2_FHEAP_HUGE_FILT_INDIR_ID:
+ cls = H5HF_HUGE_BT2_FILT_INDIR;
+ break;
- case H5B2_FHEAP_HUGE_DIR_ID:
- cls = H5HF_HUGE_BT2_DIR;
- break;
+ case H5B2_FHEAP_HUGE_DIR_ID:
+ cls = H5HF_HUGE_BT2_DIR;
+ break;
- case H5B2_FHEAP_HUGE_FILT_DIR_ID:
- cls = H5HF_HUGE_BT2_FILT_DIR;
- break;
+ case H5B2_FHEAP_HUGE_FILT_DIR_ID:
+ cls = H5HF_HUGE_BT2_FILT_DIR;
+ break;
- case H5B2_GRP_DENSE_NAME_ID:
- cls = H5G_BT2_NAME;
- break;
+ case H5B2_GRP_DENSE_NAME_ID:
+ cls = H5G_BT2_NAME;
+ break;
- case H5B2_GRP_DENSE_CORDER_ID:
- cls = H5G_BT2_CORDER;
- break;
+ case H5B2_GRP_DENSE_CORDER_ID:
+ cls = H5G_BT2_CORDER;
+ break;
- case H5B2_SOHM_INDEX_ID:
- cls = H5SM_INDEX;
- break;
+ case H5B2_SOHM_INDEX_ID:
+ cls = H5SM_INDEX;
+ break;
- case H5B2_ATTR_DENSE_NAME_ID:
- cls = H5A_BT2_NAME;
- break;
+ case H5B2_ATTR_DENSE_NAME_ID:
+ cls = H5A_BT2_NAME;
+ break;
- case H5B2_ATTR_DENSE_CORDER_ID:
- cls = H5A_BT2_CORDER;
- break;
+ case H5B2_ATTR_DENSE_CORDER_ID:
+ cls = H5A_BT2_CORDER;
+ break;
default:
HDfprintf(stderr, "Unknown B-tree subtype %u\n", (unsigned)(subtype));
@@ -122,7 +122,7 @@ get_H5B2_class(const uint8_t *sig)
return(cls);
} /* end get_H5B2_class() */
-
+
/*-------------------------------------------------------------------------
* Function: main
*
@@ -147,12 +147,12 @@ main(int argc, char *argv[])
uint8_t sig[H5F_SIGNATURE_LEN];
size_t u;
H5E_auto2_t func;
- void *edata;
+ void *edata;
herr_t status = SUCCEED;
if(argc == 1) {
- HDfprintf(stderr, "Usage: %s filename [signature-addr [extra]]\n", argv[0]);
- HDexit(1);
+ HDfprintf(stderr, "Usage: %s filename [signature-addr [extra]]\n", argv[0]);
+ HDexit(1);
} /* end if */
/* Initialize the library */
@@ -224,13 +224,15 @@ main(int argc, char *argv[])
*/
status = H5HL_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL);
- } else if(!HDmemcmp (sig, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
- /*
- * Debug a global heap collection.
- */
- status = H5HG_debug (f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL);
+ }
+ else if(!HDmemcmp (sig, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ /*
+ * Debug a global heap collection.
+ */
+ status = H5HG_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL);
- } else if(!HDmemcmp(sig, H5G_NODE_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5G_NODE_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug a symbol table node.
*/
@@ -244,7 +246,8 @@ main(int argc, char *argv[])
status = H5G_node_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra);
- } else if(!HDmemcmp(sig, H5B_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5B_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug a B-tree. B-trees are debugged through the B-tree
* subclass. The subclass identifier is the byte immediately
@@ -254,27 +257,29 @@ main(int argc, char *argv[])
unsigned ndims;
switch(subtype) {
- case H5B_SNODE_ID:
- /* Check for extra parameters */
- if(extra == 0) {
- HDfprintf(stderr, "\nWarning: Providing the group's local heap address will give more information\n");
- HDfprintf(stderr, "B-tree symbol table node usage:\n");
- HDfprintf(stderr, "\th5debug <filename> <B-tree node address> <address of local heap>\n\n");
- } /* end if */
-
- status = H5G_node_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra);
- break;
+ case H5B_SNODE_ID:
+ /* Check for extra parameters */
+ if(extra == 0) {
+ HDfprintf(stderr, "\nWarning: Providing the group's local heap address will give more information\n");
+ HDfprintf(stderr, "B-tree symbol table node usage:\n");
+ HDfprintf(stderr, "\th5debug <filename> <B-tree node address> <address of local heap>\n\n");
+ HDexit(4);
+ } /* end if */
- case H5B_CHUNK_ID:
- /* Check for extra parameters */
- if(extra == 0) {
- HDfprintf(stderr, "ERROR: Need number of dimensions of chunk in order to dump chunk B-tree node\n");
- HDfprintf(stderr, "B-tree chunked storage node usage:\n");
- HDfprintf(stderr, "\th5debug <filename> <B-tree node address> <# of dimensions>\n");
- HDexit(4);
- } /* end if */
+ status = H5G_node_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra);
+ break;
+
+ case H5B_CHUNK_ID:
+ /* Check for extra parameters */
+ if(extra == 0) {
+ HDfprintf(stderr, "ERROR: Need number of dimensions of chunk in order to dump chunk B-tree node\n");
+ HDfprintf(stderr, "B-tree chunked storage node usage:\n");
+ HDfprintf(stderr, "\th5debug <filename> <B-tree node address> <# of dimensions> <slowest chunk dim>...<fastest chunk dim>\n");
+ HDexit(4);
+ } /* end if */
- ndims = (unsigned)extra;
+ /* Build array of chunk dimensions */
+ ndims = (unsigned)extra;
status = H5D_btree_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, ndims);
break;
@@ -283,7 +288,8 @@ main(int argc, char *argv[])
HDexit(4);
}
- } else if(!HDmemcmp(sig, H5B2_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5B2_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug a v2 B-tree header.
*/
@@ -292,7 +298,8 @@ main(int argc, char *argv[])
status = H5B2_hdr_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, cls, (haddr_t)extra);
- } else if(!HDmemcmp(sig, H5B2_INT_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5B2_INT_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug a v2 B-tree internal node.
*/
@@ -310,7 +317,8 @@ main(int argc, char *argv[])
status = H5B2_int_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, cls, extra, (unsigned)extra2, (unsigned)extra3, (haddr_t)extra4);
- } else if(!HDmemcmp(sig, H5B2_LEAF_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5B2_LEAF_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug a v2 B-tree leaf node.
*/
@@ -327,13 +335,15 @@ main(int argc, char *argv[])
status = H5B2_leaf_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, cls, extra, (unsigned)extra2, (haddr_t)extra3);
- } else if(!HDmemcmp(sig, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug a fractal heap header.
*/
status = H5HF_hdr_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL);
- } else if(!HDmemcmp(sig, H5HF_DBLOCK_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5HF_DBLOCK_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug a fractal heap direct block.
*/
@@ -348,7 +358,8 @@ main(int argc, char *argv[])
status = H5HF_dblock_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra, (size_t)extra2);
- } else if(!HDmemcmp(sig, H5HF_IBLOCK_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5HF_IBLOCK_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug a fractal heap indirect block.
*/
@@ -363,14 +374,16 @@ main(int argc, char *argv[])
status = H5HF_iblock_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra, (unsigned)extra2);
- } else if(!HDmemcmp(sig, H5FS_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5FS_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug a free space header.
*/
status = H5FS_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL);
- } else if(!HDmemcmp(sig, H5FS_SINFO_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5FS_SINFO_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug free space serialized sections.
*/
@@ -385,14 +398,16 @@ main(int argc, char *argv[])
status = H5FS_sects_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra, extra2);
- } else if(!HDmemcmp(sig, H5SM_TABLE_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5SM_TABLE_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug shared message master table.
*/
status = H5SM_table_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, (unsigned) UFAIL, (unsigned) UFAIL);
- } else if(!HDmemcmp(sig, H5SM_LIST_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5SM_LIST_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug shared message list index.
*/
@@ -407,33 +422,37 @@ main(int argc, char *argv[])
status = H5SM_list_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, (unsigned) extra, (size_t) extra2);
- } else if(!HDmemcmp(sig, H5O_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
+ }
+ else if(!HDmemcmp(sig, H5EA_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) {
/*
* Debug v2 object header (which have signatures).
*/
status = H5O_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL);
- } else if(sig[0] == H5O_VERSION_1) {
+ }
+ else if(sig[0] == H5O_VERSION_1) {
/*
* This could be a v1 object header. Since they don't have a signature
* it's a somewhat "ify" detection.
*/
status = H5O_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL);
- } else {
+ }
+ else {
/*
* Got some other unrecognized signature.
*/
- printf("%-*s ", VCOL, "Signature:");
+ HDprintf("%-*s ", VCOL, "Signature:");
for (u = 0; u < sizeof(sig); u++) {
if (sig[u] > ' ' && sig[u] <= '~' && '\\' != sig[u])
HDputchar(sig[u]);
else if ('\\' == sig[u]) {
HDputchar('\\');
HDputchar('\\');
- } else
- printf("\\%03o", sig[u]);
+ }
+ else
+ HDprintf("\\%03o", sig[u]);
}
HDputchar('\n');
diff --git a/tools/misc/h5mkgrp.c b/tools/misc/h5mkgrp.c
index b7ff73c..feb60d4 100644
--- a/tools/misc/h5mkgrp.c
+++ b/tools/misc/h5mkgrp.c
@@ -251,7 +251,7 @@ main(int argc, const char *argv[])
/* Display some output if requested */
if(params.verbose)
- printf("%s: Creating groups with latest version of the format\n", h5tools_getprogname());
+ HDprintf("%s: Creating groups with latest version of the format\n", h5tools_getprogname());
} /* end if */
/* Attempt to open an existing HDF5 file first */
@@ -284,7 +284,7 @@ main(int argc, const char *argv[])
/* Display some output if requested */
if(params.verbose)
- printf("%s: Creating parent groups\n", h5tools_getprogname());
+ HDprintf("%s: Creating parent groups\n", h5tools_getprogname());
} /* end if */
/* Loop over creating requested groups */
@@ -305,7 +305,7 @@ main(int argc, const char *argv[])
/* Display some output if requested */
if(params.verbose)
- printf("%s: created group '%s'\n", h5tools_getprogname(), params.groups[curr_group]);
+ HDprintf("%s: created group '%s'\n", h5tools_getprogname(), params.groups[curr_group]);
} /* end for */
/* Close link creation property list */
diff --git a/tools/misc/h5repart.c b/tools/misc/h5repart.c
index dea27d4..461f631 100644
--- a/tools/misc/h5repart.c
+++ b/tools/misc/h5repart.c
@@ -15,11 +15,11 @@
* Programmer: Robb Matzke <matzke@llnl.gov>
* Wednesday, May 13, 1998
*
- * Purpose: Repartitions a file family. This program can be used to
- * split a single file into a family of files, join a family of
- * files into a single file, or copy one family to another while
- * changing the size of the family members. It can also be used
- * to copy a single file to a single file with holes.
+ * Purpose: Repartitions a file family. This program can be used to
+ * split a single file into a family of files, join a family of
+ * files into a single file, or copy one family to another while
+ * changing the size of the family members. It can also be used
+ * to copy a single file to a single file with holes.
*/
/* See H5private.h for how to include system headers */
@@ -50,13 +50,13 @@
/*-------------------------------------------------------------------------
- * Function: usage
+ * Function: usage
*
- * Purpose: Prints a usage message.
+ * Purpose: Prints a usage message.
*
- * Return: void
+ * Return: void
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, May 13, 1998
*
* Modifications:
@@ -67,96 +67,97 @@ static void
usage (const char *progname)
{
HDfprintf(stderr, "usage: %s [-v] [-V] [-[b|m] N[g|m|k]] [-family_to_sec2] SRC DST\n",
- progname);
+ progname);
HDfprintf(stderr, " -v Produce verbose output\n");
HDfprintf(stderr, " -V Print a version number and exit\n");
HDfprintf(stderr, " -b N The I/O block size, defaults to 1kB\n");
HDfprintf(stderr, " -m N The destination member size or 1GB\n");
HDfprintf(stderr, " -family_to_sec2 Change file driver from family to sec2\n");
HDfprintf(stderr, " SRC The name of the source file\n");
- HDfprintf(stderr, " DST The name of the destination files\n");
+ HDfprintf(stderr, " DST The name of the destination files\n");
HDfprintf(stderr, "Sizes may be suffixed with `g' for GB, `m' for MB or "
- "`k' for kB.\n");
+ "`k' for kB.\n");
HDfprintf(stderr, "File family names include an integer printf "
- "format such as `%%d'\n");
- exit (EXIT_FAILURE);
+ "format such as `%%d'\n");
+ HDexit (EXIT_FAILURE);
}
-
+
/*-------------------------------------------------------------------------
- * Function: get_size
+ * Function: get_size
*
- * Purpose: Reads a size option of the form `-XNS' where `X' is any
- * letter, `N' is a multi-character positive decimal number, and
- * `S' is an optional suffix letter in the set [GgMmk]. The
- * option may also be split among two arguments as: `-X NS'.
- * The input value of ARGNO is the argument number for the
- * switch in the ARGV vector and ARGC is the number of entries
- * in that vector.
+ * Purpose: Reads a size option of the form `-XNS' where `X' is any
+ * letter, `N' is a multi-character positive decimal number, and
+ * `S' is an optional suffix letter in the set [GgMmk]. The
+ * option may also be split among two arguments as: `-X NS'.
+ * The input value of ARGNO is the argument number for the
+ * switch in the ARGV vector and ARGC is the number of entries
+ * in that vector.
*
- * Return: Success: The value N multiplied according to the
- * suffix S. On return ARGNO will be the number
- * of the next argument to process.
+ * Return: Success: The value N multiplied according to the
+ * suffix S. On return ARGNO will be the number
+ * of the next argument to process.
*
- * Failure: Calls usage() which exits with a non-zero
- * status.
+ * Failure: Calls usage() which exits with a non-zero
+ * status.
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, May 13, 1998
- *
- * Modifications:
- *
*-------------------------------------------------------------------------
*/
static off_t
get_size (const char *progname, int *argno, int argc, char *argv[])
{
- off_t retval=-1;
- char *suffix;
+ off_t retval=-1;
+ char *suffix;
if (isdigit ((int)(argv[*argno][2]))) {
- retval = strtol (argv[*argno]+2, &suffix, 10);
- (*argno)++;
- } else if (argv[*argno][2] || *argno+1>=argc) {
- usage (progname);
- } else {
- retval = strtol (argv[*argno+1], &suffix, 0);
- if (suffix==argv[*argno+1]) usage (progname);
- *argno += 2;
+ retval = HDstrtol(argv[*argno]+2, &suffix, 10);
+ (*argno)++;
+ }
+ else if (argv[*argno][2] || *argno+1>=argc) {
+ usage (progname);
+ }
+ else {
+ retval = HDstrtol(argv[*argno+1], &suffix, 0);
+ if (suffix == argv[*argno+1])
+ usage (progname);
+ *argno += 2;
}
if (suffix && suffix[0] && !suffix[1]) {
- switch (*suffix) {
- case 'G':
- case 'g':
- retval *= 1024 * 1024 * 1024;
- break;
- case 'M':
- case 'm':
- retval *= 1024 * 1024;
- break;
- case 'k':
- retval *= 1024;
- break;
- default:
- usage (progname);
- }
- } else if (suffix && suffix[0]) {
- usage (progname);
+ switch (*suffix) {
+ case 'G':
+ case 'g':
+ retval *= 1024 * 1024 * 1024;
+ break;
+ case 'M':
+ case 'm':
+ retval *= 1024 * 1024;
+ break;
+ case 'k':
+ retval *= 1024;
+ break;
+ default:
+ usage(progname);
+ }
+ }
+ else if (suffix && suffix[0]) {
+ usage (progname);
}
return retval;
}
-
+
/*-------------------------------------------------------------------------
- * Function: main
+ * Function: main
*
- * Purpose: Split an hdf5 file
+ * Purpose: Split an hdf5 file
*
- * Return: Success:
+ * Return: Success:
*
- * Failure:
+ * Failure:
*
- * Programmer: Robb Matzke
+ * Programmer: Robb Matzke
* Wednesday, May 13, 1998
*
* Modifications:
@@ -166,36 +167,36 @@ get_size (const char *progname, int *argno, int argc, char *argv[])
int
main (int argc, char *argv[])
{
- const char *prog_name; /*program name */
- size_t blk_size=1024; /*size of each I/O block */
- char *buf=NULL; /*I/O block buffer */
- size_t n, i; /*counters */
- ssize_t nio; /*I/O return value */
- int argno=1; /*program argument number */
- int src, dst=-1; /*source & destination files */
- int need_seek=FALSE; /*destination needs to seek? */
- int need_write; /*data needs to be written? */
- h5_stat_t sb; /*temporary file stat buffer */
-
- int verbose=FALSE; /*display file names? */
-
- const char *src_gen_name; /*general source name */
- char *src_name=NULL; /*source member name */
-
- int src_is_family; /*is source name a family name? */
- int src_membno=0; /*source member number */
-
- const char *dst_gen_name; /*general destination name */
- char *dst_name=NULL; /*destination member name */
- int dst_is_family; /*is dst name a family name? */
- int dst_membno=0; /*destination member number */
-
- off_t left_overs=0; /*amount of zeros left over */
- off_t src_offset=0; /*offset in source member */
- off_t dst_offset=0; /*offset in destination member */
- off_t src_size; /*source logical member size */
- off_t src_act_size; /*source actual member size */
- off_t dst_size=1 GB; /*destination logical memb size */
+ const char *prog_name; /*program name */
+ size_t blk_size=1024; /*size of each I/O block */
+ char *buf=NULL; /*I/O block buffer */
+ size_t n, i; /*counters */
+ ssize_t nio; /*I/O return value */
+ int argno=1; /*program argument number */
+ int src, dst=-1; /*source & destination files */
+ int need_seek=FALSE; /*destination needs to seek? */
+ int need_write; /*data needs to be written? */
+ h5_stat_t sb; /*temporary file stat buffer */
+
+ int verbose=FALSE; /*display file names? */
+
+ const char *src_gen_name; /*general source name */
+ char *src_name=NULL; /*source member name */
+
+ int src_is_family; /*is source name a family name? */
+ int src_membno=0; /*source member number */
+
+ const char *dst_gen_name; /*general destination name */
+ char *dst_name=NULL; /*destination member name */
+ int dst_is_family; /*is dst name a family name? */
+ int dst_membno=0; /*destination member number */
+
+ off_t left_overs=0; /*amount of zeros left over */
+ off_t src_offset=0; /*offset in source member */
+ off_t dst_offset=0; /*offset in destination member */
+ off_t src_size; /*source logical member size */
+ off_t src_act_size; /*source actual member size */
+ off_t dst_size=1 GB; /*destination logical memb size */
hid_t fapl; /*file access property list */
hid_t file;
hsize_t hdsize; /*destination logical memb size */
@@ -204,203 +205,218 @@ main (int argc, char *argv[])
/*
* Get the program name from argv[0]. Use only the last component.
*/
- if ((prog_name=strrchr (argv[0], '/'))) prog_name++;
- else prog_name = argv[0];
+ if ((prog_name = HDstrrchr (argv[0], '/')))
+ prog_name++;
+ else
+ prog_name = argv[0];
/*
* Parse switches.
*/
- while (argno<argc && '-'==argv[argno][0]) {
- if (!strcmp (argv[argno], "-v")) {
+ while (argno < argc && '-' == argv[argno][0]) {
+ if (!HDstrcmp(argv[argno], "-v")) {
verbose = TRUE;
argno++;
- } else if (!strcmp(argv[argno], "-V")) {
- printf("This is %s version %u.%u release %u\n",
- prog_name, H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE);
- exit(EXIT_SUCCESS);
- } else if (!strcmp (argv[argno], "-family_to_sec2")) {
+ } else if (!HDstrcmp(argv[argno], "-V")) {
+ HDprintf("This is %s version %u.%u release %u\n", prog_name, H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE);
+ HDexit(EXIT_SUCCESS);
+ } else if (!HDstrcmp(argv[argno], "-family_to_sec2")) {
family_to_sec2 = TRUE;
argno++;
- } else if ('b'==argv[argno][1]) {
- blk_size = (size_t)get_size (prog_name, &argno, argc, argv);
- } else if ('m'==argv[argno][1]) {
- dst_size = get_size (prog_name, &argno, argc, argv);
+ } else if ('b' == argv[argno][1]) {
+ blk_size = (size_t) get_size(prog_name, &argno, argc, argv);
+ } else if ('m' == argv[argno][1]) {
+ dst_size = get_size(prog_name, &argno, argc, argv);
} else {
- usage (prog_name);
+ usage(prog_name);
} /* end if */
} /* end while */
/* allocate names */
if(NULL == (src_name = (char *)HDcalloc((size_t)NAMELEN, sizeof(char))))
- exit(EXIT_FAILURE);
+ HDexit(EXIT_FAILURE);
if(NULL == (dst_name = (char *)HDcalloc((size_t)NAMELEN, sizeof(char))))
- exit(EXIT_FAILURE);
+ HDexit(EXIT_FAILURE);
/*
* Get the name for the source file and open the first member. The size
* of the first member determines the logical size of all the members.
*/
- if (argno>=argc) usage (prog_name);
+ if (argno >= argc)
+ usage(prog_name);
src_gen_name = argv[argno++];
- sprintf (src_name, src_gen_name, src_membno);
+ HDsprintf(src_name, src_gen_name, src_membno);
src_is_family = strcmp (src_name, src_gen_name);
if ((src = HDopen(src_name, O_RDONLY, 0)) < 0) {
- HDperror (src_name);
- HDexit (EXIT_FAILURE);
+ HDperror(src_name);
+ HDexit(EXIT_FAILURE);
}
- if (HDfstat(src, &sb)<0) {
- perror ("fstat");
- exit (EXIT_FAILURE);
+ if (HDfstat(src, &sb) < 0) {
+ HDperror("fstat");
+ HDexit(EXIT_FAILURE);
}
src_size = src_act_size = sb.st_size;
- if (verbose) HDfprintf (stderr, "< %s\n", src_name);
+ if (verbose)
+ HDfprintf(stderr, "< %s\n", src_name);
/*
* Get the name for the destination file and open the first member.
*/
- if (argno>=argc) usage (prog_name);
+ if (argno>=argc)
+ usage (prog_name);
dst_gen_name = argv[argno++];
- sprintf (dst_name, dst_gen_name, dst_membno);
- dst_is_family = strcmp (dst_name, dst_gen_name);
+ HDsprintf(dst_name, dst_gen_name, dst_membno);
+ dst_is_family = HDstrcmp(dst_name, dst_gen_name);
- if ((dst = HDopen (dst_name, O_RDWR|O_CREAT|O_TRUNC, 0666)) < 0) {
- HDperror (dst_name);
- HDexit (EXIT_FAILURE);
+ if ((dst = HDopen(dst_name, O_RDWR|O_CREAT|O_TRUNC, 0666)) < 0) {
+ HDperror(dst_name);
+ HDexit(EXIT_FAILURE);
}
- if (verbose) HDfprintf (stderr, "> %s\n", dst_name);
+ if (verbose)
+ HDfprintf(stderr, "> %s\n", dst_name);
/* No more arguments */
- if (argno<argc) usage (prog_name);
+ if (argno < argc)
+ usage (prog_name);
/* Now the real work, split the file */
buf = (char *)HDmalloc(blk_size);
while (src_offset<src_size) {
- /* Read a block. The amount to read is the minimum of:
- * 1. The I/O block size
- * 2. What's left to write in the destination member
- * 3. Left over zeros or what's left in the source member.
- */
- n = blk_size;
- if (dst_is_family) n = (size_t)MIN((off_t)n, dst_size-dst_offset);
- if (left_overs) {
- n = (size_t)MIN ((off_t)n, left_overs);
- left_overs = left_overs - (off_t)n;
- need_write = FALSE;
- } else if (src_offset<src_act_size) {
- n = (size_t)MIN ((off_t)n, src_act_size-src_offset);
- if ((nio=HDread (src, buf, n))<0) {
- perror ("read");
- exit (EXIT_FAILURE);
- } else if ((size_t)nio!=n) {
- HDfprintf (stderr, "%s: short read\n", src_name);
- exit (EXIT_FAILURE);
- }
- for (i=0; i<n; i++) {
- if (buf[i]) break;
- }
- need_write = (i<n);
- } else {
- n = 0;
- left_overs = src_size - src_act_size;
- need_write = FALSE;
- }
-
- /*
- * If the block contains non-zero data then write it to the
- * destination, otherwise just remember that we'll have to do a seek
- * later in the destination when we finally get non-zero data.
- */
- if (need_write) {
- if (need_seek && HDlseek (dst, dst_offset, SEEK_SET)<0) {
- perror ("HDlseek");
- exit (EXIT_FAILURE);
- }
- if ((nio=HDwrite (dst, buf, n))<0) {
- perror ("write");
- exit (EXIT_FAILURE);
- } else if ((size_t)nio!=n) {
- HDfprintf (stderr, "%s: short write\n", dst_name);
- exit (EXIT_FAILURE);
- }
- need_seek = FALSE;
- } else {
- need_seek = TRUE;
- }
-
- /*
- * Update the source offset and open the next source family member if
- * necessary. The source stream ends at the first member which
- * cannot be opened because it doesn't exist. At the end of the
- * source stream, update the destination offset and break out of the
- * loop. The destination offset must be updated so we can fix
- * trailing holes.
- */
- src_offset = src_offset + (off_t)n;
- if (src_offset==src_act_size) {
- HDclose (src);
- if (!src_is_family) {
- dst_offset = dst_offset + (off_t)n;
- break;
- }
- sprintf (src_name, src_gen_name, ++src_membno);
- if ((src = HDopen (src_name, O_RDONLY, 0)) < 0 && ENOENT == errno) {
- dst_offset = dst_offset + (off_t)n;
- break;
- } else if (src<0) {
- perror (src_name);
- exit (EXIT_FAILURE);
- }
- if (HDfstat (src, &sb)<0) {
- perror ("fstat");
- exit (EXIT_FAILURE);
- }
- src_act_size = sb.st_size;
- if (src_act_size>src_size) {
- HDfprintf (stderr, "%s: member truncated to %lu bytes\n",
- src_name, (unsigned long)src_size);
- }
- src_offset = 0;
- if (verbose) HDfprintf (stderr, "< %s\n", src_name);
- }
-
- /*
- * Update the destination offset, opening a new member if one will be
- * needed. The first member is extended to the logical member size
- * but other members might be smaller if they end with a hole.
- */
- dst_offset = dst_offset + (off_t)n;
- if (dst_is_family && dst_offset==dst_size) {
- if (0==dst_membno) {
- if (HDlseek (dst, dst_size-1, SEEK_SET)<0) {
- perror ("HDHDlseek");
- exit (EXIT_FAILURE);
- }
- if (HDread (dst, buf, 1)<0) {
- perror ("read");
- exit (EXIT_FAILURE);
- }
- if (HDlseek (dst, dst_size-1, SEEK_SET)<0) {
- perror ("HDlseek");
- exit (EXIT_FAILURE);
- }
- if (HDwrite (dst, buf, 1)<0) {
- perror ("write");
- exit (EXIT_FAILURE);
- }
- }
- HDclose (dst);
- sprintf (dst_name, dst_gen_name, ++dst_membno);
- if ((dst = HDopen (dst_name, O_RDWR|O_CREAT|O_TRUNC, 0666)) < 0) {
- HDperror (dst_name);
- HDexit (EXIT_FAILURE);
- }
- dst_offset = 0;
- need_seek = FALSE;
- if (verbose) HDfprintf (stderr, "> %s\n", dst_name);
- }
+ /* Read a block. The amount to read is the minimum of:
+ * 1. The I/O block size
+ * 2. What's left to write in the destination member
+ * 3. Left over zeros or what's left in the source member.
+ */
+ n = blk_size;
+ if (dst_is_family)
+ n = (size_t)MIN((off_t)n, dst_size-dst_offset);
+ if (left_overs) {
+ n = (size_t)MIN((off_t)n, left_overs);
+ left_overs = left_overs - (off_t) n;
+ need_write = FALSE;
+ }
+ else if (src_offset < src_act_size) {
+ n = (size_t)MIN((off_t)n, src_act_size-src_offset);
+ if ((nio = HDread(src, buf, n)) < 0) {
+ HDperror("read");
+ HDexit(EXIT_FAILURE);
+ }
+ else if ((size_t) nio != n) {
+ HDfprintf(stderr, "%s: short read\n", src_name);
+ HDexit(EXIT_FAILURE);
+ }
+ for (i = 0; i < n; i++) {
+ if (buf[i])
+ break;
+ }
+ need_write = (i < n);
+ }
+ else {
+ n = 0;
+ left_overs = src_size - src_act_size;
+ need_write = FALSE;
+ }
+
+ /*
+ * If the block contains non-zero data then write it to the
+ * destination, otherwise just remember that we'll have to do a seek
+ * later in the destination when we finally get non-zero data.
+ */
+ if (need_write) {
+ if (need_seek && HDlseek (dst, dst_offset, SEEK_SET) < 0) {
+ HDperror("HDlseek");
+ HDexit(EXIT_FAILURE);
+ }
+ if ((nio = HDwrite(dst, buf, n)) < 0) {
+ HDperror("write");
+ HDexit(EXIT_FAILURE);
+ }
+ else if ((size_t) nio != n) {
+ HDfprintf(stderr, "%s: short write\n", dst_name);
+ HDexit(EXIT_FAILURE);
+ }
+ need_seek = FALSE;
+ }
+ else {
+ need_seek = TRUE;
+ }
+
+ /*
+ * Update the source offset and open the next source family member if
+ * necessary. The source stream ends at the first member which
+ * cannot be opened because it doesn't exist. At the end of the
+ * source stream, update the destination offset and break out of the
+ * loop. The destination offset must be updated so we can fix
+ * trailing holes.
+ */
+ src_offset = src_offset + (off_t) n;
+ if (src_offset == src_act_size) {
+ HDclose(src);
+ if (!src_is_family) {
+ dst_offset = dst_offset + (off_t) n;
+ break;
+ }
+ HDsprintf(src_name, src_gen_name, ++src_membno);
+ if ((src = HDopen(src_name, O_RDONLY)) < 0 && ENOENT == errno) {
+ dst_offset = dst_offset + (off_t) n;
+ break;
+ }
+ else if (src < 0) {
+ HDperror(src_name);
+ HDexit(EXIT_FAILURE);
+ }
+ if (HDfstat (src, &sb) < 0) {
+ HDperror("fstat");
+ HDexit(EXIT_FAILURE);
+ }
+ src_act_size = sb.st_size;
+ if (src_act_size > src_size) {
+ HDfprintf(stderr, "%s: member truncated to %lu bytes\n", src_name, (unsigned long) src_size);
+ }
+ src_offset = 0;
+ if (verbose)
+ HDfprintf(stderr, "< %s\n", src_name);
+ }
+
+ /*
+ * Update the destination offset, opening a new member if one will be
+ * needed. The first member is extended to the logical member size
+ * but other members might be smaller if they end with a hole.
+ */
+ dst_offset = dst_offset + (off_t) n;
+ if (dst_is_family && dst_offset == dst_size) {
+ if (0 == dst_membno) {
+ if (HDlseek (dst, dst_size-1, SEEK_SET) < 0) {
+ HDperror("HDHDlseek");
+ HDexit(EXIT_FAILURE);
+ }
+ if (HDread (dst, buf, 1) < 0) {
+ HDperror("read");
+ HDexit(EXIT_FAILURE);
+ }
+ if (HDlseek (dst, dst_size-1, SEEK_SET) < 0) {
+ HDperror("HDlseek");
+ HDexit(EXIT_FAILURE);
+ }
+ if (HDwrite (dst, buf, 1) < 0) {
+ HDperror("write");
+ HDexit(EXIT_FAILURE);
+ }
+ }
+ HDclose(dst);
+ HDsprintf(dst_name, dst_gen_name, ++dst_membno);
+ if ((dst = HDopen(dst_name, O_RDWR | O_CREAT | O_TRUNC, 0666)) < 0) {
+ HDperror(dst_name);
+ HDexit(EXIT_FAILURE);
+ }
+ dst_offset = 0;
+ need_seek = FALSE;
+ if (verbose)
+ HDfprintf(stderr, "> %s\n", dst_name);
+ }
}
/*
@@ -409,30 +425,30 @@ main (int argc, char *argv[])
* family has been truncated.
*/
if (need_seek) {
- if (HDlseek (dst, dst_offset-1, SEEK_SET)<0) {
- perror ("HDlseek");
- exit (EXIT_FAILURE);
- }
- if (HDread (dst, buf, 1)<0) {
- perror ("read");
- exit (EXIT_FAILURE);
- }
- if (HDlseek (dst, dst_offset-1, SEEK_SET)<0) {
- perror ("HDlseek");
- exit (EXIT_FAILURE);
- }
- if (HDwrite (dst, buf, 1)<0) {
- perror ("write");
- exit (EXIT_FAILURE);
- }
+ if (HDlseek (dst, dst_offset-1, SEEK_SET)<0) {
+ HDperror ("HDlseek");
+ HDexit (EXIT_FAILURE);
+ }
+ if (HDread (dst, buf, 1)<0) {
+ HDperror ("read");
+ HDexit (EXIT_FAILURE);
+ }
+ if (HDlseek (dst, dst_offset-1, SEEK_SET)<0) {
+ HDperror ("HDlseek");
+ HDexit (EXIT_FAILURE);
+ }
+ if (HDwrite (dst, buf, 1)<0) {
+ HDperror ("write");
+ HDexit (EXIT_FAILURE);
+ }
}
HDclose (dst);
/* Modify family driver information saved in superblock through private property.
* These private properties are for this tool only. */
- if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) {
- perror ("H5Pcreate");
- exit (EXIT_FAILURE);
+ if ((fapl=H5Pcreate(H5P_FILE_ACCESS)) < 0) {
+ HDperror ("H5Pcreate");
+ HDexit (EXIT_FAILURE);
}
if(family_to_sec2) {
@@ -440,23 +456,24 @@ main (int argc, char *argv[])
* with sec2 driver. This property signals the library to ignore the family
* driver information saved in the superblock. */
if(H5Pset(fapl, H5F_ACS_FAMILY_TO_SEC2_NAME, &family_to_sec2) < 0) {
- perror ("H5Pset");
- exit (EXIT_FAILURE);
+ HDperror ("H5Pset");
+ HDexit (EXIT_FAILURE);
}
- } else {
+ }
+ else {
/* Modify family size saved in superblock through private property. It signals
* library to save the new member size(specified in command line) in superblock.
* This private property is for this tool only. */
if(H5Pset_fapl_family(fapl, H5F_FAMILY_DEFAULT, H5P_DEFAULT) < 0) {
- perror ("H5Pset_fapl_family");
- exit (EXIT_FAILURE);
+ HDperror ("H5Pset_fapl_family");
+ HDexit (EXIT_FAILURE);
}
/* Set the property of the new member size as hsize_t */
hdsize = (hsize_t)dst_size;
if(H5Pset(fapl, H5F_ACS_FAMILY_NEWSIZE_NAME, &hdsize) < 0) {
- perror ("H5Pset");
- exit (EXIT_FAILURE);
+ HDperror ("H5Pset");
+ HDexit (EXIT_FAILURE);
}
}
@@ -468,19 +485,19 @@ main (int argc, char *argv[])
* new file can only be a sec2 file, reopen the new file should fail. There's
* nothing to do in this case. */
H5E_BEGIN_TRY {
- file=H5Fopen(dst_gen_name, H5F_ACC_RDWR, fapl);
+ file = H5Fopen(dst_gen_name, H5F_ACC_RDWR, fapl);
} H5E_END_TRY;
- if(file>=0) {
- if(H5Fclose(file)<0) {
- perror ("H5Fclose");
- exit (EXIT_FAILURE);
+ if(file >= 0) {
+ if(H5Fclose(file) < 0) {
+ HDperror("H5Fclose");
+ HDexit(EXIT_FAILURE);
} /* end if */
} /* end if */
- if(H5Pclose(fapl)<0) {
- perror ("H5Pclose");
- exit (EXIT_FAILURE);
+ if(H5Pclose(fapl) < 0) {
+ HDperror("H5Pclose");
+ HDexit(EXIT_FAILURE);
} /* end if */
/* Free resources and return */
diff --git a/tools/testfiles/h5dump-help.txt b/tools/testfiles/h5dump-help.txt
index bc37a8f..17c7b9d 100644
--- a/tools/testfiles/h5dump-help.txt
+++ b/tools/testfiles/h5dump-help.txt
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/help-1.ls b/tools/testfiles/help-1.ls
index 491f696..396bed3 100644
--- a/tools/testfiles/help-1.ls
+++ b/tools/testfiles/help-1.ls
@@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
-V, --version Print version number and exit
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
+ --s3-cred=C Supply S3 authentication information to "ros3" vfd.
+ Accepts tuple of "(<aws-region>,<access-id>,<access-key>)".
+ If absent or C->"(,,)", defaults to no-authentication.
+ Has no effect if vfd flag not set to "ros3".
+ --hdfs-attrs=A Supply configuration information to Hadoop VFD.
+ Accepts tuple of (<namenode name>,<namenode port>,
+ ...<kerberos cache path>,<username>,<buffer size>)
+ If absent or A == '(,,,,)', all default values are used.
+ Has no effect if vfd flag is not 'hdfs'.
file/OBJECT
Each object consists of an HDF5 file name optionally followed by a
diff --git a/tools/testfiles/help-2.ls b/tools/testfiles/help-2.ls
index 491f696..396bed3 100644
--- a/tools/testfiles/help-2.ls
+++ b/tools/testfiles/help-2.ls
@@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
-V, --version Print version number and exit
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
+ --s3-cred=C Supply S3 authentication information to "ros3" vfd.
+ Accepts tuple of "(<aws-region>,<access-id>,<access-key>)".
+ If absent or C->"(,,)", defaults to no-authentication.
+ Has no effect if vfd flag not set to "ros3".
+ --hdfs-attrs=A Supply configuration information to Hadoop VFD.
+ Accepts tuple of (<namenode name>,<namenode port>,
+ ...<kerberos cache path>,<username>,<buffer size>)
+ If absent or A == '(,,,,)', all default values are used.
+ Has no effect if vfd flag is not 'hdfs'.
file/OBJECT
Each object consists of an HDF5 file name optionally followed by a
diff --git a/tools/testfiles/help-3.ls b/tools/testfiles/help-3.ls
index 491f696..396bed3 100644
--- a/tools/testfiles/help-3.ls
+++ b/tools/testfiles/help-3.ls
@@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
-V, --version Print version number and exit
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
+ --s3-cred=C Supply S3 authentication information to "ros3" vfd.
+ Accepts tuple of "(<aws-region>,<access-id>,<access-key>)".
+ If absent or C->"(,,)", defaults to no-authentication.
+ Has no effect if vfd flag not set to "ros3".
+ --hdfs-attrs=A Supply configuration information to Hadoop VFD.
+ Accepts tuple of (<namenode name>,<namenode port>,
+ ...<kerberos cache path>,<username>,<buffer size>)
+ If absent or A == '(,,,,)', all default values are used.
+ Has no effect if vfd flag is not 'hdfs'.
file/OBJECT
Each object consists of an HDF5 file name optionally followed by a
diff --git a/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl b/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl
index bc37a8f..17c7b9d 100644
--- a/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl
+++ b/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/pbits/tpbitsIncomplete.ddl b/tools/testfiles/pbits/tpbitsIncomplete.ddl
index bc37a8f..17c7b9d 100644
--- a/tools/testfiles/pbits/tpbitsIncomplete.ddl
+++ b/tools/testfiles/pbits/tpbitsIncomplete.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/pbits/tpbitsLengthExceeded.ddl b/tools/testfiles/pbits/tpbitsLengthExceeded.ddl
index bc37a8f..17c7b9d 100644
--- a/tools/testfiles/pbits/tpbitsLengthExceeded.ddl
+++ b/tools/testfiles/pbits/tpbitsLengthExceeded.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/pbits/tpbitsLengthPositive.ddl b/tools/testfiles/pbits/tpbitsLengthPositive.ddl
index bc37a8f..17c7b9d 100644
--- a/tools/testfiles/pbits/tpbitsLengthPositive.ddl
+++ b/tools/testfiles/pbits/tpbitsLengthPositive.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/pbits/tpbitsMaxExceeded.ddl b/tools/testfiles/pbits/tpbitsMaxExceeded.ddl
index bc37a8f..17c7b9d 100644
--- a/tools/testfiles/pbits/tpbitsMaxExceeded.ddl
+++ b/tools/testfiles/pbits/tpbitsMaxExceeded.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl b/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl
index bc37a8f..17c7b9d 100644
--- a/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl
+++ b/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/pbits/tpbitsOffsetNegative.ddl b/tools/testfiles/pbits/tpbitsOffsetNegative.ddl
index bc37a8f..17c7b9d 100644
--- a/tools/testfiles/pbits/tpbitsOffsetNegative.ddl
+++ b/tools/testfiles/pbits/tpbitsOffsetNegative.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/textlinksrc-nodangle-1.ls b/tools/testfiles/textlinksrc-nodangle-1.ls
index 491f696..396bed3 100644
--- a/tools/testfiles/textlinksrc-nodangle-1.ls
+++ b/tools/testfiles/textlinksrc-nodangle-1.ls
@@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
-V, --version Print version number and exit
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
+ --s3-cred=C Supply S3 authentication information to "ros3" vfd.
+ Accepts tuple of "(<aws-region>,<access-id>,<access-key>)".
+ If absent or C->"(,,)", defaults to no-authentication.
+ Has no effect if vfd flag not set to "ros3".
+ --hdfs-attrs=A Supply configuration information to Hadoop VFD.
+ Accepts tuple of (<namenode name>,<namenode port>,
+ ...<kerberos cache path>,<username>,<buffer size>)
+ If absent or A == '(,,,,)', all default values are used.
+ Has no effect if vfd flag is not 'hdfs'.
file/OBJECT
Each object consists of an HDF5 file name optionally followed by a
diff --git a/tools/testfiles/tgroup-1.ls b/tools/testfiles/tgroup-1.ls
index 491f696..396bed3 100644
--- a/tools/testfiles/tgroup-1.ls
+++ b/tools/testfiles/tgroup-1.ls
@@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
-V, --version Print version number and exit
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
+ --s3-cred=C Supply S3 authentication information to "ros3" vfd.
+ Accepts tuple of "(<aws-region>,<access-id>,<access-key>)".
+ If absent or C->"(,,)", defaults to no-authentication.
+ Has no effect if vfd flag not set to "ros3".
+ --hdfs-attrs=A Supply configuration information to Hadoop VFD.
+ Accepts tuple of (<namenode name>,<namenode port>,
+ ...<kerberos cache path>,<username>,<buffer size>)
+ If absent or A == '(,,,,)', all default values are used.
+ Has no effect if vfd flag is not 'hdfs'.
file/OBJECT
Each object consists of an HDF5 file name optionally followed by a
diff --git a/tools/testfiles/tudlink-2.ddl b/tools/testfiles/tudlink-2.ddl
index 7f4281a..82b9f8e 100644
--- a/tools/testfiles/tudlink-2.ddl
+++ b/tools/testfiles/tudlink-2.ddl
@@ -1,6 +1,5 @@
HDF5 "tudlink.h5" {
USERDEFINED_LINK "udlink2" {
- USERDEFINED_LINK "udlink2" {
LINKCLASS 187
}
}