summaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tools')
-rw-r--r--tools/CMakeLists.txt7
-rw-r--r--tools/lib/h5tools_utils.c517
-rw-r--r--tools/lib/h5tools_utils.h10
-rw-r--r--tools/libtest/CMakeLists.txt21
-rw-r--r--tools/libtest/CMakeTests.cmake49
-rw-r--r--tools/libtest/Makefile.am34
-rw-r--r--tools/libtest/h5tools_test_utils.c1295
-rw-r--r--tools/src/h5dump/h5dump.c330
-rw-r--r--tools/src/h5ls/h5ls.c321
-rw-r--r--tools/src/h5stat/h5stat.c251
-rw-r--r--tools/test/h5stat/testfiles/h5stat_help1.ddl10
-rw-r--r--tools/test/h5stat/testfiles/h5stat_help2.ddl10
-rw-r--r--tools/test/h5stat/testfiles/h5stat_nofile.ddl10
-rw-r--r--tools/testfiles/h5dump-help.txt10
-rw-r--r--tools/testfiles/help-1.ls9
-rw-r--r--tools/testfiles/help-2.ls9
-rw-r--r--tools/testfiles/help-3.ls9
-rw-r--r--tools/testfiles/pbits/tnofilename-with-packed-bits.ddl10
-rw-r--r--tools/testfiles/pbits/tpbitsIncomplete.ddl10
-rw-r--r--tools/testfiles/pbits/tpbitsLengthExceeded.ddl10
-rw-r--r--tools/testfiles/pbits/tpbitsLengthPositive.ddl10
-rw-r--r--tools/testfiles/pbits/tpbitsMaxExceeded.ddl10
-rw-r--r--tools/testfiles/pbits/tpbitsOffsetExceeded.ddl10
-rw-r--r--tools/testfiles/pbits/tpbitsOffsetNegative.ddl10
-rw-r--r--tools/testfiles/textlinksrc-nodangle-1.ls9
-rw-r--r--tools/testfiles/tgroup-1.ls9
26 files changed, 2798 insertions, 192 deletions
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
index 400039e..aa09aa6 100644
--- a/tools/CMakeLists.txt
+++ b/tools/CMakeLists.txt
@@ -9,4 +9,11 @@ add_subdirectory (src)
#-- Add the tests
if (BUILD_TESTING)
add_subdirectory (test)
+
+# --------------------------------------------------------------------
+# If S3 or HDFS enabled, then we need to test the tools library
+# --------------------------------------------------------------------
+ if (HDF5_ENABLE_ROS3_VFD OR HDF5_ENABLE_HDFS)
+ add_subdirectory (libtest)
+ endif ()
endif ()
diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c
index e7e017f..a3cd7d9 100644
--- a/tools/lib/h5tools_utils.c
+++ b/tools/lib/h5tools_utils.c
@@ -21,6 +21,10 @@
#include "H5private.h"
#include "h5trav.h"
+#ifdef H5_HAVE_ROS3_VFD
+#include "H5FDros3.h"
+#endif
+
/* global variables */
unsigned h5tools_nCols = 80;
/* ``get_option'' variables */
@@ -97,7 +101,7 @@ parallel_print(const char* format, ...)
HDva_end(ap);
}
-
+
/*-------------------------------------------------------------------------
* Function: error_msg
*
@@ -122,7 +126,7 @@ error_msg(const char *fmt, ...)
HDva_end(ap);
}
-
+
/*-------------------------------------------------------------------------
* Function: warn_msg
*
@@ -161,7 +165,7 @@ help_ref_msg(FILE *output)
HDfprintf(output, "see the <%s> entry in the 'HDF5 Reference Manual'.\n",h5tools_getprogname());
}
-
+
/*-------------------------------------------------------------------------
* Function: get_option
*
@@ -322,7 +326,229 @@ get_option(int argc, const char **argv, const char *opts, const struct long_opti
return opt_opt;
}
-
+
+/*****************************************************************************
+ *
+ * Function: parse_tuple()
+ *
+ * Purpose:
+ *
+ * Create array of pointers to strings, identified as elements in a tuple
+ * of arbitrary length separated by provided character.
+ * ("tuple" because "nple" looks strange)
+ *
+ * * Receives pointer to start of tuple sequence string, '('.
+ * * Attempts to separate elements by token-character `sep`.
+ * * If the separator character is preceded by a backslash '\',
+ * the backslash is deleted and the separator is included in the
+ * element string as any other character.
+ * * To end an element with a backslash, escape the backslash, e.g.
+ * "(myelem\\,otherelem) -> {"myelem\", "otherelem"}
+ * * In all other cases, a backslash appearing not as part of "\\" or
+ * "\<sep>" digraph will be included berbatim.
+ * * Last two characters in the string MUST be ")\0".
+ *
+ * * Generates a copy of the input string `start`, (src..")\0"), replacing
+ * separators and close-paren with null charaters.
+ * * This string is allocated at runtime and should be freed when done.
+ * * Generates array of char pointers, and directs start of each element
+ * (each pointer) into this copy.
+ * * Each tuple element points to the start of its string (substring)
+ * and ends with a null terminator.
+ * * This array is allocated at runtime and should be freed when done.
+ * * Reallocates and expands elements array during parsing.
+ * * Initially allocated for 2 (plus one null entry), and grows by
+ * powers of 2.
+ * * The final 'slot' in the element array (elements[nelements], e.g.)
+ * always points to NULL.
+ * * The number of elements found and stored are passed out through pointer
+ * to unsigned, `nelems`.
+ *
+ * Return:
+ *
+ * FAIL If malformed--does not look like a tuple "(...)"
+ * or major error was encountered while parsing.
+ * or
+ * SUCCEED String looks properly formed "(...)" and no major errors.
+ *
+ * Stores number of elements through pointer `nelems`.
+ * Stores list of pointers to char (first char in each element
+ * string) through pointer `ptrs_out`.
+ * NOTE: `ptrs_out[nelems] == NULL` should be true.
+ * NOTE: list is malloc'd by function, and should be freed
+ * when done.
+ * Stores "source string" for element pointers through `cpy_out`.
+ * NOTE: Each element substring is null-terminated.
+ * NOTE: There may be extra characters after the last element
+ * (past its null terminator), but is guaranteed to
+ * be null-terminated.
+ * NOTE: `cpy_out` string is malloc'd by function,
+ * and should be freed when done.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-10
+ *
+ * Changes: None.
+ *
+ *****************************************************************************
+ */
+herr_t
+parse_tuple(const char *start,
+ int sep,
+ char **cpy_out,
+ unsigned *nelems,
+ char ***ptrs_out)
+{
+ char *elem_ptr = NULL;
+ char *dest_ptr = NULL;
+ unsigned elems_count = 0;
+ char **elems = NULL; /* more like *elems[], but complier... */
+ char **elems_re = NULL; /* temporary pointer, for realloc */
+ char *cpy = NULL;
+ herr_t ret_value = SUCCEED;
+ unsigned init_slots = 2;
+
+
+
+ /*****************
+ * SANITY-CHECKS *
+ *****************/
+
+ /* must start with "("
+ */
+ if (start[0] != '(') {
+ ret_value = FAIL;
+ goto done;
+ }
+
+ /* must end with ")"
+ */
+ while (start[elems_count] != '\0') {
+ elems_count++;
+ }
+ if (start[elems_count - 1] != ')') {
+ ret_value = FAIL;
+ goto done;
+ }
+
+ elems_count = 0;
+
+
+
+ /***********
+ * PREPARE *
+ ***********/
+
+ /* create list
+ */
+ elems = (char **)HDmalloc(sizeof(char *) * (init_slots + 1));
+ if (elems == NULL) { ret_value = FAIL; goto done; } /* CANTALLOC */
+
+ /* create destination string
+ */
+ start++; /* advance past opening paren '(' */
+ cpy = (char *)HDmalloc(sizeof(char) * (HDstrlen(start))); /* no +1; less '(' */
+ if (cpy == NULL) { ret_value = FAIL; goto done; } /* CANTALLOC */
+
+ /* set pointers
+ */
+ dest_ptr = cpy; /* start writing copy here */
+ elem_ptr = cpy; /* first element starts here */
+ elems[elems_count++] = elem_ptr; /* set first element pointer into list */
+
+
+
+ /*********
+ * PARSE *
+ *********/
+
+ while (*start != '\0') {
+ /* For each character in the source string...
+ */
+ if (*start == '\\') {
+ /* Possibly an escape digraph.
+ */
+ if ((*(start + 1) == '\\') ||
+ (*(start + 1) == sep) )
+ {
+ /* Valid escape digraph of "\\" or "\<sep>".
+ */
+ start++; /* advance past escape char '\' */
+ *(dest_ptr++) = *(start++); /* Copy subsequent char */
+ /* and advance pointers. */
+ } else {
+ /* Not an accepted escape digraph.
+ * Copy backslash character.
+ */
+ *(dest_ptr++) = *(start++);
+ }
+ } else if (*start == sep) {
+ /* Non-escaped separator.
+ * Terminate elements substring in copy, record element, advance.
+ * Expand elements list if appropriate.
+ */
+ *(dest_ptr++) = 0; /* Null-terminate elem substring in copy */
+ /* and advance pointer. */
+ start++; /* Advance src pointer past separator. */
+ elem_ptr = dest_ptr; /* Element pointer points to start of first */
+ /* character after null sep in copy. */
+ elems[elems_count++] = elem_ptr; /* Set elem pointer in list */
+ /* and increment count. */
+
+ /* Expand elements list, if necessary.
+ */
+ if (elems_count == init_slots) {
+ init_slots *= 2;
+ elems_re = (char **)realloc(elems, sizeof(char *) * \
+ (init_slots + 1));
+ if (elems_re == NULL) {
+ /* CANTREALLOC */
+ ret_value = FAIL;
+ goto done;
+ }
+ elems = elems_re;
+ }
+ } else if (*start == ')' && *(start + 1) == '\0') {
+ /* Found terminal, non-escaped close-paren. Last element.
+ * Write null terminator to copy.
+ * Advance source pointer to gently break from loop.
+ * Requred to prevent ")" from always being added to last element.
+ */
+ start++;
+ } else {
+ /* Copy character into destination. Advance pointers.
+ */
+ *(dest_ptr++) = *(start++);
+ }
+ }
+ *dest_ptr = '\0'; /* Null-terminate destination string. */
+ elems[elems_count] = NULL; /* Null-terminate elements list. */
+
+
+
+ /********************
+ * PASS BACK VALUES *
+ ********************/
+
+ *ptrs_out = elems;
+ *nelems = elems_count;
+ *cpy_out = cpy;
+
+done:
+ if (ret_value == FAIL) {
+ /* CLEANUP */
+ if (cpy) free(cpy);
+ if (elems) free(elems);
+ }
+
+ return ret_value;
+
+} /* parse_tuple */
+
+
+
+
+
/*-------------------------------------------------------------------------
* Function: indentation
*
@@ -344,7 +570,7 @@ indentation(unsigned x)
}
}
-
+
/*-------------------------------------------------------------------------
* Function: print_version
*
@@ -362,7 +588,7 @@ print_version(const char *progname)
((const char *)H5_VERS_SUBRELEASE)[0] ? "-" : "", H5_VERS_SUBRELEASE);
}
-
+
/*-------------------------------------------------------------------------
* Function: init_table
*
@@ -384,7 +610,7 @@ init_table(table_t **tbl)
*tbl = table;
}
-
+
/*-------------------------------------------------------------------------
* Function: free_table
*
@@ -408,7 +634,7 @@ free_table(table_t *table)
}
#ifdef H5DUMP_DEBUG
-
+
/*-------------------------------------------------------------------------
* Function: dump_table
*
@@ -429,7 +655,7 @@ dump_table(char* tablename, table_t *table)
table->objs[u].displayed, table->objs[u].recorded);
}
-
+
/*-------------------------------------------------------------------------
* Function: dump_tables
*
@@ -447,7 +673,7 @@ dump_tables(find_objs_t *info)
}
#endif /* H5DUMP_DEBUG */
-
+
/*-------------------------------------------------------------------------
* Function: search_obj
*
@@ -470,7 +696,7 @@ search_obj(table_t *table, haddr_t objno)
return NULL;
}
-
+
/*-------------------------------------------------------------------------
* Function: find_objs_cb
*
@@ -546,7 +772,7 @@ find_objs_cb(const char *name, const H5O_info_t *oinfo, const char *already_seen
return ret_value;
}
-
+
/*-------------------------------------------------------------------------
* Function: init_objs
*
@@ -591,7 +817,7 @@ done:
return ret_value;
}
-
+
/*-------------------------------------------------------------------------
* Function: add_obj
*
@@ -622,7 +848,7 @@ add_obj(table_t *table, haddr_t objno, const char *objname, hbool_t record)
table->objs[u].displayed = 0;
}
-
+
#ifndef H5_HAVE_TMPFILE
/*-------------------------------------------------------------------------
* Function: tmpfile
@@ -841,3 +1067,266 @@ done:
return ret_value;
}
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: h5tools_populate_ros3_fapl()
+ *
+ * Purpose:
+ *
+ * Set the values of a ROS3 fapl configuration object.
+ *
+ * If the values pointer is NULL, sets fapl target `fa` to a default
+ * (valid, current-version, non-authenticating) fapl config.
+ *
+ * If `values` pointer is _not_ NULL, expects `values` to contain at least
+ * three non-null pointers to null-terminated strings, corresponding to:
+ * { aws_region,
+ * secret_id,
+ * secret_key,
+ * }
+ * If all three strings are empty (""), the default fapl will be default.
+ * Both aws_region and secret_id values must be both empty or both
+ * populated. If
+ * Only secret_key is allowed to be empty (the empty string, "").
+ * All values are checked against overflow as defined in the ros3 vfd
+ * header file; if a value overruns the permitted space, FAIL is returned
+ * and the function aborts without resetting the fapl to values initially
+ * present.
+ *
+ * Return:
+ *
+ * 0 (failure) if...
+ * * Read-Only S3 VFD is not enabled.
+ * * NULL fapl pointer: (NULL, {...} )
+ * * Warning: In all cases below, fapl will be set as "default"
+ * before error occurs.
+ * * NULL value strings: (&fa, {NULL?, NULL? NULL?, ...})
+ * * Incomplete fapl info:
+ * * empty region, non-empty id, key either way
+ * * (&fa, {"", "...", "?"})
+ * * empty id, non-empty region, key either way
+ * * (&fa, {"...", "", "?"})
+ * * "non-empty key and either id or region empty
+ * * (&fa, {"", "", "...")
+ * * (&fa, {"", "...", "...")
+ * * (&fa, {"...", "", "...")
+ * * Any string would overflow allowed space in fapl definition.
+ * or
+ * 1 (success)
+ * * Sets components in fapl_t pointer, copying strings as appropriate.
+ * * "Default" fapl (valid version, authenticate->False, empty strings)
+ * * `values` pointer is NULL
+ * * (&fa, NULL)
+ * * first three strings in `values` are empty ("")
+ * * (&fa, {"", "", "", ...}
+ * * Authenticating fapl
+ * * region, id, and optional key provided
+ * * (&fa, {"...", "...", ""})
+ * * (&fa, {"...", "...", "..."})
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-13
+ *
+ * Changes: None.
+ *
+ *----------------------------------------------------------------------------
+ */
+int
+h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa,
+ const char **values)
+{
+#ifndef H5_HAVE_ROS3_VFD
+ return 0;
+#else
+ int show_progress = 0; /* set to 1 for debugging */
+ int ret_value = 1; /* 1 for success, 0 for failure */
+ /* e.g.? if (!populate()) { then failed } */
+
+ if (show_progress) {
+ HDprintf("called h5tools_populate_ros3_fapl\n");
+ }
+
+ if (fa == NULL) {
+ if (show_progress) {
+ HDprintf(" ERROR: null pointer to fapl_t\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+
+ if (show_progress) {
+ HDprintf(" preset fapl with default values\n");
+ }
+ fa->version = H5FD__CURR_ROS3_FAPL_T_VERSION;
+ fa->authenticate = FALSE;
+ *(fa->aws_region) = '\0';
+ *(fa->secret_id) = '\0';
+ *(fa->secret_key) = '\0';
+
+ /* sanity-check supplied values
+ */
+ if (values != NULL) {
+ if (values[0] == NULL) {
+ if (show_progress) {
+ HDprintf(" ERROR: aws_region value cannot be NULL\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+ if (values[1] == NULL) {
+ if (show_progress) {
+ HDprintf(" ERROR: secret_id value cannot be NULL\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+ if (values[2] == NULL) {
+ if (show_progress) {
+ HDprintf(" ERROR: secret_key value cannot be NULL\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+
+ /* if region and ID are supplied (key optional), write to fapl...
+ * fail if value would overflow
+ */
+ if (*values[0] != '\0' &&
+ *values[1] != '\0')
+ {
+ if (HDstrlen(values[0]) > H5FD__ROS3_MAX_REGION_LEN) {
+ if (show_progress) {
+ HDprintf(" ERROR: aws_region value too long\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+ HDmemcpy(fa->aws_region, values[0],
+ (HDstrlen(values[0]) + 1));
+ if (show_progress) {
+ HDprintf(" aws_region set\n");
+ }
+
+
+ if (HDstrlen(values[1]) > H5FD__ROS3_MAX_SECRET_ID_LEN) {
+ if (show_progress) {
+ HDprintf(" ERROR: secret_id value too long\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+ HDmemcpy(fa->secret_id,
+ values[1],
+ (HDstrlen(values[1]) + 1));
+ if (show_progress) {
+ HDprintf(" secret_id set\n");
+ }
+
+ if (HDstrlen(values[2]) > H5FD__ROS3_MAX_SECRET_KEY_LEN) {
+ if (show_progress) {
+ HDprintf(" ERROR: secret_key value too long\n");
+ }
+ ret_value = 0;
+ goto done;
+ }
+ HDmemcpy(fa->secret_key,
+ values[2],
+ (HDstrlen(values[2]) + 1));
+ if (show_progress) {
+ HDprintf(" secret_key set\n");
+ }
+
+ fa->authenticate = TRUE;
+ if (show_progress) {
+ HDprintf(" set to authenticate\n");
+ }
+
+ } else if (*values[0] != '\0' ||
+ *values[1] != '\0' ||
+ *values[2] != '\0')
+ {
+ if (show_progress) {
+ HDprintf(
+ " ERROR: invalid assortment of empty/non-empty values\n"
+ );
+ }
+ ret_value = 0;
+ goto done;
+ }
+ } /* values != NULL */
+
+done:
+ return ret_value;
+#endif /* H5_HAVE_ROS3_VFD */
+
+} /* h5tools_populate_ros3_fapl */
+
+
+/*-----------------------------------------------------------------------------
+ *
+ * Function: h5tools_set_configured_fapl
+ *
+ * Purpose: prepare fapl_id with the given property list, according to
+ * VFD prototype.
+ *
+ * Return: 0 on failure, 1 on success
+ *
+ * Programmer: Jacob Smith
+ * 2018-05-21
+ *
+ * Changes: None.
+ *
+ *-----------------------------------------------------------------------------
+ */
+int
+h5tools_set_configured_fapl(hid_t fapl_id,
+ const char vfd_name[],
+ void *fapl_t_ptr)
+{
+ int ret_value = 1;
+
+ if (fapl_id < 0) {
+ return 0;
+ }
+
+ if (!strcmp("", vfd_name)) {
+ goto done;
+
+#ifdef H5_HAVE_ROS3_VFD
+ } else if (!strcmp("ros3", vfd_name)) {
+ if ((fapl_id == H5P_DEFAULT) ||
+ (fapl_t_ptr == NULL) ||
+ (FAIL == H5Pset_fapl_ros3(
+ fapl_id,
+ (H5FD_ros3_fapl_t *)fapl_t_ptr)))
+ {
+ ret_value = 0;
+ goto done;
+ }
+#endif /* H5_HAVE_ROS3_VFD */
+
+#ifdef H5_HAVE_LIBHDFS
+ } else if (!strcmp("hdfs", vfd_name)) {
+ if ((fapl_id == H5P_DEFAULT) ||
+ (fapl_t_ptr == NULL) ||
+ (FAIL == H5Pset_fapl_hdfs(
+ fapl_id,
+ (H5FD_hdfs_fapl_t *)fapl_t_ptr)))
+ {
+ ret_value = 0;
+ goto done;
+ }
+#endif /* H5_HAVE_LIBHDFS */
+
+ } else {
+ ret_value = 0; /* unrecognized fapl type "name" */
+ }
+
+done:
+ return ret_value;
+
+} /* h5tools_set_configured_fapl() */
+
+
+
diff --git a/tools/lib/h5tools_utils.h b/tools/lib/h5tools_utils.h
index 4c2bf1e..1c6ba2a 100644
--- a/tools/lib/h5tools_utils.h
+++ b/tools/lib/h5tools_utils.h
@@ -123,6 +123,11 @@ H5TOOLS_DLLVAR unsigned h5tools_nCols; /*max number of columns for
H5TOOLS_DLL void indentation(unsigned);
H5TOOLS_DLL void print_version(const char *progname);
H5TOOLS_DLL void parallel_print(const char* format, ... );
+H5TOOLS_DLL herr_t parse_tuple(const char *start,
+ int sep,
+ char **cpy_out,
+ unsigned *nelems,
+ char ***ptrs_out);
H5TOOLS_DLL void error_msg(const char *fmt, ...);
H5TOOLS_DLL void warn_msg(const char *fmt, ...);
H5TOOLS_DLL void help_ref_msg(FILE *output);
@@ -174,6 +179,11 @@ H5TOOLS_DLL void h5tools_setprogname(const char*progname);
H5TOOLS_DLL int h5tools_getstatus(void);
H5TOOLS_DLL void h5tools_setstatus(int d_status);
H5TOOLS_DLL int h5tools_getenv_update_hyperslab_bufsize(void);
+H5TOOLS_DLL int h5tools_set_configured_fapl(hid_t fapl_id,
+ const char vfd_name[],
+ void *fapl_t_ptr);
+H5TOOLS_DLL int h5tools_populate_ros3_fapl(H5FD_ros3_fapl_t *fa,
+ const char **values);
#ifdef __cplusplus
}
#endif
diff --git a/tools/libtest/CMakeLists.txt b/tools/libtest/CMakeLists.txt
new file mode 100644
index 0000000..105607d
--- /dev/null
+++ b/tools/libtest/CMakeLists.txt
@@ -0,0 +1,21 @@
+cmake_minimum_required (VERSION 3.10)
+project (HDF5_TOOLS_LIBTEST C)
+
+#-----------------------------------------------------------------------------
+# Add the h5tools_utils test executables
+#-----------------------------------------------------------------------------
+add_executable (h5tools_test_utils ${HDF5_TOOLS_LIBTEST_SOURCE_DIR}/h5tools_test_utils.c)
+target_include_directories(h5tools_utils PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
+TARGET_C_PROPERTIES (h5tools_utils STATIC)
+target_link_libraries (h5tools_utils PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET})
+set_target_properties (h5tools_utils PROPERTIES FOLDER tools)
+
+if (BUILD_SHARED_LIBS)
+ add_executable (h5tools_utils-shared ${HDF5_TOOLS_LIBTEST_SOURCE_DIR}/h5tools_utils.c)
+ target_include_directories(h5tools_utils-shared PRIVATE "${HDF5_TOOLS_DIR}/lib;${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
+ TARGET_C_PROPERTIES (h5tools_utils-shared SHARED)
+ target_link_libraries (h5tools_utils-shared PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET})
+ set_target_properties (h5tools_utils-shared PROPERTIES FOLDER tools)
+endif ()
+
+include (CMakeTests.cmake)
diff --git a/tools/libtest/CMakeTests.cmake b/tools/libtest/CMakeTests.cmake
new file mode 100644
index 0000000..403969d
--- /dev/null
+++ b/tools/libtest/CMakeTests.cmake
@@ -0,0 +1,49 @@
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+
+##############################################################################
+##############################################################################
+### T E S T I N G ###
+##############################################################################
+##############################################################################
+
+
+##############################################################################
+##############################################################################
+### T H E T E S T S M A C R O S ###
+##############################################################################
+##############################################################################
+
+ macro (ADD_H5_TEST resultfile resultcode)
+ add_test (
+ NAME H5LIBTEST-${resultfile}-clear-objects
+ COMMAND ${CMAKE_COMMAND}
+ -E remove
+ ${resultfile}.out
+ ${resultfile}.out.err
+ )
+ if (NOT "${last_test}" STREQUAL "")
+ set_tests_properties (H5LIBTEST-${resultfile}-clear-objects PROPERTIES DEPENDS ${last_test})
+ endif ()
+ add_test (NAME H5LIBTEST-${resultfile} COMMAND $<TARGET_FILE:h5tools_utils> ${ARGN})
+ if (NOT "${resultcode}" STREQUAL "0")
+ set_tests_properties (H5LIBTEST-${resultfile} PROPERTIES WILL_FAIL "true")
+ endif ()
+ set_tests_properties (H5LIBTEST-${resultfile} PROPERTIES DEPENDS H5LIBTEST-${resultfile}-clear-objects)
+ endmacro ()
+
+##############################################################################
+##############################################################################
+### T H E T E S T S ###
+##############################################################################
+##############################################################################
+ ADD_H5_TEST (h5tools_utils-default 0)
diff --git a/tools/libtest/Makefile.am b/tools/libtest/Makefile.am
new file mode 100644
index 0000000..a93e25d
--- /dev/null
+++ b/tools/libtest/Makefile.am
@@ -0,0 +1,34 @@
+#
+# Read-Only S3 Virtual File Driver (VFD)
+# Copyright (c) 2017-2018, The HDF Group.
+#
+# All rights reserved.
+#
+# NOTICE:
+# All information contained herein is, and remains, the property of The HDF
+# Group. The intellectual and technical concepts contained herein are
+# proprietary to The HDF Group. Dissemination of this information or
+# reproduction of this material is strictly forbidden unless prior written
+# permission is obtained from The HDF Group.
+##
+## Makefile.am
+## Run automake to generate a Makefile.in from this file.
+#
+# HDF5 Library Makefile(.in)
+#
+
+include $(top_srcdir)/config/commence.am
+
+# Include src and tools/lib directories
+AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib
+
+# All programs depend on the hdf5 and h5tools libraries
+LDADD=$(LIBH5TOOLS) $(LIBHDF5)
+
+
+# main target
+bin_PROGRAMS=h5tools_test_utils
+# check_PROGRAMS=$(TEST_PROG)
+
+
+include $(top_srcdir)/config/conclude.am
diff --git a/tools/libtest/h5tools_test_utils.c b/tools/libtest/h5tools_test_utils.c
new file mode 100644
index 0000000..ac8e4c0
--- /dev/null
+++ b/tools/libtest/h5tools_test_utils.c
@@ -0,0 +1,1295 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/*
+ * Purpose: unit-test functionality of the routines in `tools/lib/h5tools_utils`
+ *
+ * Jacob Smith 2017-11-10
+ */
+
+#include "hdf5.h"
+#include "H5private.h"
+#include "h5tools_utils.h"
+/* #include "h5test.h" */ /* linking failure */
+
+#define UTIL_TEST_DEBUG 0
+
+#ifndef _H5TEST_
+
+#define AT() fprintf(stdout, " at %s:%d in %s()...\n", \
+ __FILE__, __LINE__, FUNC);
+
+#define FAILED(msg) { \
+ fprintf(stdout, "*FAILED*"); AT() \
+ if (msg == NULL) { \
+ fprintf(stdout,"(NULL)\n"); \
+ } else { \
+ fprintf(stdout, "%s\n", msg); \
+ } \
+ fflush(stdout); \
+}
+
+#define TESTING(msg) { \
+ fprintf(stdout, "TESTING %-62s", (msg)); \
+ fflush(stdout); \
+}
+
+#define PASSED() { \
+ fprintf(stdout, " PASSED\n"); \
+ fflush(stdout); \
+}
+
+#endif /* ifndef _H5TEST_ */
+
+#ifndef __js_test__
+
+#define __js_test__ 1L
+
+/*****************************************************************************
+ *
+ * FILE-LOCAL TESTING MACROS
+ *
+ * Purpose:
+ *
+ * 1. Upon test failure, goto-jump to single-location teardown in test
+ * function. E.g., `error:` (consistency with HDF corpus) or
+ * `failed:` (reflects purpose).
+ * >>> using "error", in part because `H5E_BEGIN_TRY` expects it.
+ * 2. Increase clarity and reduce overhead found with `TEST_ERROR`.
+ * e.g., "if(somefunction(arg, arg2) < 0) TEST_ERROR:"
+ * requires reading of entire line to know whether this if/call is
+ * part of the test setup, test operation, or a test unto itself.
+ * 3. Provide testing macros with optional user-supplied failure message;
+ * if not supplied (NULL), generate comparison output in the spirit of
+ * test-driven development. E.g., "expected 5 but was -3"
+ * User messages clarify test's purpose in code, encouraging description
+ * without relying on comments.
+ * 4. Configurable expected-actual order in generated comparison strings.
+ * Some prefer `VERIFY(expected, actual)`, others
+ * `VERIFY(actual, expected)`. Provide preprocessor ifdef switch
+ * to satifsy both parties, assuming one paradigm per test file.
+ * (One could #undef and redefine the flag through the file as desired,
+ * but _why_.)
+ *
+ * Provided as courtesy, per consideration for inclusion in the library
+ * proper.
+ *
+ * Macros:
+ *
+ * JSVERIFY_EXP_ACT - ifdef flag, configures comparison order
+ * FAIL_IF() - check condition
+ * FAIL_UNLESS() - check _not_ condition
+ * JSVERIFY() - long-int equality check; prints reason/comparison
+ * JSVERIFY_NOT() - long-int inequality check; prints
+ * JSVERIFY_STR() - string equality check; prints
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *****************************************************************************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * ifdef flag: JSVERIFY_EXP_ACT
+ *
+ * JSVERIFY macros accept arguments as (EXPECTED, ACTUAL[, reason])
+ * default, if this is undefined, is (ACTUAL, EXPECTED[, reason])
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_EXP_ACT 1L
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSFAILED_AT()
+ *
+ * Purpose:
+ *
+ * Preface a test failure by printing "*FAILED*" and location to stdout
+ * Similar to `H5_FAILED(); AT();` from h5test.h
+ *
+ * *FAILED* at somefile.c:12 in function_name()...
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSFAILED_AT() { \
+ HDprintf("*FAILED* at %s:%d in %s()...\n", __FILE__, __LINE__, FUNC); \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: FAIL_IF()
+ *
+ * Purpose:
+ *
+ * Make tests more accessible and less cluttered than
+ * `if (thing == otherthing()) TEST_ERROR`
+ * paradigm.
+ *
+ * The following lines are roughly equivalent:
+ *
+ * `if (myfunc() < 0) TEST_ERROR;` (as seen elsewhere in HDF tests)
+ * `FAIL_IF(myfunc() < 0)`
+ *
+ * Prints a generic "FAILED AT" line to stdout and jumps to `error`,
+ * similar to `TEST_ERROR` in h5test.h
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-23
+ *
+ *----------------------------------------------------------------------------
+ */
+#define FAIL_IF(condition) \
+if (condition) { \
+ JSFAILED_AT() \
+ goto error; \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: FAIL_UNLESS()
+ *
+ * Purpose:
+ *
+ * TEST_ERROR wrapper to reduce cognitive overhead from "negative tests",
+ * e.g., "a != b".
+ *
+ * Opposite of FAIL_IF; fails if the given condition is _not_ true.
+ *
+ * `FAIL_IF( 5 != my_op() )`
+ * is equivalent to
+ * `FAIL_UNLESS( 5 == my_op() )`
+ * However, `JSVERIFY(5, my_op(), "bad return")` may be even clearer.
+ * (see JSVERIFY)
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define FAIL_UNLESS(condition) \
+if (!(condition)) { \
+ JSFAILED_AT() \
+ goto error; \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSERR_LONG()
+ *
+ * Purpose:
+ *
+ * Print an failure message for long-int arguments.
+ * ERROR-AT printed first.
+ * If `reason` is given, it is printed on own line and newlined after
+ * else, prints "expected/actual" aligned on own lines.
+ *
+ * *FAILED* at myfile.c:488 in somefunc()...
+ * forest must be made of trees.
+ *
+ * or
+ *
+ * *FAILED* at myfile.c:488 in somefunc()...
+ * ! Expected 425
+ * ! Actual 3
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSERR_LONG(expected, actual, reason) { \
+ JSFAILED_AT() \
+ if (reason!= NULL) { \
+ HDprintf("%s\n", (reason)); \
+ } else { \
+ HDprintf(" ! Expected %ld\n ! Actual %ld\n", \
+ (long)(expected), (long)(actual)); \
+ } \
+}
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSERR_STR()
+ *
+ * Purpose:
+ *
+ * Print an failure message for string arguments.
+ * ERROR-AT printed first.
+ * If `reason` is given, it is printed on own line and newlined after
+ * else, prints "expected/actual" aligned on own lines.
+ *
+ * *FAILED* at myfile.c:421 in myfunc()...
+ * Blue and Red strings don't match!
+ *
+ * or
+ *
+ * *FAILED* at myfile.c:421 in myfunc()...
+ * !!! Expected:
+ * this is my expected
+ * string
+ * !!! Actual:
+ * not what I expected at all
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSERR_STR(expected, actual, reason) { \
+ JSFAILED_AT() \
+ if ((reason) != NULL) { \
+ HDprintf("%s\n", (reason)); \
+ } else { \
+ HDprintf("!!! Expected:\n%s\n!!!Actual:\n%s\n", \
+ (expected), (actual)); \
+ } \
+}
+
+#ifdef JSVERIFY_EXP_ACT
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY()
+ *
+ * Purpose:
+ *
+ * Verify that two long integers are equal.
+ * If unequal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY(expected, actual, reason) \
+if ((long)(actual) != (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY_NOT()
+ *
+ * Purpose:
+ *
+ * Verify that two long integers are _not_ equal.
+ * If equal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_NOT(expected, actual, reason) \
+if ((long)(actual) == (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY_NOT */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Macro: JSVERIFY_STR()
+ *
+ * Purpose:
+ *
+ * Verify that two strings are equal.
+ * If unequal, print failure message
+ * (with `reason`, if not NULL; expected/actual if NULL)
+ * and jump to `error` at end of function
+ *
+ * Programmer: Jacob Smith
+ * 2017-10-24
+ *
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_STR(expected, actual, reason) \
+if (HDstrcmp((actual), (expected)) != 0) { \
+ JSERR_STR((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY_STR */
+
+
+#else /* JSVERIFY_EXP_ACT not defined */
+ /* Repeats macros above, but with actual/expected parameters reversed. */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY()
+ * See: JSVERIFY documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY(actual, expected, reason) \
+if ((long)(actual) != (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY_NOT()
+ * See: JSVERIFY_NOT documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_NOT(actual, expected, reason) \
+if ((long)(actual) == (long)(expected)) { \
+ JSERR_LONG((expected), (actual), (reason)) \
+ goto error; \
+} /* JSVERIFY_NOT */
+
+
+/*----------------------------------------------------------------------------
+ * Macro: JSVERIFY_STR()
+ * See: JSVERIFY_STR documentation above.
+ * Programmer: Jacob Smith
+ * 2017-10-14
+ *----------------------------------------------------------------------------
+ */
+#define JSVERIFY_STR(actual, expected, reason) \
+if (HDstrcmp((actual), (expected)) != 0) { \
+ JSERR_STR((expected), (actual), (reason)); \
+ goto error; \
+} /* JSVERIFY_STR */
+
+#endif /* ifdef/else JSVERIFY_EXP_ACT */
+
+#endif /* __js_test__ */
+
+/* if > 0, be very verbose when performing tests */
+#define H5TOOLS_UTILS_TEST_DEBUG 0
+
+/******************/
+/* TEST FUNCTIONS */
+/******************/
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: test_parse_tuple()
+ *
+ * Purpose:
+ *
+ * Provide unit tests and specification for the `parse_tuple()` function.
+ *
+ * Return:
+ *
+ * 0 Tests passed.
+ * 1 Tests failed.
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-11
+ *
+ * Changes: None.
+ *
+ *----------------------------------------------------------------------------
+ */
+static unsigned
+test_parse_tuple(void)
+{
+ /*************************
+ * TEST-LOCAL STRUCTURES *
+ *************************/
+
+ struct testcase {
+ const char *test_msg; /* info about test case */
+ const char *in_str; /* input string */
+ int sep; /* separator "character" */
+ herr_t exp_ret; /* expected SUCCEED / FAIL */
+ unsigned exp_nelems; /* expected number of elements */
+ /* (no more than 7!) */
+ const char *exp_elems[7]; /* list of elements (no more than 7!) */
+ };
+
+ /******************
+ * TEST VARIABLES *
+ ******************/
+
+ struct testcase cases[] = {
+ { "bad start",
+ "words(before)",
+ ';',
+ FAIL,
+ 0,
+ {NULL},
+ },
+ { "tuple not closed",
+ "(not ok",
+ ',',
+ FAIL,
+ 0,
+ {NULL},
+ },
+ { "empty tuple",
+ "()",
+ '-',
+ SUCCEED,
+ 1,
+ {""},
+ },
+ { "no separator",
+ "(stuff keeps on going)",
+ ',',
+ SUCCEED,
+ 1,
+ {"stuff keeps on going"},
+ },
+ { "4-ple, escaped seperator",
+ "(elem0,elem1,el\\,em2,elem3)", /* "el\,em" */
+ ',',
+ SUCCEED,
+ 4,
+ {"elem0", "elem1", "el,em2", "elem3"},
+ },
+ { "5-ple, escaped escaped separator",
+ "(elem0,elem1,el\\\\,em2,elem3)",
+ ',',
+ SUCCEED,
+ 5,
+ {"elem0", "elem1", "el\\", "em2", "elem3"},
+ },
+ { "escaped non-comma separator",
+ "(5-2-7-2\\-6-2)",
+ '-',
+ SUCCEED,
+ 5,
+ {"5","2","7","2-6","2"},
+ },
+ { "embedded close-paren",
+ "(be;fo)re)",
+ ';',
+ SUCCEED,
+ 2,
+ {"be", "fo)re"},
+ },
+ { "embedded non-escaping backslash",
+ "(be;fo\\re)",
+ ';',
+ SUCCEED,
+ 2,
+ {"be", "fo\\re"},
+ },
+ { "double close-paren at end",
+ "(be;fore))",
+ ';',
+ SUCCEED,
+ 2,
+ {"be", "fore)"},
+ },
+ { "empty elements",
+ "(;a1;;a4;)",
+ ';',
+ SUCCEED,
+ 5,
+ {"", "a1", "", "a4", ""},
+ },
+ { "nested tuples with different separators",
+ "((4,e,a);(6,2,a))",
+ ';',
+ SUCCEED,
+ 2,
+ {"(4,e,a)","(6,2,a)"},
+ },
+ { "nested tuples with same separators",
+ "((4,e,a),(6,2,a))",
+ ',',
+ SUCCEED,
+ 6,
+ {"(4","e","a)","(6","2","a)"},
+ },
+ { "real-world use case",
+ "(us-east-2,AKIAIMC3D3XLYXLN5COA,ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+)",
+ ',',
+ SUCCEED,
+ 3,
+ {"us-east-2",
+ "AKIAIMC3D3XLYXLN5COA",
+ "ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+"},
+ }
+ };
+ struct testcase tc;
+ unsigned n_tests = 14;
+ unsigned i = 0;
+ unsigned count = 0;
+ unsigned elem_i = 0;
+ char **parsed = NULL;
+ char *cpy = NULL;
+ herr_t success = TRUE;
+ hbool_t show_progress = FALSE;
+
+
+
+ TESTING("arbitrary-count tuple parsing");
+
+#if H5TOOLS_UTILS_TEST_DEBUG > 0
+ show_progress = TRUE;
+#endif /* H5TOOLS_UTILS_TEST_DEBUG */
+
+ /*********
+ * TESTS *
+ *********/
+
+ for (i = 0; i < n_tests; i++) {
+
+ /* SETUP
+ */
+ HDassert(parsed == NULL);
+ HDassert(cpy == NULL);
+ tc = cases[i];
+ if (show_progress == TRUE) {
+ printf("testing %d: %s...\n", i, tc.test_msg);
+ }
+
+ /* VERIFY
+ */
+ success = parse_tuple(tc.in_str, tc.sep,
+ &cpy, &count, &parsed);
+
+ JSVERIFY( tc.exp_ret, success, "function returned incorrect value" )
+ JSVERIFY( tc.exp_nelems, count, NULL )
+ if (success == SUCCEED) {
+ FAIL_IF( parsed == NULL )
+ for (elem_i = 0; elem_i < count; elem_i++) {
+ JSVERIFY_STR( tc.exp_elems[elem_i], parsed[elem_i], NULL )
+ }
+ /* TEARDOWN */
+ HDassert(parsed != NULL);
+ HDassert(cpy != NULL);
+ free(parsed);
+ parsed = NULL;
+ free(cpy);
+ cpy = NULL;
+ } else {
+ FAIL_IF( parsed != NULL )
+ } /* if parse_tuple() == SUCCEED or no */
+
+ } /* for each testcase */
+
+ PASSED();
+ return 0;
+
+error:
+ /***********
+ * CLEANUP *
+ ***********/
+
+ if (parsed != NULL) free(parsed);
+ if (cpy != NULL) free(cpy);
+
+ return 1;
+
+} /* test_parse_tuple */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: test_populate_ros3_fa()
+ *
+ * Purpose: Verify behavior of `populate_ros3_fa()`
+ *
+ * Return: 0 if test passes
+ * 1 if failure
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-13
+ *
+ * Changes: None
+ *
+ *----------------------------------------------------------------------------
+ */
+static unsigned
+test_populate_ros3_fa(void)
+{
+#ifdef H5_HAVE_ROS3_VFD
+ /*************************
+ * TEST-LOCAL STRUCTURES *
+ *************************/
+
+ /************************
+ * TEST-LOCAL VARIABLES *
+ ************************/
+
+ hbool_t show_progress = FALSE;
+ int bad_version = 0xf87a; /* arbitrarily wrong version number */
+#endif /* H5_HAVE_ROS3_VFD */
+
+ TESTING("programmatic ros3 fapl population");
+
+#ifndef H5_HAVE_ROS3_VFD
+ puts(" -SKIP-");
+ puts(" Read-Only S3 VFD not enabled");
+ fflush(stdout);
+ return 0;
+#else
+#if H5TOOLS_UTILS_TEST_DEBUG > 0
+ show_progress = TRUE;
+#endif /* H5TOOLS_UTILS_TEST_DEBUG */
+
+ HDassert(bad_version != H5FD__CURR_ROS3_FAPL_T_VERSION);
+
+ /*********
+ * TESTS *
+ *********/
+
+ /* NULL fapl config pointer fails
+ */
+ {
+ const char *values[] = {"x", "y", "z"};
+
+ if (show_progress) { HDprintf("NULL fapl pointer\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(NULL, values),
+ "fapl pointer cannot be null" )
+ }
+
+ /* NULL values pointer yields default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"};
+
+ if (show_progress) { HDprintf("NULL values pointer\n"); }
+
+ JSVERIFY( 1, h5tools_populate_ros3_fapl(&fa, NULL),
+ "NULL values pointer yields \"default\" fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* all-empty values
+ * yields default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, TRUE, "u", "v", "w"};
+ const char *values[] = {"", "", ""};
+
+ if (show_progress) { HDprintf("all empty values\n"); }
+
+ JSVERIFY( 1, h5tools_populate_ros3_fapl(&fa, values),
+ "empty values yields \"default\" fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* successfully set fapl with values
+ * excess value is ignored
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"x", "y", "z", "a"};
+
+ if (show_progress) { HDprintf("successful full set\n"); }
+
+ JSVERIFY( 1, h5tools_populate_ros3_fapl(&fa, values),
+ "four values" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( TRUE, fa.authenticate, NULL )
+ JSVERIFY_STR( "x", fa.aws_region, NULL )
+ JSVERIFY_STR( "y", fa.secret_id, NULL )
+ JSVERIFY_STR( "z", fa.secret_key, NULL )
+ }
+
+ /* NULL region
+ * yeilds default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {NULL, "y", "z", NULL};
+
+ if (show_progress) { HDprintf("NULL region\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* empty region
+ * yeilds default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"", "y", "z", NULL};
+
+ if (show_progress) { HDprintf("empty region; non-empty id, key\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* region overflow
+ * yeilds default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {
+ "somewhere over the rainbow not too high " \
+ "there is another rainbow bounding some darkened sky",
+ "y",
+ "z"};
+
+ if (show_progress) { HDprintf("region overflow\n"); }
+
+ HDassert(HDstrlen(values[0]) > H5FD__ROS3_MAX_REGION_LEN);
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* NULL id
+ * yields default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"x", NULL, "z", NULL};
+
+ if (show_progress) { HDprintf("NULL id\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* empty id (non-empty region, key)
+ * yeilds default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"x", "", "z", NULL};
+
+ if (show_progress) { HDprintf("empty id; non-empty region and key\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* id overflow
+ * partial set: region
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {
+ "x",
+ "Why is it necessary to solve the problem? " \
+ "What benefits will you receive by solving the problem? " \
+ "What is the unknown? " \
+ "What is it you don't yet understand? " \
+ "What is the information you have? " \
+ "What isn't the problem? " \
+ "Is the information insufficient, redundant, or contradictory? " \
+ "Should you draw a diagram or figure of the problem? " \
+ "What are the boundaries of the problem? " \
+ "Can you separate the various parts of the problem?",
+ "z"};
+
+ if (show_progress) { HDprintf("id overflow\n"); }
+
+ HDassert(HDstrlen(values[1]) > H5FD__ROS3_MAX_SECRET_ID_LEN);
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "x", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* NULL key
+ * yields default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"x", "y", NULL, NULL};
+
+ if (show_progress) { HDprintf("NULL key\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* empty key (non-empty region, id)
+ * yeilds authenticating fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"x", "y", "", NULL};
+
+ if (show_progress) { HDprintf("empty key; non-empty region and id\n"); }
+
+ JSVERIFY( 1, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( TRUE, fa.authenticate, NULL )
+ JSVERIFY_STR( "x", fa.aws_region, NULL )
+ JSVERIFY_STR( "y", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* empty key, region (non-empty id)
+ * yeilds default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"", "y", "", NULL};
+
+ if (show_progress) { HDprintf("empty key and region; non-empty id\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* empty key, id (non-empty region)
+ * yeilds default fapl
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {"x", "", "", NULL};
+
+ if (show_progress) { HDprintf("empty key and id; non-empty region\n"); }
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "", fa.aws_region, NULL )
+ JSVERIFY_STR( "", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* key overflow
+ * partial set: region, id
+ */
+ {
+ H5FD_ros3_fapl_t fa = {bad_version, FALSE, "a", "b", "c"};
+ const char *values[] = {
+ "x",
+ "y",
+ "Why is it necessary to solve the problem? " \
+ "What benefits will you receive by solving the problem? " \
+ "What is the unknown? " \
+ "What is it you don't yet understand? " \
+ "What is the information you have? " \
+ "What isn't the problem? " \
+ "Is the information insufficient, redundant, or contradictory? " \
+ "Should you draw a diagram or figure of the problem? " \
+ "What are the boundaries of the problem? " \
+ "Can you separate the various parts of the problem?"};
+
+ if (show_progress) { HDprintf("key overflow\n"); }
+
+ HDassert(HDstrlen(values[2]) > H5FD__ROS3_MAX_SECRET_KEY_LEN);
+
+ JSVERIFY( 0, h5tools_populate_ros3_fapl(&fa, values),
+ "could not fill fapl" )
+ JSVERIFY( H5FD__CURR_ROS3_FAPL_T_VERSION, fa.version, NULL )
+ JSVERIFY( FALSE, fa.authenticate, NULL )
+ JSVERIFY_STR( "x", fa.aws_region, NULL )
+ JSVERIFY_STR( "y", fa.secret_id, NULL )
+ JSVERIFY_STR( "", fa.secret_key, NULL )
+ }
+
+ /* use case
+ */
+ {
+ H5FD_ros3_fapl_t fa = {0, 0, "", "", ""};
+ const char *values[] = {
+ "us-east-2",
+ "AKIAIMC3D3XLYXLN5COA",
+ "ugs5aVVnLFCErO/8uW14iWE3K5AgXMpsMlWneO/+"
+ };
+ JSVERIFY( 1,
+ h5tools_populate_ros3_fapl(&fa, values),
+ "unable to set use case" )
+ JSVERIFY( 1, fa.version, "version check" )
+ JSVERIFY( 1, fa.authenticate, "should authenticate" )
+ }
+
+ PASSED();
+ return 0;
+
+error :
+ /***********
+ * CLEANUP *
+ ***********/
+
+ return 1;
+
+#endif /* H5_HAVE_ROS3_VFD */
+
+} /* test_populate_ros3_fa */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: test_set_configured_fapl()
+ *
+ * Purpose: Verify `h5tools_set_configured_fapl()` with ROS3 VFD
+ *
+ * Return: 0 if test passes
+ * 1 if failure
+ *
+ * Programmer: Jacob Smith
+ * 2018-07-12
+ *
+ * Changes: None
+ *
+ *----------------------------------------------------------------------------
+ */
+static unsigned
+test_set_configured_fapl(void)
+{
+#define UTIL_TEST_NOFAPL 1
+#define UTIL_TEST_DEFAULT 2
+#define UTIL_TEST_CREATE 3
+
+ /*************************
+ * TEST-LOCAL STRUCTURES *
+ *************************/
+ typedef struct testcase {
+ const char message[88];
+ int expected;
+ int fapl_choice;
+ const char vfdname[12];
+ void *conf_fa;
+ } testcase;
+
+ typedef struct other_fa_t {
+ int a;
+ int b;
+ int c;
+ } other_fa_t;
+
+ /************************
+ * TEST-LOCAL VARIABLES *
+ ************************/
+
+ hid_t fapl_id = -1;
+ other_fa_t wrong_fa = {0x432, 0xf82, 0x9093};
+ H5FD_ros3_fapl_t ros3_anon_fa = {1, FALSE, "", "", ""};
+ H5FD_ros3_fapl_t ros3_auth_fa = {
+ 1, /* fapl version */
+ TRUE, /* authenticate */
+ "us-east-1", /* aws region */
+ "12345677890abcdef", /* simulate access key ID */
+ "oiwnerwe9u0234nJw0-aoj+dsf", /* simulate secret key */
+ };
+ H5FD_hdfs_fapl_t hdfs_fa = {
+ 1, /* fapl version */
+ "", /* namenode name */
+ 0, /* namenode port */
+ "", /* kerberos ticket cache */
+ "", /* user name */
+ 2048, /* stream buffer size */
+ };
+ unsigned n_cases = 7; /* number of common testcases */
+ testcase cases[] = {
+ { "(common) should fail: no fapl id",
+ 0,
+ UTIL_TEST_NOFAPL,
+ "",
+ NULL,
+ },
+ { "(common) should fail: no fapl id (with struct)",
+ 0,
+ UTIL_TEST_NOFAPL,
+ "",
+ &wrong_fa,
+ },
+ { "(common) H5P_DEFAULT with no struct should succeed",
+ 1,
+ UTIL_TEST_DEFAULT,
+ "",
+ NULL,
+ },
+ { "(common) H5P_DEFAULT with (ignored) struct should succeed",
+ 1,
+ UTIL_TEST_DEFAULT,
+ "",
+ &wrong_fa,
+ },
+ { "(common) provided fapl entry should not fail",
+ 1,
+ UTIL_TEST_CREATE,
+ "",
+ NULL,
+ },
+ { "(common) provided fapl entry should not fail; ignores struct",
+ 1,
+ UTIL_TEST_CREATE,
+ "",
+ &wrong_fa,
+ },
+ { "(common) should fail: unrecoginzed vfd name",
+ 0,
+ UTIL_TEST_DEFAULT,
+ "unknown",
+ NULL,
+ },
+
+#ifdef H5_HAVE_ROS3_VFD
+ /* WARNING: add number of ROS3 test cases after array definition
+ */
+ { "(ROS3) should fail: no fapl id, no struct",
+ 0,
+ UTIL_TEST_NOFAPL,
+ "ros3",
+ NULL,
+ },
+ { "(ROS3) should fail: no fapl id",
+ 0,
+ UTIL_TEST_NOFAPL,
+ "ros3",
+ &ros3_anon_fa,
+ },
+ { "(ROS3) should fail: no struct",
+ 0,
+ UTIL_TEST_CREATE,
+ "ros3",
+ NULL,
+ },
+ { "(ROS3) successful set",
+ 1,
+ UTIL_TEST_CREATE,
+ "ros3",
+ &ros3_anon_fa,
+ },
+ { "(ROS3) should fail: attempt to set DEFAULT fapl",
+ 0,
+ UTIL_TEST_DEFAULT,
+ "ros3",
+ &ros3_anon_fa,
+ },
+#endif /* H5_HAVE_ROS3_VFD */
+
+#ifdef H5_HAVE_LIBHDFS
+ /* WARNING: add number of HDFS test cases after array definition
+ */
+ { "(HDFS) should fail: no fapl id, no struct",
+ 0,
+ UTIL_TEST_NOFAPL,
+ "hdfs",
+ NULL,
+ },
+ { "(HDFS) should fail: no fapl id",
+ 0,
+ UTIL_TEST_NOFAPL,
+ "hdfs",
+ &hdfs_fa,
+ },
+ { "(HDFS) should fail: no struct",
+ 0,
+ UTIL_TEST_CREATE,
+ "hdfs",
+ NULL,
+ },
+ { "(HDFS) successful set",
+ 1,
+ UTIL_TEST_CREATE,
+ "hdfs",
+ &hdfs_fa,
+ },
+ { "(HDFS) should fail: attempt to set DEFAULT fapl",
+ 0,
+ UTIL_TEST_DEFAULT,
+ "hdfs",
+ &hdfs_fa,
+ },
+#endif /* H5_HAVE_LIBHDFS */
+
+ }; /* testcases `cases` array */
+
+#ifdef H5_HAVE_ROS3_VFD
+ n_cases += 5;
+#endif /* H5_HAVE_ROS3_VFD */
+
+#ifdef H5_HAVE_LIBHDFS
+ n_cases += 5;
+#endif /* H5_HAVE_LIBHDFS */
+
+ TESTING("programmatic fapl set");
+
+ for (unsigned i = 0; i < n_cases; i++) {
+ int result;
+ testcase C = cases[i];
+
+ fapl_id = -1;
+
+#if UTIL_TEST_DEBUG
+ HDfprintf(stderr, "setup test %d\t%s\n", i, C.message); fflush(stderr);
+#endif /* UTIL_TEST_DEBUG */
+
+ /* per-test setup */
+ if (C.fapl_choice == UTIL_TEST_DEFAULT) {
+ fapl_id = H5P_DEFAULT;
+ } else if (C.fapl_choice == UTIL_TEST_CREATE) {
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ FAIL_IF( fapl_id < 0 )
+ }
+
+#if UTIL_TEST_DEBUG
+ HDfprintf(stderr, "before test\n"); fflush(stderr);
+#endif /* UTIL_TEST_DEBUG */
+
+ /* test */
+ result = h5tools_set_configured_fapl(
+ fapl_id,
+ C.vfdname,
+ C.conf_fa);
+ JSVERIFY( result, C.expected, C.message )
+
+#if UTIL_TEST_DEBUG
+ HDfprintf(stderr, "after test\n"); fflush(stderr);
+#endif /* UTIL_TEST_DEBUG */
+
+ /* per-test-teardown */
+ if (fapl_id > 0) {
+ FAIL_IF( FAIL == H5Pclose(fapl_id) )
+ }
+ fapl_id = -1;
+
+#if UTIL_TEST_DEBUG
+ HDfprintf(stderr, "after cleanup\n"); fflush(stderr);
+#endif /* UTIL_TEST_DEBUG */
+
+ }
+
+#if UTIL_TEST_DEBUG
+ HDfprintf(stderr, "after loop\n"); fflush(stderr);
+#endif /* UTIL_TEST_DEBUG */
+
+ PASSED();
+ return 0;
+
+error :
+ /***********
+ * CLEANUP *
+ ***********/
+
+#if UTIL_TEST_DEBUG
+ HDfprintf(stderr, "ERROR\n"); fflush(stderr);
+#endif /* UTIL_TEST_DEBUG */
+
+ if (fapl_id > 0) {
+ (void)H5Pclose(fapl_id);
+ }
+
+ return 1;
+
+#undef UTIL_TEST_NOFAPL
+#undef UTIL_TEST_DEFAULT
+#undef UTIL_TEST_CREATE
+} /* test_set_configured_fapl */
+
+
+/*----------------------------------------------------------------------------
+ *
+ * Function: main()
+ *
+ * Purpose: Run all test functions.
+ *
+ * Return: 0 iff all test pass
+ * 1 iff any failures
+ *
+ * Programmer: Jacob Smith
+ * 2017-11-10
+ *
+ * Changes: None.
+ *
+ *----------------------------------------------------------------------------
+ */
+int
+main(void)
+{
+ unsigned nerrors = 0;
+
+#ifdef _H5TEST_
+ h5reset(); /* h5test? */
+#endif /* _H5TEST_ */
+
+ HDfprintf(stdout, "Testing h5tools_utils corpus.\n");
+
+ nerrors += test_parse_tuple();
+ nerrors += test_populate_ros3_fa();
+ nerrors += test_set_configured_fapl();
+
+ if (nerrors > 0) {
+ HDfprintf(stdout, "***** %d h5tools_utils TEST%s FAILED! *****\n",
+ nerrors,
+ nerrors > 1 ? "S" : "");
+ nerrors = 1;
+ } else {
+ HDfprintf(stdout, "All h5tools_utils tests passed\n");
+ }
+
+ return (int)nerrors;
+
+} /* main */
+
+
diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c
index b9e37e8..a824197 100644
--- a/tools/src/h5dump/h5dump.c
+++ b/tools/src/h5dump/h5dump.c
@@ -24,6 +24,23 @@ static int doxml = 0;
static int useschema = 1;
static const char *xml_dtd_uri = NULL;
+static H5FD_ros3_fapl_t ros3_fa = {
+ 1, /* version */
+ false, /* authenticate */
+ "", /* aws region */
+ "", /* access key id */
+ "", /* secret access key */
+};
+
+static H5FD_hdfs_fapl_t hdfs_fa = {
+ 1, /* fapl version */
+ "localhost", /* namenode name */
+ 0, /* namenode port */
+ "", /* kerberos ticket cache */
+ "", /* user name */
+ 2048, /* stream buffer size */
+};
+
/* module-scoped variables for XML option */
#define DEFAULT_XSD "http://www.hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd"
#define DEFAULT_DTD "http://www.hdfgroup.org/HDF5/XML/DTD/HDF5-File.dtd"
@@ -188,6 +205,8 @@ static struct long_options l_opts[] = {
{ "any_path", require_arg, 'N' },
{ "vds-view-first-missing", no_arg, 'v' },
{ "vds-gap-size", require_arg, 'G' },
+ { "s3-cred", require_arg, '$' },
+ { "hdfs-attrs", require_arg, '#' },
{ NULL, 0, '\0' }
};
@@ -241,6 +260,16 @@ usage(const char *prog)
PRINTVALSTREAM(rawoutstream, " -b B, --binary=B Binary file output, of form B\n");
PRINTVALSTREAM(rawoutstream, " -O F, --ddl=F Output ddl text into file F\n");
PRINTVALSTREAM(rawoutstream, " Use blank(empty) filename F to suppress ddl display\n");
+ PRINTVALSTREAM(rawoutstream, " --s3-cred=<cred> Supply S3 authentication information to \"ros3\" vfd.\n");
+ PRINTVALSTREAM(rawoutstream, " <cred> :: \"(<aws-region>,<access-id>,<access-key>)\"\n");
+ PRINTVALSTREAM(rawoutstream, " If absent or <cred> -> \"(,,)\", no authentication.\n");
+ PRINTVALSTREAM(rawoutstream, " Has no effect is filedriver is not `ros3'.\n");
+ PRINTVALSTREAM(rawoutstream, " --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.\n");
+ PRINTVALSTREAM(rawoutstream, " For use with \"--filedriver=hdfs\"\n");
+ PRINTVALSTREAM(rawoutstream, " <attrs> :: (<namenode name>,<namenode port>,\n");
+ PRINTVALSTREAM(rawoutstream, " <kerberos cache path>,<username>,\n");
+ PRINTVALSTREAM(rawoutstream, " <buffer size>)\n");
+ PRINTVALSTREAM(rawoutstream, " Any absent attribute will use a default value.\n");
PRINTVALSTREAM(rawoutstream, "--------------- Object Options ---------------\n");
PRINTVALSTREAM(rawoutstream, " -a P, --attribute=P Print the specified attribute\n");
PRINTVALSTREAM(rawoutstream, " If an attribute name contains a slash (/), escape the\n");
@@ -1282,6 +1311,126 @@ end_collect:
hand = NULL;
h5tools_setstatus(EXIT_SUCCESS);
goto done;
+
+ case '$':
+#ifndef H5_HAVE_ROS3_VFD
+ error_msg("Read-Only S3 VFD not enabled.\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+#else
+ /* s3 credential */
+ {
+ char **s3_cred = NULL;
+ char *s3_cred_string = NULL;
+ const char *ccred[3];
+ unsigned nelems = 0;
+ if ( FAIL ==
+ parse_tuple(opt_arg, ',',
+ &s3_cred_string, &nelems, &s3_cred))
+ {
+ error_msg("unable to parse malformed s3 credentials\n");
+ usage(h5tools_getprogname());
+ free_handler(hand, argc);
+ hand= NULL;
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ if (nelems != 3) {
+ error_msg("s3 credentials expects 3 elements\n");
+ usage(h5tools_getprogname());
+ free_handler(hand, argc);
+ hand= NULL;
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ ccred[0] = (const char *)s3_cred[0];
+ ccred[1] = (const char *)s3_cred[1];
+ ccred[2] = (const char *)s3_cred[2];
+ if (0 == h5tools_populate_ros3_fapl(&ros3_fa, ccred)) {
+ error_msg("Invalid S3 credentials\n");
+ usage(h5tools_getprogname());
+ free_handler(hand, argc);
+ hand= NULL;
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ HDfree(s3_cred);
+ HDfree(s3_cred_string);
+ } /* s3 credential block */
+ break;
+#endif /* H5_HAVE_ROS3_VFD */
+
+ case '#':
+#ifndef H5_HAVE_LIBHDFS
+ error_msg("HDFS VFD is not enabled.\n");
+ goto error;
+#else
+ {
+ /* read hdfs properties tuple and store values in `hdfs_fa`
+ */
+ unsigned nelems = 0;
+ char *props_src = NULL;
+ char **props = NULL;
+ unsigned long k = 0;
+ if (FAIL == parse_tuple(
+ (const char *)opt_arg,
+ ',',
+ &props_src,
+ &nelems,
+ &props))
+ {
+ error_msg("unable to parse hdfs properties tuple\n");
+ goto error;
+ }
+ /* sanity-check tuple count
+ */
+ if (nelems != 5) {
+ h5tools_setstatus(EXIT_FAILURE);
+ goto error;
+ }
+ /* Populate fapl configuration structure with given
+ * properties.
+ * WARNING: No error-checking is done on length of input
+ * strings... Silent overflow is possible, albeit
+ * unlikely.
+ */
+ if (strncmp(props[0], "", 1)) {
+ HDstrncpy(hdfs_fa.namenode_name,
+ (const char *)props[0],
+ HDstrlen(props[0]));
+ }
+ if (strncmp(props[1], "", 1)) {
+ k = strtoul((const char *)props[1], NULL, 0);
+ if (errno == ERANGE) {
+ h5tools_setstatus(EXIT_FAILURE);
+ goto error;
+ }
+ hdfs_fa.namenode_port = (int32_t)k;
+ }
+ if (strncmp(props[2], "", 1)) {
+ HDstrncpy(hdfs_fa.kerberos_ticket_cache,
+ (const char *)props[2],
+ HDstrlen(props[2]));
+ }
+ if (strncmp(props[3], "", 1)) {
+ HDstrncpy(hdfs_fa.user_name,
+ (const char *)props[3],
+ HDstrlen(props[3]));
+ }
+ if (strncmp(props[4], "", 1)) {
+ k = strtoul((const char *)props[4], NULL, 0);
+ if (errno == ERANGE) {
+ h5tools_setstatus(EXIT_FAILURE);
+ goto error;
+ }
+ hdfs_fa.stream_buffer_size = (int32_t)k;
+ }
+ HDfree(props);
+ HDfree(props_src);
+ }
+#endif /* H5_HAVE_LIBHDFS */
+ break;
+
case '?':
default:
usage(h5tools_getprogname());
@@ -1354,6 +1503,7 @@ main(int argc, const char *argv[])
{
hid_t fid = -1;
hid_t gid = -1;
+ hid_t fapl_id = H5P_DEFAULT;
H5E_auto2_t func;
H5E_auto2_t tools_func;
H5O_info_t oi;
@@ -1440,10 +1590,60 @@ main(int argc, const char *argv[])
/* Initialize indexing options */
h5trav_set_index(sort_by, sort_order);
+ if (driver != NULL) {
+ void *conf_fa = NULL;
+
+ if (!strcmp(driver, "ros3")) {
+#ifndef H5_HAVE_ROS3_VFD
+ error_msg("Read-Only S3 VFD not enabled.\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+#else
+ conf_fa = (void *)&ros3_fa;
+#endif /* H5_HAVE_ROS3_VFD */
+ } else if (!HDstrcmp(driver, "hdfs")) {
+#ifndef H5_HAVE_LIBHDFS
+ error_msg("HDFS VFD is not enabled.\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+#else
+ conf_fa = (void *)&hdfs_fa;
+#endif /* H5_HAVE_LIBHDFS */
+ }
+
+ if (conf_fa != NULL) {
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ if (fapl_id < 0) {
+ error_msg("unable to create fapl entry\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ if (0 == h5tools_set_configured_fapl(
+ fapl_id,
+ driver, /* guaranteed "ros3" or "hdfs" */
+ conf_fa)) /* appropriate to driver */
+ {
+ error_msg("unable to set fapl\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ }
+ } /* driver defined */
+
while(opt_ind < argc) {
fname = HDstrdup(argv[opt_ind++]);
- fid = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, driver, NULL, 0);
+ if (fapl_id != H5P_DEFAULT) {
+ fid = H5Fopen(fname, H5F_ACC_RDONLY, fapl_id);
+ } else {
+ fid = h5tools_fopen(
+ fname,
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT,
+ driver,
+ NULL,
+ 0);
+ }
if (fid < 0) {
error_msg("unable to open file \"%s\"\n", fname);
@@ -1624,6 +1824,11 @@ done:
/* Free tables for objects */
table_list_free();
+ if (fapl_id != H5P_DEFAULT && 0 < H5Pclose(fapl_id)) {
+ error_msg("Can't close fapl entry\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ }
+
if(fid >=0)
if (H5Fclose(fid) < 0)
h5tools_setstatus(EXIT_FAILURE);
@@ -1645,127 +1850,7 @@ done:
H5Eset_auto2(H5E_DEFAULT, func, edata);
leave(h5tools_getstatus());
-}
-
-/*-------------------------------------------------------------------------
- * Function: h5_fileaccess
- *
- * Purpose: Returns a file access template which is the default template
- * but with a file driver set according to the constant or
- * environment variable HDF5_DRIVER
- *
- * Return: Success: A file access property list
- *
- * Failure: -1
- *
- * Programmer: Robb Matzke
- * Thursday, November 19, 1998
- *
- * Modifications:
- *
- *-------------------------------------------------------------------------
- */
-hid_t
-h5_fileaccess(void)
-{
- static const char *multi_letters = "msbrglo";
- const char *val = NULL;
- const char *name;
- char s[1024];
- hid_t fapl = -1;
-
- /* First use the environment variable, then the constant */
- val = HDgetenv("HDF5_DRIVER");
-#ifdef HDF5_DRIVER
- if (!val) val = HDF5_DRIVER;
-#endif
-
- if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) return -1;
- if (!val || !*val) return fapl; /*use default*/
-
- HDstrncpy(s, val, sizeof s);
- s[sizeof(s)-1] = '\0';
- if (NULL==(name=HDstrtok(s, " \t\n\r"))) return fapl;
-
- if (!HDstrcmp(name, "sec2")) {
- /* Unix read() and write() system calls */
- if (H5Pset_fapl_sec2(fapl)<0) return -1;
- }
- else if (!HDstrcmp(name, "stdio")) {
- /* Standard C fread() and fwrite() system calls */
- if (H5Pset_fapl_stdio(fapl)<0) return -1;
- }
- else if (!HDstrcmp(name, "core")) {
- /* In-core temporary file with 1MB increment */
- if (H5Pset_fapl_core(fapl, 1024*1024, FALSE)<0) return -1;
- }
- else if (!HDstrcmp(name, "split")) {
- /* Split meta data and raw data each using default driver */
- if (H5Pset_fapl_split(fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0)
- return -1;
- }
- else if (!HDstrcmp(name, "multi")) {
- /* Multi-file driver, general case of the split driver */
- H5FD_mem_t memb_map[H5FD_MEM_NTYPES];
- hid_t memb_fapl[H5FD_MEM_NTYPES];
- const char *memb_name[H5FD_MEM_NTYPES];
- char sv[H5FD_MEM_NTYPES][1024];
- haddr_t memb_addr[H5FD_MEM_NTYPES];
- H5FD_mem_t mt;
-
- HDmemset(memb_map, 0, sizeof memb_map);
- HDmemset(memb_fapl, 0, sizeof memb_fapl);
- HDmemset(memb_name, 0, sizeof memb_name);
- HDmemset(memb_addr, 0, sizeof memb_addr);
-
- if(HDstrlen(multi_letters)==H5FD_MEM_NTYPES) {
- for (mt=H5FD_MEM_DEFAULT; mt<H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t,mt)) {
- memb_fapl[mt] = H5P_DEFAULT;
- memb_map[mt] = mt;
- sprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]);
- memb_name[mt] = sv[mt];
- memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10);
- }
- }
- else {
- error_msg("Bad multi_letters list\n");
- return FAIL;
- }
-
- if (H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name, memb_addr, FALSE) < 0)
- return -1;
- }
- else if (!HDstrcmp(name, "family")) {
- hsize_t fam_size = 100*1024*1024; /*100 MB*/
-
- /* Family of files, each 1MB and using the default driver */
- if ((val=HDstrtok(NULL, " \t\n\r")))
- fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024);
- if (H5Pset_fapl_family(fapl, fam_size, H5P_DEFAULT)<0)
- return -1;
- }
- else if (!HDstrcmp(name, "log")) {
- long log_flags = H5FD_LOG_LOC_IO;
-
- /* Log file access */
- if ((val = HDstrtok(NULL, " \t\n\r")))
- log_flags = HDstrtol(val, NULL, 0);
-
- if (H5Pset_fapl_log(fapl, NULL, (unsigned)log_flags, 0) < 0)
- return -1;
- }
- else if (!HDstrcmp(name, "direct")) {
- /* Substitute Direct I/O driver with sec2 driver temporarily because
- * some output has sec2 driver as the standard. */
- if (H5Pset_fapl_sec2(fapl)<0) return -1;
- }
- else {
- /* Unknown driver */
- return -1;
- }
-
- return fapl;
-}
+} /* main */
/*-------------------------------------------------------------------------
@@ -1813,3 +1898,4 @@ add_prefix(char **prfx, size_t *prfx_len, const char *name)
HDstrcat(HDstrcat(*prfx, "/"), name);
} /* end add_prefix */
+
diff --git a/tools/src/h5ls/h5ls.c b/tools/src/h5ls/h5ls.c
index 4bc1526..c81da1e 100644
--- a/tools/src/h5ls/h5ls.c
+++ b/tools/src/h5ls/h5ls.c
@@ -158,7 +158,7 @@ static hbool_t print_int_type(h5tools_str_t *buffer, hid_t type, int ind);
static hbool_t print_float_type(h5tools_str_t *buffer, hid_t type, int ind);
static herr_t visit_obj(hid_t file, const char *oname, iter_t *iter);
-
+
/*-------------------------------------------------------------------------
* Function: usage
*
@@ -216,6 +216,15 @@ usage (void)
PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n");
PRINTVALSTREAM(rawoutstream, " --vfd=DRIVER Use the specified virtual file driver\n");
PRINTVALSTREAM(rawoutstream, " -x, --hexdump Show raw data in hexadecimal format\n");
+ PRINTVALSTREAM(rawoutstream, " --s3-cred=C Supply S3 authentication information to \"ros3\" vfd.\n");
+ PRINTVALSTREAM(rawoutstream, " Accepts tuple of \"(<aws-region>,<access-id>,<access-key>)\".\n");
+ PRINTVALSTREAM(rawoutstream, " If absent or C->\"(,,)\", defaults to no-authentication.\n");
+ PRINTVALSTREAM(rawoutstream, " Has no effect if vfd flag not set to \"ros3\".\n");
+ PRINTVALSTREAM(rawoutstream, " --hdfs-attrs=A Supply configuration information to Hadoop VFD.\n");
+ PRINTVALSTREAM(rawoutstream, " Accepts tuple of (<namenode name>,<namenode port>,\n");
+ PRINTVALSTREAM(rawoutstream, " ...<kerberos cache path>,<username>,<buffer size>)\n");
+ PRINTVALSTREAM(rawoutstream, " If absent or A == '(,,,,)', all default values are used.\n");
+ PRINTVALSTREAM(rawoutstream, " Has no effect if vfd flag is not 'hdfs'.\n");
PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, " file/OBJECT\n");
PRINTVALSTREAM(rawoutstream, " Each object consists of an HDF5 file name optionally followed by a\n");
@@ -237,7 +246,7 @@ usage (void)
PRINTVALSTREAM(rawoutstream, " Replaced by --enable-error-stack.\n");
}
-
+
/*-------------------------------------------------------------------------
* Function: print_string
@@ -315,7 +324,7 @@ print_string(h5tools_str_t *buffer, const char *s, hbool_t escape_spaces)
return nprint;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_obj_name
*
@@ -364,7 +373,7 @@ print_obj_name(h5tools_str_t *buffer, const iter_t *iter, const char *oname,
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_native_type
*
@@ -489,7 +498,7 @@ print_native_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_ieee_type
*
@@ -527,7 +536,7 @@ print_ieee_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_precision
*
@@ -619,7 +628,7 @@ print_precision(h5tools_str_t *buffer, hid_t type, int ind)
}
}
-
+
/*-------------------------------------------------------------------------
* Function: print_int_type
*
@@ -693,7 +702,7 @@ print_int_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_float_type
*
@@ -807,7 +816,7 @@ print_float_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_cmpd_type
*
@@ -860,7 +869,7 @@ print_cmpd_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_enum_type
*
@@ -985,7 +994,7 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_string_type
*
@@ -1086,7 +1095,7 @@ print_string_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_reference_type
*
@@ -1124,7 +1133,7 @@ print_reference_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_opaque_type
*
@@ -1160,7 +1169,7 @@ print_opaque_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_vlen_type
*
@@ -1190,7 +1199,7 @@ print_vlen_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*---------------------------------------------------------------------------
* Purpose: Print information about an array type
*
@@ -1237,7 +1246,7 @@ print_array_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_bitfield_type
*
@@ -1345,7 +1354,7 @@ print_type(h5tools_str_t *buffer, hid_t type, int ind)
(unsigned long)H5Tget_size(type), (unsigned)data_class);
}
-
+
/*-------------------------------------------------------------------------
* Function: dump_dataset_values
*
@@ -1475,7 +1484,7 @@ dump_dataset_values(hid_t dset)
PRINTVALSTREAM(rawoutstream, "\n");
}
-
+
/*-------------------------------------------------------------------------
* Function: list_attr
*
@@ -1662,7 +1671,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain
return 0;
}
-
+
/*-------------------------------------------------------------------------
* Function: dataset_list1
*
@@ -1727,7 +1736,7 @@ dataset_list1(hid_t dset)
return 0;
}
-
+
/*-------------------------------------------------------------------------
* Function: dataset_list2
*
@@ -1962,7 +1971,7 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name)
return 0;
} /* end dataset_list2() */
-
+
/*-------------------------------------------------------------------------
* Function: datatype_list2
*
@@ -2004,7 +2013,7 @@ datatype_list2(hid_t type, const char H5_ATTR_UNUSED *name)
return 0;
}
-
+
/*-------------------------------------------------------------------------
* Function: list_obj
*
@@ -2160,7 +2169,7 @@ done:
} /* end list_obj() */
-
+
/*-------------------------------------------------------------------------
* Function: list_lnk
*
@@ -2354,7 +2363,7 @@ done:
return 0;
} /* end list_lnk() */
-
+
/*-------------------------------------------------------------------------
* Function: visit_obj
*
@@ -2434,7 +2443,7 @@ done:
return retval;
}
-
+
/*-------------------------------------------------------------------------
* Function: get_width
*
@@ -2550,7 +2559,7 @@ out:
return ret;
}
-
+
/*-------------------------------------------------------------------------
* Function: leave
*
@@ -2573,7 +2582,7 @@ leave(int ret)
HDexit(ret);
}
-
+
/*-------------------------------------------------------------------------
* Function: main
*
@@ -2602,6 +2611,26 @@ main(int argc, const char *argv[])
char drivername[50];
const char *preferred_driver = NULL;
int err_exit = 0;
+ hid_t fapl_id = H5P_DEFAULT;
+
+ /* default "anonymous" s3 configuration */
+ H5FD_ros3_fapl_t ros3_fa = {
+ 1, /* fapl version */
+ false, /* authenticate */
+ "", /* aws region */
+ "", /* access key id */
+ "", /* secret access key */
+ };
+
+ /* "default" HDFS configuration */
+ H5FD_hdfs_fapl_t hdfs_fa = {
+ 1, /* fapl version */
+ "localhost", /* namenode name */
+ 0, /* namenode port */
+ "", /* kerberos ticket cache */
+ "", /* user name */
+ 2048, /* stream buffer size */
+ };
h5tools_setprogname(PROGRAMNAME);
h5tools_setstatus(EXIT_SUCCESS);
@@ -2701,6 +2730,185 @@ main(int argc, const char *argv[])
usage();
leave(EXIT_FAILURE);
}
+
+ } else if (!HDstrncmp(argv[argno], "--s3-cred=", (size_t)10)) {
+#ifndef H5_HAVE_ROS3_VFD
+ HDfprintf(rawerrorstream,
+ "Error: Read-Only S3 VFD is not enabled\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+#else
+ unsigned nelems = 0;
+ char *start = NULL;
+ char *s3cred_src = NULL;
+ char **s3cred = NULL;
+ char const *ccred[3];
+ /* try to parse s3 credentials tuple
+ */
+ start = strchr(argv[argno], '=');
+ if (start == NULL) {
+ HDfprintf(rawerrorstream,
+ "Error: Unable to parse null credentials tuple\n"
+ " For anonymous access, omit \"--s3-cred\" and use"
+ "only \"--vfd=ros3\"\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ start++;
+ if (FAIL ==
+ parse_tuple((const char *)start, ',',
+ &s3cred_src, &nelems, &s3cred))
+ {
+ HDfprintf(rawerrorstream,
+ "Error: Unable to parse S3 credentials\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ /* sanity-check tuple count
+ */
+ if (nelems != 3) {
+ HDfprintf(rawerrorstream,
+ "Error: Invalid S3 credentials\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ ccred[0] = (const char *)s3cred[0];
+ ccred[1] = (const char *)s3cred[1];
+ ccred[2] = (const char *)s3cred[2];
+ if (0 == h5tools_populate_ros3_fapl(&ros3_fa, ccred)) {
+ HDfprintf(rawerrorstream,
+ "Error: Invalid S3 credentials\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ HDfree(s3cred);
+ HDfree(s3cred_src);
+#endif /* H5_HAVE_ROS3_VFD */
+
+ } else if (!HDstrncmp(argv[argno], "--hdfs-attrs=", (size_t)13)) {
+#ifndef H5_HAVE_LIBHDFS
+ PRINTVALSTREAM(rawoutstream, "The HDFS VFD is not enabled.\n");
+ leave(EXIT_FAILURE);
+#else
+ /* Parse received configuration data and set fapl config struct
+ */
+
+ hbool_t _debug = FALSE;
+ unsigned nelems = 0;
+ char const *start = NULL;
+ char *props_src = NULL;
+ char **props = NULL;
+ unsigned long k = 0;
+
+ /* try to parse tuple
+ */
+ if (_debug) {
+ HDfprintf(stderr, "configuring hdfs...\n");
+ }
+ start = argv[argno]+13; /* should never segfault: worst case of */
+ if (*start != '(') /* null-termintor after '='. */
+ {
+ if (_debug) {
+ HDfprintf(stderr, " no tuple.\n");
+ }
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ if (FAIL ==
+ parse_tuple((const char *)start, ',',
+ &props_src, &nelems, &props))
+ {
+ HDfprintf(stderr,
+ " unable to parse tuple.\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+
+ /* sanity-check tuple count
+ */
+ if (nelems != 5) {
+ HDfprintf(stderr,
+ " expected 5-ple, got `%d`\n",
+ nelems);
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ if (_debug) {
+ HDfprintf(stderr,
+ " got hdfs-attrs tuple: `(%s,%s,%s,%s,%s)`\n",
+ props[0],
+ props[1],
+ props[2],
+ props[3],
+ props[4]);
+ }
+
+ /* Populate fapl configuration structure with given properties.
+ * WARNING: No error-checking is done on length of input strings...
+ * Silent overflow is possible, albeit unlikely.
+ */
+ if (HDstrncmp(props[0], "", 1)) {
+ if (_debug) {
+ HDfprintf(stderr,
+ " setting namenode name: %s\n",
+ props[0]);
+ }
+ HDstrncpy(hdfs_fa.namenode_name,
+ (const char *)props[0],
+ HDstrlen(props[0]));
+ }
+ if (HDstrncmp(props[1], "", 1)) {
+ k = strtoul((const char *)props[1], NULL, 0);
+ if (errno == ERANGE) {
+ HDfprintf(stderr,
+ " supposed port number wasn't.\n");
+ leave(EXIT_FAILURE);
+ }
+ if (_debug) {
+ HDfprintf(stderr,
+ " setting namenode port: %lu\n",
+ k);
+ }
+ hdfs_fa.namenode_port = (int32_t)k;
+ }
+ if (HDstrncmp(props[2], "", 1)) {
+ if (_debug) {
+ HDfprintf(stderr,
+ " setting kerb cache path: %s\n",
+ props[2]);
+ }
+ HDstrncpy(hdfs_fa.kerberos_ticket_cache,
+ (const char *)props[2],
+ HDstrlen(props[2]));
+ }
+ if (HDstrncmp(props[3], "", 1)) {
+ if (_debug) {
+ HDfprintf(stderr,
+ " setting username: %s\n",
+ props[3]);
+ }
+ HDstrncpy(hdfs_fa.user_name,
+ (const char *)props[3],
+ HDstrlen(props[3]));
+ }
+ if (HDstrncmp(props[4], "", 1)) {
+ k = HDstrtoul((const char *)props[4], NULL, 0);
+ if (errno == ERANGE) {
+ HDfprintf(stderr,
+ " supposed buffersize number wasn't.\n");
+ leave(EXIT_FAILURE);
+ }
+ if (_debug) {
+ HDfprintf(stderr,
+ " setting stream buffer size: %lu\n",
+ k);
+ }
+ hdfs_fa.stream_buffer_size = (int32_t)k;
+ }
+ HDfree(props);
+ HDfree(props_src);
+#endif /* H5_HAVE_LIBHDFS */
+
} else if('-'!=argv[argno][1]) {
/* Single-letter switches */
for(s = argv[argno] + 1; *s; s++) {
@@ -2772,6 +2980,7 @@ main(int argc, const char *argv[])
} /* end switch */
} /* end for */
} else {
+ HDfprintf(stderr, "Unknown argument: %s\n", argv[argno]);
usage();
leave(EXIT_FAILURE);
}
@@ -2791,6 +3000,49 @@ main(int argc, const char *argv[])
leave(EXIT_FAILURE);
}
+ if (preferred_driver) {
+ void *conf_fa = NULL;
+
+ if (!HDstrcmp(preferred_driver, "ros3")) {
+#ifndef H5_HAVE_ROS3_VFD
+ HDfprintf(rawerrorstream,
+ "Error: Read-Only S3 VFD not enabled.\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+#else
+ conf_fa = (void *)&ros3_fa;
+#endif /* H5_HAVE_ROS3_VFD */
+
+ } else if (!HDstrcmp(preferred_driver, "hdfs")) {
+#ifndef H5_HAVE_LIBHDFS
+ PRINTVALSTREAM(rawoutstream, "The HDFS VFD is not enabled.\n");
+ leave(EXIT_FAILURE);
+#else
+ conf_fa = (void *)&hdfs_fa;
+#endif /* H5_HAVE_LIBHDFS */
+ }
+
+ if (conf_fa != NULL) {
+ HDassert(fapl_id == H5P_DEFAULT);
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ if (fapl_id < 0) {
+ HDfprintf(rawerrorstream,
+ "Error: Unable to create fapl entry\n\n");
+ leave(EXIT_FAILURE);
+ }
+ if (0 == h5tools_set_configured_fapl(
+ fapl_id,
+ preferred_driver,
+ conf_fa))
+ {
+ HDfprintf(rawerrorstream,
+ "Error: Unable to set fapl\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ }
+ } /* preferred_driver defined */
+
/* Turn off HDF5's automatic error printing unless you're debugging h5ls */
if(!show_errors_g)
H5Eset_auto2(H5E_DEFAULT, NULL, NULL);
@@ -2820,7 +3072,12 @@ main(int argc, const char *argv[])
file = -1;
while(fname && *fname) {
- file = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, preferred_driver, drivername, sizeof drivername);
+ if (fapl_id != H5P_DEFAULT) {
+ file = H5Fopen(fname, H5F_ACC_RDONLY, fapl_id);
+ }
+ else {
+ file = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, preferred_driver, drivername, sizeof drivername);
+ }
if(file >= 0) {
if(verbose_g)
@@ -2933,6 +3190,14 @@ main(int argc, const char *argv[])
err_exit = 1;
} /* end while */
+ if (fapl_id != H5P_DEFAULT) {
+ if (0 < H5Pclose(fapl_id)) {
+ HDfprintf(rawerrorstream,
+ "Error: Unable to set close fapl entry\n\n");
+ leave(EXIT_FAILURE);
+ }
+ }
+
if (err_exit)
leave(EXIT_FAILURE);
else
diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c
index 450f731..59038a0 100644
--- a/tools/src/h5stat/h5stat.c
+++ b/tools/src/h5stat/h5stat.c
@@ -74,7 +74,7 @@ typedef struct iter_t {
ohdr_info_t group_ohdr_info; /* Object header information for groups */
hsize_t max_attrs; /* Maximum attributes from a group */
- unsigned long *num_small_attrs; /* Size of small attributes tracked */
+ unsigned long *num_small_attrs; /* Size of small attributes tracked */
unsigned attr_nbins; /* Number of bins for attribute counts */
unsigned long *attr_bins; /* Pointer to array of bins for attribute counts */
@@ -118,6 +118,29 @@ typedef struct iter_t {
} iter_t;
+static const char *drivername = "";
+
+/* default "anonymous" s3 configuration
+ */
+static H5FD_ros3_fapl_t ros3_fa = {
+ 1, /* fapl version */
+ false, /* authenticate */
+ "", /* aws region */
+ "", /* access key id */
+ "", /* secret access key */
+};
+
+/* default HDFS access configuration
+ */
+static H5FD_hdfs_fapl_t hdfs_fa = {
+ 1, /* fapl version */
+ "localhost", /* namenode name */
+ 0, /* namenode port */
+ "", /* kerberos ticket cache */
+ "", /* user name */
+ 2048, /* stream buffer size */
+};
+
static int display_all = TRUE;
/* Enable the printing of selected statistics */
@@ -146,7 +169,7 @@ struct handler_t {
char **obj;
};
-static const char *s_opts ="Aa:Ddm:EFfhGgl:sSTO:V";
+static const char *s_opts ="Aa:Ddm:EFfhGgl:sSTO:Vw:";
/* e.g. "filemetadata" has to precede "file"; "groupmetadata" has to precede "group" etc. */
static struct long_options l_opts[] = {
{"help", no_arg, 'h'},
@@ -246,6 +269,8 @@ static struct long_options l_opts[] = {
{ "summ", no_arg, 'S' },
{ "sum", no_arg, 'S' },
{ "su", no_arg, 'S' },
+ { "s3-cred", require_arg, 'w' },
+ { "hdfs-attrs", require_arg, 'H' },
{ NULL, 0, '\0' }
};
@@ -257,7 +282,7 @@ leave(int ret)
}
-
+
/*-------------------------------------------------------------------------
* Function: usage
*
@@ -295,9 +320,19 @@ static void usage(const char *prog)
HDfprintf(stdout, " -s, --freespace Print free space information\n");
HDfprintf(stdout, " -S, --summary Print summary of file space information\n");
HDfprintf(stdout, " --enable-error-stack Prints messages from the HDF5 error stack as they occur\n");
+ HDfprintf(stdout, " --s3-cred=<cred> Access file on S3, using provided credential\n");
+ HDfprintf(stdout, " <cred> :: (region,id,key)\n");
+ HDfprintf(stdout, " If <cred> == \"(,,)\", no authentication is used.\n");
+ HDfprintf(stdout, " --hdfs-attrs=<attrs> Access a file on HDFS with given configuration\n");
+ HDfprintf(stdout, " attributes.\n");
+ HDfprintf(stdout, " <attrs> :: (<namenode name>,<namenode port>,\n");
+ HDfprintf(stdout, " <kerberos cache path>,<username>,\n");
+ HDfprintf(stdout, " <buffer size>)\n");
+ HDfprintf(stdout, " If an attribute is empty, a default value will be\n");
+ HDfprintf(stdout, " used.\n");
}
-
+
/*-------------------------------------------------------------------------
* Function: ceil_log10
*
@@ -324,7 +359,7 @@ ceil_log10(unsigned long x)
return ret;
} /* ceil_log10() */
-
+
/*-------------------------------------------------------------------------
* Function: attribute_stats
*
@@ -374,7 +409,7 @@ attribute_stats(iter_t *iter, const H5O_info_t *oi)
return 0;
} /* end attribute_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: group_stats
*
@@ -456,7 +491,7 @@ done:
return ret_value;
} /* end group_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: dataset_stats
*
@@ -647,7 +682,7 @@ done:
return ret_value;
} /* end dataset_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: datatype_stats
*
@@ -679,7 +714,7 @@ done:
return ret_value;
} /* end datatype_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: obj_stats
*
@@ -735,7 +770,7 @@ done:
return ret_value;
} /* end obj_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: lnk_stats
*
@@ -833,7 +868,7 @@ freespace_stats(hid_t fid, iter_t *iter)
return 0;
} /* end freespace_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: hand_free
*
@@ -862,7 +897,7 @@ hand_free(struct handler_t *hand)
} /* end if */
} /* end hand_free() */
-
+
/*-------------------------------------------------------------------------
* Function: parse_command_line
*
@@ -1014,6 +1049,119 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret)
} /* end if */
break;
+ case 'w':
+#ifndef H5_HAVE_ROS3_VFD
+ error_msg("Read-Only S3 VFD not enabled.\n");
+ goto error;
+#else
+ {
+ char *cred_str = NULL;
+ unsigned nelems = 0;
+ char **cred = NULL;
+ char const *ccred[3];
+
+ if (FAIL == parse_tuple((const char *)opt_arg, ',',
+ &cred_str, &nelems, &cred)) {
+ error_msg("Unable to parse s3 credential\n");
+ goto error;
+ }
+ if (nelems != 3) {
+ error_msg("s3 credential must have three elements\n");
+ goto error;
+ }
+ ccred[0] = (const char *)cred[0];
+ ccred[1] = (const char *)cred[1];
+ ccred[2] = (const char *)cred[2];
+ if (0 ==
+ h5tools_populate_ros3_fapl(&ros3_fa, ccred))
+ {
+ error_msg("Unable to set ros3 fapl config\n");
+ goto error;
+ }
+ HDfree(cred);
+ HDfree(cred_str);
+ } /* parse s3-cred block */
+ drivername = "ros3";
+ break;
+#endif /* H5_HAVE_ROS3_VFD */
+
+ case 'H':
+#ifndef H5_HAVE_LIBHDFS
+ error_msg("HDFS VFD is not enabled.\n");
+ goto error;
+#else
+ {
+ unsigned nelems = 0;
+ char *props_src = NULL;
+ char **props = NULL;
+ unsigned long k = 0;
+ if (FAIL == parse_tuple(
+ (const char *)opt_arg,
+ ',',
+ &props_src,
+ &nelems,
+ &props))
+ {
+ error_msg("unable to parse hdfs properties tuple\n");
+ goto error;
+ }
+ /* sanity-check tuple count
+ */
+ if (nelems != 5) {
+ char str[64] = "";
+ sprintf(str,
+ "expected 5 elements in hdfs properties tuple "
+ "but found %u\n",
+ nelems);
+ HDfree(props);
+ HDfree(props_src);
+ error_msg(str);
+ goto error;
+ }
+ /* Populate fapl configuration structure with given
+ * properties.
+ * TODO/WARNING: No error-checking is done on length of
+ * input strings... Silent overflow is possible,
+ * albeit unlikely.
+ */
+ if (strncmp(props[0], "", 1)) {
+ HDstrncpy(hdfs_fa.namenode_name,
+ (const char *)props[0],
+ HDstrlen(props[0]));
+ }
+ if (strncmp(props[1], "", 1)) {
+ k = strtoul((const char *)props[1], NULL, 0);
+ if (errno == ERANGE) {
+ error_msg("supposed port number wasn't.\n");
+ goto error;
+ }
+ hdfs_fa.namenode_port = (int32_t)k;
+ }
+ if (strncmp(props[2], "", 1)) {
+ HDstrncpy(hdfs_fa.kerberos_ticket_cache,
+ (const char *)props[2],
+ HDstrlen(props[2]));
+ }
+ if (strncmp(props[3], "", 1)) {
+ HDstrncpy(hdfs_fa.user_name,
+ (const char *)props[3],
+ HDstrlen(props[3]));
+ }
+ if (strncmp(props[4], "", 1)) {
+ k = strtoul((const char *)props[4], NULL, 0);
+ if (errno == ERANGE) {
+ error_msg("supposed buffersize number wasn't.\n");
+ goto error;
+ }
+ hdfs_fa.stream_buffer_size = (int32_t)k;
+ }
+ HDfree(props);
+ HDfree(props_src);
+ drivername = "hdfs";
+ }
+ break;
+#endif /* H5_HAVE_LIBHDFS */
+
default:
usage(h5tools_getprogname());
goto error;
@@ -1040,7 +1188,7 @@ error:
return -1;
}
-
+
/*-------------------------------------------------------------------------
* Function: iter_free
*
@@ -1105,7 +1253,7 @@ iter_free(iter_t *iter)
} /* end if */
} /* end iter_free() */
-
+
/*-------------------------------------------------------------------------
* Function: print_file_info
*
@@ -1137,7 +1285,7 @@ print_file_info(const iter_t *iter)
return 0;
} /* print_file_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_file_metadata
*
@@ -1197,7 +1345,7 @@ print_file_metadata(const iter_t *iter)
return 0;
} /* print_file_metadata() */
-
+
/*-------------------------------------------------------------------------
* Function: print_group_info
*
@@ -1254,7 +1402,7 @@ print_group_info(const iter_t *iter)
return 0;
} /* print_group_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_group_metadata
*
@@ -1281,7 +1429,7 @@ print_group_metadata(const iter_t *iter)
return 0;
} /* print_group_metadata() */
-
+
/*-------------------------------------------------------------------------
* Function: print_dataset_info
*
@@ -1368,7 +1516,7 @@ print_dataset_info(const iter_t *iter)
return 0;
} /* print_dataset_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_dataset_metadata
*
@@ -1397,7 +1545,7 @@ print_dset_metadata(const iter_t *iter)
return 0;
} /* print_dset_metadata() */
-
+
/*-------------------------------------------------------------------------
* Function: print_dset_dtype_meta
*
@@ -1438,7 +1586,7 @@ print_dset_dtype_meta(const iter_t *iter)
return 0;
} /* print_dset_dtype_meta() */
-
+
/*-------------------------------------------------------------------------
* Function: print_attr_info
*
@@ -1487,7 +1635,7 @@ print_attr_info(const iter_t *iter)
return 0;
} /* print_attr_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_freespace_info
*
@@ -1537,7 +1685,7 @@ print_freespace_info(const iter_t *iter)
return 0;
} /* print_freespace_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_storage_summary
*
@@ -1601,7 +1749,7 @@ print_storage_summary(const iter_t *iter)
return 0;
} /* print_storage_summary() */
-
+
/*-------------------------------------------------------------------------
* Function: print_file_statistics
*
@@ -1648,7 +1796,7 @@ print_file_statistics(const iter_t *iter)
if(display_summary) print_storage_summary(iter);
} /* print_file_statistics() */
-
+
/*-------------------------------------------------------------------------
* Function: print_object_statistics
*
@@ -1671,7 +1819,7 @@ print_object_statistics(const char *name)
printf("Object name %s\n", name);
} /* print_object_statistics() */
-
+
/*-------------------------------------------------------------------------
* Function: print_statistics
*
@@ -1697,7 +1845,7 @@ print_statistics(const char *name, const iter_t *iter)
print_file_statistics(iter);
} /* print_statistics() */
-
+
/*-------------------------------------------------------------------------
* Function: main
*
@@ -1718,6 +1866,7 @@ main(int argc, const char *argv[])
void *edata;
void *tools_edata;
struct handler_t *hand = NULL;
+ hid_t fapl_id = H5P_DEFAULT;
h5tools_setprogname(PROGRAMNAME);
h5tools_setstatus(EXIT_SUCCESS);
@@ -1738,6 +1887,45 @@ main(int argc, const char *argv[])
if(parse_command_line(argc, argv, &hand) < 0)
goto done;
+ /* if drivername is not null, probably need to set the fapl */
+ if (HDstrcmp(drivername, "")) {
+ void *conf_fa = NULL;
+
+ if (!HDstrcmp(drivername, "ros3")) {
+#ifndef H5_HAVE_ROS3_VFD
+ error_msg("Read-Only S3 VFD not enabled.\n\n");
+ goto done;
+#else
+ conf_fa = (void *)&ros3_fa;
+#endif /* H5_HAVE_ROS3_VFD */
+
+ } else if (!HDstrcmp(drivername, "hdfs")) {
+#ifndef H5_HAVE_LIBHDFS
+ error_msg("HDFS VFD not enabled.\n\n");
+ goto done;
+#else
+ conf_fa = (void *)&hdfs_fa;
+#endif /* H5_HAVE_LIBHDFS */
+ }
+
+ if (conf_fa != NULL) {
+ HDassert(fapl_id == H5P_DEFAULT);
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ if (fapl_id < 0) {
+ error_msg("Unable to create fapl entry\n");
+ goto done;
+ }
+ if (1 > h5tools_set_configured_fapl(
+ fapl_id,
+ drivername,
+ conf_fa))
+ {
+ error_msg("Unable to set fapl\n");
+ goto done;
+ }
+ }
+ } /* drivername set */
+
fname = argv[opt_ind];
if(enable_error_stack > 0) {
@@ -1752,7 +1940,7 @@ main(int argc, const char *argv[])
printf("Filename: %s\n", fname);
- fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
+ fid = H5Fopen(fname, H5F_ACC_RDONLY, fapl_id);
if(fid < 0) {
error_msg("unable to open file \"%s\"\n", fname);
h5tools_setstatus(EXIT_FAILURE);
@@ -1833,6 +2021,13 @@ done:
/* Free iter structure */
iter_free(&iter);
+ if (fapl_id != H5P_DEFAULT) {
+ if (0 < H5Pclose(fapl_id)) {
+ error_msg("unable to close fapl entry\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ }
+ }
+
if(fid >= 0 && H5Fclose(fid) < 0) {
error_msg("unable to close file \"%s\"\n", fname);
h5tools_setstatus(EXIT_FAILURE);
diff --git a/tools/test/h5stat/testfiles/h5stat_help1.ddl b/tools/test/h5stat/testfiles/h5stat_help1.ddl
index 01e39af..2ba7772 100644
--- a/tools/test/h5stat/testfiles/h5stat_help1.ddl
+++ b/tools/test/h5stat/testfiles/h5stat_help1.ddl
@@ -23,3 +23,13 @@ Usage: h5stat [OPTIONS] file
-s, --freespace Print free space information
-S, --summary Print summary of file space information
--enable-error-stack Prints messages from the HDF5 error stack as they occur
+ --s3-cred=<cred> Access file on S3, using provided credential
+ <cred> :: (region,id,key)
+ If <cred> == "(,,)", no authentication is used.
+ --hdfs-attrs=<attrs> Access a file on HDFS with given configuration
+ attributes.
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ If an attribute is empty, a default value will be
+ used.
diff --git a/tools/test/h5stat/testfiles/h5stat_help2.ddl b/tools/test/h5stat/testfiles/h5stat_help2.ddl
index 01e39af..2ba7772 100644
--- a/tools/test/h5stat/testfiles/h5stat_help2.ddl
+++ b/tools/test/h5stat/testfiles/h5stat_help2.ddl
@@ -23,3 +23,13 @@ Usage: h5stat [OPTIONS] file
-s, --freespace Print free space information
-S, --summary Print summary of file space information
--enable-error-stack Prints messages from the HDF5 error stack as they occur
+ --s3-cred=<cred> Access file on S3, using provided credential
+ <cred> :: (region,id,key)
+ If <cred> == "(,,)", no authentication is used.
+ --hdfs-attrs=<attrs> Access a file on HDFS with given configuration
+ attributes.
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ If an attribute is empty, a default value will be
+ used.
diff --git a/tools/test/h5stat/testfiles/h5stat_nofile.ddl b/tools/test/h5stat/testfiles/h5stat_nofile.ddl
index 01e39af..2ba7772 100644
--- a/tools/test/h5stat/testfiles/h5stat_nofile.ddl
+++ b/tools/test/h5stat/testfiles/h5stat_nofile.ddl
@@ -23,3 +23,13 @@ Usage: h5stat [OPTIONS] file
-s, --freespace Print free space information
-S, --summary Print summary of file space information
--enable-error-stack Prints messages from the HDF5 error stack as they occur
+ --s3-cred=<cred> Access file on S3, using provided credential
+ <cred> :: (region,id,key)
+ If <cred> == "(,,)", no authentication is used.
+ --hdfs-attrs=<attrs> Access a file on HDFS with given configuration
+ attributes.
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ If an attribute is empty, a default value will be
+ used.
diff --git a/tools/testfiles/h5dump-help.txt b/tools/testfiles/h5dump-help.txt
index 19de76f..95dfc3b 100644
--- a/tools/testfiles/h5dump-help.txt
+++ b/tools/testfiles/h5dump-help.txt
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/help-1.ls b/tools/testfiles/help-1.ls
index 491f696..396bed3 100644
--- a/tools/testfiles/help-1.ls
+++ b/tools/testfiles/help-1.ls
@@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
-V, --version Print version number and exit
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
+ --s3-cred=C Supply S3 authentication information to "ros3" vfd.
+ Accepts tuple of "(<aws-region>,<access-id>,<access-key>)".
+ If absent or C->"(,,)", defaults to no-authentication.
+ Has no effect if vfd flag not set to "ros3".
+ --hdfs-attrs=A Supply configuration information to Hadoop VFD.
+ Accepts tuple of (<namenode name>,<namenode port>,
+ ...<kerberos cache path>,<username>,<buffer size>)
+ If absent or A == '(,,,,)', all default values are used.
+ Has no effect if vfd flag is not 'hdfs'.
file/OBJECT
Each object consists of an HDF5 file name optionally followed by a
diff --git a/tools/testfiles/help-2.ls b/tools/testfiles/help-2.ls
index 491f696..396bed3 100644
--- a/tools/testfiles/help-2.ls
+++ b/tools/testfiles/help-2.ls
@@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
-V, --version Print version number and exit
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
+ --s3-cred=C Supply S3 authentication information to "ros3" vfd.
+ Accepts tuple of "(<aws-region>,<access-id>,<access-key>)".
+ If absent or C->"(,,)", defaults to no-authentication.
+ Has no effect if vfd flag not set to "ros3".
+ --hdfs-attrs=A Supply configuration information to Hadoop VFD.
+ Accepts tuple of (<namenode name>,<namenode port>,
+ ...<kerberos cache path>,<username>,<buffer size>)
+ If absent or A == '(,,,,)', all default values are used.
+ Has no effect if vfd flag is not 'hdfs'.
file/OBJECT
Each object consists of an HDF5 file name optionally followed by a
diff --git a/tools/testfiles/help-3.ls b/tools/testfiles/help-3.ls
index 491f696..396bed3 100644
--- a/tools/testfiles/help-3.ls
+++ b/tools/testfiles/help-3.ls
@@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
-V, --version Print version number and exit
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
+ --s3-cred=C Supply S3 authentication information to "ros3" vfd.
+ Accepts tuple of "(<aws-region>,<access-id>,<access-key>)".
+ If absent or C->"(,,)", defaults to no-authentication.
+ Has no effect if vfd flag not set to "ros3".
+ --hdfs-attrs=A Supply configuration information to Hadoop VFD.
+ Accepts tuple of (<namenode name>,<namenode port>,
+ ...<kerberos cache path>,<username>,<buffer size>)
+ If absent or A == '(,,,,)', all default values are used.
+ Has no effect if vfd flag is not 'hdfs'.
file/OBJECT
Each object consists of an HDF5 file name optionally followed by a
diff --git a/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl b/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl
index 19de76f..95dfc3b 100644
--- a/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl
+++ b/tools/testfiles/pbits/tnofilename-with-packed-bits.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/pbits/tpbitsIncomplete.ddl b/tools/testfiles/pbits/tpbitsIncomplete.ddl
index 19de76f..95dfc3b 100644
--- a/tools/testfiles/pbits/tpbitsIncomplete.ddl
+++ b/tools/testfiles/pbits/tpbitsIncomplete.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/pbits/tpbitsLengthExceeded.ddl b/tools/testfiles/pbits/tpbitsLengthExceeded.ddl
index 19de76f..95dfc3b 100644
--- a/tools/testfiles/pbits/tpbitsLengthExceeded.ddl
+++ b/tools/testfiles/pbits/tpbitsLengthExceeded.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/pbits/tpbitsLengthPositive.ddl b/tools/testfiles/pbits/tpbitsLengthPositive.ddl
index 19de76f..95dfc3b 100644
--- a/tools/testfiles/pbits/tpbitsLengthPositive.ddl
+++ b/tools/testfiles/pbits/tpbitsLengthPositive.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/pbits/tpbitsMaxExceeded.ddl b/tools/testfiles/pbits/tpbitsMaxExceeded.ddl
index 19de76f..95dfc3b 100644
--- a/tools/testfiles/pbits/tpbitsMaxExceeded.ddl
+++ b/tools/testfiles/pbits/tpbitsMaxExceeded.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl b/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl
index 19de76f..95dfc3b 100644
--- a/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl
+++ b/tools/testfiles/pbits/tpbitsOffsetExceeded.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/pbits/tpbitsOffsetNegative.ddl b/tools/testfiles/pbits/tpbitsOffsetNegative.ddl
index 19de76f..95dfc3b 100644
--- a/tools/testfiles/pbits/tpbitsOffsetNegative.ddl
+++ b/tools/testfiles/pbits/tpbitsOffsetNegative.ddl
@@ -12,6 +12,16 @@ usage: h5dump [OPTIONS] files
-b B, --binary=B Binary file output, of form B
-O F, --ddl=F Output ddl text into file F
Use blank(empty) filename F to suppress ddl display
+ --s3-cred=<cred> Supply S3 authentication information to "ros3" vfd.
+ <cred> :: "(<aws-region>,<access-id>,<access-key>)"
+ If absent or <cred> -> "(,,)", no authentication.
+ Has no effect is filedriver is not `ros3'.
+ --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.
+ For use with "--filedriver=hdfs"
+ <attrs> :: (<namenode name>,<namenode port>,
+ <kerberos cache path>,<username>,
+ <buffer size>)
+ Any absent attribute will use a default value.
--------------- Object Options ---------------
-a P, --attribute=P Print the specified attribute
If an attribute name contains a slash (/), escape the
diff --git a/tools/testfiles/textlinksrc-nodangle-1.ls b/tools/testfiles/textlinksrc-nodangle-1.ls
index 491f696..396bed3 100644
--- a/tools/testfiles/textlinksrc-nodangle-1.ls
+++ b/tools/testfiles/textlinksrc-nodangle-1.ls
@@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
-V, --version Print version number and exit
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
+ --s3-cred=C Supply S3 authentication information to "ros3" vfd.
+ Accepts tuple of "(<aws-region>,<access-id>,<access-key>)".
+ If absent or C->"(,,)", defaults to no-authentication.
+ Has no effect if vfd flag not set to "ros3".
+ --hdfs-attrs=A Supply configuration information to Hadoop VFD.
+ Accepts tuple of (<namenode name>,<namenode port>,
+ ...<kerberos cache path>,<username>,<buffer size>)
+ If absent or A == '(,,,,)', all default values are used.
+ Has no effect if vfd flag is not 'hdfs'.
file/OBJECT
Each object consists of an HDF5 file name optionally followed by a
diff --git a/tools/testfiles/tgroup-1.ls b/tools/testfiles/tgroup-1.ls
index 491f696..396bed3 100644
--- a/tools/testfiles/tgroup-1.ls
+++ b/tools/testfiles/tgroup-1.ls
@@ -37,6 +37,15 @@ usage: h5ls [OPTIONS] file[/OBJECT] [file[/[OBJECT]...]
-V, --version Print version number and exit
--vfd=DRIVER Use the specified virtual file driver
-x, --hexdump Show raw data in hexadecimal format
+ --s3-cred=C Supply S3 authentication information to "ros3" vfd.
+ Accepts tuple of "(<aws-region>,<access-id>,<access-key>)".
+ If absent or C->"(,,)", defaults to no-authentication.
+ Has no effect if vfd flag not set to "ros3".
+ --hdfs-attrs=A Supply configuration information to Hadoop VFD.
+ Accepts tuple of (<namenode name>,<namenode port>,
+ ...<kerberos cache path>,<username>,<buffer size>)
+ If absent or A == '(,,,,)', all default values are used.
+ Has no effect if vfd flag is not 'hdfs'.
file/OBJECT
Each object consists of an HDF5 file name optionally followed by a