Squashed commit of the following:

    Merge changes from update_merged_S3_HDFS branch into develop.

commit d5034315aea88629929ac0c9c59ebfafd5f21a31
Merge: 9c48823 d3fdcd8
Author: Larry Knox <lrknox@hdfgroup.org>
Date:   Thu Jul 25 08:24:53 2019 -0500

    Merge branch 'develop' into update_merged_S3_HDFS

1 files changed, 208 insertions, 122 deletions

diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c
index b9e37e8..a824197 100644
--- a/tools/src/h5dump/h5dump.c
+++ b/tools/src/h5dump/h5dump.c
@@ -24,6 +24,23 @@ static int           doxml = 0;
 static int           useschema = 1;
 static const char   *xml_dtd_uri = NULL;
 
+static H5FD_ros3_fapl_t ros3_fa = {
+    1,     /* version           */
+    false, /* authenticate      */
+    "",    /* aws region        */
+    "",    /* access key id     */
+    "",    /* secret access key */
+};
+
+static H5FD_hdfs_fapl_t hdfs_fa = {
+    1,           /* fapl version          */
+    "localhost", /* namenode name         */
+    0,           /* namenode port         */
+    "",          /* kerberos ticket cache */
+    "",          /* user name             */
+    2048,        /* stream buffer size    */
+};
+
 /* module-scoped variables for XML option */
 #define DEFAULT_XSD     "http://www.hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd"
 #define DEFAULT_DTD     "http://www.hdfgroup.org/HDF5/XML/DTD/HDF5-File.dtd"
@@ -188,6 +205,8 @@ static struct long_options l_opts[] = {
     { "any_path", require_arg, 'N' },
     { "vds-view-first-missing", no_arg, 'v' },
     { "vds-gap-size", require_arg, 'G' },
+    { "s3-cred", require_arg, '$' },
+    { "hdfs-attrs", require_arg, '#' },
     { NULL, 0, '\0' }
 };
 
@@ -241,6 +260,16 @@ usage(const char *prog)
     PRINTVALSTREAM(rawoutstream, "     -b B, --binary=B     Binary file output, of form B\n");
     PRINTVALSTREAM(rawoutstream, "     -O F, --ddl=F        Output ddl text into file F\n");
     PRINTVALSTREAM(rawoutstream, "                          Use blank(empty) filename F to suppress ddl display\n");
+    PRINTVALSTREAM(rawoutstream, "     --s3-cred=<cred>     Supply S3 authentication information to \"ros3\" vfd.\n");
+    PRINTVALSTREAM(rawoutstream, "                          <cred> :: \"(<aws-region>,<access-id>,<access-key>)\"\n");
+    PRINTVALSTREAM(rawoutstream, "                          If absent or <cred> -> \"(,,)\", no authentication.\n");
+    PRINTVALSTREAM(rawoutstream, "                          Has no effect is filedriver is not `ros3'.\n");
+    PRINTVALSTREAM(rawoutstream, "     --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.\n");
+    PRINTVALSTREAM(rawoutstream, "                          For use with \"--filedriver=hdfs\"\n");
+    PRINTVALSTREAM(rawoutstream, "                          <attrs> :: (<namenode name>,<namenode port>,\n");
+    PRINTVALSTREAM(rawoutstream, "                                      <kerberos cache path>,<username>,\n");
+    PRINTVALSTREAM(rawoutstream, "                                      <buffer size>)\n");
+    PRINTVALSTREAM(rawoutstream, "                          Any absent attribute will use a default value.\n");
     PRINTVALSTREAM(rawoutstream, "--------------- Object Options ---------------\n");
     PRINTVALSTREAM(rawoutstream, "     -a P, --attribute=P  Print the specified attribute\n");
     PRINTVALSTREAM(rawoutstream, "                          If an attribute name contains a slash (/), escape the\n");
@@ -1282,6 +1311,126 @@ end_collect:
             hand = NULL;
             h5tools_setstatus(EXIT_SUCCESS);
             goto done;
+
+        case '$':
+#ifndef H5_HAVE_ROS3_VFD
+            error_msg("Read-Only S3 VFD not enabled.\n");
+            h5tools_setstatus(EXIT_FAILURE);
+            goto done;
+#else
+            /* s3 credential */
+            {
+                char       **s3_cred = NULL;
+                char        *s3_cred_string = NULL;
+                const char  *ccred[3];
+                unsigned     nelems = 0;
+                if ( FAIL ==
+                     parse_tuple(opt_arg, ',',
+                                 &s3_cred_string, &nelems, &s3_cred))
+                {
+                    error_msg("unable to parse malformed s3 credentials\n");
+                    usage(h5tools_getprogname());
+                    free_handler(hand, argc);
+                    hand= NULL;
+                    h5tools_setstatus(EXIT_FAILURE);
+                    goto done;
+                }
+                if (nelems != 3) {
+                    error_msg("s3 credentials expects 3 elements\n");
+                    usage(h5tools_getprogname());
+                    free_handler(hand, argc);
+                    hand= NULL;
+                    h5tools_setstatus(EXIT_FAILURE);
+                    goto done;
+                }
+                ccred[0] = (const char *)s3_cred[0];
+                ccred[1] = (const char *)s3_cred[1];
+                ccred[2] = (const char *)s3_cred[2];
+                if (0 == h5tools_populate_ros3_fapl(&ros3_fa, ccred)) {
+                    error_msg("Invalid S3 credentials\n");
+                    usage(h5tools_getprogname());
+                    free_handler(hand, argc);
+                    hand= NULL;
+                    h5tools_setstatus(EXIT_FAILURE);
+                    goto done;
+                }
+                HDfree(s3_cred);
+                HDfree(s3_cred_string);
+            } /* s3 credential block */
+            break;
+#endif /* H5_HAVE_ROS3_VFD */
+
+        case '#':
+#ifndef H5_HAVE_LIBHDFS
+            error_msg("HDFS VFD is not enabled.\n");
+            goto error;
+#else
+            {
+                /* read hdfs properties tuple and store values in `hdfs_fa`
+                 */
+                unsigned         nelems    = 0;
+                char            *props_src = NULL;
+                char           **props     = NULL;
+                unsigned long    k         = 0;
+                if (FAIL == parse_tuple(
+                        (const char *)opt_arg,
+                        ',',
+                        &props_src,
+                        &nelems,
+                        &props))
+                {
+                    error_msg("unable to parse hdfs properties tuple\n");
+                    goto error;
+                }
+                /* sanity-check tuple count
+                 */
+                if (nelems != 5) {
+                    h5tools_setstatus(EXIT_FAILURE);
+                    goto error;
+                }
+                /* Populate fapl configuration structure with given
+                 * properties.
+                 * WARNING: No error-checking is done on length of input
+                 *          strings... Silent overflow is possible, albeit
+                 *          unlikely.
+                 */
+                if (strncmp(props[0], "", 1)) {
+                    HDstrncpy(hdfs_fa.namenode_name,
+                            (const char *)props[0],
+                            HDstrlen(props[0]));
+                }
+                if (strncmp(props[1], "", 1)) {
+                    k = strtoul((const char *)props[1], NULL, 0);
+                    if (errno == ERANGE) {
+                        h5tools_setstatus(EXIT_FAILURE);
+                        goto error;
+                    }
+                    hdfs_fa.namenode_port = (int32_t)k;
+                }
+                if (strncmp(props[2], "", 1)) {
+                    HDstrncpy(hdfs_fa.kerberos_ticket_cache,
+                            (const char *)props[2],
+                            HDstrlen(props[2]));
+                }
+                if (strncmp(props[3], "", 1)) {
+                    HDstrncpy(hdfs_fa.user_name,
+                            (const char *)props[3],
+                            HDstrlen(props[3]));
+                }
+                if (strncmp(props[4], "", 1)) {
+                    k = strtoul((const char *)props[4], NULL, 0);
+                    if (errno == ERANGE) {
+                        h5tools_setstatus(EXIT_FAILURE);
+                        goto error;
+                    }
+                    hdfs_fa.stream_buffer_size = (int32_t)k;
+                }
+                HDfree(props);
+                HDfree(props_src);
+            }
+#endif /* H5_HAVE_LIBHDFS */
+            break;
+
         case '?':
         default:
             usage(h5tools_getprogname());
@@ -1354,6 +1503,7 @@ main(int argc, const char *argv[])
 {
     hid_t               fid = -1;
     hid_t               gid = -1;
+    hid_t               fapl_id = H5P_DEFAULT;
     H5E_auto2_t         func;
     H5E_auto2_t         tools_func;
     H5O_info_t          oi;
@@ -1440,10 +1590,60 @@ main(int argc, const char *argv[])
     /* Initialize indexing options */
     h5trav_set_index(sort_by, sort_order);
 
+    if (driver != NULL) {
+        void *conf_fa = NULL;
+
+        if (!strcmp(driver, "ros3")) {
+#ifndef H5_HAVE_ROS3_VFD
+            error_msg("Read-Only S3 VFD not enabled.\n");
+            h5tools_setstatus(EXIT_FAILURE);
+            goto done;
+#else
+            conf_fa = (void *)&ros3_fa;
+#endif /* H5_HAVE_ROS3_VFD */
+        } else if (!HDstrcmp(driver, "hdfs")) {
+#ifndef H5_HAVE_LIBHDFS
+            error_msg("HDFS VFD is not enabled.\n");
+            h5tools_setstatus(EXIT_FAILURE);
+            goto done;
+#else
+            conf_fa = (void *)&hdfs_fa;
+#endif /* H5_HAVE_LIBHDFS */
+        }
+
+        if (conf_fa != NULL) {
+            fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+            if (fapl_id < 0) {
+                error_msg("unable to create fapl entry\n");
+                h5tools_setstatus(EXIT_FAILURE);
+                goto done;
+            }
+            if (0 == h5tools_set_configured_fapl(
+                    fapl_id,
+                    driver,   /* guaranteed "ros3" or "hdfs" */
+                    conf_fa)) /* appropriate to driver */
+            {
+                error_msg("unable to set fapl\n");
+                h5tools_setstatus(EXIT_FAILURE);
+                goto done;
+            }
+        }
+    } /* driver defined */
+
     while(opt_ind < argc) {
         fname = HDstrdup(argv[opt_ind++]);
 
-        fid = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, driver, NULL, 0);
+        if (fapl_id != H5P_DEFAULT) {
+            fid = H5Fopen(fname, H5F_ACC_RDONLY, fapl_id);
+        } else {
+            fid = h5tools_fopen(
+                    fname,
+                    H5F_ACC_RDONLY,
+                    H5P_DEFAULT,
+                    driver,
+                    NULL,
+                    0);
+        }
 
         if (fid < 0) {
             error_msg("unable to open file \"%s\"\n", fname);
@@ -1624,6 +1824,11 @@ done:
     /* Free tables for objects */
     table_list_free();
 
+    if (fapl_id != H5P_DEFAULT && 0 < H5Pclose(fapl_id)) {
+        error_msg("Can't close fapl entry\n");
+        h5tools_setstatus(EXIT_FAILURE);
+    }
+
     if(fid >=0)
         if (H5Fclose(fid) < 0)
             h5tools_setstatus(EXIT_FAILURE);
@@ -1645,127 +1850,7 @@ done:
     H5Eset_auto2(H5E_DEFAULT, func, edata);
 
     leave(h5tools_getstatus());
-}
-
-/*-------------------------------------------------------------------------
- * Function:    h5_fileaccess
- *
- * Purpose: Returns a file access template which is the default template
- *      but with a file driver set according to the constant or
- *      environment variable HDF5_DRIVER
- *
- * Return:  Success:    A file access property list
- *
- *      Failure:    -1
- *
- * Programmer:  Robb Matzke
- *              Thursday, November 19, 1998
- *
- * Modifications:
- *
- *-------------------------------------------------------------------------
- */
-hid_t
-h5_fileaccess(void)
-{
-    static const char *multi_letters = "msbrglo";
-    const char  *val = NULL;
-    const char  *name;
-    char         s[1024];
-    hid_t        fapl = -1;
-
-    /* First use the environment variable, then the constant */
-    val = HDgetenv("HDF5_DRIVER");
-#ifdef HDF5_DRIVER
-    if (!val) val = HDF5_DRIVER;
-#endif
-
-    if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) return -1;
-    if (!val || !*val) return fapl; /*use default*/
-
-    HDstrncpy(s, val, sizeof s);
-    s[sizeof(s)-1] = '\0';
-    if (NULL==(name=HDstrtok(s, " \t\n\r"))) return fapl;
-
-    if (!HDstrcmp(name, "sec2")) {
-        /* Unix read() and write() system calls */
-        if (H5Pset_fapl_sec2(fapl)<0) return -1;
-    }
-    else if (!HDstrcmp(name, "stdio")) {
-        /* Standard C fread() and fwrite() system calls */
-        if (H5Pset_fapl_stdio(fapl)<0) return -1;
-    }
-    else if (!HDstrcmp(name, "core")) {
-        /* In-core temporary file with 1MB increment */
-        if (H5Pset_fapl_core(fapl, 1024*1024, FALSE)<0) return -1;
-    }
-    else if (!HDstrcmp(name, "split")) {
-        /* Split meta data and raw data each using default driver */
-        if (H5Pset_fapl_split(fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0)
-            return -1;
-    }
-    else if (!HDstrcmp(name, "multi")) {
-        /* Multi-file driver, general case of the split driver */
-        H5FD_mem_t      memb_map[H5FD_MEM_NTYPES];
-        hid_t           memb_fapl[H5FD_MEM_NTYPES];
-        const char     *memb_name[H5FD_MEM_NTYPES];
-        char            sv[H5FD_MEM_NTYPES][1024];
-        haddr_t         memb_addr[H5FD_MEM_NTYPES];
-        H5FD_mem_t      mt;
-
-        HDmemset(memb_map, 0, sizeof memb_map);
-        HDmemset(memb_fapl, 0, sizeof memb_fapl);
-        HDmemset(memb_name, 0, sizeof memb_name);
-        HDmemset(memb_addr, 0, sizeof memb_addr);
-
-        if(HDstrlen(multi_letters)==H5FD_MEM_NTYPES) {
-            for (mt=H5FD_MEM_DEFAULT; mt<H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t,mt)) {
-                memb_fapl[mt] = H5P_DEFAULT;
-                memb_map[mt] = mt;
-                sprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]);
-                memb_name[mt] = sv[mt];
-                memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10);
-            }
-        }
-        else {
-            error_msg("Bad multi_letters list\n");
-            return FAIL;
-        }
-
-        if (H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name, memb_addr, FALSE) < 0)
-            return -1;
-    }
-    else if (!HDstrcmp(name, "family")) {
-        hsize_t fam_size = 100*1024*1024; /*100 MB*/
-
-        /* Family of files, each 1MB and using the default driver */
-        if ((val=HDstrtok(NULL, " \t\n\r")))
-            fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024);
-        if (H5Pset_fapl_family(fapl, fam_size, H5P_DEFAULT)<0)
-            return -1;
-    }
-    else if (!HDstrcmp(name, "log")) {
-        long log_flags = H5FD_LOG_LOC_IO;
-
-        /* Log file access */
-        if ((val = HDstrtok(NULL, " \t\n\r")))
-            log_flags = HDstrtol(val, NULL, 0);
-
-        if (H5Pset_fapl_log(fapl, NULL, (unsigned)log_flags, 0) < 0)
-            return -1;
-    }
-    else if (!HDstrcmp(name, "direct")) {
-        /* Substitute Direct I/O driver with sec2 driver temporarily because
-         * some output has sec2 driver as the standard. */
-        if (H5Pset_fapl_sec2(fapl)<0) return -1;
-    }
-    else {
-        /* Unknown driver */
-        return -1;
-    }
-
-    return fapl;
-}
+} /* main */
 
 
 /*-------------------------------------------------------------------------
@@ -1813,3 +1898,4 @@ add_prefix(char **prfx, size_t *prfx_len, const char *name)
     HDstrcat(HDstrcat(*prfx, "/"), name);
 } /* end add_prefix */
 
+