Merge S3 code from develop to 1.10

1 files changed, 184 insertions, 2 deletions

diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c
index 5e9339f..5b47481 100644
--- a/tools/src/h5stat/h5stat.c
+++ b/tools/src/h5stat/h5stat.c
@@ -118,6 +118,33 @@ typedef struct iter_t {
 } iter_t;
 
 
+static const char *drivername = "";
+
+#ifdef H5_HAVE_ROS3_VFD
+/* default "anonymous" s3 configuration
+ */
+static H5FD_ros3_fapl_t ros3_fa = {
+    1,     /* fapl version      */
+    false, /* authenticate      */
+    "",    /* aws region        */
+    "",    /* access key id     */
+    "",    /* secret access key */
+};
+#endif /* H5_HAVE_ROS3_VFD */
+
+#ifdef H5_HAVE_LIBHDFS
+/* default HDFS access configuration
+ */
+static H5FD_hdfs_fapl_t hdfs_fa = {
+    1,           /* fapl version          */
+    "localhost", /* namenode name         */
+    0,           /* namenode port         */
+    "",          /* kerberos ticket cache */
+    "",          /* user name             */
+    2048,        /* stream buffer size    */
+};
+#endif /* H5_HAVE_LIBHDFS */
+
 static int        display_all = TRUE;
 
 /* Enable the printing of selected statistics */
@@ -146,7 +173,7 @@ struct handler_t {
     char **obj;
 };
 
-static const char *s_opts ="Aa:Ddm:EFfhGgl:sSTO:V";
+static const char *s_opts ="Aa:Ddm:EFfhGgl:sSTO:Vw:";
 /* e.g. "filemetadata" has to precede "file"; "groupmetadata" has to precede "group" etc. */
 static struct long_options l_opts[] = {
     {"help", no_arg, 'h'},
@@ -246,6 +273,8 @@ static struct long_options l_opts[] = {
     { "summ", no_arg, 'S' },
     { "sum", no_arg, 'S' },
     { "su", no_arg, 'S' },
+    { "s3-cred", require_arg, 'w' },
+    { "hdfs-attrs", require_arg, 'H' },
     { NULL, 0, '\0' }
 };
 
@@ -295,6 +324,16 @@ static void usage(const char *prog)
      HDfprintf(stdout, "     -s, --freespace       Print free space information\n");
      HDfprintf(stdout, "     -S, --summary         Print summary of file space information\n");
      HDfprintf(stdout, "     --enable-error-stack  Prints messages from the HDF5 error stack as they occur\n");
+     HDfprintf(stdout, "     --s3-cred=<cred>      Access file on S3, using provided credential\n");
+     HDfprintf(stdout, "                           <cred> :: (region,id,key)\n");
+     HDfprintf(stdout, "                           If <cred> == \"(,,)\", no authentication is used.\n");
+     HDfprintf(stdout, "     --hdfs-attrs=<attrs>  Access a file on HDFS with given configuration\n");
+     HDfprintf(stdout, "                           attributes.\n");
+     HDfprintf(stdout, "                           <attrs> :: (<namenode name>,<namenode port>,\n");
+     HDfprintf(stdout, "                                       <kerberos cache path>,<username>,\n");
+     HDfprintf(stdout, "                                       <buffer size>)\n");
+     HDfprintf(stdout, "                           If an attribute is empty, a default value will be\n");
+     HDfprintf(stdout, "                           used.\n");
 }
 
 
@@ -1019,6 +1058,105 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret)
                     } /* end if */
                 break;
 
+            case 'w':
+#ifndef H5_HAVE_ROS3_VFD
+                error_msg("Read-Only S3 VFD not enabled.\n");
+                goto error;
+#else
+                {
+                    char        *cred_str = NULL;
+                    unsigned     nelems   = 0;
+                    char       **cred     = NULL;
+                    char const  *ccred[3];
+
+                    if (FAIL == parse_tuple((const char *)opt_arg, ',', &cred_str, &nelems, &cred)) {
+                        error_msg("Unable to parse s3 credential\n");
+                        goto error;
+                    }
+                    if (nelems != 3) {
+                        error_msg("s3 credential must have three elements\n");
+                        goto error;
+                    }
+                    ccred[0] = (const char *)cred[0];
+                    ccred[1] = (const char *)cred[1];
+                    ccred[2] = (const char *)cred[2];
+                    if (0 == h5tools_populate_ros3_fapl(&ros3_fa, ccred)) {
+                        error_msg("Unable to set ros3 fapl config\n");
+                        goto error;
+                    }
+                    HDfree(cred);
+                    HDfree(cred_str);
+                } /* parse s3-cred block */
+                drivername = "ros3";
+                break;
+#endif /* H5_HAVE_ROS3_VFD */
+
+            case 'H':
+#ifndef H5_HAVE_LIBHDFS
+                error_msg("HDFS VFD is not enabled.\n");
+                goto error;
+#else
+                {
+                    unsigned         nelems    = 0;
+                    char            *props_src = NULL;
+                    char           **props     = NULL;
+                    unsigned long    k         = 0;
+                    if (FAIL == parse_tuple((const char *)opt_arg,
+                            ',', &props_src, &nelems, &props)) {
+                        error_msg("unable to parse hdfs properties tuple\n");
+                        goto error;
+                    }
+                    /* sanity-check tuple count
+                     */
+                    if (nelems != 5) {
+                        char str[64] = "";
+                        HDsprintf(str,
+                                "expected 5 elements in hdfs properties tuple "
+                                "but found %u\n",
+                                nelems);
+                        HDfree(props);
+                        HDfree(props_src);
+                        error_msg(str);
+                        goto error;
+                    }
+                    /* Populate fapl configuration structure with given
+                     * properties.
+                     * TODO/WARNING: No error-checking is done on length of
+                     *         input strings... Silent overflow is possible,
+                     *         albeit unlikely.
+                     */
+                    if (HDstrncmp(props[0], "", 1)) {
+                        HDstrncpy(hdfs_fa.namenode_name,(const char *)props[0], HDstrlen(props[0]));
+                    }
+                    if (HDstrncmp(props[1], "", 1)) {
+                        k = strtoul((const char *)props[1], NULL, 0);
+                        if (errno == ERANGE) {
+                            error_msg("supposed port number wasn't.\n");
+                            goto error;
+                        }
+                        hdfs_fa.namenode_port = (int32_t)k;
+                    }
+                    if (HDstrncmp(props[2], "", 1)) {
+                        HDstrncpy(hdfs_fa.kerberos_ticket_cache, (const char *)props[2], HDstrlen(props[2]));
+                    }
+                    if (HDstrncmp(props[3], "", 1)) {
+                        HDstrncpy(hdfs_fa.user_name, (const char *)props[3], HDstrlen(props[3]));
+                    }
+                    if (strncmp(props[4], "", 1)) {
+                        k = HDstrtoul((const char *)props[4], NULL, 0);
+                        if (errno == ERANGE) {
+                            error_msg("supposed buffersize number wasn't.\n");
+                            goto error;
+                        }
+                        hdfs_fa.stream_buffer_size = (int32_t)k;
+                    }
+                    HDfree(props);
+                    HDfree(props_src);
+                    drivername = "hdfs";
+                }
+                break;
+#endif /* H5_HAVE_LIBHDFS */
+
             default:
                 usage(h5tools_getprogname());
                 goto error;
@@ -1720,6 +1858,7 @@ main(int argc, const char *argv[])
     void               *edata;
     void               *tools_edata;
     struct handler_t   *hand = NULL;
+    hid_t               fapl_id = H5P_DEFAULT;
 
     h5tools_setprogname(PROGRAMNAME);
     h5tools_setstatus(EXIT_SUCCESS);
@@ -1740,6 +1879,42 @@ main(int argc, const char *argv[])
     if(parse_command_line(argc, argv, &hand) < 0)
         goto done;
 
+    /* if drivername is not null, probably need to set the fapl */
+    if (HDstrcmp(drivername, "")) {
+        void *conf_fa = NULL;
+
+        if (!HDstrcmp(drivername, "ros3")) {
+#ifndef H5_HAVE_ROS3_VFD
+            error_msg("Read-Only S3 VFD not enabled.\n\n");
+            goto done;
+#else
+            conf_fa = (void *)&ros3_fa;
+#endif /* H5_HAVE_ROS3_VFD */
+
+        }
+        else if (!HDstrcmp(drivername, "hdfs")) {
+#ifndef H5_HAVE_LIBHDFS
+            error_msg("HDFS VFD not enabled.\n\n");
+            goto done;
+#else
+            conf_fa = (void *)&hdfs_fa;
+#endif /* H5_HAVE_LIBHDFS */
+        }
+
+        if (conf_fa != NULL) {
+            HDassert(fapl_id == H5P_DEFAULT);
+            fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+            if (fapl_id < 0) {
+                error_msg("Unable to create fapl entry\n");
+                goto done;
+            }
+            if (1 > h5tools_set_configured_fapl(fapl_id, drivername, conf_fa)) {
+                error_msg("Unable to set fapl\n");
+                goto done;
+            }
+        }
+    } /* drivername set */
+
     fname = argv[opt_ind];
 
     if(enable_error_stack > 0) {
@@ -1754,7 +1929,7 @@ main(int argc, const char *argv[])
 
         HDprintf("Filename: %s\n", fname);
 
-        fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
+        fid = H5Fopen(fname, H5F_ACC_RDONLY, fapl_id);
         if(fid < 0) {
             error_msg("unable to open file \"%s\"\n", fname);
             h5tools_setstatus(EXIT_FAILURE);
@@ -1837,6 +2012,13 @@ done:
     /* Free iter structure */
     iter_free(&iter);
 
+    if (fapl_id != H5P_DEFAULT) {
+        if (0 < H5Pclose(fapl_id)) {
+            error_msg("unable to close fapl entry\n");
+            h5tools_setstatus(EXIT_FAILURE);
+        }
+    }
+
     if(fid >= 0 && H5Fclose(fid) < 0) {
         error_msg("unable to close file \"%s\"\n", fname);
         h5tools_setstatus(EXIT_FAILURE);