summaryrefslogtreecommitdiffstats
path: root/tools/src
diff options
context:
space:
mode:
Diffstat (limited to 'tools/src')
-rw-r--r--tools/src/h5dump/h5dump.c330
-rw-r--r--tools/src/h5ls/h5ls.c321
-rw-r--r--tools/src/h5stat/h5stat.c251
3 files changed, 724 insertions, 178 deletions
diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c
index b9e37e8..a824197 100644
--- a/tools/src/h5dump/h5dump.c
+++ b/tools/src/h5dump/h5dump.c
@@ -24,6 +24,23 @@ static int doxml = 0;
static int useschema = 1;
static const char *xml_dtd_uri = NULL;
+static H5FD_ros3_fapl_t ros3_fa = {
+ 1, /* version */
+ false, /* authenticate */
+ "", /* aws region */
+ "", /* access key id */
+ "", /* secret access key */
+};
+
+static H5FD_hdfs_fapl_t hdfs_fa = {
+ 1, /* fapl version */
+ "localhost", /* namenode name */
+ 0, /* namenode port */
+ "", /* kerberos ticket cache */
+ "", /* user name */
+ 2048, /* stream buffer size */
+};
+
/* module-scoped variables for XML option */
#define DEFAULT_XSD "http://www.hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd"
#define DEFAULT_DTD "http://www.hdfgroup.org/HDF5/XML/DTD/HDF5-File.dtd"
@@ -188,6 +205,8 @@ static struct long_options l_opts[] = {
{ "any_path", require_arg, 'N' },
{ "vds-view-first-missing", no_arg, 'v' },
{ "vds-gap-size", require_arg, 'G' },
+ { "s3-cred", require_arg, '$' },
+ { "hdfs-attrs", require_arg, '#' },
{ NULL, 0, '\0' }
};
@@ -241,6 +260,16 @@ usage(const char *prog)
PRINTVALSTREAM(rawoutstream, " -b B, --binary=B Binary file output, of form B\n");
PRINTVALSTREAM(rawoutstream, " -O F, --ddl=F Output ddl text into file F\n");
PRINTVALSTREAM(rawoutstream, " Use blank(empty) filename F to suppress ddl display\n");
+ PRINTVALSTREAM(rawoutstream, " --s3-cred=<cred> Supply S3 authentication information to \"ros3\" vfd.\n");
+ PRINTVALSTREAM(rawoutstream, " <cred> :: \"(<aws-region>,<access-id>,<access-key>)\"\n");
+ PRINTVALSTREAM(rawoutstream, " If absent or <cred> -> \"(,,)\", no authentication.\n");
+ PRINTVALSTREAM(rawoutstream, " Has no effect is filedriver is not `ros3'.\n");
+ PRINTVALSTREAM(rawoutstream, " --hdfs-attrs=<attrs> Supply configuration information for HDFS file access.\n");
+ PRINTVALSTREAM(rawoutstream, " For use with \"--filedriver=hdfs\"\n");
+ PRINTVALSTREAM(rawoutstream, " <attrs> :: (<namenode name>,<namenode port>,\n");
+ PRINTVALSTREAM(rawoutstream, " <kerberos cache path>,<username>,\n");
+ PRINTVALSTREAM(rawoutstream, " <buffer size>)\n");
+ PRINTVALSTREAM(rawoutstream, " Any absent attribute will use a default value.\n");
PRINTVALSTREAM(rawoutstream, "--------------- Object Options ---------------\n");
PRINTVALSTREAM(rawoutstream, " -a P, --attribute=P Print the specified attribute\n");
PRINTVALSTREAM(rawoutstream, " If an attribute name contains a slash (/), escape the\n");
@@ -1282,6 +1311,126 @@ end_collect:
hand = NULL;
h5tools_setstatus(EXIT_SUCCESS);
goto done;
+
+ case '$':
+#ifndef H5_HAVE_ROS3_VFD
+ error_msg("Read-Only S3 VFD not enabled.\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+#else
+ /* s3 credential */
+ {
+ char **s3_cred = NULL;
+ char *s3_cred_string = NULL;
+ const char *ccred[3];
+ unsigned nelems = 0;
+ if ( FAIL ==
+ parse_tuple(opt_arg, ',',
+ &s3_cred_string, &nelems, &s3_cred))
+ {
+ error_msg("unable to parse malformed s3 credentials\n");
+ usage(h5tools_getprogname());
+ free_handler(hand, argc);
+ hand= NULL;
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ if (nelems != 3) {
+ error_msg("s3 credentials expects 3 elements\n");
+ usage(h5tools_getprogname());
+ free_handler(hand, argc);
+ hand= NULL;
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ ccred[0] = (const char *)s3_cred[0];
+ ccred[1] = (const char *)s3_cred[1];
+ ccred[2] = (const char *)s3_cred[2];
+ if (0 == h5tools_populate_ros3_fapl(&ros3_fa, ccred)) {
+ error_msg("Invalid S3 credentials\n");
+ usage(h5tools_getprogname());
+ free_handler(hand, argc);
+ hand= NULL;
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ HDfree(s3_cred);
+ HDfree(s3_cred_string);
+ } /* s3 credential block */
+ break;
+#endif /* H5_HAVE_ROS3_VFD */
+
+ case '#':
+#ifndef H5_HAVE_LIBHDFS
+ error_msg("HDFS VFD is not enabled.\n");
+ goto error;
+#else
+ {
+ /* read hdfs properties tuple and store values in `hdfs_fa`
+ */
+ unsigned nelems = 0;
+ char *props_src = NULL;
+ char **props = NULL;
+ unsigned long k = 0;
+ if (FAIL == parse_tuple(
+ (const char *)opt_arg,
+ ',',
+ &props_src,
+ &nelems,
+ &props))
+ {
+ error_msg("unable to parse hdfs properties tuple\n");
+ goto error;
+ }
+ /* sanity-check tuple count
+ */
+ if (nelems != 5) {
+ h5tools_setstatus(EXIT_FAILURE);
+ goto error;
+ }
+ /* Populate fapl configuration structure with given
+ * properties.
+ * WARNING: No error-checking is done on length of input
+ * strings... Silent overflow is possible, albeit
+ * unlikely.
+ */
+ if (strncmp(props[0], "", 1)) {
+ HDstrncpy(hdfs_fa.namenode_name,
+ (const char *)props[0],
+ HDstrlen(props[0]));
+ }
+ if (strncmp(props[1], "", 1)) {
+ k = strtoul((const char *)props[1], NULL, 0);
+ if (errno == ERANGE) {
+ h5tools_setstatus(EXIT_FAILURE);
+ goto error;
+ }
+ hdfs_fa.namenode_port = (int32_t)k;
+ }
+ if (strncmp(props[2], "", 1)) {
+ HDstrncpy(hdfs_fa.kerberos_ticket_cache,
+ (const char *)props[2],
+ HDstrlen(props[2]));
+ }
+ if (strncmp(props[3], "", 1)) {
+ HDstrncpy(hdfs_fa.user_name,
+ (const char *)props[3],
+ HDstrlen(props[3]));
+ }
+ if (strncmp(props[4], "", 1)) {
+ k = strtoul((const char *)props[4], NULL, 0);
+ if (errno == ERANGE) {
+ h5tools_setstatus(EXIT_FAILURE);
+ goto error;
+ }
+ hdfs_fa.stream_buffer_size = (int32_t)k;
+ }
+ HDfree(props);
+ HDfree(props_src);
+ }
+#endif /* H5_HAVE_LIBHDFS */
+ break;
+
case '?':
default:
usage(h5tools_getprogname());
@@ -1354,6 +1503,7 @@ main(int argc, const char *argv[])
{
hid_t fid = -1;
hid_t gid = -1;
+ hid_t fapl_id = H5P_DEFAULT;
H5E_auto2_t func;
H5E_auto2_t tools_func;
H5O_info_t oi;
@@ -1440,10 +1590,60 @@ main(int argc, const char *argv[])
/* Initialize indexing options */
h5trav_set_index(sort_by, sort_order);
+ if (driver != NULL) {
+ void *conf_fa = NULL;
+
+ if (!strcmp(driver, "ros3")) {
+#ifndef H5_HAVE_ROS3_VFD
+ error_msg("Read-Only S3 VFD not enabled.\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+#else
+ conf_fa = (void *)&ros3_fa;
+#endif /* H5_HAVE_ROS3_VFD */
+ } else if (!HDstrcmp(driver, "hdfs")) {
+#ifndef H5_HAVE_LIBHDFS
+ error_msg("HDFS VFD is not enabled.\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+#else
+ conf_fa = (void *)&hdfs_fa;
+#endif /* H5_HAVE_LIBHDFS */
+ }
+
+ if (conf_fa != NULL) {
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ if (fapl_id < 0) {
+ error_msg("unable to create fapl entry\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ if (0 == h5tools_set_configured_fapl(
+ fapl_id,
+ driver, /* guaranteed "ros3" or "hdfs" */
+ conf_fa)) /* appropriate to driver */
+ {
+ error_msg("unable to set fapl\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ goto done;
+ }
+ }
+ } /* driver defined */
+
while(opt_ind < argc) {
fname = HDstrdup(argv[opt_ind++]);
- fid = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, driver, NULL, 0);
+ if (fapl_id != H5P_DEFAULT) {
+ fid = H5Fopen(fname, H5F_ACC_RDONLY, fapl_id);
+ } else {
+ fid = h5tools_fopen(
+ fname,
+ H5F_ACC_RDONLY,
+ H5P_DEFAULT,
+ driver,
+ NULL,
+ 0);
+ }
if (fid < 0) {
error_msg("unable to open file \"%s\"\n", fname);
@@ -1624,6 +1824,11 @@ done:
/* Free tables for objects */
table_list_free();
+ if (fapl_id != H5P_DEFAULT && 0 < H5Pclose(fapl_id)) {
+ error_msg("Can't close fapl entry\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ }
+
if(fid >=0)
if (H5Fclose(fid) < 0)
h5tools_setstatus(EXIT_FAILURE);
@@ -1645,127 +1850,7 @@ done:
H5Eset_auto2(H5E_DEFAULT, func, edata);
leave(h5tools_getstatus());
-}
-
-/*-------------------------------------------------------------------------
- * Function: h5_fileaccess
- *
- * Purpose: Returns a file access template which is the default template
- * but with a file driver set according to the constant or
- * environment variable HDF5_DRIVER
- *
- * Return: Success: A file access property list
- *
- * Failure: -1
- *
- * Programmer: Robb Matzke
- * Thursday, November 19, 1998
- *
- * Modifications:
- *
- *-------------------------------------------------------------------------
- */
-hid_t
-h5_fileaccess(void)
-{
- static const char *multi_letters = "msbrglo";
- const char *val = NULL;
- const char *name;
- char s[1024];
- hid_t fapl = -1;
-
- /* First use the environment variable, then the constant */
- val = HDgetenv("HDF5_DRIVER");
-#ifdef HDF5_DRIVER
- if (!val) val = HDF5_DRIVER;
-#endif
-
- if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) return -1;
- if (!val || !*val) return fapl; /*use default*/
-
- HDstrncpy(s, val, sizeof s);
- s[sizeof(s)-1] = '\0';
- if (NULL==(name=HDstrtok(s, " \t\n\r"))) return fapl;
-
- if (!HDstrcmp(name, "sec2")) {
- /* Unix read() and write() system calls */
- if (H5Pset_fapl_sec2(fapl)<0) return -1;
- }
- else if (!HDstrcmp(name, "stdio")) {
- /* Standard C fread() and fwrite() system calls */
- if (H5Pset_fapl_stdio(fapl)<0) return -1;
- }
- else if (!HDstrcmp(name, "core")) {
- /* In-core temporary file with 1MB increment */
- if (H5Pset_fapl_core(fapl, 1024*1024, FALSE)<0) return -1;
- }
- else if (!HDstrcmp(name, "split")) {
- /* Split meta data and raw data each using default driver */
- if (H5Pset_fapl_split(fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0)
- return -1;
- }
- else if (!HDstrcmp(name, "multi")) {
- /* Multi-file driver, general case of the split driver */
- H5FD_mem_t memb_map[H5FD_MEM_NTYPES];
- hid_t memb_fapl[H5FD_MEM_NTYPES];
- const char *memb_name[H5FD_MEM_NTYPES];
- char sv[H5FD_MEM_NTYPES][1024];
- haddr_t memb_addr[H5FD_MEM_NTYPES];
- H5FD_mem_t mt;
-
- HDmemset(memb_map, 0, sizeof memb_map);
- HDmemset(memb_fapl, 0, sizeof memb_fapl);
- HDmemset(memb_name, 0, sizeof memb_name);
- HDmemset(memb_addr, 0, sizeof memb_addr);
-
- if(HDstrlen(multi_letters)==H5FD_MEM_NTYPES) {
- for (mt=H5FD_MEM_DEFAULT; mt<H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t,mt)) {
- memb_fapl[mt] = H5P_DEFAULT;
- memb_map[mt] = mt;
- sprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]);
- memb_name[mt] = sv[mt];
- memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10);
- }
- }
- else {
- error_msg("Bad multi_letters list\n");
- return FAIL;
- }
-
- if (H5Pset_fapl_multi(fapl, memb_map, memb_fapl, memb_name, memb_addr, FALSE) < 0)
- return -1;
- }
- else if (!HDstrcmp(name, "family")) {
- hsize_t fam_size = 100*1024*1024; /*100 MB*/
-
- /* Family of files, each 1MB and using the default driver */
- if ((val=HDstrtok(NULL, " \t\n\r")))
- fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024);
- if (H5Pset_fapl_family(fapl, fam_size, H5P_DEFAULT)<0)
- return -1;
- }
- else if (!HDstrcmp(name, "log")) {
- long log_flags = H5FD_LOG_LOC_IO;
-
- /* Log file access */
- if ((val = HDstrtok(NULL, " \t\n\r")))
- log_flags = HDstrtol(val, NULL, 0);
-
- if (H5Pset_fapl_log(fapl, NULL, (unsigned)log_flags, 0) < 0)
- return -1;
- }
- else if (!HDstrcmp(name, "direct")) {
- /* Substitute Direct I/O driver with sec2 driver temporarily because
- * some output has sec2 driver as the standard. */
- if (H5Pset_fapl_sec2(fapl)<0) return -1;
- }
- else {
- /* Unknown driver */
- return -1;
- }
-
- return fapl;
-}
+} /* main */
/*-------------------------------------------------------------------------
@@ -1813,3 +1898,4 @@ add_prefix(char **prfx, size_t *prfx_len, const char *name)
HDstrcat(HDstrcat(*prfx, "/"), name);
} /* end add_prefix */
+
diff --git a/tools/src/h5ls/h5ls.c b/tools/src/h5ls/h5ls.c
index 4bc1526..c81da1e 100644
--- a/tools/src/h5ls/h5ls.c
+++ b/tools/src/h5ls/h5ls.c
@@ -158,7 +158,7 @@ static hbool_t print_int_type(h5tools_str_t *buffer, hid_t type, int ind);
static hbool_t print_float_type(h5tools_str_t *buffer, hid_t type, int ind);
static herr_t visit_obj(hid_t file, const char *oname, iter_t *iter);
-
+
/*-------------------------------------------------------------------------
* Function: usage
*
@@ -216,6 +216,15 @@ usage (void)
PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n");
PRINTVALSTREAM(rawoutstream, " --vfd=DRIVER Use the specified virtual file driver\n");
PRINTVALSTREAM(rawoutstream, " -x, --hexdump Show raw data in hexadecimal format\n");
+ PRINTVALSTREAM(rawoutstream, " --s3-cred=C Supply S3 authentication information to \"ros3\" vfd.\n");
+ PRINTVALSTREAM(rawoutstream, " Accepts tuple of \"(<aws-region>,<access-id>,<access-key>)\".\n");
+ PRINTVALSTREAM(rawoutstream, " If absent or C->\"(,,)\", defaults to no-authentication.\n");
+ PRINTVALSTREAM(rawoutstream, " Has no effect if vfd flag not set to \"ros3\".\n");
+ PRINTVALSTREAM(rawoutstream, " --hdfs-attrs=A Supply configuration information to Hadoop VFD.\n");
+ PRINTVALSTREAM(rawoutstream, " Accepts tuple of (<namenode name>,<namenode port>,\n");
+ PRINTVALSTREAM(rawoutstream, " ...<kerberos cache path>,<username>,<buffer size>)\n");
+ PRINTVALSTREAM(rawoutstream, " If absent or A == '(,,,,)', all default values are used.\n");
+ PRINTVALSTREAM(rawoutstream, " Has no effect if vfd flag is not 'hdfs'.\n");
PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, " file/OBJECT\n");
PRINTVALSTREAM(rawoutstream, " Each object consists of an HDF5 file name optionally followed by a\n");
@@ -237,7 +246,7 @@ usage (void)
PRINTVALSTREAM(rawoutstream, " Replaced by --enable-error-stack.\n");
}
-
+
/*-------------------------------------------------------------------------
* Function: print_string
@@ -315,7 +324,7 @@ print_string(h5tools_str_t *buffer, const char *s, hbool_t escape_spaces)
return nprint;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_obj_name
*
@@ -364,7 +373,7 @@ print_obj_name(h5tools_str_t *buffer, const iter_t *iter, const char *oname,
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_native_type
*
@@ -489,7 +498,7 @@ print_native_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_ieee_type
*
@@ -527,7 +536,7 @@ print_ieee_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_precision
*
@@ -619,7 +628,7 @@ print_precision(h5tools_str_t *buffer, hid_t type, int ind)
}
}
-
+
/*-------------------------------------------------------------------------
* Function: print_int_type
*
@@ -693,7 +702,7 @@ print_int_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_float_type
*
@@ -807,7 +816,7 @@ print_float_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_cmpd_type
*
@@ -860,7 +869,7 @@ print_cmpd_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_enum_type
*
@@ -985,7 +994,7 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_string_type
*
@@ -1086,7 +1095,7 @@ print_string_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_reference_type
*
@@ -1124,7 +1133,7 @@ print_reference_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_opaque_type
*
@@ -1160,7 +1169,7 @@ print_opaque_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_vlen_type
*
@@ -1190,7 +1199,7 @@ print_vlen_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*---------------------------------------------------------------------------
* Purpose: Print information about an array type
*
@@ -1237,7 +1246,7 @@ print_array_type(h5tools_str_t *buffer, hid_t type, int ind)
return TRUE;
}
-
+
/*-------------------------------------------------------------------------
* Function: print_bitfield_type
*
@@ -1345,7 +1354,7 @@ print_type(h5tools_str_t *buffer, hid_t type, int ind)
(unsigned long)H5Tget_size(type), (unsigned)data_class);
}
-
+
/*-------------------------------------------------------------------------
* Function: dump_dataset_values
*
@@ -1475,7 +1484,7 @@ dump_dataset_values(hid_t dset)
PRINTVALSTREAM(rawoutstream, "\n");
}
-
+
/*-------------------------------------------------------------------------
* Function: list_attr
*
@@ -1662,7 +1671,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain
return 0;
}
-
+
/*-------------------------------------------------------------------------
* Function: dataset_list1
*
@@ -1727,7 +1736,7 @@ dataset_list1(hid_t dset)
return 0;
}
-
+
/*-------------------------------------------------------------------------
* Function: dataset_list2
*
@@ -1962,7 +1971,7 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name)
return 0;
} /* end dataset_list2() */
-
+
/*-------------------------------------------------------------------------
* Function: datatype_list2
*
@@ -2004,7 +2013,7 @@ datatype_list2(hid_t type, const char H5_ATTR_UNUSED *name)
return 0;
}
-
+
/*-------------------------------------------------------------------------
* Function: list_obj
*
@@ -2160,7 +2169,7 @@ done:
} /* end list_obj() */
-
+
/*-------------------------------------------------------------------------
* Function: list_lnk
*
@@ -2354,7 +2363,7 @@ done:
return 0;
} /* end list_lnk() */
-
+
/*-------------------------------------------------------------------------
* Function: visit_obj
*
@@ -2434,7 +2443,7 @@ done:
return retval;
}
-
+
/*-------------------------------------------------------------------------
* Function: get_width
*
@@ -2550,7 +2559,7 @@ out:
return ret;
}
-
+
/*-------------------------------------------------------------------------
* Function: leave
*
@@ -2573,7 +2582,7 @@ leave(int ret)
HDexit(ret);
}
-
+
/*-------------------------------------------------------------------------
* Function: main
*
@@ -2602,6 +2611,26 @@ main(int argc, const char *argv[])
char drivername[50];
const char *preferred_driver = NULL;
int err_exit = 0;
+ hid_t fapl_id = H5P_DEFAULT;
+
+ /* default "anonymous" s3 configuration */
+ H5FD_ros3_fapl_t ros3_fa = {
+ 1, /* fapl version */
+ false, /* authenticate */
+ "", /* aws region */
+ "", /* access key id */
+ "", /* secret access key */
+ };
+
+ /* "default" HDFS configuration */
+ H5FD_hdfs_fapl_t hdfs_fa = {
+ 1, /* fapl version */
+ "localhost", /* namenode name */
+ 0, /* namenode port */
+ "", /* kerberos ticket cache */
+ "", /* user name */
+ 2048, /* stream buffer size */
+ };
h5tools_setprogname(PROGRAMNAME);
h5tools_setstatus(EXIT_SUCCESS);
@@ -2701,6 +2730,185 @@ main(int argc, const char *argv[])
usage();
leave(EXIT_FAILURE);
}
+
+ } else if (!HDstrncmp(argv[argno], "--s3-cred=", (size_t)10)) {
+#ifndef H5_HAVE_ROS3_VFD
+ HDfprintf(rawerrorstream,
+ "Error: Read-Only S3 VFD is not enabled\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+#else
+ unsigned nelems = 0;
+ char *start = NULL;
+ char *s3cred_src = NULL;
+ char **s3cred = NULL;
+ char const *ccred[3];
+ /* try to parse s3 credentials tuple
+ */
+ start = strchr(argv[argno], '=');
+ if (start == NULL) {
+ HDfprintf(rawerrorstream,
+ "Error: Unable to parse null credentials tuple\n"
+ " For anonymous access, omit \"--s3-cred\" and use"
+ "only \"--vfd=ros3\"\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ start++;
+ if (FAIL ==
+ parse_tuple((const char *)start, ',',
+ &s3cred_src, &nelems, &s3cred))
+ {
+ HDfprintf(rawerrorstream,
+ "Error: Unable to parse S3 credentials\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ /* sanity-check tuple count
+ */
+ if (nelems != 3) {
+ HDfprintf(rawerrorstream,
+ "Error: Invalid S3 credentials\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ ccred[0] = (const char *)s3cred[0];
+ ccred[1] = (const char *)s3cred[1];
+ ccred[2] = (const char *)s3cred[2];
+ if (0 == h5tools_populate_ros3_fapl(&ros3_fa, ccred)) {
+ HDfprintf(rawerrorstream,
+ "Error: Invalid S3 credentials\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ HDfree(s3cred);
+ HDfree(s3cred_src);
+#endif /* H5_HAVE_ROS3_VFD */
+
+ } else if (!HDstrncmp(argv[argno], "--hdfs-attrs=", (size_t)13)) {
+#ifndef H5_HAVE_LIBHDFS
+ PRINTVALSTREAM(rawoutstream, "The HDFS VFD is not enabled.\n");
+ leave(EXIT_FAILURE);
+#else
+ /* Parse received configuration data and set fapl config struct
+ */
+
+ hbool_t _debug = FALSE;
+ unsigned nelems = 0;
+ char const *start = NULL;
+ char *props_src = NULL;
+ char **props = NULL;
+ unsigned long k = 0;
+
+ /* try to parse tuple
+ */
+ if (_debug) {
+ HDfprintf(stderr, "configuring hdfs...\n");
+ }
+ start = argv[argno]+13; /* should never segfault: worst case of */
+ if (*start != '(') /* null-termintor after '='. */
+ {
+ if (_debug) {
+ HDfprintf(stderr, " no tuple.\n");
+ }
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ if (FAIL ==
+ parse_tuple((const char *)start, ',',
+ &props_src, &nelems, &props))
+ {
+ HDfprintf(stderr,
+ " unable to parse tuple.\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+
+ /* sanity-check tuple count
+ */
+ if (nelems != 5) {
+ HDfprintf(stderr,
+ " expected 5-ple, got `%d`\n",
+ nelems);
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ if (_debug) {
+ HDfprintf(stderr,
+ " got hdfs-attrs tuple: `(%s,%s,%s,%s,%s)`\n",
+ props[0],
+ props[1],
+ props[2],
+ props[3],
+ props[4]);
+ }
+
+ /* Populate fapl configuration structure with given properties.
+ * WARNING: No error-checking is done on length of input strings...
+ * Silent overflow is possible, albeit unlikely.
+ */
+ if (HDstrncmp(props[0], "", 1)) {
+ if (_debug) {
+ HDfprintf(stderr,
+ " setting namenode name: %s\n",
+ props[0]);
+ }
+ HDstrncpy(hdfs_fa.namenode_name,
+ (const char *)props[0],
+ HDstrlen(props[0]));
+ }
+ if (HDstrncmp(props[1], "", 1)) {
+ k = strtoul((const char *)props[1], NULL, 0);
+ if (errno == ERANGE) {
+ HDfprintf(stderr,
+ " supposed port number wasn't.\n");
+ leave(EXIT_FAILURE);
+ }
+ if (_debug) {
+ HDfprintf(stderr,
+ " setting namenode port: %lu\n",
+ k);
+ }
+ hdfs_fa.namenode_port = (int32_t)k;
+ }
+ if (HDstrncmp(props[2], "", 1)) {
+ if (_debug) {
+ HDfprintf(stderr,
+ " setting kerb cache path: %s\n",
+ props[2]);
+ }
+ HDstrncpy(hdfs_fa.kerberos_ticket_cache,
+ (const char *)props[2],
+ HDstrlen(props[2]));
+ }
+ if (HDstrncmp(props[3], "", 1)) {
+ if (_debug) {
+ HDfprintf(stderr,
+ " setting username: %s\n",
+ props[3]);
+ }
+ HDstrncpy(hdfs_fa.user_name,
+ (const char *)props[3],
+ HDstrlen(props[3]));
+ }
+ if (HDstrncmp(props[4], "", 1)) {
+ k = HDstrtoul((const char *)props[4], NULL, 0);
+ if (errno == ERANGE) {
+ HDfprintf(stderr,
+ " supposed buffersize number wasn't.\n");
+ leave(EXIT_FAILURE);
+ }
+ if (_debug) {
+ HDfprintf(stderr,
+ " setting stream buffer size: %lu\n",
+ k);
+ }
+ hdfs_fa.stream_buffer_size = (int32_t)k;
+ }
+ HDfree(props);
+ HDfree(props_src);
+#endif /* H5_HAVE_LIBHDFS */
+
} else if('-'!=argv[argno][1]) {
/* Single-letter switches */
for(s = argv[argno] + 1; *s; s++) {
@@ -2772,6 +2980,7 @@ main(int argc, const char *argv[])
} /* end switch */
} /* end for */
} else {
+ HDfprintf(stderr, "Unknown argument: %s\n", argv[argno]);
usage();
leave(EXIT_FAILURE);
}
@@ -2791,6 +3000,49 @@ main(int argc, const char *argv[])
leave(EXIT_FAILURE);
}
+ if (preferred_driver) {
+ void *conf_fa = NULL;
+
+ if (!HDstrcmp(preferred_driver, "ros3")) {
+#ifndef H5_HAVE_ROS3_VFD
+ HDfprintf(rawerrorstream,
+ "Error: Read-Only S3 VFD not enabled.\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+#else
+ conf_fa = (void *)&ros3_fa;
+#endif /* H5_HAVE_ROS3_VFD */
+
+ } else if (!HDstrcmp(preferred_driver, "hdfs")) {
+#ifndef H5_HAVE_LIBHDFS
+ PRINTVALSTREAM(rawoutstream, "The HDFS VFD is not enabled.\n");
+ leave(EXIT_FAILURE);
+#else
+ conf_fa = (void *)&hdfs_fa;
+#endif /* H5_HAVE_LIBHDFS */
+ }
+
+ if (conf_fa != NULL) {
+ HDassert(fapl_id == H5P_DEFAULT);
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ if (fapl_id < 0) {
+ HDfprintf(rawerrorstream,
+ "Error: Unable to create fapl entry\n\n");
+ leave(EXIT_FAILURE);
+ }
+ if (0 == h5tools_set_configured_fapl(
+ fapl_id,
+ preferred_driver,
+ conf_fa))
+ {
+ HDfprintf(rawerrorstream,
+ "Error: Unable to set fapl\n\n");
+ usage();
+ leave(EXIT_FAILURE);
+ }
+ }
+ } /* preferred_driver defined */
+
/* Turn off HDF5's automatic error printing unless you're debugging h5ls */
if(!show_errors_g)
H5Eset_auto2(H5E_DEFAULT, NULL, NULL);
@@ -2820,7 +3072,12 @@ main(int argc, const char *argv[])
file = -1;
while(fname && *fname) {
- file = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, preferred_driver, drivername, sizeof drivername);
+ if (fapl_id != H5P_DEFAULT) {
+ file = H5Fopen(fname, H5F_ACC_RDONLY, fapl_id);
+ }
+ else {
+ file = h5tools_fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT, preferred_driver, drivername, sizeof drivername);
+ }
if(file >= 0) {
if(verbose_g)
@@ -2933,6 +3190,14 @@ main(int argc, const char *argv[])
err_exit = 1;
} /* end while */
+ if (fapl_id != H5P_DEFAULT) {
+ if (0 < H5Pclose(fapl_id)) {
+ HDfprintf(rawerrorstream,
+ "Error: Unable to set close fapl entry\n\n");
+ leave(EXIT_FAILURE);
+ }
+ }
+
if (err_exit)
leave(EXIT_FAILURE);
else
diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c
index 450f731..59038a0 100644
--- a/tools/src/h5stat/h5stat.c
+++ b/tools/src/h5stat/h5stat.c
@@ -74,7 +74,7 @@ typedef struct iter_t {
ohdr_info_t group_ohdr_info; /* Object header information for groups */
hsize_t max_attrs; /* Maximum attributes from a group */
- unsigned long *num_small_attrs; /* Size of small attributes tracked */
+ unsigned long *num_small_attrs; /* Size of small attributes tracked */
unsigned attr_nbins; /* Number of bins for attribute counts */
unsigned long *attr_bins; /* Pointer to array of bins for attribute counts */
@@ -118,6 +118,29 @@ typedef struct iter_t {
} iter_t;
+static const char *drivername = "";
+
+/* default "anonymous" s3 configuration
+ */
+static H5FD_ros3_fapl_t ros3_fa = {
+ 1, /* fapl version */
+ false, /* authenticate */
+ "", /* aws region */
+ "", /* access key id */
+ "", /* secret access key */
+};
+
+/* default HDFS access configuration
+ */
+static H5FD_hdfs_fapl_t hdfs_fa = {
+ 1, /* fapl version */
+ "localhost", /* namenode name */
+ 0, /* namenode port */
+ "", /* kerberos ticket cache */
+ "", /* user name */
+ 2048, /* stream buffer size */
+};
+
static int display_all = TRUE;
/* Enable the printing of selected statistics */
@@ -146,7 +169,7 @@ struct handler_t {
char **obj;
};
-static const char *s_opts ="Aa:Ddm:EFfhGgl:sSTO:V";
+static const char *s_opts ="Aa:Ddm:EFfhGgl:sSTO:Vw:";
/* e.g. "filemetadata" has to precede "file"; "groupmetadata" has to precede "group" etc. */
static struct long_options l_opts[] = {
{"help", no_arg, 'h'},
@@ -246,6 +269,8 @@ static struct long_options l_opts[] = {
{ "summ", no_arg, 'S' },
{ "sum", no_arg, 'S' },
{ "su", no_arg, 'S' },
+ { "s3-cred", require_arg, 'w' },
+ { "hdfs-attrs", require_arg, 'H' },
{ NULL, 0, '\0' }
};
@@ -257,7 +282,7 @@ leave(int ret)
}
-
+
/*-------------------------------------------------------------------------
* Function: usage
*
@@ -295,9 +320,19 @@ static void usage(const char *prog)
HDfprintf(stdout, " -s, --freespace Print free space information\n");
HDfprintf(stdout, " -S, --summary Print summary of file space information\n");
HDfprintf(stdout, " --enable-error-stack Prints messages from the HDF5 error stack as they occur\n");
+ HDfprintf(stdout, " --s3-cred=<cred> Access file on S3, using provided credential\n");
+ HDfprintf(stdout, " <cred> :: (region,id,key)\n");
+ HDfprintf(stdout, " If <cred> == \"(,,)\", no authentication is used.\n");
+ HDfprintf(stdout, " --hdfs-attrs=<attrs> Access a file on HDFS with given configuration\n");
+ HDfprintf(stdout, " attributes.\n");
+ HDfprintf(stdout, " <attrs> :: (<namenode name>,<namenode port>,\n");
+ HDfprintf(stdout, " <kerberos cache path>,<username>,\n");
+ HDfprintf(stdout, " <buffer size>)\n");
+ HDfprintf(stdout, " If an attribute is empty, a default value will be\n");
+ HDfprintf(stdout, " used.\n");
}
-
+
/*-------------------------------------------------------------------------
* Function: ceil_log10
*
@@ -324,7 +359,7 @@ ceil_log10(unsigned long x)
return ret;
} /* ceil_log10() */
-
+
/*-------------------------------------------------------------------------
* Function: attribute_stats
*
@@ -374,7 +409,7 @@ attribute_stats(iter_t *iter, const H5O_info_t *oi)
return 0;
} /* end attribute_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: group_stats
*
@@ -456,7 +491,7 @@ done:
return ret_value;
} /* end group_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: dataset_stats
*
@@ -647,7 +682,7 @@ done:
return ret_value;
} /* end dataset_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: datatype_stats
*
@@ -679,7 +714,7 @@ done:
return ret_value;
} /* end datatype_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: obj_stats
*
@@ -735,7 +770,7 @@ done:
return ret_value;
} /* end obj_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: lnk_stats
*
@@ -833,7 +868,7 @@ freespace_stats(hid_t fid, iter_t *iter)
return 0;
} /* end freespace_stats() */
-
+
/*-------------------------------------------------------------------------
* Function: hand_free
*
@@ -862,7 +897,7 @@ hand_free(struct handler_t *hand)
} /* end if */
} /* end hand_free() */
-
+
/*-------------------------------------------------------------------------
* Function: parse_command_line
*
@@ -1014,6 +1049,119 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret)
} /* end if */
break;
+ case 'w':
+#ifndef H5_HAVE_ROS3_VFD
+ error_msg("Read-Only S3 VFD not enabled.\n");
+ goto error;
+#else
+ {
+ char *cred_str = NULL;
+ unsigned nelems = 0;
+ char **cred = NULL;
+ char const *ccred[3];
+
+ if (FAIL == parse_tuple((const char *)opt_arg, ',',
+ &cred_str, &nelems, &cred)) {
+ error_msg("Unable to parse s3 credential\n");
+ goto error;
+ }
+ if (nelems != 3) {
+ error_msg("s3 credential must have three elements\n");
+ goto error;
+ }
+ ccred[0] = (const char *)cred[0];
+ ccred[1] = (const char *)cred[1];
+ ccred[2] = (const char *)cred[2];
+ if (0 ==
+ h5tools_populate_ros3_fapl(&ros3_fa, ccred))
+ {
+ error_msg("Unable to set ros3 fapl config\n");
+ goto error;
+ }
+ HDfree(cred);
+ HDfree(cred_str);
+ } /* parse s3-cred block */
+ drivername = "ros3";
+ break;
+#endif /* H5_HAVE_ROS3_VFD */
+
+ case 'H':
+#ifndef H5_HAVE_LIBHDFS
+ error_msg("HDFS VFD is not enabled.\n");
+ goto error;
+#else
+ {
+ unsigned nelems = 0;
+ char *props_src = NULL;
+ char **props = NULL;
+ unsigned long k = 0;
+ if (FAIL == parse_tuple(
+ (const char *)opt_arg,
+ ',',
+ &props_src,
+ &nelems,
+ &props))
+ {
+ error_msg("unable to parse hdfs properties tuple\n");
+ goto error;
+ }
+ /* sanity-check tuple count
+ */
+ if (nelems != 5) {
+ char str[64] = "";
+ sprintf(str,
+ "expected 5 elements in hdfs properties tuple "
+ "but found %u\n",
+ nelems);
+ HDfree(props);
+ HDfree(props_src);
+ error_msg(str);
+ goto error;
+ }
+ /* Populate fapl configuration structure with given
+ * properties.
+ * TODO/WARNING: No error-checking is done on length of
+ * input strings... Silent overflow is possible,
+ * albeit unlikely.
+ */
+ if (strncmp(props[0], "", 1)) {
+ HDstrncpy(hdfs_fa.namenode_name,
+ (const char *)props[0],
+ HDstrlen(props[0]));
+ }
+ if (strncmp(props[1], "", 1)) {
+ k = strtoul((const char *)props[1], NULL, 0);
+ if (errno == ERANGE) {
+ error_msg("supposed port number wasn't.\n");
+ goto error;
+ }
+ hdfs_fa.namenode_port = (int32_t)k;
+ }
+ if (strncmp(props[2], "", 1)) {
+ HDstrncpy(hdfs_fa.kerberos_ticket_cache,
+ (const char *)props[2],
+ HDstrlen(props[2]));
+ }
+ if (strncmp(props[3], "", 1)) {
+ HDstrncpy(hdfs_fa.user_name,
+ (const char *)props[3],
+ HDstrlen(props[3]));
+ }
+ if (strncmp(props[4], "", 1)) {
+ k = strtoul((const char *)props[4], NULL, 0);
+ if (errno == ERANGE) {
+ error_msg("supposed buffersize number wasn't.\n");
+ goto error;
+ }
+ hdfs_fa.stream_buffer_size = (int32_t)k;
+ }
+ HDfree(props);
+ HDfree(props_src);
+ drivername = "hdfs";
+ }
+ break;
+#endif /* H5_HAVE_LIBHDFS */
+
default:
usage(h5tools_getprogname());
goto error;
@@ -1040,7 +1188,7 @@ error:
return -1;
}
-
+
/*-------------------------------------------------------------------------
* Function: iter_free
*
@@ -1105,7 +1253,7 @@ iter_free(iter_t *iter)
} /* end if */
} /* end iter_free() */
-
+
/*-------------------------------------------------------------------------
* Function: print_file_info
*
@@ -1137,7 +1285,7 @@ print_file_info(const iter_t *iter)
return 0;
} /* print_file_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_file_metadata
*
@@ -1197,7 +1345,7 @@ print_file_metadata(const iter_t *iter)
return 0;
} /* print_file_metadata() */
-
+
/*-------------------------------------------------------------------------
* Function: print_group_info
*
@@ -1254,7 +1402,7 @@ print_group_info(const iter_t *iter)
return 0;
} /* print_group_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_group_metadata
*
@@ -1281,7 +1429,7 @@ print_group_metadata(const iter_t *iter)
return 0;
} /* print_group_metadata() */
-
+
/*-------------------------------------------------------------------------
* Function: print_dataset_info
*
@@ -1368,7 +1516,7 @@ print_dataset_info(const iter_t *iter)
return 0;
} /* print_dataset_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_dataset_metadata
*
@@ -1397,7 +1545,7 @@ print_dset_metadata(const iter_t *iter)
return 0;
} /* print_dset_metadata() */
-
+
/*-------------------------------------------------------------------------
* Function: print_dset_dtype_meta
*
@@ -1438,7 +1586,7 @@ print_dset_dtype_meta(const iter_t *iter)
return 0;
} /* print_dset_dtype_meta() */
-
+
/*-------------------------------------------------------------------------
* Function: print_attr_info
*
@@ -1487,7 +1635,7 @@ print_attr_info(const iter_t *iter)
return 0;
} /* print_attr_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_freespace_info
*
@@ -1537,7 +1685,7 @@ print_freespace_info(const iter_t *iter)
return 0;
} /* print_freespace_info() */
-
+
/*-------------------------------------------------------------------------
* Function: print_storage_summary
*
@@ -1601,7 +1749,7 @@ print_storage_summary(const iter_t *iter)
return 0;
} /* print_storage_summary() */
-
+
/*-------------------------------------------------------------------------
* Function: print_file_statistics
*
@@ -1648,7 +1796,7 @@ print_file_statistics(const iter_t *iter)
if(display_summary) print_storage_summary(iter);
} /* print_file_statistics() */
-
+
/*-------------------------------------------------------------------------
* Function: print_object_statistics
*
@@ -1671,7 +1819,7 @@ print_object_statistics(const char *name)
printf("Object name %s\n", name);
} /* print_object_statistics() */
-
+
/*-------------------------------------------------------------------------
* Function: print_statistics
*
@@ -1697,7 +1845,7 @@ print_statistics(const char *name, const iter_t *iter)
print_file_statistics(iter);
} /* print_statistics() */
-
+
/*-------------------------------------------------------------------------
* Function: main
*
@@ -1718,6 +1866,7 @@ main(int argc, const char *argv[])
void *edata;
void *tools_edata;
struct handler_t *hand = NULL;
+ hid_t fapl_id = H5P_DEFAULT;
h5tools_setprogname(PROGRAMNAME);
h5tools_setstatus(EXIT_SUCCESS);
@@ -1738,6 +1887,45 @@ main(int argc, const char *argv[])
if(parse_command_line(argc, argv, &hand) < 0)
goto done;
+ /* if drivername is not null, probably need to set the fapl */
+ if (HDstrcmp(drivername, "")) {
+ void *conf_fa = NULL;
+
+ if (!HDstrcmp(drivername, "ros3")) {
+#ifndef H5_HAVE_ROS3_VFD
+ error_msg("Read-Only S3 VFD not enabled.\n\n");
+ goto done;
+#else
+ conf_fa = (void *)&ros3_fa;
+#endif /* H5_HAVE_ROS3_VFD */
+
+ } else if (!HDstrcmp(drivername, "hdfs")) {
+#ifndef H5_HAVE_LIBHDFS
+ error_msg("HDFS VFD not enabled.\n\n");
+ goto done;
+#else
+ conf_fa = (void *)&hdfs_fa;
+#endif /* H5_HAVE_LIBHDFS */
+ }
+
+ if (conf_fa != NULL) {
+ HDassert(fapl_id == H5P_DEFAULT);
+ fapl_id = H5Pcreate(H5P_FILE_ACCESS);
+ if (fapl_id < 0) {
+ error_msg("Unable to create fapl entry\n");
+ goto done;
+ }
+ if (1 > h5tools_set_configured_fapl(
+ fapl_id,
+ drivername,
+ conf_fa))
+ {
+ error_msg("Unable to set fapl\n");
+ goto done;
+ }
+ }
+ } /* drivername set */
+
fname = argv[opt_ind];
if(enable_error_stack > 0) {
@@ -1752,7 +1940,7 @@ main(int argc, const char *argv[])
printf("Filename: %s\n", fname);
- fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT);
+ fid = H5Fopen(fname, H5F_ACC_RDONLY, fapl_id);
if(fid < 0) {
error_msg("unable to open file \"%s\"\n", fname);
h5tools_setstatus(EXIT_FAILURE);
@@ -1833,6 +2021,13 @@ done:
/* Free iter structure */
iter_free(&iter);
+ if (fapl_id != H5P_DEFAULT) {
+ if (0 < H5Pclose(fapl_id)) {
+ error_msg("unable to close fapl entry\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ }
+ }
+
if(fid >= 0 && H5Fclose(fid) < 0) {
error_msg("unable to close file \"%s\"\n", fname);
h5tools_setstatus(EXIT_FAILURE);
generated by cgit v0.12 at 2024-11-09 09:43:08 (GMT)