summaryrefslogtreecommitdiffstats
path: root/tools/lib
diff options
context:
space:
mode:
authorjhendersonHDF <jhenderson@hdfgroup.org>2021-09-29 18:28:12 (GMT)
committerGitHub <noreply@github.com>2021-09-29 18:28:12 (GMT)
commit3da0802c40d58759995916bf9d0880e19f0af44d (patch)
tree809ada78cec1cbaaf6ec2ace5b4429a56d0f6574 /tools/lib
parent0fa5836cc5f037dd9f2cdd7f9a1051ddcc1c9ad0 (diff)
downloadhdf5-3da0802c40d58759995916bf9d0880e19f0af44d.zip
hdf5-3da0802c40d58759995916bf9d0880e19f0af44d.tar.gz
hdf5-3da0802c40d58759995916bf9d0880e19f0af44d.tar.bz2
VFD plugins (#602)
* Implement support for loading of Virtual File Drivers as plugins Fix plugin caching for VOL connector and VFD plugins Fix plugin iteration to skip paths that can't be opened * Enable dynamic loading of VFDs with HDF5_DRIVER environment variable * Temporarily disable error reporting during H5F_open double file open * Default to using HDstat in h5_get_file_size for unknown VFDs * Use macros for some environment variables that HDF5 interprets * Update "null" and "ctl testing" VFDs
Diffstat (limited to 'tools/lib')
-rw-r--r--tools/lib/h5diff.c16
-rw-r--r--tools/lib/h5diff.h2
-rw-r--r--tools/lib/h5tools.c219
-rw-r--r--tools/lib/h5tools.h9
4 files changed, 152 insertions, 94 deletions
diff --git a/tools/lib/h5diff.c b/tools/lib/h5diff.c
index 0ad319e..74db58f 100644
--- a/tools/lib/h5diff.c
+++ b/tools/lib/h5diff.c
@@ -648,14 +648,16 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char
*-------------------------------------------------------------------------
*/
/* open file 1 */
- if (opts->custom_vol[0]) {
- if ((fapl1_id = h5tools_get_fapl(H5P_DEFAULT, &(opts->vol_info[0]), NULL)) < 0) {
+ if (opts->custom_vol[0] || opts->custom_vfd[0]) {
+ if ((fapl1_id = h5tools_get_fapl(H5P_DEFAULT, opts->custom_vol[0] ? &(opts->vol_info[0]) : NULL,
+ opts->custom_vfd[0] ? &(opts->vfd_info[0]) : NULL)) < 0) {
parallel_print("h5diff: unable to create fapl for input file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create input fapl\n");
}
}
- if ((file1_id = h5tools_fopen(fname1, H5F_ACC_RDONLY, fapl1_id, FALSE, NULL, (size_t)0)) < 0) {
+ if ((file1_id = h5tools_fopen(fname1, H5F_ACC_RDONLY, fapl1_id, (fapl1_id != H5P_DEFAULT), NULL,
+ (size_t)0)) < 0) {
parallel_print("h5diff: <%s>: unable to open file\n", fname1);
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "<%s>: unable to open file\n", fname1);
}
@@ -663,14 +665,16 @@ h5diff(const char *fname1, const char *fname2, const char *objname1, const char
/* open file 2 */
- if (opts->custom_vol[1]) {
- if ((fapl2_id = h5tools_get_fapl(H5P_DEFAULT, &(opts->vol_info[1]), NULL)) < 0) {
+ if (opts->custom_vol[1] || opts->custom_vfd[1]) {
+ if ((fapl2_id = h5tools_get_fapl(H5P_DEFAULT, opts->custom_vol[1] ? &(opts->vol_info[1]) : NULL,
+ opts->custom_vfd[1] ? &(opts->vfd_info[1]) : NULL)) < 0) {
parallel_print("h5diff: unable to create fapl for output file\n");
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "unable to create output fapl\n");
}
}
- if ((file2_id = h5tools_fopen(fname2, H5F_ACC_RDONLY, fapl2_id, FALSE, NULL, (size_t)0)) < 0) {
+ if ((file2_id = h5tools_fopen(fname2, H5F_ACC_RDONLY, fapl2_id, (fapl2_id != H5P_DEFAULT), NULL,
+ (size_t)0)) < 0) {
parallel_print("h5diff: <%s>: unable to open file\n", fname2);
H5TOOLS_GOTO_ERROR(H5DIFF_ERR, "<%s>: unable to open file\n", fname2);
}
diff --git a/tools/lib/h5diff.h b/tools/lib/h5diff.h
index 7b41538..8d7ac13 100644
--- a/tools/lib/h5diff.h
+++ b/tools/lib/h5diff.h
@@ -89,7 +89,9 @@ typedef struct {
char * obj_name[2]; /* name for object */
struct subset_t * sset[2]; /* subsetting parameters */
h5tools_vol_info_t vol_info[2]; /* VOL information for input file, output file */
+ h5tools_vfd_info_t vfd_info[2]; /* VFD information for input file, output file */
hbool_t custom_vol[2]; /* Using a custom input, output VOL? */
+ hbool_t custom_vfd[2]; /* Using a custom input, output VFD? */
} diff_opt_t;
/*-------------------------------------------------------------------------
diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c
index eee9c53..4de2c5c 100644
--- a/tools/lib/h5tools.c
+++ b/tools/lib/h5tools.c
@@ -459,8 +459,8 @@ h5tools_set_error_file(const char *fname, int is_bin)
/*-------------------------------------------------------------------------
* Function: h5tools_set_fapl_vfd
*
- * Purpose: Given a VFL driver name, sets the appropriate driver on the
- * specified FAPL.
+ * Purpose: Given a VFL driver name or ID, sets the appropriate driver on
+ * the specified FAPL.
*
* Return: positive - succeeded
* negative - failed
@@ -471,107 +471,139 @@ h5tools_set_fapl_vfd(hid_t fapl_id, h5tools_vfd_info_t *vfd_info)
{
herr_t ret_value = SUCCEED;
- /* Determine which driver the user wants to open the file with */
- if (!HDstrcmp(vfd_info->name, drivernames[SEC2_VFD_IDX])) {
- /* SEC2 Driver */
- if (H5Pset_fapl_sec2(fapl_id) < 0)
- H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_sec2 failed");
- }
- else if (!HDstrcmp(vfd_info->name, drivernames[DIRECT_VFD_IDX])) {
+ switch (vfd_info->type) {
+ case VFD_BY_NAME:
+ /* Determine which driver the user wants to open the file with */
+ if (!HDstrcmp(vfd_info->u.name, drivernames[SEC2_VFD_IDX])) {
+ /* SEC2 Driver */
+ if (H5Pset_fapl_sec2(fapl_id) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_sec2 failed");
+ }
+ else if (!HDstrcmp(vfd_info->u.name, drivernames[DIRECT_VFD_IDX])) {
#ifdef H5_HAVE_DIRECT
- /* Direct Driver */
- if (H5Pset_fapl_direct(fapl_id, 1024, 4096, 8 * 4096) < 0)
- H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_direct failed");
+ /* Direct Driver */
+ if (H5Pset_fapl_direct(fapl_id, 1024, 4096, 8 * 4096) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_direct failed");
#else
- H5TOOLS_GOTO_ERROR(FAIL, "Direct VFD is not enabled");
+ H5TOOLS_GOTO_ERROR(FAIL, "Direct VFD is not enabled");
#endif
- }
- else if (!HDstrcmp(vfd_info->name, drivernames[LOG_VFD_IDX])) {
- unsigned long long log_flags = H5FD_LOG_LOC_IO | H5FD_LOG_ALLOC;
+ }
+ else if (!HDstrcmp(vfd_info->u.name, drivernames[LOG_VFD_IDX])) {
+ unsigned long long log_flags = H5FD_LOG_LOC_IO | H5FD_LOG_ALLOC;
- /* Log Driver */
- if (H5Pset_fapl_log(fapl_id, NULL, log_flags, (size_t)0) < 0)
- H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_log failed");
- }
- else if (!HDstrcmp(vfd_info->name, drivernames[WINDOWS_VFD_IDX])) {
+ /* Log Driver */
+ if (H5Pset_fapl_log(fapl_id, NULL, log_flags, (size_t)0) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_log failed");
+ }
+ else if (!HDstrcmp(vfd_info->u.name, drivernames[WINDOWS_VFD_IDX])) {
#ifdef H5_HAVE_WINDOWS
- /* There is no Windows VFD - use SEC2 */
- if (H5Pset_fapl_sec2(fapl_id) < 0)
- H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_sec2 failed");
+ /* There is no Windows VFD - use SEC2 */
+ if (H5Pset_fapl_sec2(fapl_id) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_sec2 failed");
#else
- H5TOOLS_GOTO_ERROR(FAIL, "Windows VFD is not enabled");
+ H5TOOLS_GOTO_ERROR(FAIL, "Windows VFD is not enabled");
#endif
- }
- else if (!HDstrcmp(vfd_info->name, drivernames[STDIO_VFD_IDX])) {
- /* Stdio Driver */
- if (H5Pset_fapl_stdio(fapl_id) < 0)
- H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_stdio failed");
- }
- else if (!HDstrcmp(vfd_info->name, drivernames[CORE_VFD_IDX])) {
- /* Core Driver */
- if (H5Pset_fapl_core(fapl_id, (size_t)H5_MB, TRUE) < 0)
- H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_core failed");
- }
- else if (!HDstrcmp(vfd_info->name, drivernames[FAMILY_VFD_IDX])) {
- /* FAMILY Driver */
- /* Set member size to be 0 to indicate the current first member size
- * is the member size.
- */
- if (H5Pset_fapl_family(fapl_id, (hsize_t)0, H5P_DEFAULT) < 0)
- H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_family failed");
- }
- else if (!HDstrcmp(vfd_info->name, drivernames[SPLIT_VFD_IDX])) {
- /* SPLIT Driver */
- if (H5Pset_fapl_split(fapl_id, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0)
- H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_split failed");
- }
- else if (!HDstrcmp(vfd_info->name, drivernames[MULTI_VFD_IDX])) {
- /* MULTI Driver */
- if (H5Pset_fapl_multi(fapl_id, NULL, NULL, NULL, NULL, TRUE) < 0)
- H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_multi failed");
- }
- else if (!HDstrcmp(vfd_info->name, drivernames[MPIO_VFD_IDX])) {
+ }
+ else if (!HDstrcmp(vfd_info->u.name, drivernames[STDIO_VFD_IDX])) {
+ /* Stdio Driver */
+ if (H5Pset_fapl_stdio(fapl_id) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_stdio failed");
+ }
+ else if (!HDstrcmp(vfd_info->u.name, drivernames[CORE_VFD_IDX])) {
+ /* Core Driver */
+ if (H5Pset_fapl_core(fapl_id, (size_t)H5_MB, TRUE) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_core failed");
+ }
+ else if (!HDstrcmp(vfd_info->u.name, drivernames[FAMILY_VFD_IDX])) {
+ /* FAMILY Driver */
+ /* Set member size to be 0 to indicate the current first member size
+ * is the member size.
+ */
+ if (H5Pset_fapl_family(fapl_id, (hsize_t)0, H5P_DEFAULT) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_family failed");
+ }
+ else if (!HDstrcmp(vfd_info->u.name, drivernames[SPLIT_VFD_IDX])) {
+ /* SPLIT Driver */
+ if (H5Pset_fapl_split(fapl_id, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_split failed");
+ }
+ else if (!HDstrcmp(vfd_info->u.name, drivernames[MULTI_VFD_IDX])) {
+ /* MULTI Driver */
+ if (H5Pset_fapl_multi(fapl_id, NULL, NULL, NULL, NULL, TRUE) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_multi failed");
+ }
+ else if (!HDstrcmp(vfd_info->u.name, drivernames[MPIO_VFD_IDX])) {
#ifdef H5_HAVE_PARALLEL
- int mpi_initialized, mpi_finalized;
+ int mpi_initialized, mpi_finalized;
- /* MPI-I/O Driver */
+ /* MPI-I/O Driver */
- /* check if MPI is available. */
- MPI_Initialized(&mpi_initialized);
- MPI_Finalized(&mpi_finalized);
+ /* check if MPI is available. */
+ MPI_Initialized(&mpi_initialized);
+ MPI_Finalized(&mpi_finalized);
- if (mpi_initialized && !mpi_finalized) {
- if (H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL) < 0)
- H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_mpio failed");
- }
+ if (mpi_initialized && !mpi_finalized) {
+ if (H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_mpio failed");
+ }
#else
- H5TOOLS_GOTO_ERROR(FAIL, "MPI-I/O VFD is not enabled");
+ H5TOOLS_GOTO_ERROR(FAIL, "MPI-I/O VFD is not enabled");
#endif /* H5_HAVE_PARALLEL */
- }
- else if (!HDstrcmp(vfd_info->name, drivernames[ROS3_VFD_IDX])) {
+ }
+ else if (!HDstrcmp(vfd_info->u.name, drivernames[ROS3_VFD_IDX])) {
#ifdef H5_HAVE_ROS3_VFD
- if (!vfd_info->info)
- H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD info is invalid");
- if (H5Pset_fapl_ros3(fapl_id, (H5FD_ros3_fapl_t *)vfd_info->info) < 0)
- H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3() failed");
+ if (!vfd_info->info)
+ H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD info is invalid");
+ if (H5Pset_fapl_ros3(fapl_id, (H5FD_ros3_fapl_t *)vfd_info->info) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_ros3() failed");
#else
- H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD is not enabled");
+ H5TOOLS_GOTO_ERROR(FAIL, "Read-only S3 VFD is not enabled");
#endif
- }
- else if (!HDstrcmp(vfd_info->name, drivernames[HDFS_VFD_IDX])) {
+ }
+ else if (!HDstrcmp(vfd_info->u.name, drivernames[HDFS_VFD_IDX])) {
#ifdef H5_HAVE_LIBHDFS
- if (!vfd_info->info)
- H5TOOLS_GOTO_ERROR(FAIL, "HDFS VFD info is invalid");
- if (H5Pset_fapl_hdfs(fapl_id, (H5FD_hdfs_fapl_t *)vfd_info->info) < 0)
- H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_hdfs() failed");
+ if (!vfd_info->info)
+ H5TOOLS_GOTO_ERROR(FAIL, "HDFS VFD info is invalid");
+ if (H5Pset_fapl_hdfs(fapl_id, (H5FD_hdfs_fapl_t *)vfd_info->info) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "H5Pset_fapl_hdfs() failed");
#else
- H5TOOLS_GOTO_ERROR(FAIL, "The HDFS VFD is not enabled");
+ H5TOOLS_GOTO_ERROR(FAIL, "The HDFS VFD is not enabled");
#endif
+ }
+ else {
+ /*
+ * Try to load VFD plugin.
+ *
+ * Currently, driver configuration strings are unsupported.
+ */
+ if (H5Pset_driver_by_name(fapl_id, vfd_info->u.name, (const char *)vfd_info->info) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "can't load VFD plugin by driver name '%s'", vfd_info->u.name);
+ }
+
+ break;
+
+ case VFD_BY_VALUE:
+ /*
+ * Try to load VFD plugin.
+ *
+ * Currently, driver configuration strings are unsupported.
+ */
+ if (H5Pset_driver_by_value(fapl_id, vfd_info->u.value, (const char *)vfd_info->info) < 0)
+ H5TOOLS_GOTO_ERROR(FAIL, "can't load VFD plugin by driver value '%ld'",
+ (long int)vfd_info->u.value);
+ break;
+
+ default:
+ H5TOOLS_GOTO_ERROR(FAIL, "invalid VFD retrieval type");
}
- else
- H5TOOLS_GOTO_ERROR(FAIL, "invalid VFD name");
done:
+ if (ret_value < 0) {
+ /* Clear error message unless asked for */
+ if (enable_error_stack <= 1)
+ H5Epop(H5tools_ERR_STACK_g, 1);
+ }
+
return ret_value;
}
@@ -671,6 +703,10 @@ done:
if (ret_value < 0) {
if (connector_id >= 0 && H5Idec_ref(connector_id) < 0)
H5TOOLS_ERROR(FAIL, "failed to decrement refcount on VOL connector ID");
+
+ /* Clear error message unless asked for */
+ if (enable_error_stack <= 1)
+ H5Epop(H5tools_ERR_STACK_g, 1);
}
return ret_value;
@@ -719,9 +755,15 @@ h5tools_get_fapl(hid_t prev_fapl_id, h5tools_vol_info_t *vol_info, h5tools_vfd_i
ret_value = new_fapl_id;
done:
- if ((new_fapl_id >= 0) && (ret_value < 0)) {
- H5Pclose(new_fapl_id);
- new_fapl_id = H5I_INVALID_HID;
+ if (ret_value < 0) {
+ if (new_fapl_id >= 0) {
+ H5Pclose(new_fapl_id);
+ new_fapl_id = H5I_INVALID_HID;
+ }
+
+ /* Clear error message unless asked for */
+ if (enable_error_stack <= 1)
+ H5Epop(H5tools_ERR_STACK_g, 1);
}
return ret_value;
@@ -937,8 +979,9 @@ h5tools_fopen(const char *fname, unsigned flags, hid_t fapl_id, hbool_t use_spec
if (drivernum == LOG_VFD_IDX)
continue;
- vfd_info.info = NULL;
- vfd_info.name = drivernames[drivernum];
+ vfd_info.type = VFD_BY_NAME;
+ vfd_info.info = NULL;
+ vfd_info.u.name = drivernames[drivernum];
/* Get a fapl reflecting the selected VOL connector and VFD */
if ((tmp_fapl_id = h5tools_get_fapl(fapl_id, &vol_info, &vfd_info)) < 0)
@@ -989,6 +1032,10 @@ done:
if (tmp_fapl_id >= 0)
H5Pclose(tmp_fapl_id);
+ /* Clear error message unless asked for */
+ if (ret_value < 0 && enable_error_stack <= 1)
+ H5Epop(H5tools_ERR_STACK_g, 1);
+
return ret_value;
}
diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h
index 9d065f3..dde4026 100644
--- a/tools/lib/h5tools.h
+++ b/tools/lib/h5tools.h
@@ -553,6 +553,7 @@ typedef struct h5tools_context_t {
*/
typedef enum { VOL_BY_NAME, VOL_BY_VALUE } h5tools_vol_info_type_t;
+typedef enum { VFD_BY_NAME, VFD_BY_VALUE } h5tools_vfd_info_type_t;
typedef struct h5tools_vol_info_t {
h5tools_vol_info_type_t type;
@@ -568,12 +569,16 @@ typedef struct h5tools_vol_info_t {
} h5tools_vol_info_t;
typedef struct h5tools_vfd_info_t {
+ h5tools_vfd_info_type_t type;
/* Pointer to information to be passed to the driver for its setup */
const void *info;
- /* Name of the VFD */
- const char *name;
+ /* Field specifying either the driver's name or value (ID) */
+ union {
+ const char * name;
+ H5FD_class_value_t value;
+ } u;
} h5tools_vfd_info_t;
/* This enum should match the entries in the above 'volnames'