VFD plugins (#602)

* Implement support for loading of Virtual File Drivers as plugins

Fix plugin caching for VOL connector and VFD plugins

Fix plugin iteration to skip paths that can't be opened

* Enable dynamic loading of VFDs with HDF5_DRIVER environment variable

* Temporarily disable error reporting during H5F_open double file open

* Default to using HDstat in h5_get_file_size for unknown VFDs

* Use macros for some environment variables that HDF5 interprets

* Update "null" and "ctl testing" VFDs

4 files changed, 84 insertions, 20 deletions

diff --git a/tools/src/h5repack/h5repack.c b/tools/src/h5repack/h5repack.c
index 7cad36b..d75b1cf 100644
--- a/tools/src/h5repack/h5repack.c
+++ b/tools/src/h5repack/h5repack.c
@@ -758,8 +758,8 @@ check_objects(const char *fname, pack_opt_t *options)
      * open the file
      *-------------------------------------------------------------------------
      */
-    if ((fid = h5tools_fopen(fname, H5F_ACC_RDONLY, options->fin_fapl,
-                             (options->fin_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, 0)) < 0)
+    if ((fid = h5tools_fopen(fname, H5F_ACC_RDONLY, options->fin_fapl, (options->fin_fapl != H5P_DEFAULT),
+                             NULL, 0)) < 0)
         H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname, H5FOPENERROR);
 
     /*-------------------------------------------------------------------------
diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c
index 934b4d1..3806a4e 100644
--- a/tools/src/h5repack/h5repack_copy.c
+++ b/tools/src/h5repack/h5repack_copy.c
@@ -81,8 +81,8 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options)
      * open input file
      *-------------------------------------------------------------------------
      */
-    if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl,
-                               (options->fin_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, (size_t)0)) < 0)
+    if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl, (options->fin_fapl != H5P_DEFAULT),
+                               NULL, (size_t)0)) < 0)
         H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fnamein, H5FOPENERROR);
 
     /* get user block size and file space strategy/persist/threshold */
diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c
index 3526268..08568cd 100644
--- a/tools/src/h5repack/h5repack_main.c
+++ b/tools/src/h5repack/h5repack_main.c
@@ -31,7 +31,7 @@ const char *outfile = NULL;
  * Command-line options: The user can specify short or long-named
  * parameters.
  */
-static const char *           s_opts   = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:vz:EG:LM:P:S:T:VXW1:2:3:4:5:6:";
+static const char *s_opts = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:vz:EG:LM:P:S:T:VXWY:Z:1:2:3:4:5:6:7:8:9:0:";
 static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'},
                                           {"block", require_arg, 'b'},
                                           {"compact", require_arg, 'c'},
@@ -68,6 +68,12 @@ static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'},
                                           {"dst-vol-value", require_arg, '4'},
                                           {"dst-vol-name", require_arg, '5'},
                                           {"dst-vol-info", require_arg, '6'},
+                                          {"src-vfd-value", require_arg, '7'},
+                                          {"src-vfd-name", require_arg, '8'},
+                                          {"src-vfd-info", require_arg, '9'},
+                                          {"dst-vfd-value", require_arg, '0'},
+                                          {"dst-vfd-name", require_arg, 'Y'},
+                                          {"dst-vfd-info", require_arg, 'Z'},
                                           {NULL, 0, '\0'}};
 
 /*-------------------------------------------------------------------------
@@ -112,6 +118,24 @@ usage(const char *prog)
     PRINTVALSTREAM(rawoutstream,
                    "   --dst-vol-info          VOL-specific info to pass to the VOL connector used for\n");
     PRINTVALSTREAM(rawoutstream, "                           opening the output HDF5 file specified\n");
+    PRINTVALSTREAM(rawoutstream,
+                   "   --src-vfd-value         Value (ID) of the VFL driver to use for opening the\n");
+    PRINTVALSTREAM(rawoutstream, "                           input HDF5 file specified\n");
+    PRINTVALSTREAM(rawoutstream,
+                   "   --src-vfd-name          Name of the VFL driver to use for opening the input\n");
+    PRINTVALSTREAM(rawoutstream, "                           HDF5 file specified\n");
+    PRINTVALSTREAM(rawoutstream,
+                   "   --src-vfd-info          VFD-specific info to pass to the VFL driver used for\n");
+    PRINTVALSTREAM(rawoutstream, "                           opening the input HDF5 file specified\n");
+    PRINTVALSTREAM(rawoutstream,
+                   "   --dst-vfd-value         Value (ID) of the VFL driver to use for opening the\n");
+    PRINTVALSTREAM(rawoutstream, "                           output HDF5 file specified\n");
+    PRINTVALSTREAM(rawoutstream,
+                   "   --dst-vfd-name          Name of the VFL driver to use for opening the output\n");
+    PRINTVALSTREAM(rawoutstream, "                           HDF5 file specified\n");
+    PRINTVALSTREAM(rawoutstream,
+                   "   --dst-vfd-info          VFD-specific info to pass to the VFL driver used for\n");
+    PRINTVALSTREAM(rawoutstream, "                           opening the output HDF5 file specified\n");
     PRINTVALSTREAM(rawoutstream, "   -L, --latest            Use latest version of file format\n");
     PRINTVALSTREAM(rawoutstream,
                    "                           This option will take precedence over the options\n");
@@ -486,15 +510,21 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options)
 {
     h5tools_vol_info_t in_vol_info;
     h5tools_vol_info_t out_vol_info;
-    hbool_t            custom_in_fapl  = FALSE;
-    hbool_t            custom_out_fapl = FALSE;
-    hid_t              tmp_fapl        = H5I_INVALID_HID;
+    h5tools_vfd_info_t in_vfd_info;
+    h5tools_vfd_info_t out_vfd_info;
+    hbool_t            custom_in_vol  = FALSE;
+    hbool_t            custom_in_vfd  = FALSE;
+    hbool_t            custom_out_vol = FALSE;
+    hbool_t            custom_out_vfd = FALSE;
+    hid_t              tmp_fapl       = H5I_INVALID_HID;
     int                bound, opt;
     int                ret_value = 0;
 
     /* Initialize fapl info structs */
     HDmemset(&in_vol_info, 0, sizeof(h5tools_vol_info_t));
     HDmemset(&out_vol_info, 0, sizeof(h5tools_vol_info_t));
+    HDmemset(&in_vfd_info, 0, sizeof(h5tools_vfd_info_t));
+    HDmemset(&out_vfd_info, 0, sizeof(h5tools_vfd_info_t));
 
     /* parse command line options */
     while (EOF != (opt = H5_get_option(argc, argv, s_opts, l_opts))) {
@@ -741,13 +771,13 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options)
             case '1':
                 in_vol_info.type    = VOL_BY_VALUE;
                 in_vol_info.u.value = (H5VL_class_value_t)HDatoi(H5_optarg);
-                custom_in_fapl      = TRUE;
+                custom_in_vol       = TRUE;
                 break;
 
             case '2':
                 in_vol_info.type   = VOL_BY_NAME;
                 in_vol_info.u.name = H5_optarg;
-                custom_in_fapl     = TRUE;
+                custom_in_vol      = TRUE;
                 break;
 
             case '3':
@@ -757,19 +787,51 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options)
             case '4':
                 out_vol_info.type    = VOL_BY_VALUE;
                 out_vol_info.u.value = (H5VL_class_value_t)HDatoi(H5_optarg);
-                custom_out_fapl      = TRUE;
+                custom_out_vol       = TRUE;
                 break;
 
             case '5':
                 out_vol_info.type   = VOL_BY_NAME;
                 out_vol_info.u.name = H5_optarg;
-                custom_out_fapl     = TRUE;
+                custom_out_vol      = TRUE;
                 break;
 
             case '6':
                 out_vol_info.info_string = H5_optarg;
                 break;
 
+            case '7':
+                in_vfd_info.type    = VFD_BY_VALUE;
+                in_vfd_info.u.value = (H5FD_class_value_t)HDatoi(H5_optarg);
+                custom_in_vfd       = TRUE;
+                break;
+
+            case '8':
+                in_vfd_info.type   = VFD_BY_NAME;
+                in_vfd_info.u.name = H5_optarg;
+                custom_in_vfd      = TRUE;
+                break;
+
+            case '9':
+                in_vfd_info.info = (const void *)H5_optarg;
+                break;
+
+            case '0':
+                out_vfd_info.type    = VFD_BY_VALUE;
+                out_vfd_info.u.value = (H5FD_class_value_t)HDatoi(H5_optarg);
+                custom_out_vfd       = TRUE;
+                break;
+
+            case 'Y':
+                out_vfd_info.type   = VFD_BY_NAME;
+                out_vfd_info.u.name = H5_optarg;
+                custom_out_vfd      = TRUE;
+                break;
+
+            case 'Z':
+                out_vfd_info.info = (const void *)H5_optarg;
+                break;
+
             default:
                 break;
         } /* end switch */
@@ -803,8 +865,9 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options)
     }
 
     /* Setup FAPL for input and output file accesses */
-    if (custom_in_fapl) {
-        if ((tmp_fapl = h5tools_get_fapl(options->fin_fapl, &in_vol_info, NULL)) < 0) {
+    if (custom_in_vol || custom_in_vfd) {
+        if ((tmp_fapl = h5tools_get_fapl(options->fin_fapl, custom_in_vol ? &in_vol_info : NULL,
+                                         custom_in_vfd ? &in_vfd_info : NULL)) < 0) {
             error_msg("failed to setup FAPL for input file\n");
             h5tools_setstatus(EXIT_FAILURE);
             ret_value = -1;
@@ -823,8 +886,9 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options)
         options->fin_fapl = tmp_fapl;
     }
 
-    if (custom_out_fapl) {
-        if ((tmp_fapl = h5tools_get_fapl(options->fout_fapl, &out_vol_info, NULL)) < 0) {
+    if (custom_out_vol || custom_out_vfd) {
+        if ((tmp_fapl = h5tools_get_fapl(options->fout_fapl, custom_out_vol ? &out_vol_info : NULL,
+                                         custom_out_vfd ? &out_vfd_info : NULL)) < 0) {
             error_msg("failed to setup FAPL for output file\n");
             h5tools_setstatus(EXIT_FAILURE);
             ret_value = -1;
diff --git a/tools/src/h5repack/h5repack_verify.c b/tools/src/h5repack/h5repack_verify.c
index 0542d03..8c54dde 100644
--- a/tools/src/h5repack/h5repack_verify.c
+++ b/tools/src/h5repack/h5repack_verify.c
@@ -378,11 +378,11 @@ h5repack_cmp_pl(const char *fname1, hid_t fname1_fapl, const char *fname2, hid_t
      *-------------------------------------------------------------------------
      */
     /* Open the files */
-    if ((fid1 = h5tools_fopen(fname1, H5F_ACC_RDONLY, fname1_fapl,
-                              (fname1_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, 0)) < 0)
+    if ((fid1 = h5tools_fopen(fname1, H5F_ACC_RDONLY, fname1_fapl, (fname1_fapl != H5P_DEFAULT), NULL, 0)) <
+        0)
         H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname1, H5FOPENERROR);
-    if ((fid2 = h5tools_fopen(fname2, H5F_ACC_RDONLY, fname2_fapl,
-                              (fname2_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, 0)) < 0)
+    if ((fid2 = h5tools_fopen(fname2, H5F_ACC_RDONLY, fname2_fapl, (fname2_fapl != H5P_DEFAULT), NULL, 0)) <
+        0)
         H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fname2, H5FOPENERROR);
 
     /*-------------------------------------------------------------------------