Onion VFD (#1953)

* Onion VFD feature

* Fixes onion VFD errors with non-sec2 backing store VFDs

* Disables the onion VFD tests w/ ph5diff

* Disables non-sec2 VFDs as onion VFD backing stores

* Committing clang-format changes

* Formatted source

* Typo

* Adds onion VFD tools tests to CMake

* Fixes for v16 API compatibility

* Memset structs to avoid bad frees on errors

* H5Dwrite() calls now use H5T_NATIVE_INT as the memory type vs LE

* Properly decodes checksums on BE machines

* Be more careful about uint64_t to haddr_t/hsize_t conversions

* Another fix for BE data comparison

* Removed double underscores from onion constants

* Replace hard-coded onion header string w/ constant

* Fixes cleanup paths in H5FD__onion_ingest_history()

* Fixed use of size_t revision numbers

* Fix h5dump revision count format string

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

3 files changed, 141 insertions, 6 deletions

diff --git a/tools/src/h5diff/h5diff_common.c b/tools/src/h5diff/h5diff_common.c
index 433f0c7..87c08be 100644
--- a/tools/src/h5diff/h5diff_common.c
+++ b/tools/src/h5diff/h5diff_common.c
@@ -56,6 +56,28 @@ static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'},
                                           {"vfd-info-2", require_arg, 'Z'},
                                           {NULL, 0, '\0'}};
 
+static H5FD_onion_fapl_info_t onion_fa_g_1 = {
+    H5FD_ONION_FAPL_INFO_VERSION_CURR,
+    H5P_DEFAULT,                   /* backing_fapl_id                */
+    32,                            /* page_size                      */
+    H5FD_ONION_STORE_TARGET_ONION, /* store_target                   */
+    H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
+    0,                  /* force_write_open               */
+    0,                  /* creation_flags                 */
+    "first input file", /* comment                        */
+};
+
+static H5FD_onion_fapl_info_t onion_fa_g_2 = {
+    H5FD_ONION_FAPL_INFO_VERSION_CURR,
+    H5P_DEFAULT,                   /* backing_fapl_id                */
+    32,                            /* page_size                      */
+    H5FD_ONION_STORE_TARGET_ONION, /* store_target                   */
+    H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
+    0,                   /* force_write_open               */
+    0,                   /* creation_flags                 */
+    "second input file", /* comment                        */
+};
+
 /*-------------------------------------------------------------------------
  * Function: check_options
  *
@@ -451,7 +473,7 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const
             case '8':
                 opts->vfd_info[0].type   = VFD_BY_NAME;
                 opts->vfd_info[0].u.name = H5_optarg;
-                opts->custom_vol[0]      = TRUE;
+                opts->custom_vfd[0]      = TRUE;
                 break;
 
             case '9':
@@ -476,6 +498,40 @@ parse_command_line(int argc, const char *const *argv, const char **fname1, const
         }
     }
 
+    /* If file 1 uses the onion VFD, get the revision number */
+    if (opts->vfd_info[0].u.name && !HDstrcmp(opts->vfd_info[0].u.name, "onion")) {
+        if (opts->vfd_info[0].info) {
+            errno                     = 0;
+            onion_fa_g_1.revision_num = HDstrtoull(opts->vfd_info[0].info, NULL, 10);
+            if (errno == ERANGE) {
+                HDprintf("Invalid onion revision specified for file 1\n");
+                usage();
+                h5diff_exit(EXIT_FAILURE);
+            }
+        }
+        else
+            onion_fa_g_1.revision_num = 0;
+
+        opts->vfd_info[0].info = &onion_fa_g_1;
+    }
+
+    /* If file 2 uses the onion VFD, get the revision number */
+    if (opts->vfd_info[1].u.name && !HDstrcmp(opts->vfd_info[1].u.name, "onion")) {
+        if (opts->vfd_info[1].info) {
+            errno                     = 0;
+            onion_fa_g_2.revision_num = HDstrtoull(opts->vfd_info[1].info, NULL, 10);
+            if (errno == ERANGE) {
+                HDprintf("Invalid onion revision specified for file 2\n");
+                usage();
+                h5diff_exit(EXIT_FAILURE);
+            }
+        }
+        else
+            onion_fa_g_2.revision_num = 0;
+
+        opts->vfd_info[1].info = &onion_fa_g_2;
+    }
+
     /* check options */
     check_options(opts);
 
diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c
index be68a47..3de1fbb 100644
--- a/tools/src/h5dump/h5dump.c
+++ b/tools/src/h5dump/h5dump.c
@@ -23,10 +23,13 @@ static hbool_t     doxml_g       = FALSE;
 static hbool_t     useschema_g   = TRUE;
 static const char *xml_dtd_uri_g = NULL;
 
-static hbool_t            use_custom_vol_g = FALSE;
-static hbool_t            use_custom_vfd_g = FALSE;
-static h5tools_vol_info_t vol_info_g       = {0};
-static h5tools_vfd_info_t vfd_info_g       = {0};
+static hbool_t use_custom_vol_g = FALSE;
+static hbool_t use_custom_vfd_g = FALSE;
+
+static h5tools_vol_info_t vol_info_g = {0};
+static h5tools_vfd_info_t vfd_info_g = {0};
+
+static hbool_t get_onion_revision_count = FALSE;
 
 #ifdef H5_HAVE_ROS3_VFD
 /* Default "anonymous" S3 configuration */
@@ -51,6 +54,17 @@ static H5FD_hdfs_fapl_t hdfs_fa_g = {
 };
 #endif /* H5_HAVE_LIBHDFS */
 
+static H5FD_onion_fapl_info_t onion_fa_g = {
+    H5FD_ONION_FAPL_INFO_VERSION_CURR,
+    H5P_DEFAULT,                   /* backing_fapl_id                */
+    32,                            /* page_size                      */
+    H5FD_ONION_STORE_TARGET_ONION, /* store_target                   */
+    H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
+    0,            /* force_write_open               */
+    0,            /* creation_flags                 */
+    "input file", /* comment                        */
+};
+
 /* module-scoped variables for XML option */
 #define DEFAULT_XSD "http://www.hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd"
 #define DEFAULT_DTD "http://www.hdfgroup.org/HDF5/XML/DTD/HDF5-File.dtd"
@@ -1305,6 +1319,29 @@ end_collect:
         }
     }
 
+    /* If the file uses the onion VFD, get the revision number */
+    if (vfd_info_g.u.name && !HDstrcmp(vfd_info_g.u.name, "onion")) {
+
+        if (vfd_info_g.info) {
+            if (!HDstrcmp(vfd_info_g.info, "revision_count"))
+                get_onion_revision_count = TRUE;
+            else {
+                errno                   = 0;
+                onion_fa_g.revision_num = HDstrtoull(vfd_info_g.info, NULL, 10);
+                if (errno == ERANGE) {
+                    HDprintf("Invalid onion revision specified\n");
+                    goto error;
+                }
+
+                HDprintf("Using revision %" PRIu64 "\n", onion_fa_g.revision_num);
+            }
+        }
+        else
+            onion_fa_g.revision_num = 0;
+
+        vfd_info_g.info = &onion_fa_g;
+    }
+
 parse_end:
     /* check for file name to be processed */
     if (argc <= H5_optind) {
@@ -1424,7 +1461,21 @@ main(int argc, char *argv[])
     while (H5_optind < argc) {
         fname = HDstrdup(argv[H5_optind++]);
 
-        fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id != H5P_DEFAULT), NULL, 0);
+        /* A short cut to get the revision count of an onion file without opening the file */
+        if (get_onion_revision_count && H5FD_ONION == H5Pget_driver(fapl_id)) {
+            uint64_t revision_count = 0;
+
+            if (H5FDonion_get_revision_count(fname, fapl_id, &revision_count) < 0) {
+                error_msg("unable to create FAPL for file access\n");
+                h5tools_setstatus(EXIT_FAILURE);
+                goto done;
+            }
+
+            HDprintf("The number of revisions for the onion file is %" PRIu64 "\n", revision_count);
+            goto done;
+        }
+        else
+            fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id != H5P_DEFAULT), NULL, 0);
 
         if (fid < 0) {
             error_msg("unable to open file \"%s\"\n", fname);
diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c
index a805b3d..0d3bc9a 100644
--- a/tools/src/h5repack/h5repack_main.c
+++ b/tools/src/h5repack/h5repack_main.c
@@ -76,6 +76,17 @@ static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'},
                                           {"dst-vfd-info", require_arg, 'Z'},
                                           {NULL, 0, '\0'}};
 
+static H5FD_onion_fapl_info_t onion_fa_in_g = {
+    H5FD_ONION_FAPL_INFO_VERSION_CURR,
+    H5P_DEFAULT,                   /* backing_fapl_id                */
+    32,                            /* page_size                      */
+    H5FD_ONION_STORE_TARGET_ONION, /* store_target                   */
+    H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST,
+    0,            /* force_write_open               */
+    0,            /* creation_flags                 */
+    "input file", /* comment                        */
+};
+
 /*-------------------------------------------------------------------------
  * Function: usage
  *
@@ -875,6 +886,23 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options)
         ret_value = -1;
     }
 
+    /* If the input file uses the onion VFD, get the revision number */
+    if (in_vfd_info.u.name && !HDstrcmp(in_vfd_info.u.name, "onion")) {
+        if (in_vfd_info.info) {
+            errno                      = 0;
+            onion_fa_in_g.revision_num = HDstrtoull(in_vfd_info.info, NULL, 10);
+            if (errno == ERANGE) {
+                HDprintf("Invalid onion revision specified for the input file\n");
+                usage(h5tools_getprogname());
+                exit(EXIT_FAILURE);
+            }
+        }
+        else
+            onion_fa_in_g.revision_num = 0;
+
+        in_vfd_info.info = &onion_fa_in_g;
+    }
+
     /* Setup FAPL for input and output file accesses */
     if (custom_in_vol || custom_in_vfd) {
         if ((tmp_fapl = h5tools_get_fapl(options->fin_fapl, custom_in_vol ? &in_vol_info : NULL,