Onion VFD (#1953)

* Onion VFD feature

* Fixes onion VFD errors with non-sec2 backing store VFDs

* Disables the onion VFD tests w/ ph5diff

* Disables non-sec2 VFDs as onion VFD backing stores

* Committing clang-format changes

* Formatted source

* Typo

* Adds onion VFD tools tests to CMake

* Fixes for v16 API compatibility

* Memset structs to avoid bad frees on errors

* H5Dwrite() calls now use H5T_NATIVE_INT as the memory type vs LE

* Properly decodes checksums on BE machines

* Be more careful about uint64_t to haddr_t/hsize_t conversions

* Another fix for BE data comparison

* Removed double underscores from onion constants

* Replace hard-coded onion header string w/ constant

* Fixes cleanup paths in H5FD__onion_ingest_history()

* Fixed use of size_t revision numbers

* Fix h5dump revision count format string

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

5 files changed, 172 insertions, 0 deletions

diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake
index 09648ff..163bd00 100644
--- a/tools/test/h5repack/CMakeTests.cmake
+++ b/tools/test/h5repack/CMakeTests.cmake
@@ -122,6 +122,13 @@
       ${HDF5_TOOLS_DIR}/testfiles/vds/5_b.h5
       ${HDF5_TOOLS_DIR}/testfiles/vds/5_c.h5
       ${HDF5_TOOLS_DIR}/testfiles/vds/5_vds.h5
+      # tools/testfiles onion VFD files
+      ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5
+      ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_1d.h5.onion
+      ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5
+      ${HDF5_TOOLS_DIR}/testfiles/tst_onion_dset_ext.h5.onion
+      ${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5
+      ${HDF5_TOOLS_DIR}/testfiles/tst_onion_objs.h5.onion
   )
 
   set (LIST_OTHER_TEST_FILES
@@ -446,6 +453,50 @@
     endif ()
   endmacro ()
 
+  macro (ADD_H5_DMP_NO_OPT_TEST testname testtype resultcode resultfile)
+    if ("${testtype}" STREQUAL "SKIP")
+      if (NOT HDF5_ENABLE_USING_MEMCHECKER)
+        add_test (
+            NAME H5REPACK_DMP-${testname}
+            COMMAND ${CMAKE_COMMAND} -E echo "SKIP ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}"
+        )
+        set_property(TEST H5REPACK_DMP-${testname} PROPERTY DISABLED)
+      endif ()
+    else ()
+      add_test (
+          NAME H5REPACK_DMP-${testname}-clear-objects
+          COMMAND ${CMAKE_COMMAND} -E remove testfiles/out-${testname}.${resultfile}
+      )
+      set_tests_properties (H5REPACK_DMP-${testname}-clear-objects PROPERTIES
+          FIXTURES_REQUIRED clear_h5repack
+      )
+      add_test (
+          NAME H5REPACK_DMP-${testname}
+          COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5repack${tgt_file_ext}> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}
+      )
+      set_tests_properties (H5REPACK_DMP-${testname} PROPERTIES
+          DEPENDS H5REPACK_DMP-${testname}-clear-objects
+      )
+      if (NOT HDF5_ENABLE_USING_MEMCHECKER)
+        add_test (
+            NAME H5REPACK_DMP-h5dump-${testname}
+            COMMAND "${CMAKE_COMMAND}"
+                -D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}"
+                -D "TEST_PROGRAM=$<TARGET_FILE:h5dump${tgt_file_ext}>"
+                -D "TEST_ARGS:STRING=out-${testname}.${resultfile}"
+                -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles"
+                -D "TEST_OUTPUT=${resultfile}-${testname}.out"
+                -D "TEST_EXPECT=${resultcode}"
+                -D "TEST_REFERENCE=${testname}.${resultfile}.ddl"
+                -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake"
+        )
+        set_tests_properties (H5REPACK_DMP-h5dump-${testname} PROPERTIES
+            DEPENDS "H5REPACK_DMP-${testname}"
+        )
+      endif ()
+    endif ()
+  endmacro ()
+
   macro (ADD_H5_DIFF_TEST testname testtype resultcode testfile)
     if ("${testtype}" STREQUAL "SKIP")
       if (NOT HDF5_ENABLE_USING_MEMCHECKER)
@@ -1665,6 +1716,10 @@ ADD_H5_DMP_TEST (textlink-mergeprune "TEST" 0 textlink.h5 --merge --prune --enab
 ### HDFFV-11128 needs fixed to enable the following test
 #ADD_H5_DMP_TEST (textlinktar-mergeprune "TEST" 0 textlinktar.h5 --merge --prune --enable-error-stack)
 
+ADD_H5_DMP_NO_OPT_TEST (tst_onion_dset_1d "TEST" 0 tst_onion_dset_1d.h5 --src-vfd-name onion --src-vfd-info 1)
+ADD_H5_DMP_NO_OPT_TEST (tst_onion_dset_ext "TEST" 0 tst_onion_dset_ext.h5 --src-vfd-name onion --src-vfd-info 1)
+ADD_H5_DMP_NO_OPT_TEST (tst_onion_objs "TEST" 0 tst_onion_objs.h5 --src-vfd-name onion --src-vfd-info 1)
+
 ##############################################################################
 ###    P L U G I N  T E S T S
 ##############################################################################
diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in
index a241ea8..63d62bc 100644
--- a/tools/test/h5repack/h5repack.sh.in
+++ b/tools/test/h5repack/h5repack.sh.in
@@ -177,6 +177,13 @@ $SRC_TOOLS_TESTFILES/vds/5_a.h5
 $SRC_TOOLS_TESTFILES/vds/5_b.h5
 $SRC_TOOLS_TESTFILES/vds/5_c.h5
 $SRC_TOOLS_TESTFILES/vds/5_vds.h5
+########tools/testfiles########
+$SRC_TOOLS_TESTFILES/tst_onion_dset_1d.h5
+$SRC_TOOLS_TESTFILES/tst_onion_dset_1d.h5.onion
+$SRC_TOOLS_TESTFILES/tst_onion_dset_ext.h5
+$SRC_TOOLS_TESTFILES/tst_onion_dset_ext.h5.onion
+$SRC_TOOLS_TESTFILES/tst_onion_objs.h5
+$SRC_TOOLS_TESTFILES/tst_onion_objs.h5.onion
 "
 
 LIST_OTHER_TEST_FILES="
@@ -251,6 +258,10 @@ $SRC_H5REPACK_TESTFILES/textlinksrc-merge.textlinksrc.h5.tst
 $SRC_H5REPACK_TESTFILES/textlinktar-merge.textlinktar.h5.tst
 $SRC_H5REPACK_TESTFILES/textlink-merge.textlink.h5.tst
 $SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst
+########onion#files########
+$SRC_H5REPACK_TESTFILES/onion.tst_onion_dset_1d.h5.ddl
+$SRC_H5REPACK_TESTFILES/onion.tst_onion_dset_ext.h5.ddl
+$SRC_H5REPACK_TESTFILES/onion.tst_onion_objs.h5.ddl
 "
 
 #
@@ -1130,6 +1141,56 @@ TOOLTEST_DUMP()
    rm -f $outfile
 }
 
+# This is same as TOOLTEST_DUMP() with comparing h5dump output
+# without any option
+#
+TOOLTEST_DUMP_NO_OPT()
+{
+    infile=$2
+    outfile=out-$1.$2
+    expect="$TESTDIR/$1.$2.ddl"
+    actual="$TESTDIR/out-$1.$2.out"
+    actual_err="$TESTDIR/out-$1.$2.err"
+
+    shift
+    shift
+
+    # Run test.
+    TESTING $H5REPACK $@
+    (
+        cd $TESTDIR
+        $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
+    ) >$actual 2>$actual_err
+    RET=$?
+    if [ $RET != 0 ] ; then
+    echo "*FAILED*"
+    nerrors="`expr $nerrors + 1`"
+    else
+     echo " PASSED"
+     VERIFY h5dump output $@
+     (
+        cd $TESTDIR
+        $RUNSERIAL $H5DUMP_BIN $outfile
+     ) >$actual 2>$actual_err
+     cat $actual_err >> $actual
+
+     RET=$?
+
+    fi
+
+    if cmp -s $expect $actual; then
+     echo " PASSED"
+    else
+     echo "*FAILED*"
+     echo "    Expected result (*.ddl) differs from actual result (*.out)"
+     nerrors="`expr $nerrors + 1`"
+     test yes = "$verbose" && diff -c $expect $actual |sed 's/^/    /'
+    fi
+
+   rm -f $actual $actual_err
+   rm -f $outfile
+}
+
 # This is similar to TOOLTEST_DUMP().
 # Test h5repack with options added for paged aggregation.
 # h5stat is used on the repacked file and the expected output
@@ -1830,6 +1891,9 @@ TOOLTEST_DUMP textlink-mergeprune textlink.h5 --merge --prune --enable-error-sta
 #TOOLTEST_DUMP textlinksrc-mergeprune textlinksrc.h5 --merge --prune --enable-error-stack
 ### HDFFV-11128 needs fixed to enable the following test
 #TOOLTEST_DUMP textlinktar-mergeprune textlinktar.h5 --merge --prune --enable-error-stack
+TOOLTEST_DUMP_NO_OPT onion tst_onion_dset_1d.h5 --src-vfd-name onion --src-vfd-info 1
+TOOLTEST_DUMP_NO_OPT onion tst_onion_dset_ext.h5 --src-vfd-name onion --src-vfd-info 1
+TOOLTEST_DUMP_NO_OPT onion tst_onion_objs.h5 --src-vfd-name onion --src-vfd-info 1
 
 # Clean up temporary files/directories
 CLEAN_TESTFILES_AND_TESTDIR
diff --git a/tools/test/h5repack/testfiles/onion.tst_onion_dset_1d.h5.ddl b/tools/test/h5repack/testfiles/onion.tst_onion_dset_1d.h5.ddl
new file mode 100644
index 0000000..06ecf1a
--- /dev/null
+++ b/tools/test/h5repack/testfiles/onion.tst_onion_dset_1d.h5.ddl
@@ -0,0 +1,11 @@
+HDF5 "out-onion.tst_onion_dset_1d.h5" {
+GROUP "/" {
+   DATASET "DS1" {
+      DATATYPE  H5T_STD_I32LE
+      DATASPACE  SIMPLE { ( 1, 16 ) / ( 1, 16 ) }
+      DATA {
+      (0,0): 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
+      }
+   }
+}
+}
diff --git a/tools/test/h5repack/testfiles/onion.tst_onion_dset_ext.h5.ddl b/tools/test/h5repack/testfiles/onion.tst_onion_dset_ext.h5.ddl
new file mode 100644
index 0000000..39c1900
--- /dev/null
+++ b/tools/test/h5repack/testfiles/onion.tst_onion_dset_ext.h5.ddl
@@ -0,0 +1,18 @@
+HDF5 "out-onion.tst_onion_dset_ext.h5" {
+GROUP "/" {
+   DATASET "DS1" {
+      DATATYPE  H5T_STD_I32LE
+      DATASPACE  SIMPLE { ( 8, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
+      DATA {
+      (0,0): 0, 1, 2, 3,
+      (1,0): 1, 2, 3, 4,
+      (2,0): 2, 3, 4, 5,
+      (3,0): 3, 4, 5, 6,
+      (4,0): 0, 1, 2, 3,
+      (5,0): 1, 2, 3, 4,
+      (6,0): 2, 3, 4, 5,
+      (7,0): 3, 4, 5, 6
+      }
+   }
+}
+}
diff --git a/tools/test/h5repack/testfiles/onion.tst_onion_objs.h5.ddl b/tools/test/h5repack/testfiles/onion.tst_onion_objs.h5.ddl
new file mode 100644
index 0000000..fa4ab4f
--- /dev/null
+++ b/tools/test/h5repack/testfiles/onion.tst_onion_objs.h5.ddl
@@ -0,0 +1,24 @@
+HDF5 "out-onion.tst_onion_objs.h5" {
+GROUP "/" {
+   DATASET "DS1" {
+      DATATYPE  H5T_STD_I32LE
+      DATASPACE  SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
+      DATA {
+      (0,0): 0, 1, 2, 3,
+      (1,0): 1, 2, 3, 4,
+      (2,0): 2, 3, 4, 5,
+      (3,0): 3, 4, 5, 6
+      }
+   }
+   DATASET "DS2" {
+      DATATYPE  H5T_STD_I32LE
+      DATASPACE  SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) }
+      DATA {
+      (0,0): 0, 1, 2, 3,
+      (1,0): 1, 2, 3, 4,
+      (2,0): 2, 3, 4, 5,
+      (3,0): 3, 4, 5, 6
+      }
+   }
+}
+}