From 07ea411e8a5eac35f2d798820569235490a527aa Mon Sep 17 00:00:00 2001 From: raylu-hdf <60487644+raylu-hdf@users.noreply.github.com> Date: Wed, 29 Jun 2022 17:25:27 -0500 Subject: Enabling h5repack to support onion VFD (#1788) * Enable h5repack to support onion VFD. The input file can take options for onion VFD. But output file doesn't. * Committing clang-format changes Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- tools/src/h5repack/h5repack_main.c | 28 ++++++++++ tools/test/h5repack/h5repack.sh.in | 64 ++++++++++++++++++++++ .../testfiles/onion.tst_onion_dset_1d.h5.ddl | 11 ++++ .../testfiles/onion.tst_onion_dset_ext.h5.ddl | 18 ++++++ .../h5repack/testfiles/onion.tst_onion_objs.h5.ddl | 24 ++++++++ 5 files changed, 145 insertions(+) create mode 100644 tools/test/h5repack/testfiles/onion.tst_onion_dset_1d.h5.ddl create mode 100644 tools/test/h5repack/testfiles/onion.tst_onion_dset_ext.h5.ddl create mode 100644 tools/test/h5repack/testfiles/onion.tst_onion_objs.h5.ddl diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c index a805b3d..0d3bc9a 100644 --- a/tools/src/h5repack/h5repack_main.c +++ b/tools/src/h5repack/h5repack_main.c @@ -76,6 +76,17 @@ static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'}, {"dst-vfd-info", require_arg, 'Z'}, {NULL, 0, '\0'}}; +static H5FD_onion_fapl_info_t onion_fa_in_g = { + H5FD_ONION_FAPL_INFO_VERSION_CURR, + H5P_DEFAULT, /* backing_fapl_id */ + 32, /* page_size */ + H5FD_ONION_STORE_TARGET_ONION, /* store_target */ + H5FD_ONION_FAPL_INFO_REVISION_ID_LATEST, + 0, /* force_write_open */ + 0, /* creation_flags */ + "input file", /* comment */ +}; + /*------------------------------------------------------------------------- * Function: usage * @@ -875,6 +886,23 @@ parse_command_line(int argc, const char *const *argv, pack_opt_t *options) ret_value = -1; } + /* If the input file uses the onion VFD, get the revision number */ + if (in_vfd_info.u.name && !HDstrcmp(in_vfd_info.u.name, "onion")) { + if (in_vfd_info.info) { + errno = 0; + onion_fa_in_g.revision_num = HDstrtoull(in_vfd_info.info, NULL, 10); + if (errno == ERANGE) { + HDprintf("Invalid onion revision specified for the input file\n"); + usage(h5tools_getprogname()); + exit(EXIT_FAILURE); + } + } + else + onion_fa_in_g.revision_num = 0; + + in_vfd_info.info = &onion_fa_in_g; + } + /* Setup FAPL for input and output file accesses */ if (custom_in_vol || custom_in_vfd) { if ((tmp_fapl = h5tools_get_fapl(options->fin_fapl, custom_in_vol ? &in_vol_info : NULL, diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in index a241ea8..63d62bc 100644 --- a/tools/test/h5repack/h5repack.sh.in +++ b/tools/test/h5repack/h5repack.sh.in @@ -177,6 +177,13 @@ $SRC_TOOLS_TESTFILES/vds/5_a.h5 $SRC_TOOLS_TESTFILES/vds/5_b.h5 $SRC_TOOLS_TESTFILES/vds/5_c.h5 $SRC_TOOLS_TESTFILES/vds/5_vds.h5 +########tools/testfiles######## +$SRC_TOOLS_TESTFILES/tst_onion_dset_1d.h5 +$SRC_TOOLS_TESTFILES/tst_onion_dset_1d.h5.onion +$SRC_TOOLS_TESTFILES/tst_onion_dset_ext.h5 +$SRC_TOOLS_TESTFILES/tst_onion_dset_ext.h5.onion +$SRC_TOOLS_TESTFILES/tst_onion_objs.h5 +$SRC_TOOLS_TESTFILES/tst_onion_objs.h5.onion " LIST_OTHER_TEST_FILES=" @@ -251,6 +258,10 @@ $SRC_H5REPACK_TESTFILES/textlinksrc-merge.textlinksrc.h5.tst $SRC_H5REPACK_TESTFILES/textlinktar-merge.textlinktar.h5.tst $SRC_H5REPACK_TESTFILES/textlink-merge.textlink.h5.tst $SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst +########onion#files######## +$SRC_H5REPACK_TESTFILES/onion.tst_onion_dset_1d.h5.ddl +$SRC_H5REPACK_TESTFILES/onion.tst_onion_dset_ext.h5.ddl +$SRC_H5REPACK_TESTFILES/onion.tst_onion_objs.h5.ddl " # @@ -1130,6 +1141,56 @@ TOOLTEST_DUMP() rm -f $outfile } +# This is same as TOOLTEST_DUMP() with comparing h5dump output +# without any option +# +TOOLTEST_DUMP_NO_OPT() +{ + infile=$2 + outfile=out-$1.$2 + expect="$TESTDIR/$1.$2.ddl" + actual="$TESTDIR/out-$1.$2.out" + actual_err="$TESTDIR/out-$1.$2.err" + + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) >$actual 2>$actual_err + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + VERIFY h5dump output $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DUMP_BIN $outfile + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + RET=$? + + fi + + if cmp -s $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' + fi + + rm -f $actual $actual_err + rm -f $outfile +} + # This is similar to TOOLTEST_DUMP(). # Test h5repack with options added for paged aggregation. # h5stat is used on the repacked file and the expected output @@ -1830,6 +1891,9 @@ TOOLTEST_DUMP textlink-mergeprune textlink.h5 --merge --prune --enable-error-sta #TOOLTEST_DUMP textlinksrc-mergeprune textlinksrc.h5 --merge --prune --enable-error-stack ### HDFFV-11128 needs fixed to enable the following test #TOOLTEST_DUMP textlinktar-mergeprune textlinktar.h5 --merge --prune --enable-error-stack +TOOLTEST_DUMP_NO_OPT onion tst_onion_dset_1d.h5 --src-vfd-name onion --src-vfd-info 1 +TOOLTEST_DUMP_NO_OPT onion tst_onion_dset_ext.h5 --src-vfd-name onion --src-vfd-info 1 +TOOLTEST_DUMP_NO_OPT onion tst_onion_objs.h5 --src-vfd-name onion --src-vfd-info 1 # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR diff --git a/tools/test/h5repack/testfiles/onion.tst_onion_dset_1d.h5.ddl b/tools/test/h5repack/testfiles/onion.tst_onion_dset_1d.h5.ddl new file mode 100644 index 0000000..06ecf1a --- /dev/null +++ b/tools/test/h5repack/testfiles/onion.tst_onion_dset_1d.h5.ddl @@ -0,0 +1,11 @@ +HDF5 "out-onion.tst_onion_dset_1d.h5" { +GROUP "/" { + DATASET "DS1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 1, 16 ) / ( 1, 16 ) } + DATA { + (0,0): 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31 + } + } +} +} diff --git a/tools/test/h5repack/testfiles/onion.tst_onion_dset_ext.h5.ddl b/tools/test/h5repack/testfiles/onion.tst_onion_dset_ext.h5.ddl new file mode 100644 index 0000000..39c1900 --- /dev/null +++ b/tools/test/h5repack/testfiles/onion.tst_onion_dset_ext.h5.ddl @@ -0,0 +1,18 @@ +HDF5 "out-onion.tst_onion_dset_ext.h5" { +GROUP "/" { + DATASET "DS1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 8, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) } + DATA { + (0,0): 0, 1, 2, 3, + (1,0): 1, 2, 3, 4, + (2,0): 2, 3, 4, 5, + (3,0): 3, 4, 5, 6, + (4,0): 0, 1, 2, 3, + (5,0): 1, 2, 3, 4, + (6,0): 2, 3, 4, 5, + (7,0): 3, 4, 5, 6 + } + } +} +} diff --git a/tools/test/h5repack/testfiles/onion.tst_onion_objs.h5.ddl b/tools/test/h5repack/testfiles/onion.tst_onion_objs.h5.ddl new file mode 100644 index 0000000..fa4ab4f --- /dev/null +++ b/tools/test/h5repack/testfiles/onion.tst_onion_objs.h5.ddl @@ -0,0 +1,24 @@ +HDF5 "out-onion.tst_onion_objs.h5" { +GROUP "/" { + DATASET "DS1" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) } + DATA { + (0,0): 0, 1, 2, 3, + (1,0): 1, 2, 3, 4, + (2,0): 2, 3, 4, 5, + (3,0): 3, 4, 5, 6 + } + } + DATASET "DS2" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 4, 4 ) / ( H5S_UNLIMITED, H5S_UNLIMITED ) } + DATA { + (0,0): 0, 1, 2, 3, + (1,0): 1, 2, 3, 4, + (2,0): 2, 3, 4, 5, + (3,0): 3, 4, 5, 6 + } + } +} +} -- cgit v0.12