From a648183cc38e71f7393702c3ad11bf5ccb7a74a4 Mon Sep 17 00:00:00 2001 From: Jacob Smith Date: Thu, 30 May 2019 15:45:53 -0500 Subject: Add initial failing test for h5repack copying external data into new file. --- MANIFEST | 1 + tools/test/h5repack/Makefile.am | 2 + tools/test/h5repack/h5repack.sh.in | 93 ++++++++++++++++++++- tools/test/h5repack/h5repackgentest.c | 83 ++++++++++++++++++ tools/test/h5repack/testfiles/h5repack_int32le.h5 | Bin 0 -> 4096 bytes .../h5repack/testfiles/h5repack_int32le_ex-0.dat | Bin 0 -> 2048 bytes .../test/h5repack/testfiles/h5repack_int32le_ex.h5 | Bin 0 -> 1472 bytes 7 files changed, 178 insertions(+), 1 deletion(-) create mode 100644 tools/test/h5repack/h5repackgentest.c create mode 100644 tools/test/h5repack/testfiles/h5repack_int32le.h5 create mode 100644 tools/test/h5repack/testfiles/h5repack_int32le_ex-0.dat create mode 100644 tools/test/h5repack/testfiles/h5repack_int32le_ex.h5 diff --git a/MANIFEST b/MANIFEST index c3dc0e9..7a4257e 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1478,6 +1478,7 @@ ./tools/test/h5repack/dynlib_rpk.c ./tools/test/h5repack/dynlib_vrpk.c ./tools/test/h5repack/h5repack.sh.in +./tools/test/h5repack/h5repackgentest.c ./tools/test/h5repack/h5repack_plugin.sh.in ./tools/test/h5repack/h5repacktst.c ./tools/test/h5repack/testh5repack_detect_szip.c diff --git a/tools/test/h5repack/Makefile.am b/tools/test/h5repack/Makefile.am index 59d7a6a..ff950e4 100644 --- a/tools/test/h5repack/Makefile.am +++ b/tools/test/h5repack/Makefile.am @@ -22,10 +22,12 @@ include $(top_srcdir)/config/commence.am AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib -I$(top_srcdir)/tools/src/h5repack # Test programs and scripts +TEST_PROG=h5repackgentest TEST_SCRIPT=h5repack.sh noinst_PROGRAMS=h5repacktst testh5repack_detect_szip +check_PROGRAMS=$(TEST_PROG) SCRIPT_DEPEND=../../src/h5repack/h5repack$(EXEEXT) h5repacktst$(EXEEXT) if HAVE_SHARED_CONDITIONAL if USE_PLUGINS_CONDITIONAL diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in index 282ba76..9851e89 100644 --- a/tools/test/h5repack/h5repack.sh.in +++ b/tools/test/h5repack/h5repack.sh.in @@ -115,6 +115,8 @@ $SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5 $SRC_H5REPACK_TESTFILES/h5repack_none.h5 $SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5 $SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_int32le.h5 +$SRC_H5REPACK_TESTFILES/h5repack_int32le_ex.h5 $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5 $SRC_TOOLS_TESTFILES/tfamily00000.h5 $SRC_TOOLS_TESTFILES/tfamily00001.h5 @@ -160,6 +162,7 @@ $SRC_H5REPACK_TESTFILES/ublock.bin $SRC_H5REPACK_TESTFILES/h5repack.info $SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl $SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl +$SRC_H5REPACK_TESTFILES/h5repack_int32le_ex-0.dat $SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl $SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst $SRC_H5REPACK_TESTFILES/h5repack_layout.h5-dset2_chunk_20x10-errstk.tst @@ -599,8 +602,91 @@ VERIFY_INVALIDBOUNDS() } # end of VERIFY_INVALIDBOUNDS +# ----------------------------------------------------------------------------- +# Expect h5diff to fail +# ----------------------------------------------------------------------------- +DIFFFAIL() +{ + VERIFY h5diff unequal $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DIFF_BIN -q "$@" + ) + RET=$? + if [ $RET == 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + fi +} + +# ----------------------------------------------------------------------------- +# Catchall test for repacking with external files +# Loops over all (internally-listed) cases and applies the given arguments +# to h5repack. +# Expects case files to be of format: +# + h5repack_.h5 +# + h5repack__ex.h5 +# + h5repack__ex-.dat +# ...where NAME is part of the internal list, and N is a positive decimal +# number; multiple external files (*.dat) are allowed per file, but they must +# follow the same pattern and be in numerical sequence starting at 0. +# ----------------------------------------------------------------------------- +VERIFY_EXTERNAL_CONSOLIDATION() +{ + for name in int32le + do + basename="h5repack_${name}" + base_ext="${basename}_ex" + base_rpkg="${basename}_rp" + canonical="${basename}.h5" + external="${base_ext}.h5" + ex_dat="${base_ext}-0.dat" + ex_dat_mv="__${ext_dat}" + repacked="${base_rpkg}.h5" + prev_nerrors=$nerrors + + TESTING external consolidation on $name with h5repack $@ + echo " . . ." + + # baseline comparison of known files + DIFFTEST $canonical $external + + # repack the external file, verify integrity + ( cd $TESTDIR + $H5REPACK_BIN $@ $external $repacked + ) + DIFFTEST $repacked $canonical + DIFFTEST $repacked $external + + # invalidate external file by moving its data file; verify + ( cd $TESTDIR + mv $ex_dat $ex_dat_mv + ) + DIFFFAIL $canonical $external + DIFFFAIL $repacked $external + DIFFTEST $canonical $repacked + + # reset test state for re-runs with different h5repack arguments + ( cd $TESTDIR + mv $ex_dat_mv $ex_dat + rm $repacked + ) + + # flag if any of the above checks failed + if [ $prev_nerrors != $nerrors ]; then + echo ". . . *FAILED*" + nerrors="`expr $prev_nerrors + 1`" + else + echo ". . . PASSED" + fi + done +} # end VERIFY_EXTERNAL_CONSOLIDATION + +# ----------------------------------------------------------------------------- # same as TOOLTEST, but it uses the old syntax -i input_file -o output_file -# +# ----------------------------------------------------------------------------- TOOLTEST0() { infile=$2 @@ -1464,6 +1550,11 @@ VERIFY_SUPERBLOCK 2 2 3 h5repack_layout.h5 -j 2 -k 2 h5repack_layout.h5 # -j 0 -k 1, file cannot be opened VERIFY_INVALIDBOUNDS 0 1 bounds_latest_latest.h5 +# +# Testing external storage +# +VERIFY_EXTERNAL_CONSOLIDATION + # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR diff --git a/tools/test/h5repack/h5repackgentest.c b/tools/test/h5repack/h5repackgentest.c new file mode 100644 index 0000000..8076b42 --- /dev/null +++ b/tools/test/h5repack/h5repackgentest.c @@ -0,0 +1,83 @@ +#include "hdf5.h" +#include "H5private.h" + +#define MAX_NAME_SIZE 256 +#define FILE_INT32LE "h5repack_int32le" + + +/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + */ +static void +generate_external_int32le(hbool_t external) +{ + char filename[MAX_NAME_SIZE]; + hid_t file; + hid_t dspace; + hid_t dset; + hid_t dcpl = H5P_DEFAULT; + hsize_t dims[] = {8, 8, 8}; + int32_t wdata[512]; /* 8^3 */ + int32_t n; + int i; + int j; + int k; + + /* generate values, alternating positive and negative + */ + for (i=0, n=0; i < 8; i++) + for (j=0; j < 8; j++) + for (k=0; k < 8; k++, n++) + wdata[n] = (k + j*512 + i*4096) * ((n&1) ? (-1) : (1)); + + snprintf(filename, + MAX_NAME_SIZE, + "%s%s.h5", + FILE_INT32LE, + (external) ? "_ex" : ""); + + file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + HDassert(file >= 0); + + dspace = H5Screate_simple(3, dims, NULL); + HDassert(dspace >= 0); + + if (external) { + char name[MAX_NAME_SIZE]; + dcpl = H5Pcreate(H5P_DATASET_CREATE); + HDassert(dcpl >= 0); + snprintf(name, MAX_NAME_SIZE, "%s_ex-0.dat", FILE_INT32LE); + HDassert(H5Pset_external(dcpl, name, 0, H5F_UNLIMITED) >= 0); + } + + dset = H5Dcreate2( + file, + "dset", + H5T_STD_I32LE, + dspace, + H5P_DEFAULT, + dcpl, + H5P_DEFAULT); + HDassert(dset >= 0); + + HDassert(H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata) + >= 0); + + if (external) + HDassert(H5Pclose(dcpl) >= 0); + HDassert(H5Dclose(dset) >= 0); + HDassert(H5Sclose(dspace) >= 0); + HDassert(H5Fclose(file) >= 0); +} /* end generate_external_int32le() */ + + +/* ---------------------------------------------------------------------------- + */ +int +main(void) +{ + generate_external_int32le(FALSE); + generate_external_int32le(TRUE); + return 0; +} /* end main() */ + + diff --git a/tools/test/h5repack/testfiles/h5repack_int32le.h5 b/tools/test/h5repack/testfiles/h5repack_int32le.h5 new file mode 100644 index 0000000..fb00579 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_int32le.h5 differ diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_ex-0.dat b/tools/test/h5repack/testfiles/h5repack_int32le_ex-0.dat new file mode 100644 index 0000000..b505eb4 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_int32le_ex-0.dat differ diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_ex.h5 b/tools/test/h5repack/testfiles/h5repack_int32le_ex.h5 new file mode 100644 index 0000000..337b297 Binary files /dev/null and b/tools/test/h5repack/testfiles/h5repack_int32le_ex.h5 differ -- cgit v0.12