Add initial failing test for h5repack copying external data into new file.

6 files changed, 177 insertions, 1 deletions

diff --git a/tools/test/h5repack/Makefile.am b/tools/test/h5repack/Makefile.am
index 59d7a6a..ff950e4 100644
--- a/tools/test/h5repack/Makefile.am
+++ b/tools/test/h5repack/Makefile.am
@@ -22,10 +22,12 @@ include $(top_srcdir)/config/commence.am
 AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib -I$(top_srcdir)/tools/src/h5repack
 
 # Test programs and scripts
+TEST_PROG=h5repackgentest
 TEST_SCRIPT=h5repack.sh
 
 noinst_PROGRAMS=h5repacktst testh5repack_detect_szip
 
+check_PROGRAMS=$(TEST_PROG)
 SCRIPT_DEPEND=../../src/h5repack/h5repack$(EXEEXT) h5repacktst$(EXEEXT)
 if HAVE_SHARED_CONDITIONAL
 if USE_PLUGINS_CONDITIONAL
diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in
index 282ba76..9851e89 100644
--- a/tools/test/h5repack/h5repack.sh.in
+++ b/tools/test/h5repack/h5repack.sh.in
@@ -115,6 +115,8 @@ $SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5
 $SRC_H5REPACK_TESTFILES/h5repack_none.h5
 $SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5
 $SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_ex.h5
 $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5
 $SRC_TOOLS_TESTFILES/tfamily00000.h5
 $SRC_TOOLS_TESTFILES/tfamily00001.h5
@@ -160,6 +162,7 @@ $SRC_H5REPACK_TESTFILES/ublock.bin
 $SRC_H5REPACK_TESTFILES/h5repack.info
 $SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl
 $SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_ex-0.dat
 $SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl
 $SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst
 $SRC_H5REPACK_TESTFILES/h5repack_layout.h5-dset2_chunk_20x10-errstk.tst
@@ -599,8 +602,91 @@ VERIFY_INVALIDBOUNDS()
 
 } # end of VERIFY_INVALIDBOUNDS
 
+# -----------------------------------------------------------------------------
+# Expect h5diff to fail
+# -----------------------------------------------------------------------------
+DIFFFAIL()
+{
+    VERIFY h5diff unequal $@
+    (
+        cd $TESTDIR
+        $RUNSERIAL $H5DIFF_BIN -q  "$@"
+    )
+    RET=$?
+    if [ $RET == 0 ] ; then
+         echo "*FAILED*"
+         nerrors="`expr $nerrors + 1`"
+    else
+         echo " PASSED"
+    fi
+}
+
+# -----------------------------------------------------------------------------
+# Catchall test for repacking with external files
+# Loops over all (internally-listed) cases and applies the given arguments
+# to h5repack.
+# Expects case files to be of format:
+# + h5repack_<NAME>.h5
+# + h5repack_<NAME>_ex.h5
+# + h5repack_<NAME>_ex-<N>.dat
+# ...where NAME is part of the internal list, and N is a positive decimal
+# number; multiple external files (*.dat) are allowed per file, but they must
+# follow the same pattern and be in numerical sequence starting at 0.
+# -----------------------------------------------------------------------------
+VERIFY_EXTERNAL_CONSOLIDATION()
+{
+    for name in int32le
+    do
+        basename="h5repack_${name}"
+        base_ext="${basename}_ex"
+        base_rpkg="${basename}_rp"
+        canonical="${basename}.h5"
+        external="${base_ext}.h5"
+        ex_dat="${base_ext}-0.dat"
+        ex_dat_mv="__${ext_dat}"
+        repacked="${base_rpkg}.h5"
+        prev_nerrors=$nerrors
+
+        TESTING external consolidation on $name with h5repack $@
+        echo " . . ."
+
+        # baseline comparison of known files
+        DIFFTEST $canonical $external
+
+        # repack the external file, verify integrity
+        (   cd $TESTDIR
+            $H5REPACK_BIN $@ $external $repacked
+        )
+        DIFFTEST $repacked $canonical
+        DIFFTEST $repacked $external
+
+        # invalidate external file by moving its data file; verify
+        (   cd $TESTDIR
+            mv $ex_dat $ex_dat_mv
+        )
+        DIFFFAIL $canonical $external
+        DIFFFAIL $repacked  $external
+        DIFFTEST $canonical $repacked
+
+        # reset test state for re-runs with different h5repack arguments
+        (   cd $TESTDIR
+            mv $ex_dat_mv $ex_dat
+            rm $repacked
+        )
+
+        # flag if any of the above checks failed
+        if [ $prev_nerrors != $nerrors ]; then
+             echo ". . . *FAILED*"
+             nerrors="`expr $prev_nerrors + 1`"
+        else
+             echo ". . . PASSED"
+        fi
+    done
+} # end VERIFY_EXTERNAL_CONSOLIDATION
+
+# -----------------------------------------------------------------------------
 # same as TOOLTEST, but it uses the old syntax -i input_file -o output_file
-#
+# -----------------------------------------------------------------------------
 TOOLTEST0()
 {
     infile=$2
@@ -1464,6 +1550,11 @@ VERIFY_SUPERBLOCK 2 2 3 h5repack_layout.h5 -j 2 -k 2 h5repack_layout.h5
 # -j 0 -k 1, file cannot be opened
 VERIFY_INVALIDBOUNDS 0 1 bounds_latest_latest.h5 
 
+#
+# Testing external storage
+#
+VERIFY_EXTERNAL_CONSOLIDATION
+
 # Clean up temporary files/directories
 CLEAN_TESTFILES_AND_TESTDIR
 
diff --git a/tools/test/h5repack/h5repackgentest.c b/tools/test/h5repack/h5repackgentest.c
new file mode 100644
index 0000000..8076b42
--- /dev/null
+++ b/tools/test/h5repack/h5repackgentest.c
@@ -0,0 +1,83 @@
+#include "hdf5.h"
+#include "H5private.h"
+
+#define MAX_NAME_SIZE 256
+#define FILE_INT32LE "h5repack_int32le"
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ */
+static void
+generate_external_int32le(hbool_t external)
+{
+    char    filename[MAX_NAME_SIZE];
+    hid_t   file;
+    hid_t   dspace;
+    hid_t   dset;
+    hid_t   dcpl = H5P_DEFAULT;
+    hsize_t dims[] = {8, 8, 8};
+    int32_t wdata[512]; /* 8^3 */
+    int32_t n;
+    int     i;
+    int     j;
+    int     k;
+
+    /* generate values, alternating positive and negative
+     */
+    for (i=0, n=0; i < 8; i++)
+        for (j=0; j < 8; j++)
+            for (k=0; k < 8; k++, n++)
+                wdata[n] = (k + j*512 + i*4096) * ((n&1) ? (-1) : (1));
+
+    snprintf(filename,
+            MAX_NAME_SIZE,
+            "%s%s.h5",
+            FILE_INT32LE,
+            (external) ? "_ex" : "");
+
+    file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+    HDassert(file >= 0);
+
+    dspace = H5Screate_simple(3, dims, NULL);
+    HDassert(dspace >= 0);
+
+    if (external) {
+        char name[MAX_NAME_SIZE];
+        dcpl = H5Pcreate(H5P_DATASET_CREATE);
+        HDassert(dcpl >= 0);
+        snprintf(name, MAX_NAME_SIZE, "%s_ex-0.dat", FILE_INT32LE);
+        HDassert(H5Pset_external(dcpl, name, 0, H5F_UNLIMITED) >= 0);
+    }
+
+    dset = H5Dcreate2(
+            file,
+            "dset",
+            H5T_STD_I32LE,
+            dspace,
+            H5P_DEFAULT,
+            dcpl,
+            H5P_DEFAULT);
+    HDassert(dset >= 0);
+
+    HDassert(H5Dwrite(dset, H5T_STD_I32LE, H5S_ALL, H5S_ALL, H5P_DEFAULT, wdata)
+            >= 0);
+
+    if (external)
+        HDassert(H5Pclose(dcpl) >= 0);
+    HDassert(H5Dclose(dset) >= 0);
+    HDassert(H5Sclose(dspace) >= 0);
+    HDassert(H5Fclose(file) >= 0);
+} /* end generate_external_int32le() */
+
+
+/* ----------------------------------------------------------------------------
+ */
+int
+main(void)
+{
+    generate_external_int32le(FALSE);
+    generate_external_int32le(TRUE);
+    return 0;
+} /* end main() */
+
+
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le.h5 b/tools/test/h5repack/testfiles/h5repack_int32le.h5
new file mode 100644
index 0000000..fb00579
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le.h5
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_ex-0.dat b/tools/test/h5repack/testfiles/h5repack_int32le_ex-0.dat
new file mode 100644
index 0000000..b505eb4
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le_ex-0.dat
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_ex.h5 b/tools/test/h5repack/testfiles/h5repack_int32le_ex.h5
new file mode 100644
index 0000000..337b297
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le_ex.h5