Merge pull request #1756 in HDFFV/hdf5 from ~JAKE.SMITH/hdf5:bugfix/repack_external_storage to develop

* commit '9a388325ebaec339f67ddd210ca31ddcc3ede973':
  Update to current external file population.
  Revert else-if to single line.
  Change print calls to HD-wrapped. Minor formatting tweaks (inc. regressions and in-file style-matching).
  Refactor h5repackgentest to have more modular internals, for easier addition of new tests. Update manifest and test script for new hdf5 external-storage .h5 files.
  Refactoring.
  Add copyright/disclaimer to repack gentest. Minor formatting tweaks.
  Minor formatting adjustments.
  Undo some formatting modifications. Update MANIFEST.
  Add forgotten files to include from h5repack testfiles for tests.
  Fix failing automated reporting. Add 8-bit big-endian testfile.
  First pass at repack consolidating external storage if layout given. Checks out manually, tests still complain.
  Formatting changes, mostly.
  Add initial failing test for h5repack copying external data into new file.
  Modify code comments for clarity. Some minor reformatting.
  Some minor reformatting. Replace tabs with spaces.
  Add test that uses the family member FAPL setting.
  refactor h5repack_main.c -- remove some redundancy, formatting
  Fit help message to 80 character width and clarify some details.

24 files changed, 595 insertions, 29 deletions

diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake
index a28a282..072dad2 100644
--- a/tools/test/h5repack/CMakeTests.cmake
+++ b/tools/test/h5repack/CMakeTests.cmake
@@ -26,10 +26,23 @@
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_deflate.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_early.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_ext.h5
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_f32le.h5
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_f32le_ex-0.dat
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_f32le_ex.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fill.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_filters.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fletcher.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_hlink.h5
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_1d.h5
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_1d_ex-0.dat
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_1d_ex-1.dat
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_1d_ex.h5
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_2d.h5
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_2d_ex-0.dat
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_2d_ex.h5
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_3d.h5
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_3d_ex-0.dat
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_3d_ex.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layouto.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout2.h5
@@ -44,6 +57,12 @@
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_shuffle.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_soffset.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_szip.h5
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_uint8be.h5
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_uint8be_ex-0.dat
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_uint8be_ex-1.dat
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_uint8be_ex-2.dat
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_uint8be_ex-3.dat
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_uint8be_ex.h5
       # fsm
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_aggr.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fsm_aggr_nopersist.h5
diff --git a/tools/test/h5repack/Makefile.am b/tools/test/h5repack/Makefile.am
index 59d7a6a..38f7b2f 100644
--- a/tools/test/h5repack/Makefile.am
+++ b/tools/test/h5repack/Makefile.am
@@ -22,10 +22,12 @@ include $(top_srcdir)/config/commence.am
 AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib -I$(top_srcdir)/tools/src/h5repack
 
 # Test programs and scripts
+TEST_PROG=h5repackgentest
 TEST_SCRIPT=h5repack.sh
 
 noinst_PROGRAMS=h5repacktst testh5repack_detect_szip
 
+check_PROGRAMS=$(TEST_PROG)
 SCRIPT_DEPEND=../../src/h5repack/h5repack$(EXEEXT) h5repacktst$(EXEEXT)
 if HAVE_SHARED_CONDITIONAL
 if USE_PLUGINS_CONDITIONAL
@@ -57,7 +59,8 @@ endif
 
 # Temporary files.  *.h5 are generated by h5repack.  They should
 # copied to the testfiles/ directory if update is required.
-CHECK_CLEANFILES+=*.h5 *.bin testfiles/h5diff_attr1.h5 testfiles/tfamily*.h5
+# *.dat files may be created by h5repackgentest.
+CHECK_CLEANFILES+=*.h5 *.dat *.bin testfiles/h5diff_attr1.h5 testfiles/tfamily*.h5
 DISTCLEANFILES=h5repack.sh h5repack_plugin.sh
 
 include $(top_srcdir)/config/conclude.am
diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in
index c515786..6e63136 100644
--- a/tools/test/h5repack/h5repack.sh.in
+++ b/tools/test/h5repack/h5repack.sh.in
@@ -115,6 +115,16 @@ $SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5
 $SRC_H5REPACK_TESTFILES/h5repack_none.h5
 $SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5
 $SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5
+$SRC_H5REPACK_TESTFILES/h5repack_f32le.h5
+$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex.h5
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be.h5
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex.h5
 $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5
 $SRC_TOOLS_TESTFILES/tfamily00000.h5
 $SRC_TOOLS_TESTFILES/tfamily00001.h5
@@ -160,10 +170,19 @@ $SRC_H5REPACK_TESTFILES/ublock.bin
 $SRC_H5REPACK_TESTFILES/h5repack.info
 $SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl
 $SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl
+$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex-0.dat
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-0.dat
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-1.dat
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex-0.dat
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex-0.dat
 $SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl
 $SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst
 $SRC_H5REPACK_TESTFILES/h5repack_layout.h5-dset2_chunk_20x10-errstk.tst
 $SRC_H5REPACK_TESTFILES/h5repack_layout.h5-plugin_test.ddl
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-0.dat
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-1.dat
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-2.dat
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-3.dat
 $SRC_H5REPACK_TESTFILES/plugin_test.h5repack_layout.h5.tst
 $SRC_H5REPACK_TESTFILES/1_vds.h5-vds_dset_chunk20x10x5-v.ddl
 $SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl
@@ -599,8 +618,94 @@ VERIFY_INVALIDBOUNDS()
 
 } # end of VERIFY_INVALIDBOUNDS
 
+# -----------------------------------------------------------------------------
+# Expect h5diff to fail
+# -----------------------------------------------------------------------------
+DIFFFAIL()
+{
+    VERIFY h5diff unequal $@
+    (
+        cd $TESTDIR
+        $RUNSERIAL $H5DIFF_BIN -q  "$@"
+    )
+    RET=$?
+    if [ $RET == 0 ] ; then
+         echo "*FAILED*"
+         nerrors="`expr $nerrors + 1`"
+    else
+         echo " PASSED"
+    fi
+}
+
+# -----------------------------------------------------------------------------
+# Catchall test for repacking with external files
+# Loops over all (internally-listed) cases and applies the given arguments
+# to h5repack.
+# Each test file family must be listed in the for loop at the start of the
+# 'function'.
+# Expects case files to be of format:
+# + h5repack_<NAME>.h5
+# + h5repack_<NAME>_ex.h5
+# + h5repack_<NAME>_ex-<N>.dat
+# ...where NAME is part of the internal list, and N is a positive decimal
+# number; multiple external files (*.dat) are allowed per file, but they must
+# follow the pattern and be in contiguous numerical sequence starting at 0.
+# -----------------------------------------------------------------------------
+VERIFY_EXTERNAL_CONSOLIDATION()
+{
+    for name in f32le int32le_1d int32le_2d int32le_3d uint8be
+    do
+        basename="h5repack_${name}"
+        base_ext="${basename}_ex"
+        base_rpkg="${basename}_rp"
+        canonical="${basename}.h5"
+        external="${base_ext}.h5"
+        ex_dat="${base_ext}-0.dat"
+        ex_dat_mv="__${ext_dat}"
+        repacked="${base_rpkg}.h5"
+        prev_nerrors=$nerrors
+
+        TESTING external consolidation on $name with h5repack $@
+        echo " . . ."
+
+        # baseline comparison of known files
+        DIFFTEST $canonical $external
+
+        # repack the external file, verify integrity
+        (   cd $TESTDIR
+            $H5REPACK_BIN $@ $external $repacked
+        )
+        DIFFTEST $repacked $canonical
+        DIFFTEST $repacked $external
+
+        # invalidate external file by moving its first data file; verify
+        (   cd $TESTDIR
+            mv $ex_dat $ex_dat_mv
+        )
+        DIFFFAIL $canonical $external
+        DIFFFAIL $repacked  $external
+        DIFFTEST $canonical $repacked
+
+        # reset test state for re-runs with different h5repack arguments
+        (   cd $TESTDIR
+            mv $ex_dat_mv $ex_dat
+            rm $repacked
+        )
+
+        # flag if any of the above checks failed
+        if [ $prev_nerrors != $nerrors ]
+        then
+             echo ". . . *FAILED*"
+             nerrors="`expr $prev_nerrors + 1`"
+        else
+             echo ". . . PASSED"
+        fi
+    done
+} # end VERIFY_EXTERNAL_CONSOLIDATION
+
+# -----------------------------------------------------------------------------
 # same as TOOLTEST, but it uses the old syntax -i input_file -o output_file
-#
+# -----------------------------------------------------------------------------
 TOOLTEST0()
 {
     infile=$2
@@ -1469,6 +1574,11 @@ VERIFY_SUPERBLOCK 2 2 3 h5repack_layout.h5 -j 2 -k 2 h5repack_layout.h5
 # -j 0 -k 1, file cannot be opened
 VERIFY_INVALIDBOUNDS 0 1 bounds_latest_latest.h5
 
+########################################
+# Testing external storage
+########################################
+VERIFY_EXTERNAL_CONSOLIDATION -l CONTI
+
 # Clean up temporary files/directories
 CLEAN_TESTFILES_AND_TESTDIR
 
diff --git a/tools/test/h5repack/h5repackgentest.c b/tools/test/h5repack/h5repackgentest.c
new file mode 100644
index 0000000..12883b7
--- /dev/null
+++ b/tools/test/h5repack/h5repackgentest.c
@@ -0,0 +1,413 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group.                                               *
+ * Copyright by the Board of Trustees of the University of Illinois.         *
+ * All rights reserved.                                                      *
+ *                                                                           *
+ * This file is part of HDF5.  The full HDF5 copyright notice, including     *
+ * terms governing use, modification, and redistribution, is contained in    *
+ * the COPYING file, which can be found at the root of the source code       *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.  *
+ * If you do not have access to either file, you may request a copy from     *
+ * help@hdfgroup.org.                                                        *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/* For generating files for verifying h5repack with external storage. . .
+ *
+ * Each case file should follow the format of:
+ * + h5repack_<NAME>.h5
+ * + h5repack_<NAME>_ex.h5
+ * + h5repack_<NAME>_ex-<N>.dat
+ * ...where NAME idenfities the type, and N is a positive decimal number;
+ * multiple external files (*.dat) are allowed per file, but they must
+ * follow the pattern and be in contiguous numerical sequence starting at 0.
+ *
+ * Each file typename must be added to the listing for
+ * `VERIFY_EXTERNAL_CONSOLIDATION` in h5repack.sh
+ *
+ * There is no restriction on the name, number, or structure of datasets and
+ * groups in HDF5 file.
+ *
+ * The inluded datatypes should be more than adequate to verify the correctness
+ * of the behavior -- if one type can be consolidated from external storage,
+ * then thay all can.
+ */
+
+#include "hdf5.h"
+#include "H5private.h"
+
+#define MAX_NAME_SIZE 256
+#define FILE_INT32LE_1 "h5repack_int32le_1d"
+#define FILE_INT32LE_2 "h5repack_int32le_2d"
+#define FILE_INT32LE_3 "h5repack_int32le_3d"
+#define FILE_UINT8BE "h5repack_uint8be"
+#define FILE_F32LE   "h5repack_f32le"
+
+#define H5REPACKGENTEST_OOPS ret_value = -1; goto done;
+
+#define H5REPACKGENTEST_COMMON_CLEANUP(dcpl, file, space)  {                  \
+    if ((dcpl) != H5P_DEFAULT && (dcpl) != H5I_INVALID_HID) {                 \
+        (void)H5Pclose((dcpl));                                               \
+    }                                                                         \
+    if ((file) != H5I_INVALID_HID) { (void)H5Fclose((file)); }                \
+    if ((space) != H5I_INVALID_HID) { (void)H5Sclose((space)); }              \
+}
+
+struct external_def {
+    hsize_t  type_size;
+    unsigned n_elts_per_file;
+    unsigned n_elts_total;
+};
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Helper function to create and write a dataset to file.
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+__make_dataset(
+        hid_t       file_id,
+        const char *dset_name,
+        hid_t       mem_type_id,
+        hid_t       space_id,
+        hid_t       dcpl_id,
+        void       *wdata)
+{
+    hid_t dset_id   = H5I_INVALID_HID;
+    int   ret_value = 0;
+
+    dset_id = H5Dcreate2(
+            file_id,
+            dset_name,
+            mem_type_id,
+            space_id,
+            H5P_DEFAULT,
+            dcpl_id,
+            H5P_DEFAULT);
+    if (dset_id == H5I_INVALID_HID) {
+        H5REPACKGENTEST_OOPS;
+    }
+
+    if (H5Dwrite(
+            dset_id,
+            mem_type_id,
+            H5S_ALL,
+            H5S_ALL,
+            H5P_DEFAULT,
+            wdata)
+        < 0)
+    {
+        H5REPACKGENTEST_OOPS;
+    }
+
+done:
+    if (dset_id != H5I_INVALID_HID) { (void)H5Dclose(dset_id); }
+    return ret_value;
+} /* end __make_dataset() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Helper function to populate the DCPL external storage list.
+ * Creates external files for the DCPL, with each file name following the
+ * convention "<filename>_ex-<num>.dat". Will append `n_external_files` to
+ * the filename list, with each file having space for `n_elts` items of the
+ * type (of size `elt_size`). The numeric inputs are not sanity-checked.
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+__set_dcpl_external_list(
+        hid_t       dcpl,
+        const char *filename,
+        unsigned    n_elts_per_file,
+        unsigned    n_elts_total,
+        hsize_t     elt_size)
+{
+    char     name[MAX_NAME_SIZE];
+    unsigned n_external_files = 0;
+    unsigned i = 0;
+
+    if (NULL == filename || '\0' == *filename) {
+        return -1;
+    }
+
+    n_external_files = n_elts_total / n_elts_per_file;
+    if (n_elts_total != (n_external_files * n_elts_per_file)) {
+        return -1;
+    }
+
+    for (i = 0; i < n_external_files; i++) {
+        if (snprintf(name, MAX_NAME_SIZE, "%s_ex-%u.dat", filename, i) >= MAX_NAME_SIZE) {
+            return -1;
+        }
+        if (H5Pset_external(dcpl, name, 0, n_elts_per_file * elt_size) < 0) {
+            return -1;
+        }
+    }
+    return 0;
+} /* end __set_dcpl_external_list() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Generalized utility function to write a file with the specified data and
+ * dataset configuration. If `ext` is provided, will attempt to use external
+ * storage.
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+__make_file(
+        const char          *basename,
+        struct external_def *ext,
+        hid_t                type_id,
+        hsize_t              rank,
+        hsize_t             *dims,
+        void                *wdata)
+{
+    char    filename[MAX_NAME_SIZE];
+    hid_t   file_id   = H5I_INVALID_HID;
+    hid_t   dcpl_id   = H5P_DEFAULT;
+    hid_t   space_id  = H5I_INVALID_HID;
+    int     ret_value = 0;
+
+    if (snprintf(filename,
+            MAX_NAME_SIZE,
+            "%s%s.h5",
+            basename,
+            (NULL != ext) ? "_ex" : "")
+        >= MAX_NAME_SIZE)
+    {
+        H5REPACKGENTEST_OOPS;
+    }
+
+    if (NULL != ext) {
+        dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
+        if (dcpl_id == H5I_INVALID_HID) {
+            H5REPACKGENTEST_OOPS;
+        }
+        if (__set_dcpl_external_list(
+                dcpl_id,
+                basename,
+                ext->n_elts_per_file,
+                ext->n_elts_total,
+                ext->type_size)
+            < 0)
+        {
+            H5REPACKGENTEST_OOPS;
+        }
+    }
+
+    space_id = H5Screate_simple(rank, dims, NULL);
+    if (space_id == H5I_INVALID_HID) {
+        H5REPACKGENTEST_OOPS;
+    }
+
+    file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+    if (file_id == H5I_INVALID_HID) {
+        H5REPACKGENTEST_OOPS;
+    }
+
+    if (__make_dataset(
+            file_id,
+            "dset",
+            type_id,
+            space_id,
+            dcpl_id,
+            wdata)
+        < 0)
+    {
+        H5REPACKGENTEST_OOPS;
+    }
+
+done:
+    H5REPACKGENTEST_COMMON_CLEANUP(dcpl_id, file_id, space_id);
+    return ret_value;
+} /* end __make_file() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+generate_int32le_1d(hbool_t external)
+{
+    int32_t              wdata[12];
+    hsize_t              dims[]    = {12};
+    struct external_def *def_ptr   = NULL;
+    struct external_def  def       = { (hsize_t)sizeof(int32_t), 6, 12 };
+    int32_t              n         = 0;
+    int                  ret_value = 0;
+
+    /* Generate values
+     */
+    for (n = 0; n < 12; n++) {
+        wdata[n] = n-6;
+    }
+
+    def_ptr = (TRUE == external) ? (&def) : NULL;
+    if (__make_file(FILE_INT32LE_1, def_ptr, H5T_STD_I32LE, 1, dims, wdata) < 0) {
+        ret_value = -1;
+    }
+
+    return ret_value;
+} /* end generate_int32le_1d() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+generate_int32le_2d(hbool_t external)
+{
+    int32_t              wdata[64];
+    hsize_t              dims[]    = {8, 8};
+    struct external_def *def_ptr   = NULL;
+    struct external_def  def       = { (hsize_t)sizeof(int32_t), 64, 64 };
+    int32_t              n         = 0;
+    int                  ret_value = 0;
+
+    /* Generate values
+     */
+    for (n = 0; n < 64; n++) {
+        wdata[n] = n-32;
+    }
+
+    def_ptr = (TRUE == external) ? (&def) : NULL;
+    if (__make_file(FILE_INT32LE_2, def_ptr, H5T_STD_I32LE, 2, dims, wdata) < 0) {
+        ret_value = -1;
+    }
+
+    return ret_value;
+} /* end generate_int32le_2d() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+generate_int32le_3d(hbool_t external)
+{
+    hsize_t              dims[] = {8, 8, 8};
+    int32_t              wdata[512]; /* 8^3, from dims */
+    struct external_def *def_ptr = NULL;
+    struct external_def  def     = { (hsize_t)sizeof(int32_t), 512, 512 };
+    int32_t n = 0;
+    int     i = 0;
+    int     j = 0;
+    int     k = 0;
+    int     ret_value = 0;
+
+    /* generate values, alternating positive and negative
+     */
+    for (i=0, n=0; i < dims[0]; i++) {
+        for (j=0; j < dims[1]; j++) {
+            for (k=0; k < dims[2]; k++, n++) {
+                wdata[n] = (k + j*512 + i*4096) * ((n&1) ? (-1) : (1));
+            }
+        }
+    }
+
+    def_ptr = (TRUE == external) ? (&def) : NULL;
+    if (__make_file(FILE_INT32LE_3, def_ptr, H5T_STD_I32LE, 3, dims, wdata) < 0) {
+        ret_value = -1;
+    }
+
+    return ret_value;
+} /* end generate_int32le_3d() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+generate_uint8be(hbool_t external)
+{
+    hsize_t              dims[] = {4, 8, 8};
+    uint8_t              wdata[256]; /* 4*8*8, from dims */
+    struct external_def *def_ptr = NULL;
+    struct external_def  def     = { (hsize_t)sizeof(uint8_t), 64, 256 };
+    uint8_t n = 0;
+    int     i = 0;
+    int     j = 0;
+    int     k = 0;
+    int     ret_value = 0;
+
+    /* Generate values, ping-pong from ends of range
+     */
+    for (i=0, n=0; i < dims[0]; i++) {
+        for (j=0; j < dims[1]; j++) {
+            for (k=0; k < dims[2]; k++, n++) {
+                wdata[n] = n * ((n&1) ? (-1) : (1));
+            }
+        }
+    }
+
+    def_ptr = (TRUE == external) ? (&def) : NULL;
+    if (__make_file(FILE_UINT8BE, def_ptr, H5T_STD_U8BE, 3, dims, wdata) < 0) {
+        ret_value = -1;
+    }
+
+    return ret_value;
+} /* end generate_uint8be() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+generate_f32le(hbool_t external)
+{
+    hsize_t              dims[] = {12, 6};
+    float                wdata[72]; /* 12*6, from dims */
+    struct external_def *def_ptr = NULL;
+    struct external_def  def     = {
+        (hsize_t)sizeof(float),
+        72,
+        72
+    };
+    float   n = 0;
+    int     i = 0;
+    int     j = 0;
+    int     k = 0;
+    int     ret_value = 0;
+
+    /* Generate values */
+    for (i=0, k=0, n=0; i < dims[0]; i++) {
+        for (j=0; j < dims[1]; j++, k++, n++) {
+            wdata[k] = n * 801.1 * ((k % 5 == 1) ? (-1) : (1));
+        }
+    }
+
+    def_ptr = (TRUE == external) ? (&def) : NULL;
+    if (__make_file(FILE_F32LE, def_ptr, H5T_IEEE_F32LE, 2, dims, wdata) < 0) {
+        ret_value = -1;
+    }
+
+    return ret_value;
+} /* end generate_f32le() */
+
+
+/* ----------------------------------------------------------------------------
+ * Create files.
+ * Return 0 on success, nonzero on failure.
+ */
+int
+main(void)
+{
+    int i         = 0;
+    int ret_value = 0;
+
+    for (i=0; i < 2; i++) {
+        hbool_t external = (i&1) ? TRUE : FALSE;
+        if (ret_value == 0) { ret_value -= generate_int32le_1d(external); }
+        if (ret_value == 0) { ret_value -= generate_int32le_2d(external); }
+        if (ret_value == 0) { ret_value -= generate_int32le_3d(external); }
+        if (ret_value == 0) { ret_value -= generate_uint8be(external); }
+        if (ret_value == 0) { ret_value -= generate_f32le(external); }
+    } /* end for external data storage or not */
+
+    if (ret_value != 0) {
+        HDprintf("A problem occurred!\n");
+    }
+
+    return ret_value;
+} /* end main() */
+
+
diff --git a/tools/test/h5repack/testfiles/h5repack-help.txt b/tools/test/h5repack/testfiles/h5repack-help.txt
index 2381865..e55eed9 100644
--- a/tools/test/h5repack/testfiles/h5repack-help.txt
+++ b/tools/test/h5repack/testfiles/h5repack-help.txt
@@ -6,13 +6,17 @@ usage: h5repack [OPTIONS] file1 file2
    -v, --verbose           Verbose mode, print object information
    -V, --version           Print version number and exit
    -n, --native            Use a native HDF5 type when repacking
-   --enable-error-stack    Prints messages from the HDF5 error stack as they occur
+   --enable-error-stack    Prints messages from the HDF5 error stack as they
+                           occur
    -L, --latest            Use latest version of file format
-                           This option will take precedence over the -j and -k options
-   --low=BOUND             The low bound for library release versions to use when creating
-                           objects in the file (default is H5F_LIBVER_EARLIEST)
-   --high=BOUND            The high bound for library release versions to use when creating
-                           objects in the file (default is H5F_LIBVER_LATEST)
+                           This option will take precedence over the options
+                           --low and --high
+   --low=BOUND             The low bound for library release versions to use
+                           when creating objects in the file
+                           (default is H5F_LIBVER_EARLIEST)
+   --high=BOUND            The high bound for library release versions to use
+                           when creating objects in the file
+                           (default is H5F_LIBVER_LATEST)
    -c L1, --compact=L1     Maximum number of links in header messages
    -d L2, --indexed=L2     Minimum number of links in the indexed format
    -s S[:F], --ssize=S[:F] Shared object header message minimum size
@@ -27,10 +31,14 @@ usage: h5repack [OPTIONS] file1 file2
    -z Z, --sort_order=Z    Sort groups and attributes by order Z
    -f FILT, --filter=FILT  Filter type
    -l LAYT, --layout=LAYT  Layout type
-   -S FS_STRATEGY, --fs_strategy=FS_STRATEGY  File space management strategy for H5Pset_file_space_strategy
-   -P FS_PERSIST, --fs_persist=FS_PERSIST  Persisting or not persisting free-space for H5Pset_file_space_strategy
-   -T FS_THRESHOLD, --fs_threshold=FS_THRESHOLD   Free-space section threshold for H5Pset_file_space_strategy
-   -G FS_PAGESIZE, --fs_pagesize=FS_PAGESIZE   File space page size for H5Pset_file_space_page_size
+   -S FS_STRATEGY, --fs_strategy=FS_STRATEGY  File space management strategy for
+                           H5Pset_file_space_strategy
+   -P FS_PERSIST, --fs_persist=FS_PERSIST  Persisting or not persisting free-
+                           space for H5Pset_file_space_strategy
+   -T FS_THRESHOLD, --fs_threshold=FS_THRESHOLD   Free-space section threshold
+                           for H5Pset_file_space_strategy
+   -G FS_PAGESIZE, --fs_pagesize=FS_PAGESIZE   File space page size for
+                           H5Pset_file_space_page_size
 
     M - is an integer greater than 1, size of dataset in bytes (default is 0)
     E - is a filename.
@@ -38,15 +46,17 @@ usage: h5repack [OPTIONS] file1 file2
     U - is a filename.
     T - is an integer
     A - is an integer greater than zero
-    Q - is the sort index type for the input file. It can be "name" or "creation_order" (default)
-    Z - is the sort order type for the input file. It can be "descending" or "ascending" (default)
+    Q - is the sort index type for the input file. It can be "name" or
+        "creation_order" (default)
+    Z - is the sort order type for the input file. It can be "descending" or
+        "ascending" (default)
     B - is the user block size, any value that is 512 or greater and is
         a power of 2 (1024 default)
     F - is the shared object header message type, any of <dspace|dtype|fill|
         pline|attr>. If F is not specified, S applies to all messages
 
-    BOUND is an integer indicating the library release versions to use when creating
-          objects in the file (see H5Pset_libver_bounds()):
+    BOUND is an integer indicating the library release versions to use when
+          creating objects in the file (see H5Pset_libver_bounds()):
         0: This is H5F_LIBVER_EARLIEST in H5F_libver_t struct
         1: This is H5F_LIBVER_V18 in H5F_libver_t struct
         2: This is H5F_LIBVER_V110 in H5F_libver_t struct
@@ -55,24 +65,31 @@ usage: h5repack [OPTIONS] file1 file2
 
     FS_STRATEGY is a string indicating the file space strategy used:
         FSM_AGGR:
-               The mechanisms used in managing file space are free-space managers, aggregators and virtual file driver.
+               The mechanisms used in managing file space are free-space
+               managers, aggregators and virtual file driver.
         PAGE:
-               The mechanisms used in managing file space are free-space managers with embedded paged aggregation and virtual file driver.
+               The mechanisms used in managing file space are free-space
+               managers with embedded paged aggregation and virtual file driver.
         AGGR:
-               The mechanisms used in managing file space are aggregators and virtual file driver.
+               The mechanisms used in managing file space are aggregators and
+               virtual file driver.
         NONE:
-               The mechanisms used in managing file space are virtual file driver.
-        The default strategy when not set is FSM_AGGR without persisting free-space.
+               The mechanisms used in managing file space are virtual file
+               driver.
+        The default strategy when not set is FSM_AGGR without persisting free-
+        space.
 
     FS_PERSIST is 1 to persisting free-space or 0 to not persisting free-space.
       The default when not set is not persisting free-space.
       The value is ignored for AGGR and NONE strategies.
 
-    FS_THRESHOLD is the minimum size (in bytes) of free-space sections to be tracked by the library.
+    FS_THRESHOLD is the minimum size (in bytes) of free-space sections to be
+        tracked by the library.
       The default when not set is 1.
       The value is ignored for AGGR and NONE strategies.
 
-    FS_PAGESIZE is the size (in bytes) >=512 that is used by the library when the file space strategy PAGE is used.
+    FS_PAGESIZE is the size (in bytes) >=512 that is used by the library when
+        the file space strategy PAGE is used.
       The default when not set is 4096.
 
     FILT - is a string with the format:
@@ -100,9 +117,9 @@ usage: h5repack [OPTIONS] file1 file2
         NBIT (no parameter)
         SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type
             is either IN or DS
-        UD=<filter_number,filter_flag,cd_value_count,value_1[,value_2,...,value_N]>
-            required values for filter_number,filter_flag,cd_value_count,value_1
-            optional values for value_2 to value_N
+        UD=<filter_number,filter_flag,cd_value_count,value1[,value2,...,valueN]>
+            Required values: filter_number, filter_flag, cd_value_count, value1
+            Optional values: value2 to valueN
         NONE (no parameter)
 
     LAYT - is a string with the format:
@@ -139,12 +156,12 @@ Examples of use:
 4) h5repack -L -c 10 -s 20:dtype file1 file2
 
    Using latest file format with maximum compact group size of 10 and
-   and minimum shared datatype size of 20
+   minimum shared datatype size of 20
 
 5) h5repack --low=0 --high=1 file1 file2
 
-   Set low=H5F_LIBVER_EARLIEST and high=H5F_LIBVER_V18 via H5Pset_libver_bounds() when
-   creating the repacked file: file2
+   Set low=H5F_LIBVER_EARLIEST and high=H5F_LIBVER_V18 via
+   H5Pset_libver_bounds() when creating the repacked file, file2
 
 5) h5repack -f SHUF -f GZIP=1 file1 file2
 
diff --git a/tools/test/h5repack/testfiles/h5repack_f32le.h5 b/tools/test/h5repack/testfiles/h5repack_f32le.h5
new file mode 100644
index 0000000..03a9b57
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_f32le.h5
diff --git a/tools/test/h5repack/testfiles/h5repack_f32le_ex-0.dat b/tools/test/h5repack/testfiles/h5repack_f32le_ex-0.dat
new file mode 100644
index 0000000..8378fc2
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_f32le_ex-0.dat
diff --git a/tools/test/h5repack/testfiles/h5repack_f32le_ex.h5 b/tools/test/h5repack/testfiles/h5repack_f32le_ex.h5
new file mode 100644
index 0000000..ca6c49d
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_f32le_ex.h5
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_1d.h5 b/tools/test/h5repack/testfiles/h5repack_int32le_1d.h5
new file mode 100644
index 0000000..7d60566
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le_1d.h5
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_1d_ex-0.dat b/tools/test/h5repack/testfiles/h5repack_int32le_1d_ex-0.dat
new file mode 100644
index 0000000..f75ada5
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le_1d_ex-0.dat
@@ -0,0 +1 @@
+������������������������
\ No newline at end of file
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_1d_ex-1.dat b/tools/test/h5repack/testfiles/h5repack_int32le_1d_ex-1.dat
new file mode 100644
index 0000000..1f8f3b6
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le_1d_ex-1.dat
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_1d_ex.h5 b/tools/test/h5repack/testfiles/h5repack_int32le_1d_ex.h5
new file mode 100644
index 0000000..ba4b534
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le_1d_ex.h5
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_2d.h5 b/tools/test/h5repack/testfiles/h5repack_int32le_2d.h5
new file mode 100644
index 0000000..2f32bd1
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le_2d.h5
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_2d_ex-0.dat b/tools/test/h5repack/testfiles/h5repack_int32le_2d_ex-0.dat
new file mode 100644
index 0000000..c6bfc64
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le_2d_ex-0.dat
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_2d_ex.h5 b/tools/test/h5repack/testfiles/h5repack_int32le_2d_ex.h5
new file mode 100644
index 0000000..f6cce59
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le_2d_ex.h5
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_3d.h5 b/tools/test/h5repack/testfiles/h5repack_int32le_3d.h5
new file mode 100644
index 0000000..7a61ef5
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le_3d.h5
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_3d_ex-0.dat b/tools/test/h5repack/testfiles/h5repack_int32le_3d_ex-0.dat
new file mode 100644
index 0000000..b505eb4
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le_3d_ex-0.dat
diff --git a/tools/test/h5repack/testfiles/h5repack_int32le_3d_ex.h5 b/tools/test/h5repack/testfiles/h5repack_int32le_3d_ex.h5
new file mode 100644
index 0000000..7c4b750
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_int32le_3d_ex.h5
diff --git a/tools/test/h5repack/testfiles/h5repack_uint8be.h5 b/tools/test/h5repack/testfiles/h5repack_uint8be.h5
new file mode 100644
index 0000000..0e25211
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_uint8be.h5
diff --git a/tools/test/h5repack/testfiles/h5repack_uint8be_ex-0.dat b/tools/test/h5repack/testfiles/h5repack_uint8be_ex-0.dat
new file mode 100644
index 0000000..f95b90a
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_uint8be_ex-0.dat
diff --git a/tools/test/h5repack/testfiles/h5repack_uint8be_ex-1.dat b/tools/test/h5repack/testfiles/h5repack_uint8be_ex-1.dat
new file mode 100644
index 0000000..011ab48
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_uint8be_ex-1.dat
@@ -0,0 +1 @@
+@�B�D�F�H�J�L�N�P�R�T�V�X�Z�\�^�`�b�d�f�h�j�l�n�p�r�t�v�x�z�|�~�
\ No newline at end of file
diff --git a/tools/test/h5repack/testfiles/h5repack_uint8be_ex-2.dat b/tools/test/h5repack/testfiles/h5repack_uint8be_ex-2.dat
new file mode 100644
index 0000000..069b96d
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_uint8be_ex-2.dat
@@ -0,0 +1 @@
+��}�{�y�w�u�s�q�o�m�k�i�g�e�c�a�_�]�[�Y�W�U�S�Q�O�M�K�I�G�E�C�A
\ No newline at end of file
diff --git a/tools/test/h5repack/testfiles/h5repack_uint8be_ex-3.dat b/tools/test/h5repack/testfiles/h5repack_uint8be_ex-3.dat
new file mode 100644
index 0000000..240d8ca
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_uint8be_ex-3.dat
@@ -0,0 +1 @@
+�?�=�;�9�7�5�3�1�/�-�+�)�'�%�#�!����������
��	����

\ No newline at end of file
diff --git a/tools/test/h5repack/testfiles/h5repack_uint8be_ex.h5 b/tools/test/h5repack/testfiles/h5repack_uint8be_ex.h5
new file mode 100644
index 0000000..8787188
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_uint8be_ex.h5