Merge pull request #2203 in HDFFV/hdf5 from ~LRKNOX/hdf5_lrk:1.10/master to 1.10/masterhdf5-1_10_6

* commit 'e0fa36bfe008c03365ce8ea94d705f8fdebccdf0': (166 commits)
  Update release date strings in README.txt and release_docs/RELEASE.txt
  Add release note about using ompio instead of romio when collective writes fail with OpenMPI.
  Fixed typos, grammatical errors, etc. Expanded entry for S3 and HDFS VFDs. Moved Mac 10.11 and 10.12 from Supported Platforms to More Platforms Tested.
  Add qualification to RELEASE.txt regarding performance improvements
  Standalone doesn't use h5test implementation.
  Add missing note
  Update COPYING file with new HDF5 license.
  Update bin/config.guess and config.sub to current version available from git.savannah.gnu.org.
  Fix include to correct memory calls - big-endian issue.
  Revert BLDLIBDIR value in junit.sh.in.
  Corrections to CMake functions
  Update version for HDF5 1.10.6 release. Remove autom4te.cache directory that shouldn't have been added.
  Remove duplicate instance
  Fix merge issue
  HDFFV-10979 - fix global name clash
  Correct clang search and java include
  Latest date first in RELEASE.txt
  Update RELEASE.txt with performance improvements and Steven Varga's bugfix.
  Update version numbers for shared lib files according to api compatibility report.
  Created hdf5_1_10_6 branch for HDF5 1.10.6 release preparation. Set version to 1.10.6-pre1. Switched default  maintainer mode to disabled and default build mode to production. Added files produced by autogen.sh to commit except for src/H5config.h.in~ and src/H5public.h~.
  ...

1 files changed, 118 insertions, 3 deletions

diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in
index a36eb08..67056fb 100644
--- a/tools/test/h5repack/h5repack.sh.in
+++ b/tools/test/h5repack/h5repack.sh.in
@@ -111,6 +111,16 @@ $SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5
 $SRC_H5REPACK_TESTFILES/h5repack_none.h5
 $SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5
 $SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5
+$SRC_H5REPACK_TESTFILES/h5repack_f32le.h5
+$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex.h5
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be.h5
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex.h5
 $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5
 $SRC_TOOLS_TESTFILES/tfamily00000.h5
 $SRC_TOOLS_TESTFILES/tfamily00001.h5
@@ -156,10 +166,19 @@ $SRC_H5REPACK_TESTFILES/ublock.bin
 $SRC_H5REPACK_TESTFILES/h5repack.info
 $SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl
 $SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl
+$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex-0.dat
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-0.dat
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-1.dat
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex-0.dat
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex-0.dat
 $SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl
 $SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst
 $SRC_H5REPACK_TESTFILES/h5repack_layout.h5-dset2_chunk_20x10-errstk.tst
 $SRC_H5REPACK_TESTFILES/h5repack_layout.h5-plugin_test.ddl
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-0.dat
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-1.dat
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-2.dat
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-3.dat
 $SRC_H5REPACK_TESTFILES/plugin_test.h5repack_layout.h5.tst
 $SRC_H5REPACK_TESTFILES/1_vds.h5-vds_dset_chunk20x10x5-v.ddl
 $SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl
@@ -595,8 +614,94 @@ VERIFY_INVALIDBOUNDS()
 
 } # end of VERIFY_INVALIDBOUNDS
 
+# -----------------------------------------------------------------------------
+# Expect h5diff to fail
+# -----------------------------------------------------------------------------
+DIFFFAIL()
+{
+    VERIFY h5diff unequal $@
+    (
+        cd $TESTDIR
+        $RUNSERIAL $H5DIFF_BIN -q  "$@"
+    )
+    RET=$?
+    if [ $RET == 0 ] ; then
+         echo "*FAILED*"
+         nerrors="`expr $nerrors + 1`"
+    else
+         echo " PASSED"
+    fi
+}
+
+# -----------------------------------------------------------------------------
+# Catchall test for repacking with external files
+# Loops over all (internally-listed) cases and applies the given arguments
+# to h5repack.
+# Each test file family must be listed in the for loop at the start of the
+# 'function'.
+# Expects case files to be of format:
+# + h5repack_<NAME>.h5
+# + h5repack_<NAME>_ex.h5
+# + h5repack_<NAME>_ex-<N>.dat
+# ...where NAME is part of the internal list, and N is a positive decimal
+# number; multiple external files (*.dat) are allowed per file, but they must
+# follow the pattern and be in contiguous numerical sequence starting at 0.
+# -----------------------------------------------------------------------------
+VERIFY_EXTERNAL_CONSOLIDATION()
+{
+    for name in f32le int32le_1d int32le_2d int32le_3d uint8be
+    do
+        basename="h5repack_${name}"
+        base_ext="${basename}_ex"
+        base_rpkg="${basename}_rp"
+        canonical="${basename}.h5"
+        external="${base_ext}.h5"
+        ex_dat="${base_ext}-0.dat"
+        ex_dat_mv="__${ext_dat}"
+        repacked="${base_rpkg}.h5"
+        prev_nerrors=$nerrors
+
+        TESTING external consolidation on $name with h5repack $@
+        echo " . . ."
+
+        # baseline comparison of known files
+        DIFFTEST $canonical $external
+
+        # repack the external file, verify integrity
+        (   cd $TESTDIR
+            $H5REPACK_BIN $@ $external $repacked
+        )
+        DIFFTEST $repacked $canonical
+        DIFFTEST $repacked $external
+
+        # invalidate external file by moving its first data file; verify
+        (   cd $TESTDIR
+            mv $ex_dat $ex_dat_mv
+        )
+        DIFFFAIL $canonical $external
+        DIFFFAIL $repacked  $external
+        DIFFTEST $canonical $repacked
+
+        # reset test state for re-runs with different h5repack arguments
+        (   cd $TESTDIR
+            mv $ex_dat_mv $ex_dat
+            rm $repacked
+        )
+
+        # flag if any of the above checks failed
+        if [ $prev_nerrors != $nerrors ]
+        then
+             echo ". . . *FAILED*"
+             nerrors="`expr $prev_nerrors + 1`"
+        else
+             echo ". . . PASSED"
+        fi
+    done
+} # end VERIFY_EXTERNAL_CONSOLIDATION
+
+# -----------------------------------------------------------------------------
 # same as TOOLTEST, but it uses the old syntax -i input_file -o output_file
-#
+# -----------------------------------------------------------------------------
 TOOLTEST0()
 {
     infile=$2
@@ -1387,7 +1492,12 @@ TOOLTEST add_alignment $arg
 TOOLTEST upgrade_layout h5repack_layouto.h5
 
 # test for datum size > H5TOOLS_MALLOCSIZE
-TOOLTEST gt_mallocsize h5repack_objs.h5 -f GZIP=1
+arg="h5repack_objs.h5 -f GZIP=1"
+if test $USE_FILTER_DEFLATE != "yes" ; then
+ SKIP $arg
+else
+ TOOLTEST gt_mallocsize $arg
+fi
 
 # Check repacking file with committed datatypes in odd configurations
 TOOLTEST committed_dt h5repack_named_dtypes.h5
@@ -1458,7 +1568,12 @@ VERIFY_SUPERBLOCK 1 2 2 h5repack_layout.h5 -j 1 -k 2 h5repack_layout.h5
 # -j 2 -k 2, superblock will be 3
 VERIFY_SUPERBLOCK 2 2 3 h5repack_layout.h5 -j 2 -k 2 h5repack_layout.h5
 # -j 0 -k 1, file cannot be opened
-VERIFY_INVALIDBOUNDS 0 1 bounds_latest_latest.h5 
+VERIFY_INVALIDBOUNDS 0 1 bounds_latest_latest.h5
+
+########################################
+# Testing external storage
+########################################
+VERIFY_EXTERNAL_CONSOLIDATION -l CONTI
 
 # Clean up temporary files/directories
 CLEAN_TESTFILES_AND_TESTDIR