1 10 merge hdffv 10590 (#1128)

* Merge pull request #60 from bmribler/develop

Fix HDFFV-10590

* H5repack tests should fail if a corrupted file causes h5repack to
segfault/core dump.

* Remove test file, duplicate of h5repack_CVE-2018-17432.h5.

3 files changed, 54 insertions, 10 deletions

diff --git a/src/H5Osdspace.c b/src/H5Osdspace.c
index 6a2557f..8e28c85 100644
--- a/src/H5Osdspace.c
+++ b/src/H5Osdspace.c
@@ -139,8 +139,11 @@ H5O__sdspace_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UN
     flags = *p++;
 
     /* Get or determine the type of the extent */
-    if (version >= H5O_SDSPACE_VERSION_2)
+    if (version >= H5O_SDSPACE_VERSION_2) {
         sdim->type = (H5S_class_t)*p++;
+        if (sdim->type != H5S_SIMPLE && sdim->rank > 0)
+            HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "invalid rank for scalar or NULL dataspace")
+    } /* end if */
     else {
         /* Set the dataspace type to be simple or scalar as appropriate */
         if (sdim->rank > 0)
@@ -274,13 +277,15 @@ H5O_sdspace_encode(H5F_t *f, uint8_t *p, const void *_mesg)
         *p++ = 0; /*reserved*/
     }             /* end else */
 
-    /* Current & maximum dimensions */
-    if (sdim->rank > 0) {
-        for (u = 0; u < sdim->rank; u++)
-            H5F_ENCODE_LENGTH(f, p, sdim->size[u]);
-        if (flags & H5S_VALID_MAX) {
+    /* Encode dataspace dimensions for simple dataspaces */
+    if (H5S_SIMPLE == sdim->type) {
+        /* Encode current & maximum dimensions */
+        if (sdim->rank > 0) {
             for (u = 0; u < sdim->rank; u++)
-                H5F_ENCODE_LENGTH(f, p, sdim->max[u]);
+                H5F_ENCODE_LENGTH(f, p, sdim->size[u]);
+            if (flags & H5S_VALID_MAX)
+                for (u = 0; u < sdim->rank; u++)
+                    H5F_ENCODE_LENGTH(f, p, sdim->max[u]);
         } /* end if */
     }     /* end if */
 
diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake
index 91d729b..1d0a9e9 100644
--- a/tools/test/h5repack/CMakeTests.cmake
+++ b/tools/test/h5repack/CMakeTests.cmake
@@ -1538,7 +1538,7 @@
 # the references in attribute of compund or vlen datatype
   ADD_H5_TEST (HDFFV-5932 "TEST" ${FILE_ATTR_REF})
 
-# Add test for memory leak in attirbute. This test is verified by CTEST.
+# Add test for memory leak in attribute. This test is verified by CTEST.
 # 1. leak from vlen string
 # 2. leak from compound type without reference member
 # (HDFFV-7840, )
@@ -1546,12 +1546,12 @@
   ADD_H5_TEST (HDFFV-7840 "TEST" h5diff_attr1.h5)
 
 # test CVE-2018-17432 fix
-  set (arg h5repack_CVE-2018-17432.h5 h5repack__CVE-2018-17432_out.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6)
+  set (arg h5repack_CVE-2018-17432.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6)
   set (TESTTYPE "TEST")
   ADD_H5_FILTER_TEST (HDFFV-10590 "" ${TESTTYPE} 1 ${arg})
 
 # test CVE-2018-14460 fix
-  set (arg h5repack_CVE-2018-14460.h5 h5repack_CVE-2018-14460_out.h5)
+  set (arg h5repack_CVE-2018-14460.h5)
   set (TESTTYPE "TEST")
   ADD_H5_FILTER_TEST (HDFFV-11223 "" ${TESTTYPE} 1 ${arg})
 
diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in
index 3f8e722..18be371 100644
--- a/tools/test/h5repack/h5repack.sh.in
+++ b/tools/test/h5repack/h5repack.sh.in
@@ -862,6 +862,45 @@ TOOLTESTV()
    rm -f $outfile
 }
 
+# Same as TOOLTEST, but expects h5repack fails
+#
+TOOLTEST_FAIL()
+{
+    infile=$1
+    outfile=$2
+    expect="$TESTDIR/$2-$1.ddl"
+    actual="$TESTDIR/$2-$1.out"
+    actual_err="$TESTDIR/$2-$1.err"
+    shift
+    shift
+
+    # Run test.
+    TESTING $H5REPACK $@
+    (
+        cd $TESTDIR
+        $ENVCMD $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile
+    ) >&$actual
+    RET=$?
+
+    # Normally h5repack of files tested with this function are expected
+    # to return not 0, but if the command results in "Segmentation fault"
+    # or "core dumped" it is a failure regardless of the return value.
+    failure=`grep -e 'Segmentation fault' -e 'core dumped' $actual`
+    if [ "$failure" != "" ]; then
+        nerrors="`expr $nerrors + 1`"
+        echo " FAILED"
+        echo "    $failure"
+    else
+        if [ $RET == 0 ] ; then
+            nerrors="`expr $nerrors + 1`"
+            echo " FAILED"
+        else
+            echo " PASSED"
+        fi
+    fi
+    rm -f $outfile
+}
+
 # This is same as TOOLTESTV() with comparing display output
 # with actual filename swapped
 #