Bmr dev hdffv 11223 (#640)

* Fixed HDFFV-11223 (CVE-2018-14460)

Description
    - Added checks against buffer size to prevent segfault, in case of data
      corruption, for sdim->size and sdim->max.
    - Renamed data files in an existing test to shorten their length
      as agreed with other developers previously.
Platforms tested:
    Linux/64 (jelly)

* Committing clang-format changes

* Updated for test files

* Updated for HDFFV-11223

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

7 files changed, 47 insertions, 11 deletions

diff --git a/MANIFEST b/MANIFEST
index 9ecef83..9832159 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -2836,7 +2836,8 @@
 ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_test.ddl
 ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl
 ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_zero.ddl
-./tools/test/h5repack/testfiles/h5repack_HDFFV-10590_CVE-2018-17432.h5
+./tools/test/h5repack/testfiles/h5repack_CVE-2018-17432.h5
+./tools/test/h5repack/testfiles/h5repack_CVE-2018-14460.h5
 ./tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl
 ./tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl
 ./tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index be8440f..9e99e66 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -823,7 +823,18 @@ Bug Fixes since HDF5-1.12.0 release
 ===================================
     Library
     -------
-    - Fixed CVE-2018-17435
+    - Fixed CVE-2018-14460
+
+        The tool h5repack produced a segfault when the rank in dataspace
+        message was corrupted, causing invalid read while decoding the
+        dimension sizes.
+
+        The problem was fixed by ensuring that decoding the dimension sizes
+        and max values will not go beyong the end of the buffer.
+
+        (BMR - 2021/05/12, HDFFV-11223)
+
+    - Fixed CVE-2018-11206
 
         The tool h5dump produced a segfault when the size of a fill value
         message was corrupted and caused a buffer overflow.
diff --git a/src/H5Osdspace.c b/src/H5Osdspace.c
index 2cdf6ec..dab989f 100644
--- a/src/H5Osdspace.c
+++ b/src/H5Osdspace.c
@@ -106,12 +106,13 @@ H5FL_ARR_EXTERN(hsize_t);
 --------------------------------------------------------------------------*/
 static void *
 H5O__sdspace_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags,
-                    unsigned H5_ATTR_UNUSED *ioflags, size_t H5_ATTR_UNUSED p_size, const uint8_t *p)
+                    unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, const uint8_t *p)
 {
-    H5S_extent_t *sdim = NULL; /* New extent dimensionality structure */
-    unsigned      flags, version;
-    unsigned      i;                /* Local counting variable */
-    void *        ret_value = NULL; /* Return value */
+    H5S_extent_t * sdim = NULL; /* New extent dimensionality structure */
+    unsigned       flags, version;
+    unsigned       i;                          /* Local counting variable */
+    const uint8_t *p_end     = p + p_size - 1; /* End of the p buffer */
+    void *         ret_value = NULL;           /* Return value */
 
     FUNC_ENTER_STATIC
 
@@ -161,6 +162,13 @@ H5O__sdspace_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UN
 
     /* Decode dimension sizes */
     if (sdim->rank > 0) {
+        /* Ensure that rank doesn't cause reading passed buffer's end,
+           due to possible data corruption */
+        uint8_t sizeof_size = H5F_SIZEOF_SIZE(f);
+        if (p + (sizeof_size * sdim->rank - 1) > p_end) {
+            HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "rank might cause reading passed buffer's end")
+        }
+
         if (NULL == (sdim->size = (hsize_t *)H5FL_ARR_MALLOC(hsize_t, (size_t)sdim->rank)))
             HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
 
@@ -170,6 +178,11 @@ H5O__sdspace_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UN
         if (flags & H5S_VALID_MAX) {
             if (NULL == (sdim->max = (hsize_t *)H5FL_ARR_MALLOC(hsize_t, (size_t)sdim->rank)))
                 HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
+
+            /* Ensure that rank doesn't cause reading passed buffer's end */
+            if (p + (sizeof_size * sdim->rank - 1) > p_end)
+                HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "rank might cause reading passed buffer's end")
+
             for (i = 0; i < sdim->rank; i++)
                 H5F_DECODE_LENGTH(f, p, sdim->max[i]);
         } /* end if */
diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake
index 528ee1c..037287d 100644
--- a/tools/test/h5repack/CMakeTests.cmake
+++ b/tools/test/h5repack/CMakeTests.cmake
@@ -51,7 +51,8 @@
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_named_dtypes.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_nested_8bit_enum.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_nested_8bit_enum_deflated.h5
-      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_HDFFV-10590_CVE-2018-17432.h5
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_CVE-2018-17432.h5
+      ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_CVE-2018-14460.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_nbit.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_objs.h5
       ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_refs.h5
@@ -1551,10 +1552,15 @@
   ADD_H5_TEST (HDFFV-7840 "TEST" h5diff_attr1.h5)
 
 # test CVE-2018-17432 fix
-  set (arg h5repack_HDFFV-10590_CVE-2018-17432.h5 h5repack_HDFFV-10590_CVE-2018-17432_out.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6)
+  set (arg h5repack_CVE-2018-17432.h5 h5repack__CVE-2018-17432_out.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6)
   set (TESTTYPE "TEST")
   ADD_H5_FILTER_TEST (HDFFV-10590 "" ${TESTTYPE} 1 ${arg})
 
+# test CVE-2018-14460 fix
+  set (arg h5repack_CVE-2018-14460.h5 h5repack_CVE-2018-14460_out.h5)
+  set (TESTTYPE "TEST")
+  ADD_H5_FILTER_TEST (HDFFV-11223 "" ${TESTTYPE} 1 ${arg})
+
 # tests for metadata block size option ('-M')
   ADD_H5_TEST_META (meta_short h5repack_layout.h5 -M 8192)
   ADD_H5_TEST_META (meta_long h5repack_layout.h5 --metadata_block_size=8192)
diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in
index f881b0a..3756a95 100644
--- a/tools/test/h5repack/h5repack.sh.in
+++ b/tools/test/h5repack/h5repack.sh.in
@@ -129,7 +129,8 @@ $SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5
 ########h5diff/testfile########
 $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5
 ########test#HDFFV-10590########
-$SRC_H5REPACK_TESTFILES/h5repack_HDFFV-10590_CVE-2018-17432.h5
+$SRC_H5REPACK_TESTFILES/h5repack_CVE-2018-17432.h5
+$SRC_H5REPACK_TESTFILES/h5repack_CVE-2018-14460.h5
 ########tools/testfiles#for#external#links########
 $SRC_TOOLS_TESTFILES/tsoftlinks.h5
 $SRC_TOOLS_TESTFILES/textlinkfar.h5
@@ -1712,7 +1713,11 @@ TOOLTEST HDFFV-5932 h5repack_attr_refs.h5
 TOOLTEST HDFFV-7840 h5diff_attr1.h5
 
 # test HDFFV-10590
-arg="h5repack_HDFFV-10590_CVE-2018-17432.h5 h5repack_HDFFV-10590_CVE-2018-17432_out.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6"
+arg="h5repack_CVE-2018-17432.h5 h5repack_CVE-2018-17432_out.h5 --low=1 --high=2 -f GZIP=8 -l dset1:CHUNK=5x6"
+TOOLTEST_FAIL $arg
+
+# test HDFFV-11223
+arg="h5repack_CVE-2018-14460.h5 h5repack_CVE-2018-14460_out.h5"
 TOOLTEST_FAIL $arg
 
 # tests for metadata block size option
diff --git a/tools/test/h5repack/testfiles/h5repack_CVE-2018-14460.h5 b/tools/test/h5repack/testfiles/h5repack_CVE-2018-14460.h5
new file mode 100644
index 0000000..f4093b5
--- /dev/null
+++ b/tools/test/h5repack/testfiles/h5repack_CVE-2018-14460.h5
diff --git a/tools/test/h5repack/testfiles/h5repack_HDFFV-10590_CVE-2018-17432.h5 b/tools/test/h5repack/testfiles/h5repack_CVE-2018-17432.h5
index 7a815ba..7a815ba 100644
--- a/tools/test/h5repack/testfiles/h5repack_HDFFV-10590_CVE-2018-17432.h5
+++ b/tools/test/h5repack/testfiles/h5repack_CVE-2018-17432.h5