Fixed HDFFV-10480 (CVE-2018-11206) and HDFFV-11159 (CVE-2018-14033) (#405)

* Fixed HDFFV-10480 (CVE-2018-11206) and HDFFV-11159 (CVE-2018-14033)
Description
    Checked against buffer size to prevent segfault, in case of data corruption.

    + HDFFV-11159 CVE-2018-14033 Buffer over-read in H5O_layout_decode
    + HDFFV-10480 CVE-2018-11206 Buffer over-read in H5O_fill_new[/old]_decode
Platforms tested:
    Linux/64 (jelly)

* Accidentally left in another occurrence of the previous patch from user
   after a more correct fix was applied, that is the check now accounted
   for the previous advance of the buffer pointer.  Removed it.

* Typo

* Fixed format issues.

* Added test.

* Changed arguments to ADD_H5_TEST

* Fixing arguments to ADD_H5_TEST again.

* Fixing arguments again.

* Took out the CMake changes until Allen can help.

* Added files:

tCVE_2018_11206_fill_old.h5
tCVE_2018_11206_fill_new.h5

* Revert "Took out the CMake changes until Allen can help."

This reverts commit c21324d6e0044994c5cd24b0671e7d1dd41096cc.

* Revert "Fixing arguments again."

This reverts commit 5832a70674339e4b524749adde5a181f8c3a446a.

* Revert "Fixing arguments to ADD_H5_TEST again."

This reverts commit b45de823c22ce83a388d46466ef7c04b66ff05ed.

* Revert "Changed arguments to ADD_H5_TEST"

This reverts commit 16719824f57e52158451ddd261788c0dcaa3ec55.

* Added first argument to ADD_H5_TEST for HDFFV-10480 fix.

* Changed argument 0 to 1

* Revert "Changed argument 0 to 1"

This reverts commit b343d6613ba681b43248dd5820e96389984ebcf7.

* Revert "Added first argument to ADD_H5_TEST for HDFFV-10480 fix."

This reverts commit b8a0f9a9e8ec8e6c6ff38d33195d63edff76a563.

* Added first argument and corrected the second.

* Updated fixes for HDFFV-10480 and HDFFV-11159/HDFFV-11049

* Improved error messages.

1 files changed, 17 insertions, 10 deletions

diff --git a/src/H5Ofill.c b/src/H5Ofill.c
index 695673f..5068039 100644
--- a/src/H5Ofill.c
+++ b/src/H5Ofill.c
@@ -196,8 +196,9 @@ H5O__fill_new_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh,
                      unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, size_t p_size,
                      const uint8_t *p)
 {
-    H5O_fill_t *fill      = NULL;
-    void *      ret_value = NULL; /* Return value */
+    H5O_fill_t *   fill      = NULL;
+    const uint8_t *p_end     = p + p_size - 1; /* End of the p buffer */
+    void *         ret_value = NULL;           /* Return value */
 
     FUNC_ENTER_STATIC
 
@@ -228,8 +229,11 @@ H5O__fill_new_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh,
             INT32DECODE(p, fill->size);
             if (fill->size > 0) {
                 H5_CHECK_OVERFLOW(fill->size, ssize_t, size_t);
-                if ((size_t)fill->size > p_size)
-                    HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "destination buffer too small")
+
+                /* Ensure that fill size doesn't exceed buffer size, due to possible data corruption */
+                if (p + fill->size - 1 > p_end)
+                    HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "fill size exceeds buffer size")
+
                 if (NULL == (fill->buf = H5MM_malloc((size_t)fill->size)))
                     HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for fill value")
                 H5MM_memcpy(fill->buf, p, (size_t)fill->size);
@@ -311,10 +315,11 @@ static void *
 H5O__fill_old_decode(H5F_t *f, H5O_t *open_oh, unsigned H5_ATTR_UNUSED mesg_flags,
                      unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, const uint8_t *p)
 {
-    H5O_fill_t *fill      = NULL; /* Decoded fill value message */
-    htri_t      exists    = FALSE;
-    H5T_t *     dt        = NULL;
-    void *      ret_value = NULL; /* Return value */
+    H5O_fill_t *   fill      = NULL; /* Decoded fill value message */
+    htri_t         exists    = FALSE;
+    H5T_t *        dt        = NULL;
+    const uint8_t *p_end     = p + p_size - 1; /* End of the p buffer */
+    void *         ret_value = NULL;           /* Return value */
 
     FUNC_ENTER_STATIC
 
@@ -335,8 +340,10 @@ H5O__fill_old_decode(H5F_t *f, H5O_t *open_oh, unsigned H5_ATTR_UNUSED mesg_flag
     /* Only decode the fill value itself if there is one */
     if (fill->size > 0) {
         H5_CHECK_OVERFLOW(fill->size, ssize_t, size_t);
-        if ((size_t)fill->size > p_size)
-            HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "destination buffer too small")
+
+        /* Ensure that fill size doesn't exceed buffer size, due to possible data corruption */
+        if (p + fill->size - 1 > p_end)
+            HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "fill size exceeds buffer size")
 
         /* Get the datatype message  */
         if ((exists = H5O_msg_exists_oh(open_oh, H5O_DTYPE_ID)) < 0)