Fixed HDFFV-10480 (CVE-2018-11206) and HDFFV-11159 (CVE-2018-14033) (#405)

* Fixed HDFFV-10480 (CVE-2018-11206) and HDFFV-11159 (CVE-2018-14033)
Description
    Checked against buffer size to prevent segfault, in case of data corruption.

    + HDFFV-11159 CVE-2018-14033 Buffer over-read in H5O_layout_decode
    + HDFFV-10480 CVE-2018-11206 Buffer over-read in H5O_fill_new[/old]_decode
Platforms tested:
    Linux/64 (jelly)

* Accidentally left in another occurrence of the previous patch from user
   after a more correct fix was applied, that is the check now accounted
   for the previous advance of the buffer pointer.  Removed it.

* Typo

* Fixed format issues.

* Added test.

* Changed arguments to ADD_H5_TEST

* Fixing arguments to ADD_H5_TEST again.

* Fixing arguments again.

* Took out the CMake changes until Allen can help.

* Added files:

tCVE_2018_11206_fill_old.h5
tCVE_2018_11206_fill_new.h5

* Revert "Took out the CMake changes until Allen can help."

This reverts commit c21324d6e0044994c5cd24b0671e7d1dd41096cc.

* Revert "Fixing arguments again."

This reverts commit 5832a70674339e4b524749adde5a181f8c3a446a.

* Revert "Fixing arguments to ADD_H5_TEST again."

This reverts commit b45de823c22ce83a388d46466ef7c04b66ff05ed.

* Revert "Changed arguments to ADD_H5_TEST"

This reverts commit 16719824f57e52158451ddd261788c0dcaa3ec55.

* Added first argument to ADD_H5_TEST for HDFFV-10480 fix.

* Changed argument 0 to 1

* Revert "Changed argument 0 to 1"

This reverts commit b343d6613ba681b43248dd5820e96389984ebcf7.

* Revert "Added first argument to ADD_H5_TEST for HDFFV-10480 fix."

This reverts commit b8a0f9a9e8ec8e6c6ff38d33195d63edff76a563.

* Added first argument and corrected the second.

* Updated fixes for HDFFV-10480 and HDFFV-11159/HDFFV-11049

* Improved error messages.

1 files changed, 19 insertions, 10 deletions

diff --git a/src/H5Olayout.c b/src/H5Olayout.c
index 4020b23..651e317 100644
--- a/src/H5Olayout.c
+++ b/src/H5Olayout.c
@@ -17,7 +17,7 @@
  * Purpose:     Messages related to data layout.
  */
 
-#define H5D_FRIEND     /*suppress error about including H5Dpkg	  */
+#define H5D_FRIEND     /*suppress error about including H5Dpkg       */
 #include "H5Omodule.h" /* This source code file is part of the H5O module */
 
 #include "H5private.h"   /* Generic Functions                        */
@@ -90,12 +90,13 @@ H5FL_DEFINE(H5O_layout_t);
  */
 static void *
 H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags,
-                   unsigned H5_ATTR_UNUSED *ioflags, size_t H5_ATTR_UNUSED p_size, const uint8_t *p)
+                   unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, const uint8_t *p)
 {
-    H5O_layout_t *mesg       = NULL;
-    uint8_t *     heap_block = NULL;
-    unsigned      u;
-    void *        ret_value = NULL; /* Return value */
+    H5O_layout_t * mesg       = NULL;
+    uint8_t *      heap_block = NULL;
+    unsigned       u;
+    const uint8_t *p_end     = p + p_size - 1; /* End of the p buffer */
+    void *         ret_value = NULL;           /* Return value */
 
     FUNC_ENTER_STATIC
 
@@ -179,6 +180,10 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU
         if (mesg->type == H5D_COMPACT) {
             UINT32DECODE(p, mesg->storage.u.compact.size);
             if (mesg->storage.u.compact.size > 0) {
+                /* Ensure that size doesn't exceed buffer size, due to possible data corruption */
+                if (p + mesg->storage.u.compact.size - 1 > p_end)
+                    HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "storage size exceeds buffer size")
+
                 if (NULL == (mesg->storage.u.compact.buf = H5MM_malloc(mesg->storage.u.compact.size)))
                     HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL,
                                 "memory allocation failed for compact data buffer")
@@ -198,6 +203,10 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU
                 UINT16DECODE(p, mesg->storage.u.compact.size);
 
                 if (mesg->storage.u.compact.size > 0) {
+                    /* Ensure that size doesn't exceed buffer size, due to possible data corruption */
+                    if (p + mesg->storage.u.compact.size - 1 > p_end)
+                        HGOTO_ERROR(H5E_OHDR, H5E_OVERFLOW, NULL, "storage size exceeds buffer size")
+
                     /* Allocate space for compact data */
                     if (NULL == (mesg->storage.u.compact.buf = H5MM_malloc(mesg->storage.u.compact.size)))
                         HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, NULL,
@@ -887,13 +896,13 @@ done:
 } /* end H5O__layout_reset() */
 
 /*-------------------------------------------------------------------------
- * Function:	H5O__layout_free
+ * Function:    H5O__layout_free
  *
- * Purpose:	Free's the message
+ * Purpose:     Free's the message
  *
- * Return:	Non-negative on success/Negative on failure
+ * Return:      Non-negative on success/Negative on failure
  *
- * Programmer:	Quincey Koziol
+ * Programmer:  Quincey Koziol
  *              Saturday, March 11, 2000
  *
  *-------------------------------------------------------------------------