Harden the v1 B-tree and local heap cache clients (#2803)

* Hardens v1 B-tree deserialize function
* Harden the H5HL deserialize functionality

1 files changed, 37 insertions, 46 deletions

diff --git a/src/H5Bcache.c b/src/H5Bcache.c
index cd0a0ba..437bc1b 100644
--- a/src/H5Bcache.c
+++ b/src/H5Bcache.c
@@ -13,10 +13,8 @@
 /*-------------------------------------------------------------------------
  *
  * Created:		H5Bcache.c
- *			Oct 31 2005
- *			Quincey Koziol
  *
- * Purpose:		Implement B-tree metadata cache methods.
+ * Purpose:		Implement B-tree metadata cache methods
  *
  *-------------------------------------------------------------------------
  */
@@ -83,13 +81,9 @@ const H5AC_class_t H5AC_BT[1] = {{
 /*-------------------------------------------------------------------------
  * Function:    H5B__cache_get_initial_load_size
  *
- * Purpose:     Compute the size of the data structure on disk.
- *
- * Return:      Non-negative on success/Negative on failure
- *
- * Programmer:  Quincey Koziol
- *              May 18, 2010
+ * Purpose:     Compute the size of the data structure on disk
  *
+ * Return:      SUCCEED/FAIL
  *-------------------------------------------------------------------------
  */
 static herr_t
@@ -117,24 +111,20 @@ H5B__cache_get_initial_load_size(void *_udata, size_t *image_len)
 /*-------------------------------------------------------------------------
  * Function:    H5B__cache_deserialize
  *
- * Purpose:     Deserialize the data structure from disk.
- *
- * Return:	Success:	Pointer to a new B-tree node.
- *		Failure:	NULL
- *
- * Programmer:  Quincey Koziol
- *              Mar 24, 2008
+ * Purpose:     Deserialize the data structure from disk
  *
+ * Return:      Success:    Pointer to a new B-tree node
+ *              Failure:    NULL
  *-------------------------------------------------------------------------
  */
 static void *
-H5B__cache_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata,
-                       hbool_t H5_ATTR_UNUSED *dirty)
+H5B__cache_deserialize(const void *_image, size_t len, void *_udata, hbool_t H5_ATTR_UNUSED *dirty)
 {
     H5B_t          *bt    = NULL;                     /* Pointer to the deserialized B-tree node */
     H5B_cache_ud_t *udata = (H5B_cache_ud_t *)_udata; /* User data for callback */
     H5B_shared_t   *shared;                           /* Pointer to shared B-tree info */
     const uint8_t  *image = (const uint8_t *)_image;  /* Pointer into image buffer */
+    const uint8_t  *p_end = image + len - 1;          /* End of image buffer */
     uint8_t        *native;                           /* Pointer to native keys */
     unsigned        u;                                /* Local index variable */
     H5B_t          *ret_value = NULL;                 /* Return value */
@@ -156,7 +146,8 @@ H5B__cache_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, void *_uda
 
     /* Get a pointer to the shared info, for convenience */
     shared = (H5B_shared_t *)H5UC_GET_OBJ(bt->rc_shared);
-    HDassert(shared);
+    if (NULL == shared)
+        HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, NULL, "can't get a pointer to shared data")
 
     /* Allocate space for the native keys and child addresses */
     if (NULL == (bt->native = H5FL_BLK_MALLOC(native_block, shared->sizeof_keys)))
@@ -164,49 +155,61 @@ H5B__cache_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, void *_uda
     if (NULL == (bt->child = H5FL_SEQ_MALLOC(haddr_t, (size_t)shared->two_k)))
         HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, NULL, "can't allocate buffer for child addresses")
 
-    /* magic number */
+    /* Magic number */
+    if (H5_IS_BUFFER_OVERFLOW(image, H5_SIZEOF_MAGIC, p_end))
+        HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
     if (HDmemcmp(image, H5B_MAGIC, (size_t)H5_SIZEOF_MAGIC) != 0)
         HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "wrong B-tree signature")
     image += H5_SIZEOF_MAGIC;
 
-    /* node type and level */
+    /* Node type and level */
+    if (H5_IS_BUFFER_OVERFLOW(image, 2, p_end))
+        HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
     if (*image++ != (uint8_t)udata->type->id)
         HGOTO_ERROR(H5E_BTREE, H5E_CANTLOAD, NULL, "incorrect B-tree node type")
     bt->level = *image++;
 
-    /* entries used */
+    /* Entries used */
+    if (H5_IS_BUFFER_OVERFLOW(image, 2, p_end))
+        HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
     UINT16DECODE(image, bt->nchildren);
 
     /* Check if bt->nchildren is greater than two_k */
     if (bt->nchildren > shared->two_k)
         HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "number of children is greater than maximum")
 
-    /* sibling pointers */
+    /* Sibling pointers */
+    if (H5_IS_BUFFER_OVERFLOW(image, H5F_sizeof_addr(udata->f), p_end))
+        HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
     H5F_addr_decode(udata->f, (const uint8_t **)&image, &(bt->left));
+
+    if (H5_IS_BUFFER_OVERFLOW(image, H5F_sizeof_addr(udata->f), p_end))
+        HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
     H5F_addr_decode(udata->f, (const uint8_t **)&image, &(bt->right));
 
-    /* the child/key pairs */
+    /* Child/key pairs */
     native = bt->native;
     for (u = 0; u < bt->nchildren; u++) {
         /* Decode native key value */
+        if (H5_IS_BUFFER_OVERFLOW(image, shared->sizeof_rkey, p_end))
+            HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
         if ((udata->type->decode)(shared, image, native) < 0)
             HGOTO_ERROR(H5E_BTREE, H5E_CANTDECODE, NULL, "unable to decode key")
         image += shared->sizeof_rkey;
         native += udata->type->sizeof_nkey;
 
         /* Decode address value */
+        if (H5_IS_BUFFER_OVERFLOW(image, H5F_sizeof_addr(udata->f), p_end))
+            HGOTO_ERROR(H5E_BTREE, H5E_OVERFLOW, NULL, "ran off end of input buffer while decoding");
         H5F_addr_decode(udata->f, (const uint8_t **)&image, bt->child + u);
-    } /* end for */
+    }
 
-    /* Decode final key */
+    /* Final key */
     if (bt->nchildren > 0) {
         /* Decode native key value */
         if ((udata->type->decode)(shared, image, native) < 0)
             HGOTO_ERROR(H5E_BTREE, H5E_CANTDECODE, NULL, "unable to decode key")
-    } /* end if */
-
-    /* Sanity check */
-    HDassert((size_t)((const uint8_t *)image - (const uint8_t *)_image) <= len);
+    }
 
     /* Set return value */
     ret_value = bt;
@@ -224,11 +227,7 @@ done:
  *
  * Purpose:     Compute the size of the data structure on disk.
  *
- * Return:      Non-negative on success/Negative on failure
- *
- * Programmer:  Quincey Koziol
- *              May 20, 2010
- *
+ * Return:      SUCCEED/FAIL
  *-------------------------------------------------------------------------
  */
 static herr_t
@@ -256,13 +255,9 @@ H5B__cache_image_len(const void *_thing, size_t *image_len)
 /*-------------------------------------------------------------------------
  * Function:    H5B__cache_serialize
  *
- * Purpose:     Serialize the data structure for writing to disk.
- *
- * Return:      Non-negative on success/Negative on failure
- *
- * Programmer:  Quincey Koziol
- *              Mar 24, 2008
+ * Purpose:     Serialize the data structure for writing to disk
  *
+ * Return:      SUCCEED/FAIL
  *-------------------------------------------------------------------------
  */
 static herr_t
@@ -341,11 +336,7 @@ done:
  *
  * Purpose:     Destroy/release an "in core representation" of a data structure
  *
- * Return:      Non-negative on success/Negative on failure
- *
- * Programmer:  Quincey Koziol
- *              Mar 26, 2008
- *
+ * Return:      SUCCEED/FAIL
  *-------------------------------------------------------------------------
  */
 static herr_t