Prevent buffer overrun in H5S_select_deserialize (#2931)

* Prevent buffer overrun in H5S_select_deserialize

The call to H5S_select_deserialize from H5S_decode doesn't have
the buffer size available to it, so to allow decoding there
I set it to assume a max size buffer for now.

Making the buffer size known in H5S_decode could be done by
modifying the external API's H5Sdecode, or splitting H5Sdecode
into two functions using a macro (similar to H5Sencode), with the
macro taking one argument and assuming a max buffer size.

* Conditional buffer check in H5S_select_deserialize

Moved and renamed a macro for only checking buffer overflow when
buffer size is known from H5Odtype.c to H5private.h,
so it can be used throughout the library.

Also silenced some build warnings about types.

1 files changed, 15 insertions, 4 deletions

diff --git a/src/H5Rint.c b/src/H5Rint.c
index 4b3d692..7b22150 100644
--- a/src/H5Rint.c
+++ b/src/H5Rint.c
@@ -1177,6 +1177,7 @@ static herr_t
 H5R__decode_region(const unsigned char *buf, size_t *nbytes, H5S_t **space_ptr)
 {
     const uint8_t *p        = (const uint8_t *)buf;
+    const uint8_t *p_end    = p + *nbytes - 1;
     size_t         buf_size = 0;
     unsigned       rank;
     H5S_t         *space;
@@ -1209,7 +1210,11 @@ H5R__decode_region(const unsigned char *buf, size_t *nbytes, H5S_t **space_ptr)
         HGOTO_ERROR(H5E_REFERENCE, H5E_CANTDECODE, FAIL, "Buffer size is too small")
     if (H5S_set_extent_simple(space, rank, NULL, NULL) < 0)
         HGOTO_ERROR(H5E_REFERENCE, H5E_CANTSET, FAIL, "can't set extent rank for selection")
-    if (H5S_SELECT_DESERIALIZE(&space, &p) < 0)
+
+    if (p - 1 > p_end)
+        HGOTO_ERROR(H5E_REFERENCE, H5E_CANTDECODE, FAIL, "Ran off end of buffer while decoding")
+
+    if (H5S_SELECT_DESERIALIZE(&space, &p, (size_t)(p_end - p + 1)) < 0)
         HGOTO_ERROR(H5E_REFERENCE, H5E_CANTDECODE, FAIL, "can't deserialize selection")
 
     *nbytes    = buf_size;
@@ -1472,7 +1477,8 @@ H5R__decode_token_region_compat(H5F_t *f, const unsigned char *buf, size_t *nbyt
     unsigned char *data  = NULL;
     H5O_token_t    token = {0};
     size_t         data_size;
-    const uint8_t *p;
+    const uint8_t *p         = NULL;
+    const uint8_t *p_end     = NULL;
     H5S_t         *space     = NULL;
     herr_t         ret_value = SUCCEED;
 
@@ -1488,7 +1494,8 @@ H5R__decode_token_region_compat(H5F_t *f, const unsigned char *buf, size_t *nbyt
         HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid location identifier")
 
     /* Get object address */
-    p = (const uint8_t *)data;
+    p     = (const uint8_t *)data;
+    p_end = p + data_size - 1;
     H5MM_memcpy(&token, p, token_size);
     p += token_size;
 
@@ -1508,7 +1515,11 @@ H5R__decode_token_region_compat(H5F_t *f, const unsigned char *buf, size_t *nbyt
             HGOTO_ERROR(H5E_REFERENCE, H5E_NOTFOUND, FAIL, "not found")
 
         /* Unserialize the selection */
-        if (H5S_SELECT_DESERIALIZE(&space, &p) < 0)
+
+        if (p - 1 >= p_end)
+            HGOTO_ERROR(H5E_REFERENCE, H5E_CANTDECODE, FAIL, "Ran off end of buffer while deserializing")
+
+        if (H5S_SELECT_DESERIALIZE(&space, &p, (size_t)(p_end - p + 1)) < 0)
             HGOTO_ERROR(H5E_REFERENCE, H5E_CANTDECODE, FAIL, "can't deserialize selection")
 
         *space_ptr = space;