Fix for HDFFV-9947

Fix to return error when encoding dataspace selection exceeds 32 bit integer limit.

4 files changed, 160 insertions, 19 deletions

diff --git a/src/H5Shyper.c b/src/H5Shyper.c
index 32f6200..09d450f 100644
--- a/src/H5Shyper.c
+++ b/src/H5Shyper.c
@@ -2071,21 +2071,53 @@ H5S_hyper_serialize (const H5S_t *space, uint8_t *buf)
 {
     const H5S_hyper_dim_t *diminfo;         /* Alias for dataspace's diminfo information */
     hsize_t tmp_count[H5O_LAYOUT_NDIMS];    /* Temporary hyperslab counts */
-    hsize_t offset[H5O_LAYOUT_NDIMS];      /* Offset of element in dataspace */
-    hsize_t start[H5O_LAYOUT_NDIMS];   /* Location of start of hyperslab */
-    hsize_t end[H5O_LAYOUT_NDIMS];     /* Location of end of hyperslab */
-    hsize_t temp_off;            /* Offset in a given dimension */
+    hsize_t offset[H5O_LAYOUT_NDIMS];       /* Offset of element in dataspace */
+    hsize_t start[H5O_LAYOUT_NDIMS];        /* Location of start of hyperslab */
+    hsize_t end[H5O_LAYOUT_NDIMS];          /* Location of end of hyperslab */
+    hsize_t bounds_start[H5S_MAX_RANK];     /* Selection bounds */
+    hsize_t bounds_end[H5S_MAX_RANK];       /* Selection bounds */
+    hsize_t temp_off;       /* Offset in a given dimension */
     uint8_t *lenp;          /* pointer to length location for later storage */
     uint32_t len = 0;       /* number of bytes used */
     hsize_t block_count;    /* block counter for regular hyperslabs */
     unsigned fast_dim;      /* Rank of the fastest changing dimension for the dataspace */
     unsigned ndims;         /* Rank of the dataspace */
     int done;               /* Whether we are done with the iteration */
+    unsigned u;             /* Local index variable */
+    herr_t ret_value = SUCCEED;             /* Return value */
 
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
+    FUNC_ENTER_NOAPI_NOINIT
 
     HDassert(space);
 
+    /* Set some convienence values */
+    ndims = space->extent.rank;
+    fast_dim = ndims - 1;
+    diminfo=space->select.sel_info.hslab->opt_diminfo;
+
+    /* Calculate the # of blocks */
+    if(H5S_hyper_is_regular(space)) {
+        /* Check each dimension */
+        for(block_count = 1, u = 0; u < ndims; u++)
+            block_count *= diminfo[u].count;
+    } /* end if */
+    else
+        /* Spin through hyperslab spans */
+        block_count = H5S_hyper_span_nblocks(space->select.sel_info.hslab->span_lst);
+
+    /* Get bounding box */
+    if(H5S_hyper_bounds(space, bounds_start, bounds_end) < 0)
+        HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get selection bounds")
+
+    /* Determine whether the number of blocks or the high bounds in the selection exceed (2^32 - 1) */
+    if(block_count > H5S_UINT32_MAX)
+        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of blocks in selection")
+    else {
+        for(u = 0; u < ndims; u++)
+            if(bounds_end[u] > H5S_UINT32_MAX)
+                HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid hyperslab selection")
+    }
+
     /* Store the preamble information */
     UINT32ENCODE(buf, (uint32_t)H5S_GET_SELECT_TYPE(space));  /* Store the type of selection */
     UINT32ENCODE(buf, (uint32_t)1);  /* Store the version number */
@@ -2101,15 +2133,6 @@ H5S_hyper_serialize (const H5S_t *space, uint8_t *buf)
     if(space->select.sel_info.hslab->diminfo_valid) {
         unsigned u;     /* Local counting variable */
 
-        /* Set some convienence values */
-        ndims = space->extent.rank;
-        fast_dim = ndims - 1;
-        diminfo=space->select.sel_info.hslab->opt_diminfo;
-
-        /* Check each dimension */
-        for(block_count = 1, u = 0; u < ndims; u++)
-            block_count *= diminfo[u].count;
-
         /* Encode number of hyperslabs */
         H5_CHECK_OVERFLOW(block_count, hsize_t, uint32_t);
         UINT32ENCODE(buf, (uint32_t)block_count);
@@ -2188,7 +2211,6 @@ H5S_hyper_serialize (const H5S_t *space, uint8_t *buf)
     } /* end if */
     else {
         /* Encode number of hyperslabs */
-        block_count = H5S_hyper_span_nblocks(space->select.sel_info.hslab->span_lst);
         H5_CHECK_OVERFLOW(block_count, hsize_t, uint32_t);
         UINT32ENCODE(buf, (uint32_t)block_count);
         len+=4;
@@ -2204,7 +2226,8 @@ H5S_hyper_serialize (const H5S_t *space, uint8_t *buf)
     /* Encode length */
     UINT32ENCODE(lenp, (uint32_t)len);  /* Store the length of the extra information */
 
-    FUNC_LEAVE_NOAPI(SUCCEED)
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
 }   /* H5S_hyper_serialize() */
 
 
diff --git a/src/H5Spkg.h b/src/H5Spkg.h
index 2fa872b..0597994 100644
--- a/src/H5Spkg.h
+++ b/src/H5Spkg.h
@@ -46,6 +46,9 @@
  *      and 'size' callbacks for places to change when updating this. */
 #define H5O_SDSPACE_VERSION_LATEST H5O_SDSPACE_VERSION_2
 
+/* Maximum values for uint16_t and uint32_t */
+#define H5S_UINT16_MAX      65535       /* 2^16 - 1 */
+#define H5S_UINT32_MAX      4294967295  /* 2^32 - 1 */
 
 /*
  * Dataspace extent information
diff --git a/src/H5Spoint.c b/src/H5Spoint.c
index 36684ea..e1dbb4d 100644
--- a/src/H5Spoint.c
+++ b/src/H5Spoint.c
@@ -821,12 +821,29 @@ H5S_point_serialize (const H5S_t *space, uint8_t *buf)
     H5S_pnt_node_t *curr;   /* Point information nodes */
     uint8_t *lenp;          /* pointer to length location for later storage */
     uint32_t len=0;         /* number of bytes used */
-    unsigned u;                /* local counting variable */
+    unsigned u;                 /* local counting variable */
+    hsize_t bounds_start[H5S_MAX_RANK]; /* Selection bounding box */
+    hsize_t bounds_end[H5S_MAX_RANK];   /* Selection bounding box */
+    herr_t ret_value = SUCCEED; /* Return value */
 
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
+    FUNC_ENTER_NOAPI_NOINIT
 
     HDassert(space);
 
+     /* Get bounding box for the selection */
+    HDmemset(bounds_end, 0, sizeof(bounds_end));
+    if(H5S_point_bounds(space, bounds_start, bounds_end) < 0)
+        HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get selection bounds")
+
+    /* Determine whether the number of points or the high bounds in the selection exceed (2^32 - 1) */
+    if(space->select.num_elem > H5S_UINT32_MAX)
+        HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid number of points in selection")
+    else {
+        for(u = 0; u < space->extent.rank; u++)
+            if(bounds_end[u] > H5S_UINT32_MAX)
+                HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "invalid points selection")
+    }
+
     /* Store the preamble information */
     UINT32ENCODE(buf, (uint32_t)H5S_GET_SELECT_TYPE(space));  /* Store the type of selection */
     UINT32ENCODE(buf, (uint32_t)1);  /* Store the version number */
@@ -858,7 +875,8 @@ H5S_point_serialize (const H5S_t *space, uint8_t *buf)
     /* Encode length */
     UINT32ENCODE(lenp, (uint32_t)len);  /* Store the length of the extra information */
 
-    FUNC_LEAVE_NOAPI(SUCCEED)
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
 }   /* H5S_point_serialize() */
 
 
diff --git a/test/th5s.c b/test/th5s.c
index 98be156..c56f0cc 100644
--- a/test/th5s.c
+++ b/test/th5s.c
@@ -1317,6 +1317,102 @@ test_h5s_encode(void)
 
 /****************************************************************
 **
+**  test_h5s_encode_exceed32():
+**    Verify that encoding selection that exceeds (2^32 - 1)
+**    (32 bit integer limit) will return error.
+**    See HDFFV-9947 and the RFC for "H5Sencode/H5Sdecode Format Change"
+**
+****************************************************************/
+static void
+test_h5s_encode_exceed32(void)
+{
+    hid_t sid;                      /* Dataspace ID */
+    size_t hyper_buf_size=0, pt_buf_size=0;         /* Buffer size for H5Sencode */
+    unsigned char *hyper_buf=NULL, *pt_buf=NULL;    /* Buffers for H5Sencode */
+    hsize_t numparticles = 8388608;
+    unsigned num_dsets = 513;
+    hsize_t total_particles = numparticles * num_dsets;
+    hsize_t vdsdims[1] = {total_particles};
+    hsize_t start, count, block;    /* Hyperslab selection specification */
+    hsize_t coord[4];               /* Coordinates for point selection */
+    herr_t ret;                     /* Generic return value */
+
+    /* Output message about test being performed */
+    MESSAGE(5, ("Testing Dataspace Encoding Exceeding 32 bits integer limit\n"));
+
+    /*-------------------------------------------------------------------------
+     * Test encoding and decoding of simple dataspace and hyperslab selection.
+     *-------------------------------------------------------------------------
+     */
+
+    /* Create dataspace */
+    sid = H5Screate_simple(1, vdsdims, NULL);
+    CHECK(sid, FAIL, "H5Screate_simple");
+
+    start = 0;
+    block = total_particles;    /* 4303355904 (exceeds 2^32) */
+    count = 1;
+
+    ret = H5Sselect_hyperslab(sid, H5S_SELECT_SET, &start, NULL, &count, &block);
+    CHECK(ret, FAIL, "H5Sselect_hyperslab");
+
+    /* Encode data space in a buffer */
+    ret = H5Sencode(sid, NULL, &hyper_buf_size);
+    CHECK(ret, FAIL, "H5Sencode");
+
+    /* Allocate buffer */
+    if(hyper_buf_size > 0) {
+        hyper_buf = (unsigned char*)HDcalloc((size_t)1, hyper_buf_size);
+        CHECK(hyper_buf, NULL, "HDcalloc");
+    }
+
+    /* H5Sencode should fail because block exceeds (2^32 - 1) */
+    H5E_BEGIN_TRY {
+        ret = H5Sencode(sid, hyper_buf, &hyper_buf_size);
+    } H5E_END_TRY
+    VERIFY(ret, FAIL, "H5Sencode");
+
+    /*-------------------------------------------------------------------------
+     * Test encoding and decoding of simple dataspace and points selection.
+     *-------------------------------------------------------------------------
+     */
+
+    /* Select points in dataspace */
+    coord[0] = 5;
+    coord[1] = 15;
+    coord[2] = 4294967296;      /* 2^32 */
+    coord[3] = 19;
+    ret = H5Sselect_elements(sid, H5S_SELECT_SET, (size_t)4, coord);
+    CHECK(ret, FAIL, "H5Sselect_elements");
+
+    /* Encode data space in a buffer */
+    ret = H5Sencode(sid, NULL, &pt_buf_size);
+    CHECK(ret, FAIL, "H5Sencode");
+
+    /* Allocate buffer */
+    if(pt_buf_size > 0)
+        pt_buf = (unsigned char*)HDcalloc((size_t)1, pt_buf_size);
+
+    /* H5Sencode should fail because coord[2] exceeds (2^32 - 1) */
+    H5E_BEGIN_TRY {
+        ret = H5Sencode(sid, pt_buf, &pt_buf_size);
+    } H5E_END_TRY
+    VERIFY(ret, FAIL, "H5Sencode");
+
+    /* Close the dataspace */
+    ret = H5Sclose(sid);
+    CHECK(ret, FAIL, "H5Sclose");
+
+    /* Free the buffers */
+    if(hyper_buf)
+        HDfree(hyper_buf);
+    if(pt_buf)
+        HDfree(pt_buf);
+
+} /* test_h5s_encode_exceed32() */
+
+/****************************************************************
+**
 **  test_h5s_scalar_write(): Test scalar H5S (dataspace) writing code.
 **
 ****************************************************************/
@@ -2384,6 +2480,7 @@ test_h5s(void)
     test_h5s_null();		/* Test Null dataspace H5S code */
     test_h5s_zero_dim();        /* Test dataspace with zero dimension size */
     test_h5s_encode();          /* Test encoding and decoding */
+    test_h5s_encode_exceed32(); /* Testing encoding when selection exceeds 32 bits limit */
     test_h5s_scalar_write();	/* Test scalar H5S writing code */
     test_h5s_scalar_read();	/* Test scalar H5S reading code */