Merging in latest from upstream (HDFFV/hdf5:refs/heads/develop)

* commit 'd8723ac1178a69c4dff7164203d999d61c2159bb':
  Fix for HDFFV-10357 (CVE-2017-17508).
  Fix for HDFFV-10355 (CVE-2017-17506).
  Fix for HDFFV-10354 (CVE-2017-17505).

1 files changed, 57 insertions, 36 deletions

diff --git a/src/H5Opline.c b/src/H5Opline.c
index e817f9a..e59e192 100644
--- a/src/H5Opline.c
+++ b/src/H5Opline.c
@@ -34,7 +34,7 @@
 /* PRIVATE PROTOTYPES */
 static herr_t H5O_pline_encode(H5F_t *f, uint8_t *p, const void *mesg);
 static void *H5O_pline_decode(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh,
-    unsigned mesg_flags, unsigned *ioflags, const uint8_t *p);
+    unsigned mesg_flags, unsigned *ioflags, size_t p_size, const uint8_t *p);
 static void *H5O_pline_copy(const void *_mesg, void *_dest);
 static size_t H5O_pline_size(const H5F_t *f, const void *_mesg);
 static herr_t H5O_pline_reset(void *_mesg);
@@ -115,12 +115,14 @@ H5FL_DEFINE(H5O_pline_t);
  */
 static void *
 H5O_pline_decode(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED *open_oh,
-    unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, const uint8_t *p)
+    unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags,
+    size_t p_size, const uint8_t *p)
 {
     H5O_pline_t		*pline = NULL;          /* Pipeline message */
     H5Z_filter_info_t   *filter;                /* Filter to decode */
     size_t		name_length;            /* Length of filter name */
     size_t		i;                      /* Local index variable */
+    const uint8_t *p_end = p + p_size - 1;  /* End of the p buffer */
     void		*ret_value = NULL;      /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -130,17 +132,24 @@ H5O_pline_decode(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5
 
     /* Allocate space for I/O pipeline message */
     if(NULL == (pline = H5FL_CALLOC(H5O_pline_t)))
-	HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
+        HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
 
     /* Version */
     pline->version = *p++;
     if(pline->version < H5O_PLINE_VERSION_1 || pline->version > H5O_PLINE_VERSION_LATEST)
-	HGOTO_ERROR(H5E_PLINE, H5E_CANTLOAD, NULL, "bad version number for filter pipeline message")
+        HGOTO_ERROR(H5E_PLINE, H5E_CANTLOAD, NULL, "bad version number for filter pipeline message")
 
     /* Number of filters */
     pline->nused = *p++;
-    if(pline->nused > H5Z_MAX_NFILTERS)
-	HGOTO_ERROR(H5E_PLINE, H5E_CANTLOAD, NULL, "filter pipeline message has too many filters")
+    if(pline->nused > H5Z_MAX_NFILTERS) {
+
+        /* Reset the number of filters used to avoid array traversal in error
+         * handling code.
+         */
+        pline->nused = 0;
+
+        HGOTO_ERROR(H5E_PLINE, H5E_CANTLOAD, NULL, "filter pipeline message has too many filters")
+    }
 
     /* Reserved */
     if(pline->version == H5O_PLINE_VERSION_1)
@@ -149,12 +158,12 @@ H5O_pline_decode(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5
     /* Allocate array for filters */
     pline->nalloc = pline->nused;
     if(NULL == (pline->filter = (H5Z_filter_info_t *)H5MM_calloc(pline->nalloc * sizeof(pline->filter[0]))))
-	HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
+        HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
 
     /* Decode filters */
     for(i = 0, filter = &pline->filter[0]; i < pline->nused; i++, filter++) {
         /* Filter ID */
-	UINT16DECODE(p, filter->id);
+        UINT16DECODE(p, filter->id);
 
         /* Length of filter name */
         if(pline->version > H5O_PLINE_VERSION_1 && filter->id < H5Z_FILTER_RESERVED)
@@ -166,18 +175,18 @@ H5O_pline_decode(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5
         } /* end if */
 
         /* Filter flags */
-	UINT16DECODE(p, filter->flags);
+        UINT16DECODE(p, filter->flags);
 
         /* Number of filter parameters ("client data elements") */
-	UINT16DECODE(p, filter->cd_nelmts);
+        UINT16DECODE(p, filter->cd_nelmts);
 
         /* Filter name, if there is one */
-	if(name_length) {
+        if(name_length) {
             size_t actual_name_length;          /* Actual length of name */
 
             /* Determine actual name length (without padding, but with null terminator) */
-	    actual_name_length = HDstrlen((const char *)p) + 1;
-	    HDassert(actual_name_length <= name_length);
+            actual_name_length = HDstrlen((const char *)p) + 1;
+            HDassert(actual_name_length <= name_length);
 
             /* Allocate space for the filter name, or use the internal buffer */
             if(actual_name_length > H5Z_COMMON_NAME_LEN) {
@@ -188,12 +197,12 @@ H5O_pline_decode(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5
             else
                 filter->name = filter->_name;
 
-	    HDstrncpy(filter->name, (const char *)p, actual_name_length);
-	    p += name_length;
-	} /* end if */
+            HDstrncpy(filter->name, (const char *)p, actual_name_length);
+            p += name_length;
+        } /* end if */
 
         /* Filter parameters */
-	if(filter->cd_nelmts) {
+        if(filter->cd_nelmts) {
             size_t	j;              /* Local index variable */
 
             /* Allocate space for the client data elements, or use the internal buffer */
@@ -205,15 +214,20 @@ H5O_pline_decode(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5
             else
                 filter->cd_values = filter->_cd_values;
 
-	    /*
-	     * Read the client data values and the padding
-	     */
-	    for(j = 0; j < filter->cd_nelmts; j++)
-		UINT32DECODE(p, filter->cd_values[j]);
+            /*
+             * Read the client data values and the padding
+             */
+            for (j = 0; j < filter->cd_nelmts; j++) {
+                if (p + 4 - 1 <= p_end)
+                    UINT32DECODE(p, filter->cd_values[j])
+                else
+                    HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "ran off the end of the buffer: current p = %p, p_size = %zu, p_end = %p", p, p_size, p_end)
+            }
+
             if(pline->version == H5O_PLINE_VERSION_1)
                 if(filter->cd_nelmts % 2)
                     p += 4; /*padding*/
-	} /* end if */
+        } /* end if */
     } /* end for */
 
     /* Set return value */
@@ -502,23 +516,30 @@ H5O_pline_reset(void *mesg)
 
     FUNC_ENTER_NOAPI_NOINIT_NOERR
 
+    /* NOTE: This function can be called during error processing from
+     *       other API calls so DO NOT ASSUME THAT ANY VALUES ARE SANE.
+     */
+
     HDassert(pline);
 
-    /* Free information for each filter */
-    for(i = 0; i < pline->nused; i++) {
-        if(pline->filter[i].name && pline->filter[i].name != pline->filter[i]._name)
-            HDassert((HDstrlen(pline->filter[i].name) + 1) > H5Z_COMMON_NAME_LEN);
-        if(pline->filter[i].name != pline->filter[i]._name)
-            pline->filter[i].name = (char *)H5MM_xfree(pline->filter[i].name);
-        if(pline->filter[i].cd_values && pline->filter[i].cd_values != pline->filter[i]._cd_values)
-            HDassert(pline->filter[i].cd_nelmts > H5Z_COMMON_CD_VALUES);
-        if(pline->filter[i].cd_values != pline->filter[i]._cd_values)
-            pline->filter[i].cd_values = (unsigned *)H5MM_xfree(pline->filter[i].cd_values);
-    } /* end for */
+    /* Free the filter information and array */
+    if (pline->filter) {
+
+        /* Free information for each filter */
+        for(i = 0; i < pline->nused; i++) {
+            if(pline->filter[i].name && pline->filter[i].name != pline->filter[i]._name)
+                HDassert((HDstrlen(pline->filter[i].name) + 1) > H5Z_COMMON_NAME_LEN);
+            if(pline->filter[i].name != pline->filter[i]._name)
+                pline->filter[i].name = (char *)H5MM_xfree(pline->filter[i].name);
+            if(pline->filter[i].cd_values && pline->filter[i].cd_values != pline->filter[i]._cd_values)
+                HDassert(pline->filter[i].cd_nelmts > H5Z_COMMON_CD_VALUES);
+            if(pline->filter[i].cd_values != pline->filter[i]._cd_values)
+                pline->filter[i].cd_values = (unsigned *)H5MM_xfree(pline->filter[i].cd_values);
+        } /* end for */
 
-    /* Free filter array */
-    if(pline->filter)
+        /* Free filter array */
         pline->filter = (H5Z_filter_info_t *)H5MM_xfree(pline->filter);
+    }
 
     /* Reset # of filters */
     pline->nused = pline->nalloc = 0;