Merging in latest from upstream (HDFFV/hdf5:refs/heads/hdf5_1_8)

* commit '8b4633e99d6e447327526663acd30352b5840446': (29 commits)
  Correct usuage of cmake variable
  Correct name of windows variable
  Add STGZ to cpack generator list for OSX
  Add missing exists check from master project
  Check file length before compare
  Change check for number of dimensions for old-style arrays in datatype decoding routine from an assertion to an if/HGOTO_ERROR check, since it is inappropriate to assert the contents of a file will be what we expect.
  Fix mistakes in porting TALOS-177 fix to 1.8.
  Fix bug in "nooptype" decode in fix for TALOS-0177.
  Fix issues in H5Znbit.c where the decompression algorithm would not check the compressed data for validity, potentially causing a buffer overflow.
  Replace assertion in H5O_dtype_decode_helper for number of array dimensions with a check and error. The assertion was inappropriate because it is operating on data read from the file, which the library does not always have direct control of.
  HDFFV-9991: Remove unneeded file
  Add new test file to MANIFEST
  Fix error reporting
  Update new test file references
  revert merge change to cleanup files
  Fix incorrect use of H5Rdereference2
  remove merge typo
  Remove unsupported function
  Merge/align from trunk(develop)
  Merge/align with trunk
  ...

2 files changed, 82 insertions, 28 deletions

diff --git a/src/H5Odtype.c b/src/H5Odtype.c
index 622a18e..67f2fca 100644
--- a/src/H5Odtype.c
+++ b/src/H5Odtype.c
@@ -310,7 +310,11 @@ H5O_dtype_decode_helper(H5F_t *f, unsigned *ioflags/*in,out*/, const uint8_t **p
                     if(version == H5O_DTYPE_VERSION_1) {
                         /* Decode the number of dimensions */
                         ndims = *(*pp)++;
-                        HDassert(ndims <= 4);
+
+                        /* Check that ndims is valid */
+                        if(ndims > 4)
+                            HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "invalid number of dimensions for array")
+
                         *pp += 3;		/*reserved bytes */
 
                         /* Skip dimension permutation */
@@ -518,7 +522,8 @@ H5O_dtype_decode_helper(H5F_t *f, unsigned *ioflags/*in,out*/, const uint8_t **p
             dt->shared->u.array.ndims = *(*pp)++;
 
             /* Double-check the number of dimensions */
-            HDassert(dt->shared->u.array.ndims <= H5S_MAX_RANK);
+            if(dt->shared->u.array.ndims > H5S_MAX_RANK)
+                HGOTO_ERROR(H5E_DATATYPE, H5E_CANTLOAD, FAIL, "too many dimensions for array datatype")
 
             /* Skip reserved bytes, if version has them */
             if(version < H5O_DTYPE_VERSION_3)
diff --git a/src/H5Znbit.c b/src/H5Znbit.c
index 38610b6..ddcf8fd 100644
--- a/src/H5Znbit.c
+++ b/src/H5Znbit.c
@@ -60,11 +60,11 @@ static void H5Z_nbit_decompress_one_nooptype(unsigned char *data, size_t data_of
                        unsigned char *buffer, size_t *j, int *buf_len, unsigned size);
 static void H5Z_nbit_decompress_one_atomic(unsigned char *data, size_t data_offset,
                     unsigned char *buffer, size_t *j, int *buf_len, parms_atomic p);
-static void H5Z_nbit_decompress_one_array(unsigned char *data, size_t data_offset,
+static herr_t H5Z__nbit_decompress_one_array(unsigned char *data, size_t data_offset,
            unsigned char *buffer, size_t *j, int *buf_len, const unsigned parms[]);
-static void H5Z_nbit_decompress_one_compound(unsigned char *data, size_t data_offset,
+static herr_t H5Z__nbit_decompress_one_compound(unsigned char *data, size_t data_offset,
               unsigned char *buffer, size_t *j, int *buf_len, const unsigned parms[]);
-static void H5Z_nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer,
+static herr_t H5Z__nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer,
                                 const unsigned parms[]);
 static void H5Z_nbit_compress_one_nooptype(unsigned char *data, size_t data_offset,
                      unsigned char *buffer, size_t *j, int *buf_len, unsigned size);
@@ -897,7 +897,8 @@ H5Z_filter_nbit(unsigned flags, size_t cd_nelmts, const unsigned cd_values[],
             HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for nbit decompression")
 
         /* decompress the buffer */
-        H5Z_nbit_decompress(outbuf, d_nelmts, (unsigned char *)*buf, cd_values);
+        if(H5Z__nbit_decompress(outbuf, d_nelmts, (unsigned char *)*buf, cd_values) < 0)
+            HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, 0, "can't decompress buffer")
     } /* end if */
     /* output; compress */
     else {
@@ -1046,12 +1047,15 @@ H5Z_nbit_decompress_one_atomic(unsigned char *data, size_t data_offset,
    }
 }
 
-static void
-H5Z_nbit_decompress_one_array(unsigned char *data, size_t data_offset,
+static herr_t
+H5Z__nbit_decompress_one_array(unsigned char *data, size_t data_offset,
            unsigned char *buffer, size_t *j, int *buf_len, const unsigned parms[])
 {
    unsigned i, total_size, base_class, base_size, n, begin_index;
    parms_atomic p;
+   herr_t ret_value = SUCCEED; /* Return value */
+
+   FUNC_ENTER_STATIC
 
    total_size = parms[parms_index++];
    base_class = parms[parms_index++];
@@ -1062,7 +1066,12 @@ H5Z_nbit_decompress_one_array(unsigned char *data, size_t data_offset,
            p.order = parms[parms_index++];
            p.precision = parms[parms_index++];
            p.offset = parms[parms_index++];
-           n = total_size/p.size;
+
+           /* Check values of precision and offset */
+           if(p.precision > p.size * 8 || (p.precision + p.offset) > p.size * 8)
+              HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "invalid datatype precision/offset")
+
+           n = total_size / p.size;
            for(i = 0; i < n; i++)
               H5Z_nbit_decompress_one_atomic(data, data_offset + i*p.size,
                                              buffer, j, buf_len, p);
@@ -1072,8 +1081,9 @@ H5Z_nbit_decompress_one_array(unsigned char *data, size_t data_offset,
            n = total_size/base_size; /* number of base_type elements inside the array datatype */
            begin_index = parms_index;
            for(i = 0; i < n; i++) {
-              H5Z_nbit_decompress_one_array(data, data_offset + i*base_size,
-                                            buffer, j, buf_len, parms);
+              if(H5Z__nbit_decompress_one_array(data, data_offset + i * base_size,
+                                            buffer, j, buf_len, parms) < 0)
+                 HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress array")
               parms_index = begin_index;
            }
            break;
@@ -1082,8 +1092,9 @@ H5Z_nbit_decompress_one_array(unsigned char *data, size_t data_offset,
            n = total_size/base_size; /* number of base_type elements inside the array datatype */
            begin_index = parms_index;
            for(i = 0; i < n; i++) {
-              H5Z_nbit_decompress_one_compound(data, data_offset + i*base_size,
-                                               buffer, j, buf_len, parms);
+              if(H5Z__nbit_decompress_one_compound(data, data_offset + i * base_size,
+                                               buffer, j, buf_len, parms) < 0)
+                 HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress compound")
               parms_index = begin_index;
            }
            break;
@@ -1094,51 +1105,76 @@ H5Z_nbit_decompress_one_array(unsigned char *data, size_t data_offset,
       default:
           HDassert(0 && "This Should never be executed!");
    } /* end switch */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
 }
 
-static void
-H5Z_nbit_decompress_one_compound(unsigned char *data, size_t data_offset,
+static herr_t
+H5Z__nbit_decompress_one_compound(unsigned char *data, size_t data_offset,
               unsigned char *buffer, size_t *j, int *buf_len, const unsigned parms[])
 {
-   unsigned i, nmembers, member_offset, member_class, size;
+   unsigned i, nmembers, member_offset, member_class, member_size, used_size = 0, size;
    parms_atomic p;
+   herr_t ret_value = SUCCEED; /* Return value */
 
-   parms_index++; /* skip total size of compound datatype */
+   FUNC_ENTER_STATIC
+
+   size = parms[parms_index++];
    nmembers = parms[parms_index++];
 
    for(i = 0; i < nmembers; i++) {
       member_offset = parms[parms_index++];
       member_class = parms[parms_index++];
+
+      /* Check for overflow */
+      member_size = parms[parms_index];
+      used_size += member_size;
+      if(used_size > size)
+         HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "compound member offset overflowed compound size")
       switch(member_class) {
          case H5Z_NBIT_ATOMIC:
-              p.size = parms[parms_index++];
+              p.size = member_size;
+              /* Advance past member size */
+              parms_index++;
               p.order = parms[parms_index++];
               p.precision = parms[parms_index++];
               p.offset = parms[parms_index++];
+
+              /* Check values of precision and offset */
+              if(p.precision > p.size * 8 || (p.precision + p.offset) > p.size * 8)
+                 HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "invalid datatype precision/offset")
+
               H5Z_nbit_decompress_one_atomic(data, data_offset + member_offset,
                                              buffer, j, buf_len, p);
               break;
          case H5Z_NBIT_ARRAY:
-              H5Z_nbit_decompress_one_array(data, data_offset + member_offset,
-                                            buffer, j, buf_len, parms);
+              if(H5Z__nbit_decompress_one_array(data, data_offset + member_offset,
+                                            buffer, j, buf_len, parms) < 0)
+                 HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress array")
               break;
          case H5Z_NBIT_COMPOUND:
-              H5Z_nbit_decompress_one_compound(data, data_offset+member_offset,
-                                               buffer, j, buf_len, parms);
+              if(H5Z__nbit_decompress_one_compound(data, data_offset+member_offset,
+                                            buffer, j, buf_len, parms) < 0)
+                 HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress compound")
               break;
          case H5Z_NBIT_NOOPTYPE:
-              size = parms[parms_index++];
+              /* Advance past member size */
+              parms_index++;
               H5Z_nbit_decompress_one_nooptype(data, data_offset+member_offset,
-                                               buffer, j, buf_len, size);
+                                               buffer, j, buf_len, member_size);
               break;
           default:
               HDassert(0 && "This Should never be executed!");
       } /* end switch */
    }
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
 }
 
-static void
-H5Z_nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer,
+static herr_t
+H5Z__nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer,
                                 const unsigned parms[])
 {
    /* i: index of data, j: index of buffer,
@@ -1146,6 +1182,9 @@ H5Z_nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffe
    size_t i, j, size;
    int buf_len;
    parms_atomic p;
+   herr_t ret_value = SUCCEED; /* Return value */
+
+   FUNC_ENTER_STATIC
 
    /* may not have to initialize to zeros */
    for(i = 0; i < d_nelmts*parms[4]; i++) data[i] = 0;
@@ -1161,6 +1200,11 @@ H5Z_nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffe
            p.order = parms[5];
            p.precision = parms[6];
            p.offset = parms[7];
+
+           /* Check values of precision and offset */
+           if(p.precision > p.size * 8 || (p.precision + p.offset) > p.size * 8)
+              HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "invalid datatype precision/offset")
+
            for(i = 0; i < d_nelmts; i++) {
               H5Z_nbit_decompress_one_atomic(data, i*p.size, buffer, &j, &buf_len, p);
            }
@@ -1169,7 +1213,8 @@ H5Z_nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffe
            size = parms[4];
            parms_index = 4;
            for(i = 0; i < d_nelmts; i++) {
-              H5Z_nbit_decompress_one_array(data, i*size, buffer, &j, &buf_len, parms);
+              if(H5Z__nbit_decompress_one_array(data, i*size, buffer, &j, &buf_len, parms) < 0)
+                 HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress array")
               parms_index = 4;
            }
            break;
@@ -1177,13 +1222,17 @@ H5Z_nbit_decompress(unsigned char *data, unsigned d_nelmts, unsigned char *buffe
            size = parms[4];
            parms_index = 4;
            for(i = 0; i < d_nelmts; i++) {
-              H5Z_nbit_decompress_one_compound(data, i*size, buffer, &j, &buf_len, parms);
+              if(H5Z__nbit_decompress_one_compound(data, i*size, buffer, &j, &buf_len, parms) < 0)
+                 HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "can't decompress compound")
               parms_index = 4;
            }
            break;
       default:
           HDassert(0 && "This Should never be executed!");
    } /* end switch */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
 }
 
 static void H5Z_nbit_compress_one_byte(unsigned char *data, size_t data_offset, int k, int begin_i,