Merge branch 'develop' of https://bitbucket.hdfgroup.org/scm/~lrknox/hdf5_lrk into develop

15 files changed, 277 insertions, 4 deletions

diff --git a/tools/test/h5dump/CMakeTests.cmake b/tools/test/h5dump/CMakeTests.cmake
index cdd3e6d..132fc2b 100644
--- a/tools/test/h5dump/CMakeTests.cmake
+++ b/tools/test/h5dump/CMakeTests.cmake
@@ -21,6 +21,7 @@
   # --------------------------------------------------------------------
   set (HDF5_REFERENCE_FILES
       ${HDF5_TOOLS_DIR}/testfiles/charsets.ddl
+      ${HDF5_TOOLS_DIR}/testfiles/err_attr_dspace.ddl
       ${HDF5_TOOLS_DIR}/testfiles/file_space.ddl
       ${HDF5_TOOLS_DIR}/testfiles/filter_fail.ddl
       ${HDF5_TOOLS_DIR}/testfiles/non_existing.ddl
@@ -217,6 +218,7 @@
   )
   set (HDF5_REFERENCE_TEST_FILES
       ${HDF5_TOOLS_DIR}/testfiles/charsets.h5
+      ${HDF5_TOOLS_DIR}/testfiles/err_attr_dspace.h5
       ${HDF5_TOOLS_DIR}/testfiles/file_space.h5
       ${HDF5_TOOLS_DIR}/testfiles/filter_fail.h5
       ${HDF5_TOOLS_DIR}/testfiles/packedbits.h5
@@ -1533,6 +1535,9 @@
   # test for non-existing file
   ADD_H5_TEST (non_existing 1 --enable-error-stack tgroup.h5 non_existing.h5)
 
+  # test to verify HDFFV-10333: error similar to H5O_attr_decode in the jira issue
+  ADD_H5_TEST (err_attr_dspace 1 err_attr_dspace.h5)
+
 ##############################################################################
 ###    P L U G I N  T E S T S
 ##############################################################################
diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c
index 9358fbb..9e9f6f1 100644
--- a/tools/test/h5dump/h5dumpgentest.c
+++ b/tools/test/h5dump/h5dumpgentest.c
@@ -113,6 +113,7 @@
 #define FILE83  "tvlenstr_array.h5"
 #define FILE84  "tudfilter.h5"
 #define FILE85  "tgrpnullspace.h5"
+#define FILE86  "err_attr_dspace.h5"
 
 /*-------------------------------------------------------------------------
  * prototypes
@@ -10476,6 +10477,89 @@ static void gent_null_space_group(void)
     H5Fclose(fid);
 }
 
+/*-------------------------------------------------------------------------
+ * Function: gent_err_attr_dspace
+ *
+ * Purpose: Generate a file with shared dataspace message.
+ *          Then write an illegal version to the shared dataspace message
+ *          to trigger the error.
+ *          This is to verify HDFFV-10333 that h5dump will exit
+ *          gracefully when encountered error similar to 
+ *          H5O_attr_decode in the jira issue.
+ *
+ *-------------------------------------------------------------------------
+ */
+static void 
+gent_err_attr_dspace()
+{
+    hid_t fid = -1;         /* File identifier */
+    hid_t fcpl = -1;        /* File access property list */
+    hid_t sid = -1;         /* Dataspace identifier */
+    hid_t aid = -1;         /* Attribute identifier */
+    hsize_t dims = 2;       /* Dimensino size */
+    int wdata[2] = {7, 42}; /* The buffer to write */
+    int fd = -1;            /* The file descriptor */
+    char val = 6;           /* An invalid version */
+
+    /* Create an fcpl */
+    if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0)
+        goto error;
+
+    /* Set up the dataspace message to be shared */
+    if(H5Pset_shared_mesg_nindexes(fcpl, 1) < 0)
+        goto error;
+    if(H5Pset_shared_mesg_index(fcpl, 0, H5O_SHMESG_SDSPACE_FLAG, 1) < 0)
+        goto error;
+
+    /* Create the file with the shared message setting */
+    if((fid = H5Fcreate(FILE86, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0)
+        goto error;
+
+    /* Create the dataspace */
+    if((sid = H5Screate_simple(1, &dims, &dims)) < 0)
+        goto error;
+
+    /* Create an attribute with shared dataspace  */
+    if((aid = H5Acreate2(fid, "attribute", H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+        goto error;
+    if(H5Awrite(aid, H5T_NATIVE_INT, wdata) < 0)
+        goto error;
+
+    /* Closing */
+    if(H5Aclose(aid) < 0)
+        goto error;
+    if(H5Sclose(sid) < 0)
+        goto error;
+    if(H5Pclose(fcpl) < 0)
+        goto error;
+    if(H5Fclose(fid) < 0)
+        goto error;
+
+    /* This section of code will write an illegal version to the "version" field 
+       of the shared dataspace message */
+    if((fd = HDopen(FILE86, O_RDWR, 0633)) < 0)
+        goto error;
+
+    /* Offset of the "version" field to modify is as follows: */
+    /* 1916: offset of the object header containing the attribute message */
+    /* 32: offset of the attribute message in the object header */
+    /* 30: offset in the attribute message containing the version of the shared dataspace message */
+    if(HDlseek(fd, 1916+32+30, SEEK_SET) < 0)
+        goto error;
+    if(HDwrite(fd, &val, 1) < 0)
+        goto error;
+    if(HDclose(fd) < 0)
+        goto error;
+
+error:
+    H5E_BEGIN_TRY {
+        H5Pclose(fcpl);
+        H5Aclose(aid);
+        H5Sclose(sid);
+        H5Fclose(fid);
+    } H5E_END_TRY;
+} /* gen_err_attr_dspace() */
+
 int main(void)
 {
     gent_group();
@@ -10569,6 +10653,8 @@ int main(void)
 
     gent_udfilter();
 
+    gent_err_attr_dspace();
+
     return 0;
 }
 
diff --git a/tools/test/h5dump/testh5dump.sh.in b/tools/test/h5dump/testh5dump.sh.in
index 1935b0d..42e4b07 100644
--- a/tools/test/h5dump/testh5dump.sh.in
+++ b/tools/test/h5dump/testh5dump.sh.in
@@ -175,6 +175,7 @@ $SRC_H5DUMP_TESTFILES/tvldtypes5.h5
 $SRC_H5DUMP_TESTFILES/tvlenstr_array.h5
 $SRC_H5DUMP_TESTFILES/tvlstr.h5
 $SRC_H5DUMP_TESTFILES/tvms.h5
+$SRC_H5DUMP_TESTFILES/err_attr_dspace.h5
 "
 
 LIST_OTHER_TEST_FILES="
@@ -360,6 +361,7 @@ $SRC_H5DUMP_TESTFILES/twithddlfile.exp
 $SRC_H5DUMP_TESTFILES/h5dump-help.txt
 $SRC_H5DUMP_TESTFILES/out3.h5import
 $SRC_H5DUMP_TESTFILES/tbinregR.exp
+$SRC_H5DUMP_TESTFILES/err_attr_dspace.ddl
 "
 
 LIST_ERROR_TEST_FILES="
@@ -1364,6 +1366,9 @@ TOOLTEST2 tall-6.exp --enable-error-stack -y -o tall-6.txt -d /g1/g1.1/dset1.1.1
 # test for non-existing file
 TOOLTEST3 non_existing.ddl --enable-error-stack tgroup.h5 non_existing.h5
 
+# test to verify HDFFV-10333: error similar to H5O_attr_decode in the jira issue
+TOOLTEST err_attr_dspace.ddl err_attr_dspace.h5
+
 # Clean up temporary files/directories
 CLEAN_TESTFILES_AND_TESTDIR
 
diff --git a/tools/test/h5stat/CMakeTests.cmake b/tools/test/h5stat/CMakeTests.cmake
index 69bab20..bd55ac1 100644
--- a/tools/test/h5stat/CMakeTests.cmake
+++ b/tools/test/h5stat/CMakeTests.cmake
@@ -20,6 +20,9 @@
   # Copy all the HDF5 files from the test directory into the source directory
   # --------------------------------------------------------------------
   set (HDF5_REFERENCE_FILES
+      h5stat_err_refcount
+      h5stat_err_old_layout
+      h5stat_err_old_fill
       h5stat_help1
       h5stat_help2
       h5stat_notexist
@@ -54,6 +57,9 @@
       h5stat_numattrs4
   )
   set (HDF5_REFERENCE_TEST_FILES
+      h5stat_err_refcount.h5
+      h5stat_err_old_layout.h5
+      h5stat_err_old_fill.h5
       h5stat_filters.h5
       h5stat_idx.h5
       h5stat_tsohm.h5
@@ -160,9 +166,7 @@
   ADD_H5_TEST (h5stat_newgrat-UG 0 -G h5stat_newgrat.h5)
   ADD_H5_TEST (h5stat_newgrat-UA 0 -A h5stat_newgrat.h5)
 # h5stat_idx.h5 is generated by h5stat_gentest.c
-  if (HDF5_BUILD_GENERATORS)
-    ADD_H5_TEST (h5stat_idx 0 h5stat_idx.h5)
-  endif ()
+  ADD_H5_TEST (h5stat_idx 0 h5stat_idx.h5)
 #
 # Tests for -l (--links) option on h5stat_threshold.h5:
 #   -l 0 (incorrect threshold value)
@@ -204,3 +208,11 @@
 #   -A -a 100
   ADD_H5_TEST (h5stat_numattrs4 0 -A -a 100 h5stat_newgrat.h5)
 #
+# Tests to verify HDFFV-10333:
+# h5stat_err_refcount.h5 is generated by h5stat_gentest.c
+# h5stat_err_old_layout.h5 and h5stat_err_old_fill.h5: see explanation in h5stat_gentest.c
+  ADD_H5_TEST (h5stat_err_refcount 1 h5stat_err_refcount.h5)
+  ADD_H5_TEST (h5stat_err_old_layout 1 h5stat_err_old_layout.h5)
+  ADD_H5_TEST (h5stat_err_old_fill 1 h5stat_err_old_fill.h5)
+#
+#
diff --git a/tools/test/h5stat/h5stat_gentest.c b/tools/test/h5stat/h5stat_gentest.c
index 0f696d0..2daf24b 100644
--- a/tools/test/h5stat/h5stat_gentest.c
+++ b/tools/test/h5stat/h5stat_gentest.c
@@ -21,6 +21,7 @@
  * of the expected output and update the corresponding *.ddl files.
  */
 #include "hdf5.h"
+#include "H5private.h"
 
 /* For gen_newgrat_file() */
 #define NEWGRAT_FILE 	"h5stat_newgrat.h5"
@@ -43,6 +44,9 @@
 #define THRES_NUM		10
 #define THRES_NUM_25		25
 
+/* For gen_err_refcount() */
+#define ERR_REFCOUNT_FILE   "h5stat_err_refcount.h5"
+
 /*
  * Generate HDF5 file with latest format with
  * NUM_GRPS groups and NUM_ATTRS attributes for the dataset
@@ -434,6 +438,140 @@ error:
 
 } /* gen_idx_file() */
 
+/*
+ * Function: gen_err_refcount_file
+ *
+ * Purpose: Create a file with a refcount message ID.
+ *          Then a refcount message ID is written to a
+ *          message in a version 1 object header.
+ *          This will trigger the error as a version 1
+ *          object header does not support a refcount message.
+ *          This is to verify HDFFV-10333 that h5stat will exit
+ *          gracefully when encountered error similar to
+ *          H5O_refcount_decode in the jira issue.
+ *
+ */
+static void 
+gen_err_refcount(const char *fname)
+{
+    hid_t fid = -1;     /* File identifier */
+    hid_t sid = -1;     /* Dataspace message */
+    hid_t did = -1;     /* Dataset identifier */
+    hid_t gid = -1;     /* Group identifier */
+    hid_t aid1 = -1, aid2 = -1;     /* Attribute identifier */
+    hid_t tid = -1;     /* Datatype identifier */
+    int i, n;           /* Local index variables */
+    int buf[10];        /* Data buffer */
+    hsize_t dims[1];    /* Dimension size */
+    int fd = -1;        /* File descriptor */
+    unsigned short val = 22;        /* The refcount message ID */
+
+    /* Initialize data buffer */
+    n = 0;
+    for(i = 0; i < 10; i++)
+        buf[i] = n++;
+
+    /* Create the file */
+    if((fid = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+        goto error;
+
+    /* Create a group */
+    if((gid = H5Gcreate2(fid, "group", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+        goto error;
+
+    /* Create a committed datatype in the group */
+    if((tid = H5Tcopy(H5T_NATIVE_INT)) < 0)
+        goto error;
+    if(H5Tcommit2(gid, "dtype", tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0)
+        goto error;
+
+    /* Create the dataspace */
+    dims[0] = 10;
+    if((sid = H5Screate_simple(1, dims, NULL)) < 0)
+        goto error;
+
+    /* Create a dataset with the committed datatype in the file */
+    if((did = H5Dcreate2(fid, "dset", tid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+        goto error;
+    /* Write to the dataset */
+    if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0)
+        goto error;
+
+    /* Attach an attribute with the committed datatype to the group */
+    if((aid1 = H5Acreate2(gid, "attr", tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+        goto error;
+    /* Attach an attribute with the committed datatype to the dataset */
+    if((aid2 = H5Acreate2(did, "attr", tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+        goto error;
+
+    /* Closing */
+    if(H5Aclose(aid1) < 0)
+        goto error;
+    if(H5Aclose(aid2) < 0)
+        goto error;
+    if(H5Sclose(sid) < 0)
+        goto error;
+    if(H5Dclose(did) < 0)
+        goto error;
+    if(H5Gclose(gid) < 0)
+        goto error;
+    if(H5Tclose(tid) < 0)
+        goto error;
+    if(H5Fclose(fid) < 0)
+        goto error;
+
+    /* This section of code will write a refcount message ID to a message in the
+       version 1 object header which does not support a refcount message */
+    /* Offset of the message ID to modify is as follows: */
+    /* 4520: the offset of the object header containing the attribute message
+             with the committed datatype */
+    /* 24: the offset in the object header containing the version of the 
+           attribute message */
+    if((fd = HDopen(fname, O_RDWR, 0633)) < 0)
+        goto error;
+    if(HDlseek(fd, 4520+24, SEEK_SET) < 0)
+        goto error;
+    if(HDwrite(fd, &val, 2) < 0)
+        goto error;
+    if(HDclose(fd) < 0)
+        goto error;
+
+error:
+    H5E_BEGIN_TRY {
+        H5Gclose(gid);
+        H5Dclose(did);
+        H5Tclose(tid);
+        H5Sclose(sid);
+        H5Aclose(aid1);
+        H5Aclose(aid2);
+        H5Fclose(fid);
+    } H5E_END_TRY;
+} /* gen_err_refcount() */
+
+/* 
+ * The following two test files are generated with older versions 
+ * of the library for HDFFV-10333.  They are used for testing in
+ * testh5stat.sh.in.
+ *
+ * (1) h5stat_err_old_layout.h5
+ *     This file is generated with the 1.6 library so that a file
+ *     with a version 2 layout message is created.
+ *     Then a "0" is written to the "dimension" field in the layout
+ *     message to trigger the error.
+ *     This is to verify HDFFV-10333 that h5stat will exit gracefully
+ *     when encountered error similar to H5O__layout_decode in the 
+ *     jira issue.
+ *
+ * (2) h5stat_err_old_fill.h5
+ *     This file is generated with the 1.4 library so that a file
+ *     with an old fill value message is created.
+ *     Then an illegal size is written to the "size" fild in the
+ *     fill value message to trigger the error.
+ *     This is to verify HDFFV-10333 that h5stat will exit gracefully
+ *     when encountered error similar to H5O_fill_old_decode in the 
+ *     jira issue.
+ */
+
 int main(void)
 {
     gen_newgrat_file(NEWGRAT_FILE);
@@ -442,6 +580,9 @@ int main(void)
     /* Generate an HDF file to test for datasets with Fixed Array indexing */
     gen_idx_file(IDX_FILE);
 
+    /* Generate a file with a refcount message ID */
+    gen_err_refcount(ERR_REFCOUNT_FILE);
+
     return 0;
 }
 
diff --git a/tools/test/h5stat/testfiles/h5stat_err_old_fill.ddl b/tools/test/h5stat/testfiles/h5stat_err_old_fill.ddl
new file mode 100644
index 0000000..e751b7f
--- /dev/null
+++ b/tools/test/h5stat/testfiles/h5stat_err_old_fill.ddl
@@ -0,0 +1,2 @@
+Filename: h5stat_err_old_fill.h5
+h5stat error: unable to traverse objects/links in file "h5stat_err_old_fill.h5"
diff --git a/tools/test/h5stat/testfiles/h5stat_err_old_fill.h5 b/tools/test/h5stat/testfiles/h5stat_err_old_fill.h5
new file mode 100644
index 0000000..aa164f0
--- /dev/null
+++ b/tools/test/h5stat/testfiles/h5stat_err_old_fill.h5
diff --git a/tools/test/h5stat/testfiles/h5stat_err_old_layout.ddl b/tools/test/h5stat/testfiles/h5stat_err_old_layout.ddl
new file mode 100644
index 0000000..a3e27e2
--- /dev/null
+++ b/tools/test/h5stat/testfiles/h5stat_err_old_layout.ddl
@@ -0,0 +1,2 @@
+Filename: h5stat_err_old_layout.h5
+h5stat error: unable to traverse objects/links in file "h5stat_err_old_layout.h5"
diff --git a/tools/test/h5stat/testfiles/h5stat_err_old_layout.h5 b/tools/test/h5stat/testfiles/h5stat_err_old_layout.h5
new file mode 100644
index 0000000..5c0b5dc
--- /dev/null
+++ b/tools/test/h5stat/testfiles/h5stat_err_old_layout.h5
diff --git a/tools/test/h5stat/testfiles/h5stat_err_refcount.ddl b/tools/test/h5stat/testfiles/h5stat_err_refcount.ddl
new file mode 100644
index 0000000..1f1b491
--- /dev/null
+++ b/tools/test/h5stat/testfiles/h5stat_err_refcount.ddl
@@ -0,0 +1,2 @@
+Filename: h5stat_err_refcount.h5
+h5stat error: unable to traverse objects/links in file "h5stat_err_refcount.h5"
diff --git a/tools/test/h5stat/testfiles/h5stat_err_refcount.h5 b/tools/test/h5stat/testfiles/h5stat_err_refcount.h5
new file mode 100644
index 0000000..5e0d5f9
--- /dev/null
+++ b/tools/test/h5stat/testfiles/h5stat_err_refcount.h5
diff --git a/tools/test/h5stat/testfiles/h5stat_help1.ddl b/tools/test/h5stat/testfiles/h5stat_help1.ddl
index d2a8715..01e39af 100644
--- a/tools/test/h5stat/testfiles/h5stat_help1.ddl
+++ b/tools/test/h5stat/testfiles/h5stat_help1.ddl
@@ -22,3 +22,4 @@ Usage: h5stat [OPTIONS] file
                            than 0.  The default threshold is 10.
      -s, --freespace       Print free space information
      -S, --summary         Print summary of file space information
+     --enable-error-stack  Prints messages from the HDF5 error stack as they occur
diff --git a/tools/test/h5stat/testfiles/h5stat_help2.ddl b/tools/test/h5stat/testfiles/h5stat_help2.ddl
index d2a8715..01e39af 100644
--- a/tools/test/h5stat/testfiles/h5stat_help2.ddl
+++ b/tools/test/h5stat/testfiles/h5stat_help2.ddl
@@ -22,3 +22,4 @@ Usage: h5stat [OPTIONS] file
                            than 0.  The default threshold is 10.
      -s, --freespace       Print free space information
      -S, --summary         Print summary of file space information
+     --enable-error-stack  Prints messages from the HDF5 error stack as they occur
diff --git a/tools/test/h5stat/testfiles/h5stat_nofile.ddl b/tools/test/h5stat/testfiles/h5stat_nofile.ddl
index d8a8b2c..7171320 100644
--- a/tools/test/h5stat/testfiles/h5stat_nofile.ddl
+++ b/tools/test/h5stat/testfiles/h5stat_nofile.ddl
@@ -22,4 +22,5 @@ Usage: h5stat [OPTIONS] file
                            than 0.  The default threshold is 10.
      -s, --freespace       Print free space information
      -S, --summary         Print summary of file space information
+     --enable-error-stack  Prints messages from the HDF5 error stack as they occur
 h5stat error: missing file name
diff --git a/tools/test/h5stat/testh5stat.sh.in b/tools/test/h5stat/testh5stat.sh.in
index ca7ca4c..5082daf 100644
--- a/tools/test/h5stat/testh5stat.sh.in
+++ b/tools/test/h5stat/testh5stat.sh.in
@@ -69,6 +69,9 @@ test -d $TESTDIR || mkdir $TESTDIR
 #       Comment '#' without space can be used.
 # --------------------------------------------------------------------
 LIST_HDF5_TEST_FILES="
+$SRC_H5STAT_TESTFILES/h5stat_err_refcount.h5
+$SRC_H5STAT_TESTFILES/h5stat_err_old_layout.h5
+$SRC_H5STAT_TESTFILES/h5stat_err_old_fill.h5
 $SRC_H5STAT_TESTFILES/h5stat_filters.h5
 $SRC_H5STAT_TESTFILES/h5stat_tsohm.h5
 $SRC_H5STAT_TESTFILES/h5stat_newgrat.h5
@@ -77,6 +80,9 @@ $SRC_H5STAT_TESTFILES/h5stat_threshold.h5
 "
 
 LIST_OTHER_TEST_FILES="
+$SRC_H5STAT_TESTFILES/h5stat_err_refcount.ddl
+$SRC_H5STAT_TESTFILES/h5stat_err_old_layout.ddl
+$SRC_H5STAT_TESTFILES/h5stat_err_old_fill.ddl
 $SRC_H5STAT_TESTFILES/h5stat_help1.ddl
 $SRC_H5STAT_TESTFILES/h5stat_help2.ddl
 $SRC_H5STAT_TESTFILES/h5stat_notexist.ddl
@@ -243,6 +249,7 @@ TOOLTEST h5stat_help2.ddl --help
 TOOLTEST h5stat_notexist.ddl notexist.h5
 TOOLTEST h5stat_nofile.ddl ''
 
+
 # Test file with groups, compressed datasets, user-applied fileters, etc.
 # h5stat_filters.h5 is a copy of ../../testfiles/tfilters.h5 as of release 1.8.0-alpha4
 TOOLTEST h5stat_filters.ddl h5stat_filters.h5
@@ -304,7 +311,15 @@ TOOLTEST h5stat_numattrs3.ddl -A --numattrs=25 h5stat_threshold.h5
 #    -A -a 100
 TOOLTEST h5stat_numattrs4.ddl -A -a 100 h5stat_newgrat.h5
 #
-
+#
+# Tests to verify HDFFV-10333
+#   h5stat_err_refcount.h5 is generated by h5stat_gentest.c
+#   h5stat_err_old_layout.h5 and h5stat_err_old_fill.h5: see explanation in h5stat_gentest.c
+TOOLTEST h5stat_err_refcount.ddl h5stat_err_refcount.h5
+TOOLTEST h5stat_err_old_layout.ddl h5stat_err_old_layout.h5
+TOOLTEST h5stat_err_old_fill.ddl h5stat_err_old_fill.h5
+#
+#
 # Clean up temporary files/directories
 CLEAN_TESTFILES_AND_TESTDIR