From 12128a478218652607e8319e3a3f4b03a4b0b65f Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Fri, 1 Jun 2018 11:27:00 -0500 Subject: Fix daily test failure. --- java/src/jni/h5oImp.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/java/src/jni/h5oImp.c b/java/src/jni/h5oImp.c index d9276da..7665c70 100644 --- a/java/src/jni/h5oImp.c +++ b/java/src/jni/h5oImp.c @@ -442,7 +442,7 @@ Java_hdf_hdf5lib_H5_H5Ovisit h5nullArgument(env, "H5Ovisit: callback_op is NULL"); } /* end if */ else { - status = H5Ovisit((hid_t)grp_id, (H5_index_t)idx_type, (H5_iter_order_t)order, (H5O_iterate_t)H5O_iterate_cb, (void*)op_data, (unsigned)fields); + status = H5Ovisit2((hid_t)grp_id, (H5_index_t)idx_type, (H5_iter_order_t)order, (H5O_iterate_t)H5O_iterate_cb, (void*)op_data, (unsigned)fields); if (status < 0) h5libraryError(env); @@ -478,7 +478,7 @@ Java_hdf_hdf5lib_H5_H5Ovisit_1by_1name else { PIN_JAVA_STRING(name, lName); if (lName != NULL) { - status = H5Ovisit_by_name((hid_t)grp_id, lName, (H5_index_t)idx_type, (H5_iter_order_t)order, (H5O_iterate_t)H5O_iterate_cb, (void*)op_data, (unsigned)fields, (hid_t)access_id); + status = H5Ovisit_by_name2((hid_t)grp_id, lName, (H5_index_t)idx_type, (H5_iter_order_t)order, (H5O_iterate_t)H5O_iterate_cb, (void*)op_data, (unsigned)fields, (hid_t)access_id); UNPIN_JAVA_STRING(name, lName); -- cgit v0.12 From cf38292064a0c3ffc6971de31573bbd1dab25b80 Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Tue, 10 Jul 2018 22:00:14 -0500 Subject: Fix for HDFFV-10333: 1) Check for valid object header version for a refcount messge 2) Check for invalid fill value size 3) Check for invalid dimension size in a layout message 4) Add --enable-error-stack option to h5stat 5) Add error checks to h5stat.c 6) Add tests to h5stat and h5dump --- src/H5Ocache.c | 28 +- src/H5Ofill.c | 22 +- src/H5Olayout.c | 4 +- src/H5Sselect.c | 2 +- tools/src/h5stat/h5stat.c | 290 +++++++++++---------- tools/test/h5dump/h5dumpgentest.c | 86 ++++++ tools/test/h5dump/testh5dump.sh.in | 5 + tools/test/h5stat/h5stat_gentest.c | 142 ++++++++++ .../test/h5stat/testfiles/h5stat_err_old_fill.ddl | 2 + tools/test/h5stat/testfiles/h5stat_err_old_fill.h5 | Bin 0 -> 2068 bytes .../h5stat/testfiles/h5stat_err_old_layout.ddl | 2 + .../test/h5stat/testfiles/h5stat_err_old_layout.h5 | Bin 0 -> 2088 bytes .../test/h5stat/testfiles/h5stat_err_refcount.ddl | 2 + tools/test/h5stat/testfiles/h5stat_err_refcount.h5 | Bin 0 -> 4640 bytes tools/test/h5stat/testfiles/h5stat_help1.ddl | 1 + tools/test/h5stat/testfiles/h5stat_help2.ddl | 1 + tools/test/h5stat/testfiles/h5stat_nofile.ddl | 1 + tools/test/h5stat/testh5stat.sh.in | 15 +- tools/testfiles/err_attr_dspace.ddl | 5 + tools/testfiles/err_attr_dspace.h5 | Bin 0 -> 3047 bytes 20 files changed, 461 insertions(+), 147 deletions(-) create mode 100644 tools/test/h5stat/testfiles/h5stat_err_old_fill.ddl create mode 100644 tools/test/h5stat/testfiles/h5stat_err_old_fill.h5 create mode 100644 tools/test/h5stat/testfiles/h5stat_err_old_layout.ddl create mode 100644 tools/test/h5stat/testfiles/h5stat_err_old_layout.h5 create mode 100644 tools/test/h5stat/testfiles/h5stat_err_refcount.ddl create mode 100644 tools/test/h5stat/testfiles/h5stat_err_refcount.h5 create mode 100644 tools/testfiles/err_attr_dspace.ddl create mode 100644 tools/testfiles/err_attr_dspace.h5 diff --git a/src/H5Ocache.c b/src/H5Ocache.c index 3607839..d65942b 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -1430,9 +1430,10 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image /* Check for combining two adjacent 'null' messages */ if((udata->file_intent & H5F_ACC_RDWR) && - H5O_NULL_ID == id && oh->nmesgs > 0 && - H5O_NULL_ID == oh->mesg[oh->nmesgs - 1].type->id && - oh->mesg[oh->nmesgs - 1].chunkno == chunkno) { + H5O_NULL_ID == id && oh->nmesgs > 0 && + H5O_NULL_ID == oh->mesg[oh->nmesgs - 1].type->id && + oh->mesg[oh->nmesgs - 1].chunkno == chunkno) { + size_t mesgno; /* Current message to operate on */ /* Combine adjacent null messages */ @@ -1467,13 +1468,13 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image /* Point unknown messages at 'unknown' message class */ /* (Usually from future versions of the library) */ - if(id >= H5O_UNKNOWN_ID || + if(id >= H5O_UNKNOWN_ID || #ifdef H5O_ENABLE_BOGUS - id == H5O_BOGUS_VALID_ID || + id == H5O_BOGUS_VALID_ID || #endif - NULL == H5O_msg_class_g[id]) { + NULL == H5O_msg_class_g[id]) { - H5O_unknown_t *unknown; /* Pointer to "unknown" message info */ + H5O_unknown_t *unknown; /* Pointer to "unknown" message info */ /* Allocate "unknown" message info */ if(NULL == (unknown = H5FL_MALLOC(H5O_unknown_t))) @@ -1490,9 +1491,9 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image /* Check for "fail if unknown" message flags */ if(((udata->file_intent & H5F_ACC_RDWR) && - (flags & H5O_MSG_FLAG_FAIL_IF_UNKNOWN_AND_OPEN_FOR_WRITE)) - || (flags & H5O_MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS)) - HGOTO_ERROR(H5E_OHDR, H5E_BADMESG, FAIL, "unknown message with 'fail if unknown' flag found") + (flags & H5O_MSG_FLAG_FAIL_IF_UNKNOWN_AND_OPEN_FOR_WRITE)) + || (flags & H5O_MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS)) + HGOTO_ERROR(H5E_OHDR, H5E_BADMESG, FAIL, "unknown message with 'fail if unknown' flag found") /* Check for "mark if unknown" message flag, etc. */ else if((flags & H5O_MSG_FLAG_MARK_IF_UNKNOWN) && !(flags & H5O_MSG_FLAG_WAS_UNKNOWN) && @@ -1543,7 +1544,8 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image H5O_refcount_t *refcount; /* Decode ref. count message */ - HDassert(oh->version > H5O_VERSION_1); + if(oh->version <= H5O_VERSION_1) + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "object header version does not support reference count message") refcount = (H5O_refcount_t *)(H5O_MSG_REFCOUNT->decode)(udata->f, NULL, 0, &ioflags, mesg->raw_size, mesg->raw); /* Save 'native' form of ref. count message */ @@ -1614,6 +1616,10 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image } /* end if */ done: + if(ret_value < 0 && udata->cont_msg_info->msgs) { + udata->cont_msg_info->msgs = (H5O_chunk_t *)H5FL_SEQ_FREE(H5O_cont_t, udata->cont_msg_info->msgs); + udata->cont_msg_info->alloc_nmsgs = 0; + } FUNC_LEAVE_NOAPI(ret_value) } /* H5O__chunk_deserialize() */ diff --git a/src/H5Ofill.c b/src/H5Ofill.c index 932241f..3ce2e4f 100644 --- a/src/H5Ofill.c +++ b/src/H5Ofill.c @@ -19,6 +19,7 @@ */ #include "H5Omodule.h" /* This source code file is part of the H5O module */ +#define H5T_FRIEND /*prevent warning from including H5Tpkg */ #include "H5private.h" /* Generic Functions */ @@ -31,6 +32,7 @@ #include "H5Pprivate.h" /* Property lists */ #include "H5Sprivate.h" /* Dataspaces */ +#include "H5Tpkg.h" /* Datatypes */ static void *H5O_fill_old_decode(H5F_t *f, H5O_t *open_oh, unsigned mesg_flags, unsigned *ioflags, size_t p_size, const uint8_t *p); @@ -307,11 +309,13 @@ done: *------------------------------------------------------------------------- */ static void * -H5O_fill_old_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, +H5O_fill_old_decode(H5F_t *f, H5O_t *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, size_t H5_ATTR_UNUSED p_size, const uint8_t *p) { H5O_fill_t *fill = NULL; /* Decoded fill value message */ + htri_t exists = FALSE; + H5T_t *dt = NULL; void *ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -332,6 +336,19 @@ H5O_fill_old_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, /* Only decode the fill value itself if there is one */ if(fill->size > 0) { + H5_CHECK_OVERFLOW(fill->size, ssize_t, size_t); + + /* Get the datatype message */ + if((exists = H5O_msg_exists_oh(open_oh, H5O_DTYPE_ID)) < 0) + HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, NULL, "unable to read object header") + if(exists) { + if((dt = H5O_msg_read_oh(f, open_oh, H5O_DTYPE_ID, NULL)) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTGET, NULL, "can't read DTYPE message") + /* Verify size */ + if(fill->size != dt->shared->size) + HGOTO_ERROR(H5E_SYM, H5E_CANTGET, NULL, "inconsistent fill value size") + } + if(NULL == (fill->buf = H5MM_malloc((size_t)fill->size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for fill value") HDmemcpy(fill->buf, p, (size_t)fill->size); @@ -344,6 +361,9 @@ H5O_fill_old_decode(H5F_t H5_ATTR_UNUSED *f, H5O_t H5_ATTR_UNUSED *open_oh, ret_value = (void*)fill; done: + if(dt) + H5O_msg_free(H5O_DTYPE_ID, dt); + if(!ret_value && fill) { if(fill->buf) H5MM_xfree(fill->buf); diff --git a/src/H5Olayout.c b/src/H5Olayout.c index d8f05f0..5f16837 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -125,8 +125,8 @@ H5O__layout_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, /* Dimensionality */ ndims = *p++; - if(ndims > H5O_LAYOUT_NDIMS) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "dimensionality is too large") + if(!ndims || ndims > H5O_LAYOUT_NDIMS) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "dimensionality is out of range") /* Layout class */ mesg->type = (H5D_layout_t)*p++; diff --git a/src/H5Sselect.c b/src/H5Sselect.c index 4462295..8cd73d9 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -160,7 +160,7 @@ H5S_select_release(H5S_t *ds) HDassert(ds); /* Call the selection type's release function */ - if((ret_value = (*ds->select.type->release)(ds)) < 0) + if((ds->select.type) && ((ret_value = (*ds->select.type->release)(ds)) < 0)) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection") done: diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index da713ac..ff67cf1 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -146,7 +146,7 @@ struct handler_t { char **obj; }; -static const char *s_opts ="Aa:Ddm:FfhGgl:sSTO:V"; +static const char *s_opts ="Aa:Ddm:EFfhGgl:sSTO:V"; /* e.g. "filemetadata" has to precede "file"; "groupmetadata" has to precede "group" etc. */ static struct long_options l_opts[] = { {"help", no_arg, 'h'}, @@ -224,6 +224,7 @@ static struct long_options l_opts[] = { { "attr", no_arg, 'A' }, { "att", no_arg, 'A' }, { "at", no_arg, 'A' }, + { "enable-error-stack", no_arg, 'E' }, { "numattrs", require_arg, 'a' }, { "numattr", require_arg, 'a' }, { "numatt", require_arg, 'a' }, @@ -293,6 +294,7 @@ static void usage(const char *prog) HDfprintf(stdout, " than 0. The default threshold is 10.\n"); HDfprintf(stdout, " -s, --freespace Print free space information\n"); HDfprintf(stdout, " -S, --summary Print summary of file space information\n"); + HDfprintf(stdout, " --enable-error-stack Prints messages from the HDF5 error stack as they occur\n"); } @@ -378,9 +380,8 @@ attribute_stats(iter_t *iter, const H5O_info_t *oi) * * Purpose: Gather statistics about the group * - * Return: Success: 0 - * - * Failure: -1 + * Return: Success: 0 + * Failure: -1 * * Programmer: Quincey Koziol * Tuesday, August 16, 2005 @@ -402,9 +403,9 @@ attribute_stats(iter_t *iter, const H5O_info_t *oi) static herr_t group_stats(iter_t *iter, const char *name, const H5O_info_t *oi) { - H5G_info_t ginfo; /* Group information */ - unsigned bin; /* "bin" the number of objects falls in */ - herr_t ret; + H5G_info_t ginfo; /* Group information */ + unsigned bin; /* "bin" the number of objects falls in */ + herr_t ret_value = SUCCEED; /* Return value */ /* Gather statistics about this type of object */ iter->uniq_groups++; @@ -414,8 +415,8 @@ group_stats(iter_t *iter, const char *name, const H5O_info_t *oi) iter->group_ohdr_info.free_size += oi->hdr.space.free; /* Get group information */ - ret = H5Gget_info_by_name(iter->fid, name, &ginfo, H5P_DEFAULT); - HDassert(ret >= 0); + if((ret_value = H5Gget_info_by_name(iter->fid, name, &ginfo, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Gget_info_by_name() failed"); /* Update link stats */ /* Collect statistics for small groups */ @@ -429,10 +430,10 @@ group_stats(iter_t *iter, const char *name, const H5O_info_t *oi) bin = ceil_log10((unsigned long)ginfo.nlinks); if((bin + 1) > iter->group_nbins) { /* Allocate more storage for info about dataset's datatype */ - iter->group_bins = (unsigned long *)HDrealloc(iter->group_bins, (bin + 1) * sizeof(unsigned long)); - HDassert(iter->group_bins); + if((iter->group_bins = (unsigned long *)HDrealloc(iter->group_bins, (bin + 1) * sizeof(unsigned long))) == NULL) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Drealloc() failed"); - /* Initialize counts for intermediate bins */ + /* Initialize counts for intermediate bins */ while(iter->group_nbins < bin) iter->group_bins[iter->group_nbins++] = 0; iter->group_nbins++; @@ -448,10 +449,11 @@ group_stats(iter_t *iter, const char *name, const H5O_info_t *oi) iter->groups_heap_storage_size += oi->meta_size.obj.heap_size; /* Update attribute metadata info */ - ret = attribute_stats(iter, oi); - HDassert(ret >= 0); + if((ret_value = attribute_stats(iter, oi)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "attribute_stats failed"); - return 0; +done: + return ret_value; } /* end group_stats() */ @@ -461,7 +463,6 @@ group_stats(iter_t *iter, const char *name, const H5O_info_t *oi) * Purpose: Gather statistics about the dataset * * Return: Success: 0 - * * Failure: -1 * * Programmer: Quincey Koziol @@ -472,22 +473,22 @@ group_stats(iter_t *iter, const char *name, const H5O_info_t *oi) static herr_t dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) { - unsigned bin; /* "bin" the number of objects falls in */ - hid_t did; /* Dataset ID */ - hid_t sid; /* Dataspace ID */ - hid_t tid; /* Datatype ID */ - hid_t dcpl; /* Dataset creation property list ID */ - hsize_t dims[H5S_MAX_RANK];/* Dimensions of dataset */ - H5D_layout_t lout; /* Layout of dataset */ - unsigned type_found; /* Whether the dataset's datatype was */ - /* already found */ - int ndims; /* Number of dimensions of dataset */ - hsize_t storage; /* Size of dataset storage */ - unsigned u; /* Local index variable */ - int num_ext; /* Number of external files for a dataset */ - int nfltr; /* Number of filters for a dataset */ - H5Z_filter_t fltr; /* Filter identifier */ - herr_t ret; + unsigned bin; /* "bin" the number of objects falls in */ + hid_t did; /* Dataset ID */ + hid_t sid; /* Dataspace ID */ + hid_t tid; /* Datatype ID */ + hid_t dcpl; /* Dataset creation property list ID */ + hsize_t dims[H5S_MAX_RANK]; /* Dimensions of dataset */ + H5D_layout_t lout; /* Layout of dataset */ + unsigned type_found; /* Whether the dataset's datatype was */ + /* already found */ + int ndims; /* Number of dimensions of dataset */ + hsize_t storage; /* Size of dataset storage */ + unsigned u; /* Local index variable */ + int num_ext; /* Number of external files for a dataset */ + int nfltr; /* Number of filters for a dataset */ + H5Z_filter_t fltr; /* Filter identifier */ + herr_t ret_value = SUCCEED; /* Return value */ /* Gather statistics about this type of object */ iter->uniq_dsets++; @@ -496,26 +497,27 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) iter->dset_ohdr_info.total_size += oi->hdr.space.total; iter->dset_ohdr_info.free_size += oi->hdr.space.free; - did = H5Dopen2(iter->fid, name, H5P_DEFAULT); - HDassert(did > 0); + if((did = H5Dopen2(iter->fid, name, H5P_DEFAULT)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dopen() failed"); /* Update dataset metadata info */ iter->datasets_index_storage_size += oi->meta_size.obj.index_size; iter->datasets_heap_storage_size += oi->meta_size.obj.heap_size; /* Update attribute metadata info */ - ret = attribute_stats(iter, oi); - HDassert(ret >= 0); + if((ret_value = attribute_stats(iter, oi)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "attribute_stats() failed"); /* Get storage info */ - storage = H5Dget_storage_size(did); + if((storage = H5Dget_storage_size(did)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_storage_size() failed"); /* Gather layout statistics */ - dcpl = H5Dget_create_plist(did); - HDassert(dcpl > 0); + if((dcpl = H5Dget_create_plist(did)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist() failed"); - lout = H5Pget_layout(dcpl); - HDassert(lout >= 0); + if((lout = H5Pget_layout(dcpl)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_layout() failed"); /* Object header's total size for H5D_COMPACT layout includes raw data size */ /* "storage" also includes H5D_COMPACT raw data size */ @@ -526,8 +528,8 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) (iter->dset_layouts[lout])++; /* Get the number of external files for the dataset */ - num_ext = H5Pget_external_count(dcpl); - assert (num_ext >= 0); + if((num_ext = H5Pget_external_count(dcpl)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pget_external_count() failed"); /* Accumulate raw data size accordingly */ if(num_ext) { @@ -537,11 +539,11 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) iter->dset_storage_size += storage; /* Gather dataspace statistics */ - sid = H5Dget_space(did); - HDassert(sid > 0); + if((sid = H5Dget_space(did)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_space() failed"); - ndims = H5Sget_simple_extent_dims(sid, dims, NULL); - HDassert(ndims >= 0); + if((ndims = H5Sget_simple_extent_dims(sid, dims, NULL)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_dims() failed"); /* Check for larger rank of dataset */ if((unsigned)ndims > iter->max_dset_rank) @@ -552,38 +554,38 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) /* Only gather dim size statistics on 1-D datasets */ if(ndims == 1) { - /* Determine maximum dimension size */ - if(dims[0] > iter->max_dset_dims) - iter->max_dset_dims = dims[0]; - /* Collect statistics for small datasets */ - if(dims[0] < (hsize_t)sdsets_threshold) - (iter->small_dset_dims[(size_t)dims[0]])++; - - /* Add dim count to proper bin */ - bin = ceil_log10((unsigned long)dims[0]); - if((bin + 1) > iter->dset_dim_nbins) { - /* Allocate more storage for info about dataset's datatype */ - iter->dset_dim_bins = (unsigned long *)HDrealloc(iter->dset_dim_bins, (bin + 1) * sizeof(unsigned long)); - HDassert(iter->dset_dim_bins); - - /* Initialize counts for intermediate bins */ - while(iter->dset_dim_nbins < bin) - iter->dset_dim_bins[iter->dset_dim_nbins++] = 0; - iter->dset_dim_nbins++; - - /* Initialize count for this bin */ - iter->dset_dim_bins[bin] = 1; + /* Determine maximum dimension size */ + if(dims[0] > iter->max_dset_dims) + iter->max_dset_dims = dims[0]; + /* Collect statistics for small datasets */ + if(dims[0] < (hsize_t)sdsets_threshold) + (iter->small_dset_dims[(size_t)dims[0]])++; + + /* Add dim count to proper bin */ + bin = ceil_log10((unsigned long)dims[0]); + if((bin + 1) > iter->dset_dim_nbins) { + /* Allocate more storage for info about dataset's datatype */ + if((iter->dset_dim_bins = (unsigned long *)HDrealloc(iter->dset_dim_bins, (bin + 1) * sizeof(unsigned long))) == NULL) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Drealloc() failed"); + + /* Initialize counts for intermediate bins */ + while(iter->dset_dim_nbins < bin) + iter->dset_dim_bins[iter->dset_dim_nbins++] = 0; + iter->dset_dim_nbins++; + + /* Initialize count for this bin */ + iter->dset_dim_bins[bin] = 1; } /* end if */ else (iter->dset_dim_bins[bin])++; } /* end if */ - ret = H5Sclose(sid); - HDassert(ret >= 0); + if(H5Sclose(sid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose() failed"); /* Gather datatype statistics */ - tid = H5Dget_type(did); - HDassert(tid > 0); + if((tid = H5Dget_type(did)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type() failed"); type_found = FALSE; for(u = 0; u < iter->dset_ntypes; u++) @@ -591,6 +593,7 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) type_found = TRUE; break; } /* end for */ + if(type_found) (iter->dset_type_info[u].count)++; else { @@ -600,12 +603,12 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) iter->dset_ntypes++; /* Allocate more storage for info about dataset's datatype */ - iter->dset_type_info = (dtype_info_t *)HDrealloc(iter->dset_type_info, iter->dset_ntypes * sizeof(dtype_info_t)); - HDassert(iter->dset_type_info); + if((iter->dset_type_info = (dtype_info_t *)HDrealloc(iter->dset_type_info, iter->dset_ntypes * sizeof(dtype_info_t))) == NULL) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Drealloc() failed"); /* Initialize information about datatype */ - iter->dset_type_info[curr_ntype].tid = H5Tcopy(tid); - HDassert(iter->dset_type_info[curr_ntype].tid > 0); + if((iter->dset_type_info[curr_ntype].tid = H5Tcopy(tid)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tcopy() failed"); iter->dset_type_info[curr_ntype].count = 1; iter->dset_type_info[curr_ntype].named = 0; @@ -617,8 +620,8 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) if(H5Tcommitted(tid) > 0) (iter->dset_type_info[u].named)++; - ret = H5Tclose(tid); - HDassert(ret >= 0); + if(H5Tclose(tid) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tclose() failed"); /* Track different filters */ if((nfltr = H5Pget_nfilters(dcpl)) >= 0) { @@ -635,13 +638,14 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) } /* end for */ } /* endif nfltr */ - ret = H5Pclose(dcpl); - HDassert(ret >= 0); + if(H5Pclose(dcpl) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose() failed"); - ret = H5Dclose(did); - HDassert(ret >= 0); + if(H5Dclose(did) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose() failed"); - return 0; +done: + return ret_value; } /* end dataset_stats() */ @@ -660,7 +664,7 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) static herr_t datatype_stats(iter_t *iter, const H5O_info_t *oi) { - herr_t ret; + herr_t ret_value = SUCCEED; /* Gather statistics about this type of object */ iter->uniq_dtypes++; @@ -670,10 +674,10 @@ datatype_stats(iter_t *iter, const H5O_info_t *oi) iter->dtype_ohdr_info.free_size += oi->hdr.space.free; /* Update attribute metadata info */ - ret = attribute_stats(iter, oi); - HDassert(ret >= 0); - - return 0; + if((ret_value = attribute_stats(iter, oi)) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "attribute_stats() failed"); +done: + return ret_value; } /* end datatype_stats() */ @@ -695,6 +699,7 @@ obj_stats(const char *path, const H5O_info_t *oi, const char *already_visited, void *_iter) { iter_t *iter = (iter_t *)_iter; + herr_t ret_value = SUCCEED; /* If the object has already been seen then just return */ if(NULL == already_visited) { @@ -704,15 +709,18 @@ obj_stats(const char *path, const H5O_info_t *oi, const char *already_visited, switch(oi->type) { case H5O_TYPE_GROUP: - group_stats(iter, path, oi); + if(group_stats(iter, path, oi) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "group_stats failed"); break; case H5O_TYPE_DATASET: - dataset_stats(iter, path, oi); + if(dataset_stats(iter, path, oi) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "dataset_stats failed"); break; case H5O_TYPE_NAMED_DATATYPE: - datatype_stats(iter, oi); + if(datatype_stats(iter, oi) < 0) + HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "datatype_stats failed"); break; case H5O_TYPE_UNKNOWN: @@ -724,7 +732,8 @@ obj_stats(const char *path, const H5O_info_t *oi, const char *already_visited, } /* end switch */ } /* end if */ - return 0; +done: + return ret_value; } /* end obj_stats() */ @@ -733,9 +742,8 @@ obj_stats(const char *path, const H5O_info_t *oi, const char *already_visited, * * Purpose: Gather statistics about a link * - * Return: Success: 0 - * - * Failure: -1 + * Return: Success: 0 + * Failure: -1 * * Programmer: Quincey Koziol * Tuesday, November 6, 2007 @@ -892,6 +900,10 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) goto done; break; + case 'E': + enable_error_stack = 1; + break; + case 'F': display_all = FALSE; display_file_metadata = TRUE; @@ -913,14 +925,14 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) break; case 'l': - if(opt_arg) { - sgroups_threshold = HDatoi(opt_arg); - if(sgroups_threshold < 1) { - error_msg("Invalid threshold for small groups\n"); - goto error; - } - } else - error_msg("Missing threshold for small groups\n"); + if(opt_arg) { + sgroups_threshold = HDatoi(opt_arg); + if(sgroups_threshold < 1) { + error_msg("Invalid threshold for small groups\n"); + goto error; + } + } else + error_msg("Missing threshold for small groups\n"); break; @@ -935,14 +947,14 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) break; case 'm': - if(opt_arg) { - sdsets_threshold = HDatoi(opt_arg); - if(sdsets_threshold < 1) { - error_msg("Invalid threshold for small datasets\n"); - goto error; - } - } else - error_msg("Missing threshold for small datasets\n"); + if(opt_arg) { + sdsets_threshold = HDatoi(opt_arg); + if(sdsets_threshold < 1) { + error_msg("Invalid threshold for small datasets\n"); + goto error; + } + } else + error_msg("Missing threshold for small datasets\n"); break; @@ -957,13 +969,13 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) break; case 'a': - if(opt_arg) { - sattrs_threshold = HDatoi(opt_arg); - if(sattrs_threshold < 1) { - error_msg("Invalid threshold for small # of attributes\n"); - goto error; - } - } else + if(opt_arg) { + sattrs_threshold = HDatoi(opt_arg); + if(sattrs_threshold < 1) { + error_msg("Invalid threshold for small # of attributes\n"); + goto error; + } + } else error_msg("Missing threshold for small # of attributes\n"); break; @@ -1249,9 +1261,8 @@ print_group_info(const iter_t *iter) * * Purpose: Prints file space information for groups' metadata * - * Return: Success: 0 - * - * Failure: Never fails + * Return: Success: 0 + * Failure: Never fails * * Programmer: Vailin Choi; October 2009 * @@ -1277,9 +1288,8 @@ print_group_metadata(const iter_t *iter) * * Purpose: Prints information about datasets in the file * - * Return: Success: 0 - * - * Failure: Never fails + * Return: Success: 0 + * Failure: Never fails * * Programmer: Elena Pourmal * Saturday, August 12, 2006 @@ -1340,7 +1350,7 @@ print_dataset_info(const iter_t *iter) printf("Dataset layout information:\n"); for(u = 0; u < H5D_NLAYOUTS; u++) - printf("\tDataset layout counts[%s]: %lu\n", (u == H5D_COMPACT ? "COMPACT" : + printf("\tDataset layout counts[%s]: %lu\n", (u == H5D_COMPACT ? "COMPACT" : (u == H5D_CONTIGUOUS ? "CONTIG" : (u == H5D_CHUNKED ? "CHUNKED" : "VIRTUAL"))), iter->dset_layouts[u]); printf("\tNumber of external files : %lu\n", iter->nexternal); @@ -1704,16 +1714,25 @@ main(int argc, const char *argv[]) iter_t iter; const char *fname = NULL; hid_t fid = -1; + H5E_auto2_t func; + H5E_auto2_t tools_func; + void *edata; + void *tools_edata; struct handler_t *hand = NULL; h5tools_setprogname(PROGRAMNAME); h5tools_setstatus(EXIT_SUCCESS); /* Disable error reporting */ + H5Eget_auto2(H5E_DEFAULT, &func, &edata); H5Eset_auto2(H5E_DEFAULT, NULL, NULL); /* Initialize h5tools lib */ h5tools_init(); + + /* Disable tools error reporting */ + H5Eget_auto2(H5tools_ERR_STACK_g, &tools_func, &tools_edata); + H5Eset_auto2(H5tools_ERR_STACK_g, NULL, NULL); HDmemset(&iter, 0, sizeof(iter)); @@ -1722,6 +1741,11 @@ main(int argc, const char *argv[]) fname = argv[opt_ind]; + if(enable_error_stack > 0) { + H5Eset_auto2(H5E_DEFAULT, func, edata); + H5Eset_auto2(H5tools_ERR_STACK_g, tools_func, tools_edata); + } + /* Check for filename given */ if(fname) { hid_t fcpl; @@ -1788,16 +1812,18 @@ main(int argc, const char *argv[]) unsigned u; for(u = 0; u < hand->obj_count; u++) { - if(h5trav_visit(fid, hand->obj[u], TRUE, TRUE, obj_stats, lnk_stats, &iter, H5O_INFO_ALL) < 0) - warn_msg("Unable to traverse object \"%s\"\n", hand->obj[u]); - else + if(h5trav_visit(fid, hand->obj[u], TRUE, TRUE, obj_stats, lnk_stats, &iter, H5O_INFO_ALL) < 0) { + error_msg("unable to traverse object \"%s\"\n", hand->obj[u]); + h5tools_setstatus(EXIT_FAILURE); + } else print_statistics(hand->obj[u], &iter); } /* end for */ } /* end if */ else { - if(h5trav_visit(fid, "/", TRUE, TRUE, obj_stats, lnk_stats, &iter, H5O_INFO_ALL) < 0) - warn_msg("Unable to traverse objects/links in file \"%s\"\n", fname); - else + if(h5trav_visit(fid, "/", TRUE, TRUE, obj_stats, lnk_stats, &iter, H5O_INFO_ALL) < 0) { + error_msg("unable to traverse objects/links in file \"%s\"\n", fname); + h5tools_setstatus(EXIT_FAILURE); + } else print_statistics("/", &iter); } /* end else */ } /* end if */ @@ -1813,6 +1839,8 @@ done: h5tools_setstatus(EXIT_FAILURE); } /* end if */ + H5Eset_auto2(H5E_DEFAULT, func, edata); + leave(h5tools_getstatus()); } /* end main() */ diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index 9358fbb..f34d479 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -113,6 +113,7 @@ #define FILE83 "tvlenstr_array.h5" #define FILE84 "tudfilter.h5" #define FILE85 "tgrpnullspace.h5" +#define FILE86 "err_attr_dspace.h5" /*------------------------------------------------------------------------- * prototypes @@ -10476,6 +10477,89 @@ static void gent_null_space_group(void) H5Fclose(fid); } +/*------------------------------------------------------------------------- + * Function: gent_err_attr_dspace + * + * Purpose: Generate a file with shared dataspace message. + * Then write an illegal version to the shared dataspace message + * to trigger the error. + * This is to verify HDFFV-10333 that h5dump will exit + * gracefully when encountered error similar to + * H5O_attr_decode in the jira issue. + * + *------------------------------------------------------------------------- + */ +static void +gent_err_attr_dspace() +{ + hid_t fid = -1; /* File identifier */ + hid_t fcpl = -1; /* File access property list */ + hid_t sid; /* Dataspace identifier */ + hid_t aid; /* Attribute identifier */ + hsize_t dims = 2; /* Dimensino size */ + int wdata[2] = {7, 42}; /* The buffer to write */ + int fd = -1; /* The file descriptor */ + char val = 6; /* An invalid version */ + + /* Create an fcpl */ + if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) + goto error; + + /* Set up the dataspace message to be shared */ + if(H5Pset_shared_mesg_nindexes(fcpl, 1) < 0) + goto error; + if(H5Pset_shared_mesg_index(fcpl, 0, H5O_SHMESG_SDSPACE_FLAG, 1) < 0) + goto error; + + /* Create the file with the shared message setting */ + if((fid = H5Fcreate(FILE86, H5F_ACC_TRUNC, fcpl, H5P_DEFAULT)) < 0) + goto error; + + /* Create the dataspace */ + if((sid = H5Screate_simple(1, &dims, &dims)) < 0) + goto error; + + /* Create an attribute with shared dataspace */ + if((aid = H5Acreate2(fid, "attribute", H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if(H5Awrite(aid, H5T_NATIVE_INT, wdata) < 0) + goto error; + + /* Closing */ + if(H5Aclose(aid) < 0) + goto error; + if(H5Sclose(sid) < 0) + goto error; + if(H5Pclose(fcpl) < 0) + goto error; + if(H5Fclose(fid) < 0) + goto error; + + /* This section of code will write an illegal version to the "version" field + of the shared dataspace message */ + if((fd = HDopen(FILE86, O_RDWR, 0633)) < 0) + goto error; + + /* Offset of the "version" field to modify is as follows: */ + /* 1916: offset of the object header containing the attribute message */ + /* 32: offset of the attribute message in the object header */ + /* 30: offset in the attribute message containing the version of the shared dataspace message */ + if(HDlseek(fd, 1916+32+30, SEEK_SET) < 0) + goto error; + if(HDwrite(fd, &val, 1) < 0) + goto error; + if(HDclose(fd) < 0) + goto error; + +error: + H5E_BEGIN_TRY { + H5Pclose(fcpl); + H5Aclose(aid); + H5Sclose(sid); + H5Fclose(fid); + } H5E_END_TRY; +} /* gen_err_attr_dspace() */ + int main(void) { gent_group(); @@ -10569,6 +10653,8 @@ int main(void) gent_udfilter(); + gent_err_attr_dspace(); + return 0; } diff --git a/tools/test/h5dump/testh5dump.sh.in b/tools/test/h5dump/testh5dump.sh.in index 1935b0d..a2ec035 100644 --- a/tools/test/h5dump/testh5dump.sh.in +++ b/tools/test/h5dump/testh5dump.sh.in @@ -175,6 +175,7 @@ $SRC_H5DUMP_TESTFILES/tvldtypes5.h5 $SRC_H5DUMP_TESTFILES/tvlenstr_array.h5 $SRC_H5DUMP_TESTFILES/tvlstr.h5 $SRC_H5DUMP_TESTFILES/tvms.h5 +$SRC_H5DUMP_TESTFILES/err_attr_dspace.h5 " LIST_OTHER_TEST_FILES=" @@ -360,6 +361,7 @@ $SRC_H5DUMP_TESTFILES/twithddlfile.exp $SRC_H5DUMP_TESTFILES/h5dump-help.txt $SRC_H5DUMP_TESTFILES/out3.h5import $SRC_H5DUMP_TESTFILES/tbinregR.exp +$SRC_H5DUMP_TESTFILES/err_attr_dspace.ddl " LIST_ERROR_TEST_FILES=" @@ -1322,6 +1324,9 @@ TOOLTEST tattrreg.ddl --enable-error-stack tattrreg.h5 TOOLTEST4 tattrregR.ddl --enable-error-stack -R tattrreg.h5 TOOLTEST2 tbinregR.exp --enable-error-stack -d /Dataset1 -s 0 -R -y -o tbinregR.txt tdatareg.h5 +# test to verify HDFFV-10333: error similar to H5O_attr_decode in the jira issue +TOOLTEST err_attr_dspace.ddl err_attr_dspace.h5 + # Clean up text output files if test -z "$HDF5_NOCLEANUP"; then rm -f tbinregR.txt diff --git a/tools/test/h5stat/h5stat_gentest.c b/tools/test/h5stat/h5stat_gentest.c index 0f696d0..01206dd 100644 --- a/tools/test/h5stat/h5stat_gentest.c +++ b/tools/test/h5stat/h5stat_gentest.c @@ -21,6 +21,7 @@ * of the expected output and update the corresponding *.ddl files. */ #include "hdf5.h" +#include "H5private.h" /* For gen_newgrat_file() */ #define NEWGRAT_FILE "h5stat_newgrat.h5" @@ -43,6 +44,9 @@ #define THRES_NUM 10 #define THRES_NUM_25 25 +/* For gen_err_refcount() */ +#define ERR_REFCOUNT_FILE "h5stat_err_refcount.h5" + /* * Generate HDF5 file with latest format with * NUM_GRPS groups and NUM_ATTRS attributes for the dataset @@ -434,6 +438,141 @@ error: } /* gen_idx_file() */ +/* + * Function: gen_err_refcount_file + * + * Purpose: Create a file with a refcount message ID. + * Then a refcount message ID is written to a + * message in a version 1 object header. + * This will trigger the error as a version 1 + * object header does not support a refcount message. + * This is to verify HDFFV-10333 that h5stat will exit + * gracefully when encountered error similar to + * H5O_refcount_decode in the jira issue. + * + */ +static void +gen_err_refcount(const char *fname) +{ + hid_t fid = -1; /* File identifier */ + hid_t fapl = -1; /* File access property list */ + hid_t sid = -1; /* Dataspace message */ + hid_t did = -1; /* Dataset identifier */ + hid_t gid = -1; /* Group identifier */ + hid_t aid1 = -1, aid2 = -1; /* Attribute identifier */ + hid_t tid = -1; /* Datatype identifier */ + int i, n; /* Local index variables */ + int buf[10]; /* Data buffer */ + hsize_t dims[1]; /* Dimension size */ + int fd = -1; /* File descriptor */ + unsigned short val = 22; /* The refcount message ID */ + + /* Initialize data buffer */ + n = 0; + for(i = 0; i < 10; i++) + buf[i] = n++; + + /* Create the file */ + if((fid = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* Create a group */ + if((gid = H5Gcreate2(fid, "group", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* Create a committed datatype in the group */ + if((tid = H5Tcopy(H5T_NATIVE_INT)) < 0) + goto error; + if(H5Tcommit2(gid, "dtype", tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) + goto error; + + /* Create the dataspace */ + dims[0] = 10; + if((sid = H5Screate_simple(1, dims, NULL)) < 0) + goto error; + + /* Create a dataset with the committed datatype in the file */ + if((did = H5Dcreate2(fid, "dset", tid, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + /* Write to the dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) + goto error; + + /* Attach an attribute with the committed datatype to the group */ + if((aid1 = H5Acreate2(gid, "attr", tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + /* Attach an attribute with the committed datatype to the dataset */ + if((aid2 = H5Acreate2(did, "attr", tid, sid, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + + /* Closing */ + if(H5Aclose(aid1) < 0) + goto error; + if(H5Aclose(aid2) < 0) + goto error; + if(H5Sclose(sid) < 0) + goto error; + if(H5Dclose(did) < 0) + goto error; + if(H5Gclose(gid) < 0) + goto error; + if(H5Tclose(tid) < 0) + goto error; + if(H5Fclose(fid) < 0) + goto error; + + /* This section of code will write a refcount message ID to a message in the + version 1 object header which does not support a refcount message */ + /* Offset of the message ID to modify is as follows: */ + /* 4520: the offset of the object header containing the attribute message + with the committed datatype */ + /* 24: the offset in the object header containing the version of the + attribute message */ + if((fd = HDopen(fname, O_RDWR, 0633)) < 0) + goto error; + if(HDlseek(fd, 4520+24, SEEK_SET) < 0) + goto error; + if(HDwrite(fd, &val, 2) < 0) + goto error; + if(HDclose(fd) < 0) + goto error; + +error: + H5E_BEGIN_TRY { + H5Gclose(gid); + H5Dclose(did); + H5Tclose(tid); + H5Sclose(sid); + H5Aclose(aid1); + H5Aclose(aid2); + H5Fclose(fid); + } H5E_END_TRY; +} /* gen_err_refcount() */ + +/* + * The following two test files are generated with older versions + * of the library for HDFFV-10333. They are used for testing in + * testh5stat.sh.in. + * + * (1) h5stat_err_old_layout.h5 + * This file is generated with the 1.6 library so that a file + * with a version 2 layout message is created. + * Then a "0" is written to the "dimension" field in the layout + * message to trigger the error. + * This is to verify HDFFV-10333 that h5stat will exit gracefully + * when encountered error similar to H5O__layout_decode in the + * jira issue. + * + * (2) h5stat_err_old_fill.h5 + * This file is generated with the 1.4 library so that a file + * with an old fill value message is created. + * Then an illegal size is written to the "size" fild in the + * fill value message to trigger the error. + * This is to verify HDFFV-10333 that h5stat will exit gracefully + * when encountered error similar to H5O_fill_old_decode in the + * jira issue. + */ + int main(void) { gen_newgrat_file(NEWGRAT_FILE); @@ -442,6 +581,9 @@ int main(void) /* Generate an HDF file to test for datasets with Fixed Array indexing */ gen_idx_file(IDX_FILE); + /* Generate a file with a refcount message ID */ + gen_err_refcount(ERR_REFCOUNT_FILE); + return 0; } diff --git a/tools/test/h5stat/testfiles/h5stat_err_old_fill.ddl b/tools/test/h5stat/testfiles/h5stat_err_old_fill.ddl new file mode 100644 index 0000000..e751b7f --- /dev/null +++ b/tools/test/h5stat/testfiles/h5stat_err_old_fill.ddl @@ -0,0 +1,2 @@ +Filename: h5stat_err_old_fill.h5 +h5stat error: unable to traverse objects/links in file "h5stat_err_old_fill.h5" diff --git a/tools/test/h5stat/testfiles/h5stat_err_old_fill.h5 b/tools/test/h5stat/testfiles/h5stat_err_old_fill.h5 new file mode 100644 index 0000000..aa164f0 Binary files /dev/null and b/tools/test/h5stat/testfiles/h5stat_err_old_fill.h5 differ diff --git a/tools/test/h5stat/testfiles/h5stat_err_old_layout.ddl b/tools/test/h5stat/testfiles/h5stat_err_old_layout.ddl new file mode 100644 index 0000000..a3e27e2 --- /dev/null +++ b/tools/test/h5stat/testfiles/h5stat_err_old_layout.ddl @@ -0,0 +1,2 @@ +Filename: h5stat_err_old_layout.h5 +h5stat error: unable to traverse objects/links in file "h5stat_err_old_layout.h5" diff --git a/tools/test/h5stat/testfiles/h5stat_err_old_layout.h5 b/tools/test/h5stat/testfiles/h5stat_err_old_layout.h5 new file mode 100644 index 0000000..5c0b5dc Binary files /dev/null and b/tools/test/h5stat/testfiles/h5stat_err_old_layout.h5 differ diff --git a/tools/test/h5stat/testfiles/h5stat_err_refcount.ddl b/tools/test/h5stat/testfiles/h5stat_err_refcount.ddl new file mode 100644 index 0000000..1f1b491 --- /dev/null +++ b/tools/test/h5stat/testfiles/h5stat_err_refcount.ddl @@ -0,0 +1,2 @@ +Filename: h5stat_err_refcount.h5 +h5stat error: unable to traverse objects/links in file "h5stat_err_refcount.h5" diff --git a/tools/test/h5stat/testfiles/h5stat_err_refcount.h5 b/tools/test/h5stat/testfiles/h5stat_err_refcount.h5 new file mode 100644 index 0000000..5e0d5f9 Binary files /dev/null and b/tools/test/h5stat/testfiles/h5stat_err_refcount.h5 differ diff --git a/tools/test/h5stat/testfiles/h5stat_help1.ddl b/tools/test/h5stat/testfiles/h5stat_help1.ddl index d2a8715..01e39af 100644 --- a/tools/test/h5stat/testfiles/h5stat_help1.ddl +++ b/tools/test/h5stat/testfiles/h5stat_help1.ddl @@ -22,3 +22,4 @@ Usage: h5stat [OPTIONS] file than 0. The default threshold is 10. -s, --freespace Print free space information -S, --summary Print summary of file space information + --enable-error-stack Prints messages from the HDF5 error stack as they occur diff --git a/tools/test/h5stat/testfiles/h5stat_help2.ddl b/tools/test/h5stat/testfiles/h5stat_help2.ddl index d2a8715..01e39af 100644 --- a/tools/test/h5stat/testfiles/h5stat_help2.ddl +++ b/tools/test/h5stat/testfiles/h5stat_help2.ddl @@ -22,3 +22,4 @@ Usage: h5stat [OPTIONS] file than 0. The default threshold is 10. -s, --freespace Print free space information -S, --summary Print summary of file space information + --enable-error-stack Prints messages from the HDF5 error stack as they occur diff --git a/tools/test/h5stat/testfiles/h5stat_nofile.ddl b/tools/test/h5stat/testfiles/h5stat_nofile.ddl index d8a8b2c..7171320 100644 --- a/tools/test/h5stat/testfiles/h5stat_nofile.ddl +++ b/tools/test/h5stat/testfiles/h5stat_nofile.ddl @@ -22,4 +22,5 @@ Usage: h5stat [OPTIONS] file than 0. The default threshold is 10. -s, --freespace Print free space information -S, --summary Print summary of file space information + --enable-error-stack Prints messages from the HDF5 error stack as they occur h5stat error: missing file name diff --git a/tools/test/h5stat/testh5stat.sh.in b/tools/test/h5stat/testh5stat.sh.in index ca7ca4c..d178b0d 100644 --- a/tools/test/h5stat/testh5stat.sh.in +++ b/tools/test/h5stat/testh5stat.sh.in @@ -74,6 +74,9 @@ $SRC_H5STAT_TESTFILES/h5stat_tsohm.h5 $SRC_H5STAT_TESTFILES/h5stat_newgrat.h5 $SRC_H5STAT_TESTFILES/h5stat_idx.h5 $SRC_H5STAT_TESTFILES/h5stat_threshold.h5 +$SRC_H5STAT_TESTFILES/h5stat_err_refcount.h5 +$SRC_H5STAT_TESTFILES/h5stat_err_old_layout.h5 +$SRC_H5STAT_TESTFILES/h5stat_err_old_fill.h5 " LIST_OTHER_TEST_FILES=" @@ -109,6 +112,9 @@ $SRC_H5STAT_TESTFILES/h5stat_numattrs1.ddl $SRC_H5STAT_TESTFILES/h5stat_numattrs2.ddl $SRC_H5STAT_TESTFILES/h5stat_numattrs3.ddl $SRC_H5STAT_TESTFILES/h5stat_numattrs4.ddl +$SRC_H5STAT_TESTFILES/h5stat_err_refcount.ddl +$SRC_H5STAT_TESTFILES/h5stat_err_old_layout.ddl +$SRC_H5STAT_TESTFILES/h5stat_err_old_fill.ddl " # @@ -243,6 +249,7 @@ TOOLTEST h5stat_help2.ddl --help TOOLTEST h5stat_notexist.ddl notexist.h5 TOOLTEST h5stat_nofile.ddl '' + # Test file with groups, compressed datasets, user-applied fileters, etc. # h5stat_filters.h5 is a copy of ../../testfiles/tfilters.h5 as of release 1.8.0-alpha4 TOOLTEST h5stat_filters.ddl h5stat_filters.h5 @@ -304,7 +311,13 @@ TOOLTEST h5stat_numattrs3.ddl -A --numattrs=25 h5stat_threshold.h5 # -A -a 100 TOOLTEST h5stat_numattrs4.ddl -A -a 100 h5stat_newgrat.h5 # - +# +# Tests to verify HDFFV-10333 +TOOLTEST h5stat_err_refcount.ddl h5stat_err_refcount.h5 +TOOLTEST h5stat_err_old_layout.ddl h5stat_err_old_layout.h5 +TOOLTEST h5stat_err_old_fill.ddl h5stat_err_old_fill.h5 +# +# # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR diff --git a/tools/testfiles/err_attr_dspace.ddl b/tools/testfiles/err_attr_dspace.ddl new file mode 100644 index 0000000..6c45322 --- /dev/null +++ b/tools/testfiles/err_attr_dspace.ddl @@ -0,0 +1,5 @@ +HDF5 "err_attr_dspace.h5" { +GROUP "/" { +} +} +h5dump error: error getting attribute information diff --git a/tools/testfiles/err_attr_dspace.h5 b/tools/testfiles/err_attr_dspace.h5 new file mode 100644 index 0000000..969c328 Binary files /dev/null and b/tools/testfiles/err_attr_dspace.h5 differ -- cgit v0.12 From d1f8ac331222351a63147ab54271ac9d5b271f0b Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Wed, 11 Jul 2018 16:02:51 -0500 Subject: Modifications made based on feedback from pull request. --- MANIFEST | 8 ++++++++ release_docs/RELEASE.txt | 11 +++++++++++ tools/test/h5dump/CMakeTests.cmake | 5 +++++ tools/test/h5dump/testh5dump.sh.in | 6 +++--- tools/test/h5stat/CMakeTests.cmake | 18 +++++++++++++++--- tools/test/h5stat/testh5stat.sh.in | 14 ++++++++------ 6 files changed, 50 insertions(+), 12 deletions(-) diff --git a/MANIFEST b/MANIFEST index 9ab179f..e5f06cf 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1562,6 +1562,12 @@ ./tools/test/h5stat/testfiles/h5stat_err1_links.ddl ./tools/test/h5stat/testfiles/h5stat_err1_numattrs.ddl ./tools/test/h5stat/testfiles/h5stat_err2_numattrs.ddl +./tools/test/h5stat/testfiles/h5stat_err_old_fill.h5 +./tools/test/h5stat/testfiles/h5stat_err_old_fill.ddl +./tools/test/h5stat/testfiles/h5stat_err_old_layout.h5 +./tools/test/h5stat/testfiles/h5stat_err_old_layout.ddl +./tools/test/h5stat/testfiles/h5stat_err_refcount.h5 +./tools/test/h5stat/testfiles/h5stat_err_refcount.ddl ./tools/test/h5stat/testfiles/h5stat_filters.ddl ./tools/test/h5stat/testfiles/h5stat_filters-d.ddl ./tools/test/h5stat/testfiles/h5stat_filters-dT.ddl @@ -1597,6 +1603,8 @@ # h5dump test files ./tools/testfiles/charsets.h5 ./tools/testfiles/charsets.ddl +./tools/testfiles/err_attr_dspace.h5 +./tools/testfiles/err_attr_dspace.ddl ./tools/testfiles/family_file00000.h5 ./tools/testfiles/family_file00001.h5 ./tools/testfiles/family_file00002.h5 diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 48fbef0..1204c0b 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -98,6 +98,17 @@ Bug Fixes since HDF5-1.10.2 release Library ------- + - Error checks in h5stat and when decoding messages + + h5stat exited with seg fault/core dumped when + errors are encountered in the internal library. + + Add error checks and --enable-error-stack option to h5stat. + Add range checks when decoding messages: old fill value, old + layout and refcount. + + (VC - 2018/07/11, HDFFV-10333) + - If an HDF5 file contains a malformed compound datatype with a suitably large offset, the type conversion code can run off the end of the type conversion buffer, causing a segmentation diff --git a/tools/test/h5dump/CMakeTests.cmake b/tools/test/h5dump/CMakeTests.cmake index cdd3e6d..132fc2b 100644 --- a/tools/test/h5dump/CMakeTests.cmake +++ b/tools/test/h5dump/CMakeTests.cmake @@ -21,6 +21,7 @@ # -------------------------------------------------------------------- set (HDF5_REFERENCE_FILES ${HDF5_TOOLS_DIR}/testfiles/charsets.ddl + ${HDF5_TOOLS_DIR}/testfiles/err_attr_dspace.ddl ${HDF5_TOOLS_DIR}/testfiles/file_space.ddl ${HDF5_TOOLS_DIR}/testfiles/filter_fail.ddl ${HDF5_TOOLS_DIR}/testfiles/non_existing.ddl @@ -217,6 +218,7 @@ ) set (HDF5_REFERENCE_TEST_FILES ${HDF5_TOOLS_DIR}/testfiles/charsets.h5 + ${HDF5_TOOLS_DIR}/testfiles/err_attr_dspace.h5 ${HDF5_TOOLS_DIR}/testfiles/file_space.h5 ${HDF5_TOOLS_DIR}/testfiles/filter_fail.h5 ${HDF5_TOOLS_DIR}/testfiles/packedbits.h5 @@ -1533,6 +1535,9 @@ # test for non-existing file ADD_H5_TEST (non_existing 1 --enable-error-stack tgroup.h5 non_existing.h5) + # test to verify HDFFV-10333: error similar to H5O_attr_decode in the jira issue + ADD_H5_TEST (err_attr_dspace 1 err_attr_dspace.h5) + ############################################################################## ### P L U G I N T E S T S ############################################################################## diff --git a/tools/test/h5dump/testh5dump.sh.in b/tools/test/h5dump/testh5dump.sh.in index a2ec035..42e4b07 100644 --- a/tools/test/h5dump/testh5dump.sh.in +++ b/tools/test/h5dump/testh5dump.sh.in @@ -1324,9 +1324,6 @@ TOOLTEST tattrreg.ddl --enable-error-stack tattrreg.h5 TOOLTEST4 tattrregR.ddl --enable-error-stack -R tattrreg.h5 TOOLTEST2 tbinregR.exp --enable-error-stack -d /Dataset1 -s 0 -R -y -o tbinregR.txt tdatareg.h5 -# test to verify HDFFV-10333: error similar to H5O_attr_decode in the jira issue -TOOLTEST err_attr_dspace.ddl err_attr_dspace.h5 - # Clean up text output files if test -z "$HDF5_NOCLEANUP"; then rm -f tbinregR.txt @@ -1369,6 +1366,9 @@ TOOLTEST2 tall-6.exp --enable-error-stack -y -o tall-6.txt -d /g1/g1.1/dset1.1.1 # test for non-existing file TOOLTEST3 non_existing.ddl --enable-error-stack tgroup.h5 non_existing.h5 +# test to verify HDFFV-10333: error similar to H5O_attr_decode in the jira issue +TOOLTEST err_attr_dspace.ddl err_attr_dspace.h5 + # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR diff --git a/tools/test/h5stat/CMakeTests.cmake b/tools/test/h5stat/CMakeTests.cmake index 69bab20..bd55ac1 100644 --- a/tools/test/h5stat/CMakeTests.cmake +++ b/tools/test/h5stat/CMakeTests.cmake @@ -20,6 +20,9 @@ # Copy all the HDF5 files from the test directory into the source directory # -------------------------------------------------------------------- set (HDF5_REFERENCE_FILES + h5stat_err_refcount + h5stat_err_old_layout + h5stat_err_old_fill h5stat_help1 h5stat_help2 h5stat_notexist @@ -54,6 +57,9 @@ h5stat_numattrs4 ) set (HDF5_REFERENCE_TEST_FILES + h5stat_err_refcount.h5 + h5stat_err_old_layout.h5 + h5stat_err_old_fill.h5 h5stat_filters.h5 h5stat_idx.h5 h5stat_tsohm.h5 @@ -160,9 +166,7 @@ ADD_H5_TEST (h5stat_newgrat-UG 0 -G h5stat_newgrat.h5) ADD_H5_TEST (h5stat_newgrat-UA 0 -A h5stat_newgrat.h5) # h5stat_idx.h5 is generated by h5stat_gentest.c - if (HDF5_BUILD_GENERATORS) - ADD_H5_TEST (h5stat_idx 0 h5stat_idx.h5) - endif () + ADD_H5_TEST (h5stat_idx 0 h5stat_idx.h5) # # Tests for -l (--links) option on h5stat_threshold.h5: # -l 0 (incorrect threshold value) @@ -204,3 +208,11 @@ # -A -a 100 ADD_H5_TEST (h5stat_numattrs4 0 -A -a 100 h5stat_newgrat.h5) # +# Tests to verify HDFFV-10333: +# h5stat_err_refcount.h5 is generated by h5stat_gentest.c +# h5stat_err_old_layout.h5 and h5stat_err_old_fill.h5: see explanation in h5stat_gentest.c + ADD_H5_TEST (h5stat_err_refcount 1 h5stat_err_refcount.h5) + ADD_H5_TEST (h5stat_err_old_layout 1 h5stat_err_old_layout.h5) + ADD_H5_TEST (h5stat_err_old_fill 1 h5stat_err_old_fill.h5) +# +# diff --git a/tools/test/h5stat/testh5stat.sh.in b/tools/test/h5stat/testh5stat.sh.in index d178b0d..5082daf 100644 --- a/tools/test/h5stat/testh5stat.sh.in +++ b/tools/test/h5stat/testh5stat.sh.in @@ -69,17 +69,20 @@ test -d $TESTDIR || mkdir $TESTDIR # Comment '#' without space can be used. # -------------------------------------------------------------------- LIST_HDF5_TEST_FILES=" +$SRC_H5STAT_TESTFILES/h5stat_err_refcount.h5 +$SRC_H5STAT_TESTFILES/h5stat_err_old_layout.h5 +$SRC_H5STAT_TESTFILES/h5stat_err_old_fill.h5 $SRC_H5STAT_TESTFILES/h5stat_filters.h5 $SRC_H5STAT_TESTFILES/h5stat_tsohm.h5 $SRC_H5STAT_TESTFILES/h5stat_newgrat.h5 $SRC_H5STAT_TESTFILES/h5stat_idx.h5 $SRC_H5STAT_TESTFILES/h5stat_threshold.h5 -$SRC_H5STAT_TESTFILES/h5stat_err_refcount.h5 -$SRC_H5STAT_TESTFILES/h5stat_err_old_layout.h5 -$SRC_H5STAT_TESTFILES/h5stat_err_old_fill.h5 " LIST_OTHER_TEST_FILES=" +$SRC_H5STAT_TESTFILES/h5stat_err_refcount.ddl +$SRC_H5STAT_TESTFILES/h5stat_err_old_layout.ddl +$SRC_H5STAT_TESTFILES/h5stat_err_old_fill.ddl $SRC_H5STAT_TESTFILES/h5stat_help1.ddl $SRC_H5STAT_TESTFILES/h5stat_help2.ddl $SRC_H5STAT_TESTFILES/h5stat_notexist.ddl @@ -112,9 +115,6 @@ $SRC_H5STAT_TESTFILES/h5stat_numattrs1.ddl $SRC_H5STAT_TESTFILES/h5stat_numattrs2.ddl $SRC_H5STAT_TESTFILES/h5stat_numattrs3.ddl $SRC_H5STAT_TESTFILES/h5stat_numattrs4.ddl -$SRC_H5STAT_TESTFILES/h5stat_err_refcount.ddl -$SRC_H5STAT_TESTFILES/h5stat_err_old_layout.ddl -$SRC_H5STAT_TESTFILES/h5stat_err_old_fill.ddl " # @@ -313,6 +313,8 @@ TOOLTEST h5stat_numattrs4.ddl -A -a 100 h5stat_newgrat.h5 # # # Tests to verify HDFFV-10333 +# h5stat_err_refcount.h5 is generated by h5stat_gentest.c +# h5stat_err_old_layout.h5 and h5stat_err_old_fill.h5: see explanation in h5stat_gentest.c TOOLTEST h5stat_err_refcount.ddl h5stat_err_refcount.h5 TOOLTEST h5stat_err_old_layout.ddl h5stat_err_old_layout.h5 TOOLTEST h5stat_err_old_fill.ddl h5stat_err_old_fill.h5 -- cgit v0.12 From f369590a1005756e2574eeb1fcf4f8d3fb8886fe Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Thu, 12 Jul 2018 13:55:45 -0500 Subject: Fix test_misc33() in test/tmisc.c Open the test file read-only so that it can be accessed for testing. --- test/tmisc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tmisc.c b/test/tmisc.c index 07234e9..1e62302 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -5499,7 +5499,7 @@ test_misc33(void) MESSAGE(5, ("Testing that bad offset into the heap returns error")); /* Open the test file */ - fid = H5Fopen(testfile, H5F_ACC_RDWR, H5P_DEFAULT); + fid = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fopen"); /* Case (1) */ -- cgit v0.12 From 92bf81a0fbad8e263c1e0d3c5791f49b78046afb Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Thu, 12 Jul 2018 18:25:49 -0500 Subject: Changes based on feedback from pull request. --- src/H5Ofill.c | 4 +--- tools/test/h5dump/h5dumpgentest.c | 4 ++-- tools/test/h5stat/h5stat_gentest.c | 1 - 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/H5Ofill.c b/src/H5Ofill.c index 3ce2e4f..da9829b 100644 --- a/src/H5Ofill.c +++ b/src/H5Ofill.c @@ -19,7 +19,6 @@ */ #include "H5Omodule.h" /* This source code file is part of the H5O module */ -#define H5T_FRIEND /*prevent warning from including H5Tpkg */ #include "H5private.h" /* Generic Functions */ @@ -32,7 +31,6 @@ #include "H5Pprivate.h" /* Property lists */ #include "H5Sprivate.h" /* Dataspaces */ -#include "H5Tpkg.h" /* Datatypes */ static void *H5O_fill_old_decode(H5F_t *f, H5O_t *open_oh, unsigned mesg_flags, unsigned *ioflags, size_t p_size, const uint8_t *p); @@ -345,7 +343,7 @@ H5O_fill_old_decode(H5F_t *f, H5O_t *open_oh, if((dt = H5O_msg_read_oh(f, open_oh, H5O_DTYPE_ID, NULL)) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTGET, NULL, "can't read DTYPE message") /* Verify size */ - if(fill->size != dt->shared->size) + if(fill->size != H5T_GET_SIZE(dt)) HGOTO_ERROR(H5E_SYM, H5E_CANTGET, NULL, "inconsistent fill value size") } diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index f34d479..9e9f6f1 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -10494,8 +10494,8 @@ gent_err_attr_dspace() { hid_t fid = -1; /* File identifier */ hid_t fcpl = -1; /* File access property list */ - hid_t sid; /* Dataspace identifier */ - hid_t aid; /* Attribute identifier */ + hid_t sid = -1; /* Dataspace identifier */ + hid_t aid = -1; /* Attribute identifier */ hsize_t dims = 2; /* Dimensino size */ int wdata[2] = {7, 42}; /* The buffer to write */ int fd = -1; /* The file descriptor */ diff --git a/tools/test/h5stat/h5stat_gentest.c b/tools/test/h5stat/h5stat_gentest.c index 01206dd..2daf24b 100644 --- a/tools/test/h5stat/h5stat_gentest.c +++ b/tools/test/h5stat/h5stat_gentest.c @@ -455,7 +455,6 @@ static void gen_err_refcount(const char *fname) { hid_t fid = -1; /* File identifier */ - hid_t fapl = -1; /* File access property list */ hid_t sid = -1; /* Dataspace message */ hid_t did = -1; /* Dataset identifier */ hid_t gid = -1; /* Group identifier */ -- cgit v0.12