diff options
| author | Quincey Koziol <koziol@hdfgroup.org> | 2009-09-24 22:14:50 (GMT) |
|---|---|---|
| committer | Quincey Koziol <koziol@hdfgroup.org> | 2009-09-24 22:14:50 (GMT) |
| commit | 426a3c1204fdcd1a999a966958a2edecc99b2d3b (patch) | |
| tree | a786821d8e38f98f8977c30bf91c14b6d3267fd9 | |
| parent | a2a61197aa18fcafef35ecc4aaef268b2e4731d5 (diff) | |
| download | hdf5-426a3c1204fdcd1a999a966958a2edecc99b2d3b.zip hdf5-426a3c1204fdcd1a999a966958a2edecc99b2d3b.tar.gz hdf5-426a3c1204fdcd1a999a966958a2edecc99b2d3b.tar.bz2 | |
[svn-r17522] Description:
Bring in the rest of Vailin's changes for testing fixed array chunk
indices, across the range of range of all the tools and the appropriate dataset
tests.
Tested on:
Mac OS X/32 10.5.8 (amazon) w/debug & production
(h5committest not required for this branch)
40 files changed, 1616 insertions, 122 deletions
@@ -804,6 +804,8 @@ ./test/be_extlink2.h5 ./test/big.c ./test/bittests.c +./test/btree_idx_1_6.h5 +./test/btree_idx_1_8.h5 ./test/btree2.c ./test/cache.c ./test/cache_api.c @@ -839,6 +841,7 @@ ./test/gen_bogus.c _DO_NOT_DISTRIBUTE_ ./test/gen_cross.c _DO_NOT_DISTRIBUTE_ ./test/gen_deflate.c _DO_NOT_DISTRIBUTE_ +./test/gen_idx.c _DO_NOT_DISTRIBUTE_ ./test/gen_mergemsg.c _DO_NOT_DISTRIBUTE_ ./test/gen_new_array.c _DO_NOT_DISTRIBUTE_ ./test/gen_new_fill.c _DO_NOT_DISTRIBUTE_ @@ -1113,6 +1116,8 @@ ./tools/h5stat/testfiles/h5stat_filters.h5 ./tools/h5stat/testfiles/h5stat_help1.ddl ./tools/h5stat/testfiles/h5stat_help2.ddl +./tools/h5stat/testfiles/h5stat_idx.h5 +./tools/h5stat/testfiles/h5stat_idx.ddl ./tools/h5stat/testfiles/h5stat_newgrat.ddl ./tools/h5stat/testfiles/h5stat_newgrat.h5 ./tools/h5stat/testfiles/h5stat_tsohm.ddl @@ -1185,6 +1190,9 @@ ./tools/testfiles/tdset-3s.ddl ./tools/testfiles/tdset.h5 ./tools/testfiles/tdset2.h5 +./tools/testfiles/tdset_idx.ls +./tools/testfiles/tdset_idx.ddl +./tools/testfiles/tdset_idx.h5 ./tools/testfiles/tempty.ddl ./tools/testfiles/tempty.h5 ./tools/testfiles/tempty.ls @@ -1521,6 +1529,7 @@ ./tools/h5diff/testfiles/h5diff_205.txt ./tools/h5diff/testfiles/h5diff_206.txt ./tools/h5diff/testfiles/h5diff_207.txt +./tools/h5diff/testfiles/h5diff_idx.txt ./tools/h5diff/testfiles/h5diff_attr1.h5 @@ -1529,6 +1538,8 @@ ./tools/h5diff/testfiles/h5diff_basic2.h5 ./tools/h5diff/testfiles/h5diff_dset1.h5 ./tools/h5diff/testfiles/h5diff_dset2.h5 +./tools/h5diff/testfiles/h5diff_dset_idx1.h5 +./tools/h5diff/testfiles/h5diff_dset_idx2.h5 ./tools/h5diff/testfiles/h5diff_dtypes.h5 ./tools/h5diff/testfiles/h5diff_empty.h5 ./tools/h5diff/testfiles/h5diff_hyper1.h5 @@ -1583,6 +1594,8 @@ # test files for h5copy ./tools/h5copy/testfiles/h5copytst.h5 ./tools/h5copy/testfiles/h5copytst.out.ls +./tools/h5copy/testfiles/h5copytst_new.h5 +./tools/h5copy/testfiles/h5copytst_new.out.ls # test files for h5mkgrp ./tools/testfiles/h5mkgrp_help.ls diff --git a/src/H5Dearray.c b/src/H5Dearray.c index b8cb580..55c0a8f 100644 --- a/src/H5Dearray.c +++ b/src/H5Dearray.c @@ -58,7 +58,7 @@ /* Extensible array create/open user data */ typedef struct H5D_earray_ctx_ud_t { const H5F_t *f; /* Pointer to file info */ - const H5O_layout_chunk_t *layout; /* Pointer to layout info */ + uint32_t chunk_size; /* Size of chunk (bytes) */ } H5D_earray_ctx_ud_t; /* Extensible array callback context */ @@ -227,7 +227,7 @@ H5D_earray_crt_context(void *_udata) /* Sanity checks */ HDassert(udata); HDassert(udata->f); - HDassert(udata->layout); + HDassert(udata->chunk_size > 0); /* Allocate new context structure */ if(NULL == (ctx = H5FL_MALLOC(H5D_earray_ctx_t))) @@ -239,7 +239,7 @@ H5D_earray_crt_context(void *_udata) /* Compute the size required for encoding the size of a chunk, allowing * for an extra byte, in case the filter makes the chunk larger. */ - ctx->chunk_size_len = 1 + ((H5V_log2_gen(udata->layout->size) + 8) / 8); + ctx->chunk_size_len = 1 + ((H5V_log2_gen(udata->chunk_size) + 8) / 8); if(ctx->chunk_size_len > 8) ctx->chunk_size_len = 8; @@ -746,7 +746,7 @@ H5D_earray_idx_open(const H5D_chk_idx_info_t *idx_info) /* Set up the user data */ udata.f = idx_info->f; - udata.layout = idx_info->layout; + udata.chunk_size = idx_info->layout->size; /* Open the extensible array for the chunk index */ cls = (idx_info->pline->nused > 0) ? H5EA_CLS_FILT_CHUNK : H5EA_CLS_CHUNK; @@ -899,7 +899,7 @@ H5D_earray_idx_create(const H5D_chk_idx_info_t *idx_info) /* Set up the user data */ udata.f = idx_info->f; - udata.layout = idx_info->layout; + udata.chunk_size = idx_info->layout->size; /* Create the extensible array for the chunk index */ if(NULL == (idx_info->storage->u.earray.ea = H5EA_create(idx_info->f, idx_info->dxpl_id, &cparam, &udata))) diff --git a/src/H5Dfarray.c b/src/H5Dfarray.c index ebe6495..7b0e8a7 100644 --- a/src/H5Dfarray.c +++ b/src/H5Dfarray.c @@ -57,7 +57,7 @@ /* Fixed array create/open user data */ typedef struct H5D_farray_ctx_ud_t { const H5F_t *f; /* Pointer to file info */ - const H5O_layout_chunk_t *layout; /* Pointer to layout info */ + uint32_t chunk_size; /* Size of chunk (bytes) */ } H5D_farray_ctx_ud_t; /* Fixed array callback context */ @@ -109,6 +109,9 @@ static herr_t H5D_farray_decode(const void *raw, void *elmt, size_t nelmts, void *ctx); static herr_t H5D_farray_debug(FILE *stream, int indent, int fwidth, hsize_t idx, const void *elmt); +static void *H5D_farray_crt_dbg_context(H5F_t *f, hid_t dxpl_id, + haddr_t obj_addr); +static herr_t H5D_farray_dst_dbg_context(void *dbg_ctx); /* Fixed array class callbacks for chunks w/filters */ /* (some shared with callbacks for chunks w/o filters) */ @@ -189,7 +192,9 @@ const H5FA_class_t H5FA_CLS_CHUNK[1]={{ H5D_farray_fill, /* Fill block of missing elements callback */ H5D_farray_encode, /* Element encoding callback */ H5D_farray_decode, /* Element decoding callback */ - H5D_farray_debug /* Element debugging callback */ + H5D_farray_debug, /* Element debugging callback */ + H5D_farray_crt_dbg_context, /* Create debugging context */ + H5D_farray_dst_dbg_context /* Destroy debugging context */ }}; /* Fixed array class callbacks for dataset chunks w/filters */ @@ -202,12 +207,16 @@ const H5FA_class_t H5FA_CLS_FILT_CHUNK[1]={{ H5D_farray_filt_fill, /* Fill block of missing elements callback */ H5D_farray_filt_encode, /* Element encoding callback */ H5D_farray_filt_decode, /* Element decoding callback */ - H5D_farray_filt_debug /* Element debugging callback */ + H5D_farray_filt_debug, /* Element debugging callback */ + H5D_farray_crt_dbg_context, /* Create debugging context */ + H5D_farray_dst_dbg_context /* Destroy debugging context */ }}; /* Declare a free list to manage the H5D_farray_ctx_t struct */ H5FL_DEFINE_STATIC(H5D_farray_ctx_t); +/* Declare a free list to manage the H5D_farray_ctx_ud_t struct */ +H5FL_DEFINE_STATIC(H5D_farray_ctx_ud_t); /*------------------------------------------------------------------------- @@ -235,7 +244,7 @@ H5D_farray_crt_context(void *_udata) /* Sanity checks */ HDassert(udata); HDassert(udata->f); - HDassert(udata->layout); + HDassert(udata->chunk_size > 0); /* Allocate new context structure */ if(NULL == (ctx = H5FL_MALLOC(H5D_farray_ctx_t))) @@ -247,7 +256,7 @@ H5D_farray_crt_context(void *_udata) /* Compute the size required for encoding the size of a chunk, allowing * for an extra byte, in case the filter makes the chunk larger. */ - ctx->chunk_size_len = 1 + ((H5V_log2_gen(udata->layout->size) + 8) / 8); + ctx->chunk_size_len = 1 + ((H5V_log2_gen(udata->chunk_size) + 8) / 8); if(ctx->chunk_size_len > 8) ctx->chunk_size_len = 8; @@ -442,6 +451,111 @@ H5D_farray_debug(FILE *stream, int indent, int fwidth, hsize_t idx, /*------------------------------------------------------------------------- + * Function: H5D_farray_crt_dbg_context + * + * Purpose: Create context for debugging callback + * (get the layout message in the specified object header) + * + * Return: Success: non-NULL + * Failure: NULL + * + * Programmer: Vailin Choi + * 5th August, 2009 + * + *------------------------------------------------------------------------- + */ +static void * +H5D_farray_crt_dbg_context(H5F_t *f, hid_t dxpl_id, haddr_t obj_addr) +{ + H5D_farray_ctx_ud_t *dbg_ctx = NULL; /* Context for fixed array callback */ + H5O_loc_t obj_loc; /* Pointer to an object's location */ + hbool_t obj_opened = FALSE; /* Flag to indicate that the object header was opened */ + H5O_layout_t layout; /* Layout message */ + void *ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_farray_crt_dbg_context) + + HDassert(f); + HDassert(H5F_addr_defined(obj_addr)); + + /* Allocate context for debugging callback */ + if(NULL == (dbg_ctx = H5FL_MALLOC(H5D_farray_ctx_ud_t))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "can't allocate fixed array client callback context") + + /* Set up the object header location info */ + H5O_loc_reset(&obj_loc); + obj_loc.file = f; + obj_loc.addr = obj_addr; + + /* Open the object header where the layout message resides */ + if(H5O_open(&obj_loc) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, NULL, "can't open object header") + obj_opened = TRUE; + + /* Read the layout message */ + if(NULL == H5O_msg_read(&obj_loc, H5O_LAYOUT_ID, &layout, dxpl_id)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "can't get layout info") + + /* close the object header */ + if(H5O_close(&obj_loc) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, NULL, "can't close object header") + + /* Create user data */ + dbg_ctx->f = f; + dbg_ctx->chunk_size = layout.u.chunk.size; + + /* Set return value */ + ret_value = dbg_ctx; + +done: + /* Cleanup on error */ + if(ret_value == NULL) { + /* Release context structure */ + if(dbg_ctx) + dbg_ctx = H5FL_FREE(H5D_farray_ctx_ud_t, dbg_ctx); + + /* Close object header */ + if(obj_opened) { + if(H5O_close(&obj_loc) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, NULL, "can't close object header") + } /* end if */ + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_farray_crt_dbg_context() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_farray_dst_dbg_context + * + * Purpose: Destroy context for debugging callback + * (free the layout message from the specified object header) + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * 24th September, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_farray_dst_dbg_context(void *_dbg_ctx) +{ + H5D_farray_ctx_ud_t *dbg_ctx = (H5D_farray_ctx_ud_t *)_dbg_ctx; /* Context for fixed array callback */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_farray_dst_dbg_context) + + HDassert(dbg_ctx); + + /* Release context structure */ + dbg_ctx = H5FL_FREE(H5D_farray_ctx_ud_t, dbg_ctx); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_farray_dst_dbg_context() */ + + +/*------------------------------------------------------------------------- * Function: H5D_farray_filt_fill * * Purpose: Fill "missing elements" in block of elements @@ -636,7 +750,7 @@ H5D_farray_idx_open(const H5D_chk_idx_info_t *idx_info) /* Set up the user data */ udata.f = idx_info->f; - udata.layout = idx_info->layout; + udata.chunk_size = idx_info->layout->size; /* Open the fixed array for the chunk index */ cls = (idx_info->pline->nused > 0) ? H5FA_CLS_FILT_CHUNK : H5FA_CLS_CHUNK; @@ -709,7 +823,7 @@ H5D_farray_idx_create(const H5D_chk_idx_info_t *idx_info) /* Set up the user data */ udata.f = idx_info->f; - udata.layout = idx_info->layout; + udata.chunk_size = idx_info->layout->size; /* Create the fixed array for the chunk index */ if(NULL == (idx_info->storage->u.farray.fa = H5FA_create(idx_info->f, idx_info->dxpl_id, &cparam, &udata))) diff --git a/src/H5FAtest.c b/src/H5FAtest.c index 70e21dc..98abd16 100644 --- a/src/H5FAtest.c +++ b/src/H5FAtest.c @@ -96,7 +96,9 @@ const H5FA_class_t H5FA_CLS_TEST[1]={{ H5FA__test_fill, /* Fill block of missing elements callback */ H5FA__test_encode, /* Element encoding callback */ H5FA__test_decode, /* Element decoding callback */ - H5FA__test_debug /* Element debugging callback */ + H5FA__test_debug, /* Element debugging callback */ + NULL, /* Create debugging context */ + NULL /* Destroy debugging context */ }}; diff --git a/test/Makefile.am b/test/Makefile.am index 7f21f74..7ef1ef0 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -61,7 +61,7 @@ check_PROGRAMS=$(TEST_PROG) error_test err_compat testmeta # --enable-build-all at configure time. # The gen_old_* files can only be compiled with older versions of the library # so do not appear in this list. -BUILD_ALL_PROGS=gen_bad_ohdr gen_bogus gen_cross gen_deflate gen_filters gen_new_array \ +BUILD_ALL_PROGS=gen_bad_ohdr gen_bogus gen_cross gen_deflate gen_filters gen_idx gen_new_array \ gen_new_fill gen_new_group gen_new_mtime gen_new_super gen_noencoder \ gen_nullspace gen_udlinks space_overflow @@ -125,7 +125,7 @@ CHECK_CLEANFILES+=cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offset.h5 \ new_move_[ab].h5 ntypes.h5 dangle.h5 error_test.h5 err_compat.h5 \ dtransform.h5 test_filters.h5 get_file_name.h5 tstint[1-2].h5 \ unlink_chunked.h5 btree2.h5 objcopy_src.h5 objcopy_dst.h5 \ - objcopy_ext.dat trefer1.h5 trefer2.h5 app_ref.h5 farray.h5 \ + objcopy_ext.dat trefer1.h5 trefer2.h5 app_ref.h5 farray.h5 \ earray.h5 swmr_data.h5 # Sources for testhdf5 executable diff --git a/test/Makefile.in b/test/Makefile.in index ed3e205..0acc722 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -89,14 +89,14 @@ am__EXEEXT_1 = testhdf5$(EXEEXT) lheap$(EXEEXT) ohdr$(EXEEXT) \ ttsafe$(EXEEXT) getname$(EXEEXT) vfd$(EXEEXT) ntypes$(EXEEXT) \ dangle$(EXEEXT) dtransform$(EXEEXT) reserved$(EXEEXT) \ cross_read$(EXEEXT) freespace$(EXEEXT) mf$(EXEEXT) \ - farray$(EXEEXT) earray$(EXEEXT) btree2$(EXEEXT) fheap$(EXEEXT) + farray$(EXEEXT) earray$(EXEEXT) btree2$(EXEEXT) fheap$(EXEEXT) am__EXEEXT_2 = gen_bad_ohdr$(EXEEXT) gen_bogus$(EXEEXT) \ gen_cross$(EXEEXT) gen_deflate$(EXEEXT) gen_filters$(EXEEXT) \ gen_new_array$(EXEEXT) gen_new_fill$(EXEEXT) \ gen_new_group$(EXEEXT) gen_new_mtime$(EXEEXT) \ gen_new_super$(EXEEXT) gen_noencoder$(EXEEXT) \ gen_nullspace$(EXEEXT) gen_udlinks$(EXEEXT) \ - space_overflow$(EXEEXT) + space_overflow$(EXEEXT) gen_idx$(EXEEXT) PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS) app_ref_SOURCES = app_ref.c app_ref_OBJECTS = app_ref.$(OBJEXT) @@ -218,6 +218,10 @@ gen_filters_SOURCES = gen_filters.c gen_filters_OBJECTS = gen_filters.$(OBJEXT) gen_filters_LDADD = $(LDADD) gen_filters_DEPENDENCIES = libh5test.la $(LIBHDF5) +gen_idx_SOURCES = gen_idx.c +gen_idx_OBJECTS = gen_idx.$(OBJEXT) +gen_idx_LDADD = $(LDADD) +gen_idx_DEPENDENCIES = libh5test.la $(LIBHDF5) gen_new_array_SOURCES = gen_new_array.c gen_new_array_OBJECTS = gen_new_array.$(OBJEXT) gen_new_array_LDADD = $(LDADD) @@ -378,11 +382,11 @@ SOURCES = $(libh5test_la_SOURCES) app_ref.c big.c bittests.c btree2.c \ dt_arith.c dtransform.c dtypes.c earray.c enum.c err_compat.c \ error_test.c extend.c external.c farray.c fheap.c fillval.c \ flush1.c flush2.c freespace.c gen_bad_ohdr.c gen_bogus.c \ - gen_cross.c gen_deflate.c gen_filters.c gen_new_array.c \ - gen_new_fill.c gen_new_group.c gen_new_mtime.c gen_new_super.c \ - gen_noencoder.c gen_nullspace.c gen_udlinks.c getname.c \ - gheap.c hyperslab.c istore.c lheap.c links.c mf.c mount.c \ - mtime.c ntypes.c objcopy.c ohdr.c pool.c reserved.c \ + gen_cross.c gen_deflate.c gen_filters.c gen_idx.c \ + gen_new_array.c gen_new_fill.c gen_new_group.c gen_new_mtime.c \ + gen_new_super.c gen_noencoder.c gen_nullspace.c gen_udlinks.c \ + getname.c gheap.c hyperslab.c istore.c lheap.c links.c mf.c \ + mount.c mtime.c ntypes.c objcopy.c ohdr.c pool.c reserved.c \ set_extent.c space_overflow.c stab.c swmr_generator.c \ swmr_reader.c swmr_writer.c $(testhdf5_SOURCES) testmeta.c \ $(ttsafe_SOURCES) unlink.c vfd.c @@ -391,7 +395,7 @@ DIST_SOURCES = $(libh5test_la_SOURCES) app_ref.c big.c bittests.c \ dsets.c dt_arith.c dtransform.c dtypes.c earray.c enum.c \ err_compat.c error_test.c extend.c external.c farray.c fheap.c \ fillval.c flush1.c flush2.c freespace.c gen_bad_ohdr.c \ - gen_bogus.c gen_cross.c gen_deflate.c gen_filters.c \ + gen_bogus.c gen_cross.c gen_deflate.c gen_filters.c gen_idx.c \ gen_new_array.c gen_new_fill.c gen_new_group.c gen_new_mtime.c \ gen_new_super.c gen_noencoder.c gen_nullspace.c gen_udlinks.c \ getname.c gheap.c hyperslab.c istore.c lheap.c links.c mf.c \ @@ -714,7 +718,7 @@ TEST_PROG = testhdf5 lheap ohdr stab gheap cache cache_api \ # so do not appear in this list. BUILD_ALL_PROGS = gen_bad_ohdr gen_bogus gen_cross gen_deflate gen_filters gen_new_array \ gen_new_fill gen_new_group gen_new_mtime gen_new_super gen_noencoder \ - gen_nullspace gen_udlinks space_overflow + gen_nullspace gen_udlinks space_overflow gen_idx # The libh5test library provides common support code for the tests. @@ -956,6 +960,9 @@ gen_deflate$(EXEEXT): $(gen_deflate_OBJECTS) $(gen_deflate_DEPENDENCIES) gen_filters$(EXEEXT): $(gen_filters_OBJECTS) $(gen_filters_DEPENDENCIES) @rm -f gen_filters$(EXEEXT) $(LINK) $(gen_filters_OBJECTS) $(gen_filters_LDADD) $(LIBS) +gen_idx$(EXEEXT): $(gen_idx_OBJECTS) $(gen_idx_DEPENDENCIES) + @rm -f gen_idx$(EXEEXT) + $(LINK) $(gen_idx_OBJECTS) $(gen_idx_LDADD) $(LIBS) gen_new_array$(EXEEXT): $(gen_new_array_OBJECTS) $(gen_new_array_DEPENDENCIES) @rm -f gen_new_array$(EXEEXT) $(LINK) $(gen_new_array_OBJECTS) $(gen_new_array_LDADD) $(LIBS) @@ -1093,6 +1100,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_cross.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_deflate.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_filters.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_idx.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_new_array.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_new_fill.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/gen_new_group.Po@am__quote@ diff --git a/test/btree_idx_1_6.h5 b/test/btree_idx_1_6.h5 Binary files differnew file mode 100644 index 0000000..b1ac4da --- /dev/null +++ b/test/btree_idx_1_6.h5 diff --git a/test/btree_idx_1_8.h5 b/test/btree_idx_1_8.h5 Binary files differnew file mode 100644 index 0000000..36662b2 --- /dev/null +++ b/test/btree_idx_1_8.h5 diff --git a/test/dsets.c b/test/dsets.c index c45b2c1..2066860 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -208,6 +208,15 @@ const char *FILENAME[] = { int points[DSET_DIM1][DSET_DIM2], check[DSET_DIM1][DSET_DIM2]; double points_dbl[DSET_DIM1][DSET_DIM2], check_dbl[DSET_DIM1][DSET_DIM2]; +/* Declarations for test_idx_compatible() */ +#define DSET "dset" +#define DSET_FILTER "dset_filter" +const char *OLD_FILENAME[] = { /* Files created under 1.6 branch and 1.8 branch */ + "btree_idx_1_6.h5", /* 1.6 HDF5 file */ + "btree_idx_1_8.h5" /* 1.8 HDF5 file */ +}; + + /* Local prototypes for filter functions */ static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); @@ -7058,7 +7067,7 @@ test_chunk_fast(hid_t fapl) /* Get the chunk index type */ if(H5D_layout_idx_type_test(dsid, &idx_type) < 0) FAIL_STACK_ERROR - /* Chunk index tyepe expected depends on whether we are using the latest version of the format */ + /* Chunk index type expected depends on whether we are using the latest version of the format */ if(low == H5F_LIBVER_LATEST) { /* Verify index type */ if(idx_type != H5D_CHUNK_IDX_EARRAY) FAIL_PUTS_ERROR("should be using extensible array as index"); @@ -7777,7 +7786,7 @@ error: * Return: Success: 0 * Failure: -1 * - * Programmer: + * Programmer: Vailin Choi; 2009 * *------------------------------------------------------------------------- */ @@ -7812,10 +7821,8 @@ test_fixed_array(hid_t fapl) int rbuf_big[POINTS_BIG]; /* read buffer for big dataset */ hsize_t chunk_dim2[2] = {4, 3}; /* Chunk dimensions */ - int chunks[12][6]; /* # of chunks for dataset dimensions */ int chunks_big[125][20]; /* # of chunks for big dataset dimensions */ - int chunk_row; /* chunk row index */ int chunk_col; /* chunk column index */ @@ -7830,6 +7837,9 @@ test_fixed_array(hid_t fapl) hbool_t compress; /* Whether chunks should be compressed */ #endif /* H5_HAVE_FILTER_DEFLATE */ + h5_stat_size_t empty_size; /* Size of an empty file */ + h5_stat_size_t file_size; /* Size of each file created */ + size_t i, j; /* local index variables */ herr_t ret; /* Generic return value */ @@ -7840,6 +7850,16 @@ test_fixed_array(hid_t fapl) /* Check if we are using the latest version of the format */ if(H5Pget_libver_bounds(fapl, &low, &high) < 0) FAIL_STACK_ERROR + /* Create and close the file to get the file size */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + STACK_ERROR + if(H5Fclose(fid) < 0) + STACK_ERROR + + /* Get the size of the empty file */ + if((empty_size = h5_get_file_size(filename, fapl)) < 0) + TEST_ERROR + #ifdef H5_HAVE_FILTER_DEFLATE /* Loop over compressing chunks */ for(compress = FALSE; compress <= TRUE; compress++) { @@ -7993,10 +8013,6 @@ test_fixed_array(hid_t fapl) if(H5Sclose(sid_big) < 0) FAIL_STACK_ERROR if(H5Sclose(big_mem_id) < 0) FAIL_STACK_ERROR if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR - if(H5Fclose(fid) < 0) FAIL_STACK_ERROR - - /* Re-open file */ - if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR /* Open the first dataset */ if((dsid = H5Dopen2(fid, DSET_FIXED_NAME, H5P_DEFAULT)) < 0) TEST_ERROR; @@ -8060,6 +8076,15 @@ test_fixed_array(hid_t fapl) /* Close everything */ if(H5Fclose(fid) < 0) FAIL_STACK_ERROR + + /* Get the size of the file */ + if((file_size = h5_get_file_size(filename, fapl)) < 0) + TEST_ERROR + + /* Verify the file is correct size */ + if(file_size != empty_size) + TEST_ERROR + } /* end for */ #ifdef H5_HAVE_FILTER_DEFLATE } /* end for */ @@ -8079,6 +8104,85 @@ error: return -1; } /* end test_fixed_array() */ +/*------------------------------------------------------------------------- + * + * test_idx_compatible(): + * Verify that the library can read datasets created with + * 1.6/1.8 library that use the B-tree indexing method. + * + * Programmer: Vailin Choi; 26th August, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_idx_compatible(void) +{ + hid_t fid; /* File id */ + hid_t did; /* Dataset id */ + ssize_t nread; /* Number of bytes read in */ + char *srcdir = HDgetenv("srcdir"); /* where the src code is located */ + char filename[FILENAME_BUF_SIZE] = ""; /* old test file name */ + unsigned j; /* Local index variable */ + H5D_chunk_index_t idx_type; /* Chunked dataset index type */ + herr_t ret; /* Return value */ + + /* Output message about test being performed */ + TESTING("compatibility for 1.6/1.8 datasets that use B-tree indexing"); + + for(j = 0; j < NELMTS(OLD_FILENAME); j++) { + + /* Generate correct name for test file by prepending the source path */ + if(srcdir && ((HDstrlen(srcdir) + HDstrlen(OLD_FILENAME[j]) + 1) < sizeof(filename))) { + HDstrcpy(filename, srcdir); + HDstrcat(filename, "/"); + } + HDstrcat(filename, OLD_FILENAME[j]); + + /* Open the file */ + if((fid = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Should be able to read the dataset w/o filter created under 1.8/1.6 */ + if((did = H5Dopen2(fid, DSET, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Get the chunk index type */ + if(H5D_layout_idx_type_test(did, &idx_type) < 0) FAIL_STACK_ERROR + + /* Verify index type */ + if(idx_type != H5D_CHUNK_IDX_BTREE) + FAIL_PUTS_ERROR("should be using v1 B-tree as index"); + + if(H5Dclose(did) < 0) FAIL_STACK_ERROR + + /* Should be able to read the dataset w/ filter created under 1.8/1.6 */ + if((did = H5Dopen2(fid, DSET_FILTER, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Get the chunk index type */ + if(H5D_layout_idx_type_test(did, &idx_type) < 0) FAIL_STACK_ERROR + + /* Verify index type */ + if(idx_type != H5D_CHUNK_IDX_BTREE) + FAIL_PUTS_ERROR("should be using v1 B-tree as index"); + + if(H5Dclose(did) < 0) FAIL_STACK_ERROR + + /* Close the file */ + if(H5Fclose(fid) < 0) FAIL_STACK_ERROR + } + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(did); + H5Fclose(fid); + } H5E_END_TRY; + return -1; +} /* test_idx_compatible */ + /*------------------------------------------------------------------------- * Function: main @@ -8206,6 +8310,7 @@ main(void) nerrors += (test_reopen_chunk_fast(my_fapl) < 0 ? 1 : 0); nerrors += (test_chunk_expand(my_fapl) < 0 ? 1 : 0); nerrors += (test_fixed_array(my_fapl) < 0 ? 1 : 0); + nerrors += (test_idx_compatible() < 0 ? 1 : 0); if(H5Fclose(file) < 0) goto error; diff --git a/test/gen_idx.c b/test/gen_idx.c new file mode 100644 index 0000000..8c24198 --- /dev/null +++ b/test/gen_idx.c @@ -0,0 +1,126 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: This program is run to generate an HDF5 data file with datasets + * that use Fixed Array indexing method. + * + * To test compatibility, compile and run this program + * which will generate a file called "fixed_idx.h5". + * Move it to the test directory in the HDF5 v1.6/1.8 source tree. + * The test: test_idx_compatible() in dsets.c will read it. + */ +#include <assert.h> +#include "hdf5.h" + +const char *FILENAME[1] = { + "fixed_idx.h5" /* file with datasets that use Fixed Array indexing method */ +}; + +#define DSET "dset" +#define DSET_FILTER "dset_filter" + +/* + * Function: gen_idx_file + * + * Purpose: Create a file with datasets that use Fixed Array indexing: + * one dataset: fixed dimension, chunked layout, w/o filters + * one dataset: fixed dimension, chunked layout, w/ filters + * + */ +static void gen_idx_file(void) +{ + hid_t fapl; /* file access property id */ + hid_t fid; /* file id */ + hid_t sid; /* space id */ + hid_t dcpl; /* dataset creation property id */ + hid_t did, did2; /* dataset id */ + hsize_t dims[1] = {10}; /* dataset dimension */ + hsize_t c_dims[1] = {2}; /* chunk dimension */ + herr_t status; /* return status */ + int i; /* local index variable */ + int buf[10]; /* data buffer */ + + + /* Get a copy of the file aaccess property */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + assert(fapl >= 0); + + /* Set the "use the latest format" bounds for creating objects in the file */ + status = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + assert(status >= 0); + + /* Create dataset */ + fid = H5Fcreate(FILENAME[0], H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + assert(fid >= 0); + + /* Create data */ + for(i = 0; i < 10; i++) + buf[i] = i; + + /* Set chunk */ + dcpl = H5Pcreate(H5P_DATASET_CREATE); + assert(dcpl >= 0); + status = H5Pset_chunk(dcpl, 1, c_dims); + assert(status >= 0); + + sid = H5Screate_simple(1, dims, NULL); + assert(sid >= 0); + + /* Create a 1D dataset */ + did = H5Dcreate2(fid, DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); + assert(did >= 0); + + /* Write to the dataset */ + status = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); + assert(status >= 0); + +#if defined (H5_HAVE_FILTER_DEFLATE) + /* set deflate data */ + status = H5Pset_deflate(dcpl, 9); + assert(status >= 0); + + /* Create and write the dataset */ + did2 = H5Dcreate2(fid, DSET_FILTER, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); + assert(did2 >= 0); + + status = H5Dwrite(did2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); + assert(status >= 0); + + /* Close the dataset */ + status = H5Dclose(did2); + assert(status >= 0); +#endif + + /* closing */ + status = H5Dclose(did); + assert(status >= 0); + status = H5Sclose(sid); + assert(status >= 0); + status = H5Pclose(dcpl); + assert(status >= 0); + status = H5Pclose(fapl); + assert(status >= 0); + status = H5Fclose(fid); + assert(status >= 0); +} /* gen_idx_file() */ + +int main(void) +{ + gen_idx_file(); + + return 0; +} + diff --git a/test/objcopy.c b/test/objcopy.c index 4bd4662..29c98b5 100755 --- a/test/objcopy.c +++ b/test/objcopy.c @@ -39,7 +39,13 @@ #define H5P_TESTING #include "H5Ppkg.h" /* Property Lists */ -#include "H5Dprivate.h" /* Datasets (for EFL property name) */ +/* + * This file needs to access private information from the H5D package. + * This file also needs to access the dataset testing code. + */ +#define H5D_PACKAGE +#define H5D_TESTING +#include "H5Dpkg.h" /* Datasets */ const char *FILENAME[] = { @@ -136,6 +142,8 @@ static int compare_datasets(hid_t did, hid_t did2, hid_t pid, const void *wbuf); static int compare_groups(hid_t gid, hid_t gid2, hid_t pid, int depth, unsigned copy_flags); +static int +compare_idx_type(hid_t fapl, hid_t did, H5D_chunk_index_t new_type, H5D_chunk_index_t old_type); /*------------------------------------------------------------------------- @@ -1380,6 +1388,47 @@ error: /*------------------------------------------------------------------------- + * Function: compare_idx_type + * + * Purpose: If using new format, the index array type should be NEW_TYPE + * If not, the index array type should be OLD_TYPE + * + * Return: TRUE if the index type retrieved for the dataset DID is + * as expected + * FALSE if not + * + * Programmer: Vailin Choi; August 2009 + * + *------------------------------------------------------------------------- + */ +static int +compare_idx_type(hid_t fapl, hid_t did, H5D_chunk_index_t new_type, H5D_chunk_index_t old_type) +{ + H5D_chunk_index_t idx_type; /* Dataset chunk index type */ + H5F_libver_t low; /* File format low bound */ + + /* Get the chunk index type */ + if(H5D_layout_idx_type_test(did, &idx_type) < 0) + FAIL_STACK_ERROR + + /* Check if we are using the latest version of the format */ + if(H5Pget_libver_bounds(fapl, &low, NULL) < 0) + FAIL_STACK_ERROR + + /* Verify index type */ + if(low == H5F_LIBVER_LATEST) { + if(idx_type != new_type) + TEST_ERROR + } else if(idx_type != old_type) + TEST_ERROR + + return TRUE; +error: + return FALSE; +} /* compare_idx_type() */ + + +/*------------------------------------------------------------------------- * Function: test_copy_named_datatype * * Purpose: Create name datatype in SRC file and copy it to DST file @@ -2261,6 +2310,10 @@ test_copy_dataset_chunked(hid_t fcpl_src, hid_t fcpl_dst, hid_t fapl) /* open the 1-D destination dataset */ if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED, H5P_DEFAULT)) < 0) TEST_ERROR + /* Check if the array index type is correct */ + if(compare_idx_type(fapl, did2, H5D_CHUNK_IDX_EARRAY, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + /* Check if the datasets are equal */ if(compare_datasets(did, did2, H5P_DEFAULT, buf1d) != TRUE) TEST_ERROR @@ -2276,6 +2329,10 @@ test_copy_dataset_chunked(hid_t fcpl_src, hid_t fcpl_dst, hid_t fapl) /* open the destination dataset */ if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED2, H5P_DEFAULT)) < 0) TEST_ERROR + /* Check if the array index type is correct */ + if(compare_idx_type(fapl, did2, H5D_CHUNK_IDX_FARRAY, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + /* Check if the datasets are equal */ if(compare_datasets(did, did2, H5P_DEFAULT, buf2d) != TRUE) TEST_ERROR @@ -2423,6 +2480,10 @@ test_copy_dataset_chunked_empty(hid_t fcpl_src, hid_t fcpl_dst, hid_t fapl) /* open the destination dataset */ if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED, H5P_DEFAULT)) < 0) TEST_ERROR + /* Check if the array index type is correct */ + if(compare_idx_type(fapl, did2, H5D_CHUNK_IDX_EARRAY, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + /* Check if the datasets are equal */ if(compare_datasets(did, did2, H5P_DEFAULT, NULL) != TRUE) TEST_ERROR @@ -2438,6 +2499,10 @@ test_copy_dataset_chunked_empty(hid_t fcpl_src, hid_t fcpl_dst, hid_t fapl) /* open the destination dataset */ if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED2, H5P_DEFAULT)) < 0) TEST_ERROR + /* Check if the array index type is correct */ + if(compare_idx_type(fapl, did2, H5D_CHUNK_IDX_FARRAY, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + /* Check if the datasets are equal */ if(compare_datasets(did, did2, H5P_DEFAULT, NULL) != TRUE) TEST_ERROR @@ -2767,6 +2832,9 @@ test_copy_dataset_compressed(hid_t fcpl_src, hid_t fcpl_dst, hid_t fapl) /* open the destination dataset */ if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED, H5P_DEFAULT)) < 0) TEST_ERROR + if(compare_idx_type(fapl, did2, H5D_CHUNK_IDX_FARRAY, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + /* Check if the datasets are equal */ if(compare_datasets(did, did2, H5P_DEFAULT, NULL) != TRUE) TEST_ERROR diff --git a/tools/h5copy/h5copygentest.c b/tools/h5copy/h5copygentest.c index 7842b38..cd4c234 100644 --- a/tools/h5copy/h5copygentest.c +++ b/tools/h5copy/h5copygentest.c @@ -20,6 +20,7 @@ #include "hdf5.h" #define FILENAME "h5copytst.h5" +#define FILENAME_NEW "h5copytst_new.h5" #define DATASET_SIMPLE "simple" #define DATASET_CHUNK "chunk" #define DATASET_COMPACT "compact" @@ -30,6 +31,8 @@ #define GROUP_EMPTY "grp_empty" #define GROUP_DATASETS "grp_dsets" #define GROUP_NESTED "grp_nested" +#define TRUE 1 +#define FALSE 0 /*------------------------------------------------------------------------- @@ -394,15 +397,29 @@ static void gent_nested_group(hid_t loc_id) int main(void) { - hid_t fid; - - /* Create source file */ - fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); - gent_datasets(fid); - gent_empty_group(fid); - gent_nested_datasets(fid); - gent_nested_group(fid); - H5Fclose(fid); + hid_t fid; /* File id */ + hid_t fapl_new; /* File access property id */ + hbool_t new_format; /* New format or old format */ + + fapl_new = H5Pcreate(H5P_FILE_ACCESS); + H5Pset_libver_bounds(fapl_new, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + + /* Test with old & new format groups */ + for(new_format = FALSE; new_format <= TRUE; new_format++) { + + /* Set the FAPL for the type of format */ + if(new_format) + fid = H5Fcreate(FILENAME_NEW, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_new); + else + fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + + gent_datasets(fid); + gent_empty_group(fid); + gent_nested_datasets(fid); + gent_nested_group(fid); + H5Fclose(fid); + } + H5Pclose(fapl_new); return 0; } diff --git a/tools/h5copy/testfiles/h5copytst.h5 b/tools/h5copy/testfiles/h5copytst.h5 Binary files differindex f407f82..0f10410 100644 --- a/tools/h5copy/testfiles/h5copytst.h5 +++ b/tools/h5copy/testfiles/h5copytst.h5 diff --git a/tools/h5copy/testfiles/h5copytst_new.h5 b/tools/h5copy/testfiles/h5copytst_new.h5 Binary files differnew file mode 100644 index 0000000..57e1805 --- /dev/null +++ b/tools/h5copy/testfiles/h5copytst_new.h5 diff --git a/tools/h5copy/testfiles/h5copytst_new.out.ls b/tools/h5copy/testfiles/h5copytst_new.out.ls new file mode 100644 index 0000000..9df6b2e --- /dev/null +++ b/tools/h5copy/testfiles/h5copytst_new.out.ls @@ -0,0 +1,502 @@ +############################# +Expected output for 'h5ls ../testfiles/h5copytst_new.out.h5' +############################# +Opened "../testfiles/h5copytst_new.out.h5" with sec2 driver. +/ Group + Location: 1:96 + Links: 1 +/A Group + Location: 1:65602 + Links: 1 +/A/B1 Group + Location: 1:66306 + Links: 1 +/A/B1/simple Dataset {6/6} + Location: 1:65509 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/A/B2 Group + Location: 1:69807 + Links: 1 +/A/B2/simple2 Dataset {6/6} + Location: 1:69714 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/C Group + Location: 1:72980 + Links: 1 +/C/D Group + Location: 1:73684 + Links: 1 +/C/D/simple Dataset {6/6} + Location: 1:72887 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/E Group + Location: 1:76217 + Links: 1 +/E/F Group + Location: 1:76921 + Links: 1 +/E/F/grp_dsets Group + Location: 1:75044 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX +/E/F/grp_dsets/chunk Dataset {6/6} + Location: 1:76014 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/E/F/grp_dsets/compact Dataset {6/6} + Location: 1:75367 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/E/F/grp_dsets/compound Dataset {2/2} + Location: 1:75470 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: struct { + "str1" +0 20-byte null-terminated ASCII string + "str2" +20 20-byte null-terminated ASCII string + } 40 bytes +/E/F/grp_dsets/compressed Dataset {6/6} + Location: 1:75683 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Filter-0: deflate-1 OPT {1} + Type: 32-bit little-endian integer +/E/F/grp_dsets/named_vl Dataset {2/2} + Location: 1:75853 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: shared-1:75793 variable length of + 32-bit little-endian integer +/E/F/grp_dsets/nested_vl Dataset {2/2} + Location: 1:76108 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: variable length of + variable length of + 32-bit little-endian integer +/E/F/grp_dsets/simple Dataset {6/6} + Location: 1:75274 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/E/F/grp_dsets/vl Type + Location: 1:75793 + Links: 2 + Modified: XXXX-XX-XX XX:XX:XX XXX + Type: shared-1:75793 variable length of + 32-bit little-endian integer +/G Group + Location: 1:85688 + Links: 1 +/G/H Group + Location: 1:86392 + Links: 1 +/G/H/grp_nested Group + Location: 1:84436 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX +/G/H/grp_nested/grp_dsets Group + Location: 1:84515 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX +/G/H/grp_nested/grp_dsets/chunk Dataset {6/6} + Location: 1:85485 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/G/H/grp_nested/grp_dsets/compact Dataset {6/6} + Location: 1:84838 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/G/H/grp_nested/grp_dsets/compound Dataset {2/2} + Location: 1:84941 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: struct { + "str1" +0 20-byte null-terminated ASCII string + "str2" +20 20-byte null-terminated ASCII string + } 40 bytes +/G/H/grp_nested/grp_dsets/compressed Dataset {6/6} + Location: 1:85154 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Filter-0: deflate-1 OPT {1} + Type: 32-bit little-endian integer +/G/H/grp_nested/grp_dsets/named_vl Dataset {2/2} + Location: 1:85324 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: shared-1:85264 variable length of + 32-bit little-endian integer +/G/H/grp_nested/grp_dsets/nested_vl Dataset {2/2} + Location: 1:85579 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: variable length of + variable length of + 32-bit little-endian integer +/G/H/grp_nested/grp_dsets/simple Dataset {6/6} + Location: 1:84745 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/G/H/grp_nested/grp_dsets/vl Type + Location: 1:85264 + Links: 2 + Modified: XXXX-XX-XX XX:XX:XX XXX + Type: shared-1:85264 variable length of + 32-bit little-endian integer +/chunk Dataset {6/6} + Location: 1:2238 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/compact Dataset {6/6} + Location: 1:4240 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/compound Dataset {2/2} + Location: 1:6391 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: struct { + "str1" +0 20-byte null-terminated ASCII string + "str2" +20 20-byte null-terminated ASCII string + } 40 bytes +/compressed Dataset {6/6} + Location: 1:6604 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Filter-0: deflate-1 OPT {1} + Type: 32-bit little-endian integer +/grp_dsets Group + Location: 1:27748 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX +/grp_dsets/chunk Dataset {6/6} + Location: 1:28718 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_dsets/compact Dataset {6/6} + Location: 1:28071 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_dsets/compound Dataset {2/2} + Location: 1:28174 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: struct { + "str1" +0 20-byte null-terminated ASCII string + "str2" +20 20-byte null-terminated ASCII string + } 40 bytes +/grp_dsets/compressed Dataset {6/6} + Location: 1:28387 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Filter-0: deflate-1 OPT {1} + Type: 32-bit little-endian integer +/grp_dsets/named_vl Dataset {2/2} + Location: 1:28557 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: shared-1:28497 variable length of + 32-bit little-endian integer +/grp_dsets/nested_vl Dataset {2/2} + Location: 1:28812 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: variable length of + variable length of + 32-bit little-endian integer +/grp_dsets/simple Dataset {6/6} + Location: 1:27978 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_dsets/simple_group Dataset {6/6} + Location: 1:46180 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_dsets/vl Type + Location: 1:28497 + Links: 2 + Modified: XXXX-XX-XX XX:XX:XX XXX + Type: shared-1:28497 variable length of + 32-bit little-endian integer +/grp_empty Group + Location: 1:27693 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX +/grp_nested Group + Location: 1:35940 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX +/grp_nested/grp_dsets Group + Location: 1:36019 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX +/grp_nested/grp_dsets/chunk Dataset {6/6} + Location: 1:36989 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_nested/grp_dsets/compact Dataset {6/6} + Location: 1:36342 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_nested/grp_dsets/compound Dataset {2/2} + Location: 1:36445 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: struct { + "str1" +0 20-byte null-terminated ASCII string + "str2" +20 20-byte null-terminated ASCII string + } 40 bytes +/grp_nested/grp_dsets/compressed Dataset {6/6} + Location: 1:36658 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Filter-0: deflate-1 OPT {1} + Type: 32-bit little-endian integer +/grp_nested/grp_dsets/named_vl Dataset {2/2} + Location: 1:36828 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: shared-1:36768 variable length of + 32-bit little-endian integer +/grp_nested/grp_dsets/nested_vl Dataset {2/2} + Location: 1:37083 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: variable length of + variable length of + 32-bit little-endian integer +/grp_nested/grp_dsets/simple Dataset {6/6} + Location: 1:36249 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_nested/grp_dsets/vl Type + Location: 1:36768 + Links: 2 + Modified: XXXX-XX-XX XX:XX:XX XXX + Type: shared-1:36768 variable length of + 32-bit little-endian integer +/grp_rename Group + Location: 1:47077 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX +/grp_rename/chunk Dataset {6/6} + Location: 1:48047 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_rename/compact Dataset {6/6} + Location: 1:47400 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_rename/compound Dataset {2/2} + Location: 1:47503 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: struct { + "str1" +0 20-byte null-terminated ASCII string + "str2" +20 20-byte null-terminated ASCII string + } 40 bytes +/grp_rename/compressed Dataset {6/6} + Location: 1:47716 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Filter-0: deflate-1 OPT {1} + Type: 32-bit little-endian integer +/grp_rename/grp_dsets Group + Location: 1:55269 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX +/grp_rename/grp_dsets/chunk Dataset {6/6} + Location: 1:56239 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_rename/grp_dsets/compact Dataset {6/6} + Location: 1:55592 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_rename/grp_dsets/compound Dataset {2/2} + Location: 1:55695 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: struct { + "str1" +0 20-byte null-terminated ASCII string + "str2" +20 20-byte null-terminated ASCII string + } 40 bytes +/grp_rename/grp_dsets/compressed Dataset {6/6} + Location: 1:55908 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Chunks: {2} 8 bytes + Storage: <details removed for portability> + Filter-0: deflate-1 OPT {1} + Type: 32-bit little-endian integer +/grp_rename/grp_dsets/named_vl Dataset {2/2} + Location: 1:56078 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: shared-1:56018 variable length of + 32-bit little-endian integer +/grp_rename/grp_dsets/nested_vl Dataset {2/2} + Location: 1:56333 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: variable length of + variable length of + 32-bit little-endian integer +/grp_rename/grp_dsets/simple Dataset {6/6} + Location: 1:55499 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_rename/grp_dsets/vl Type + Location: 1:56018 + Links: 2 + Modified: XXXX-XX-XX XX:XX:XX XXX + Type: shared-1:56018 variable length of + 32-bit little-endian integer +/grp_rename/named_vl Dataset {2/2} + Location: 1:47886 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: shared-1:47826 variable length of + 32-bit little-endian integer +/grp_rename/nested_vl Dataset {2/2} + Location: 1:48141 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: variable length of + variable length of + 32-bit little-endian integer +/grp_rename/simple Dataset {6/6} + Location: 1:47307 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/grp_rename/vl Type + Location: 1:47826 + Links: 2 + Modified: XXXX-XX-XX XX:XX:XX XXX + Type: shared-1:47826 variable length of + 32-bit little-endian integer +/named_vl Dataset {2/2} + Location: 1:8657 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: shared-1:8606 variable length of + 32-bit little-endian integer +/nested_vl Dataset {2/2} + Location: 1:22942 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: variable length of + variable length of + 32-bit little-endian integer +/rename Dataset {2/2} + Location: 1:27240 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: struct { + "str1" +0 20-byte null-terminated ASCII string + "str2" +20 20-byte null-terminated ASCII string + } 40 bytes +/simple Dataset {6/6} + Location: 1:800 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer +/simple_top Dataset {6/6} + Location: 1:25099 + Links: 1 + Modified: XXXX-XX-XX XX:XX:XX XXX + Storage: <details removed for portability> + Type: 32-bit little-endian integer diff --git a/tools/h5copy/testh5copy.sh b/tools/h5copy/testh5copy.sh index c1d0309..046440d 100644 --- a/tools/h5copy/testh5copy.sh +++ b/tools/h5copy/testh5copy.sh @@ -30,7 +30,6 @@ H5LS_BIN=`pwd`/../h5ls/$H5LS # The path of the h5ls tool binary nerrors=0 verbose=yes -SRCFILE=h5copytst.h5 INDIR=$srcdir/testfiles OUTDIR=../testfiles CMP='cmp -s' @@ -56,7 +55,7 @@ TESTING() VERIFY() { SPACES=" " - echo "Verifying h5diff output $* $SPACES" | cut -c1-70 | tr -d '\012' + echo "Verifying h5diff output `basename $1` `basename $2` `basename $3` `basename $4` $SPACES" | cut -c1-70 | tr -d '\012' } # Print a line-line message left justified in a field of 70 characters @@ -92,7 +91,7 @@ TOOLTEST() runh5diff=no fi - TESTING $H5COPY $@ + TESTING $H5COPY $5 $6 $7 $8 $9 ( echo "#############################" echo " output for '$H5COPY $@'" @@ -134,7 +133,7 @@ TOOLTEST_FAIL() runh5diff=no fi - TESTING $H5COPY $@ + TESTING $H5COPY $5 $6 $7 $8 $9 ( echo "#############################" echo " output for '$H5COPY $@'" @@ -248,12 +247,13 @@ H5LSTEST() # <none> COPYOBJECTS() { - TESTFILE="$INDIR/$SRCFILE" - FILEOUT="$OUTDIR/`basename $SRCFILE .h5`.out.h5" + TESTFILE="$INDIR/$1" + FILEOUT="$OUTDIR/`basename $1 .h5`.out.h5" # Remove any output file left over from previous test run rm -f $FILEOUT + echo "Testing from `basename $TESTFILE` to `basename $FILEOUT` for the following tests:" echo "Test copying various forms of datasets" TOOLTEST -i $TESTFILE -o $FILEOUT -v -s simple -d simple TOOLTEST -i $TESTFILE -o $FILEOUT -v -s chunk -d chunk @@ -304,8 +304,12 @@ COPYOBJECTS() ### T H E T E S T S ### ############################################################################## -COPYOBJECTS +COPYOBJECTS h5copytst.h5 +# Add newline for nicer formatting +echo " " + +COPYOBJECTS h5copytst_new.h5 if test $nerrors -eq 0 ; then echo "All h5copy tests passed." diff --git a/tools/h5diff/h5diffgentest.c b/tools/h5diff/h5diffgentest.c index cae79b6..1bfea17 100644 --- a/tools/h5diff/h5diffgentest.c +++ b/tools/h5diff/h5diffgentest.c @@ -43,6 +43,8 @@ #define FILE9 "h5diff_hyper1.h5" #define FILE10 "h5diff_hyper2.h5" #define FILE11 "h5diff_empty.h5" +#define FILE12 "h5diff_dset_idx1.h5" +#define FILE13 "h5diff_dset_idx2.h5" #define UIMAX 4294967295u /*Maximum value for a variable of type unsigned int */ #define STR_SIZE 3 #define GBLL ((unsigned long long) 1024 * 1024 *1024 ) @@ -87,6 +89,7 @@ static void gen_datareg(hid_t fid,int make_diffs); /* utilities */ static int write_attr(hid_t loc_id,int rank,hsize_t *dims,const char *name,hid_t tid,void *buf); static int write_dset(hid_t loc_id,int rank,hsize_t *dims,const char *name,hid_t tid,void *buf); +static int gen_dataset_idx(const char *file, int format); /*------------------------------------------------------------------------- @@ -118,8 +121,17 @@ int main(void) /* generate 2 files, the second call creates a similar file with differences */ test_hyperslab(FILE9,0); test_hyperslab(FILE10,1); - return 0; + /* + * Generate 2 files: FILE12 with old format; FILE13 with new format + * Create 2 datasets in each file: + * One dataset: chunked layout, w/o filters, fixed dimension + * One dataset: chunked layout, w/ filters, fixed dimension + */ + gen_dataset_idx(FILE12, 0); + gen_dataset_idx(FILE13, 1); + + return 0; } /*------------------------------------------------------------------------- @@ -893,6 +905,95 @@ int test_datasets(const char *file, } /*------------------------------------------------------------------------- +* Function: gen_dataset_idx +* +* Purpose: Create a file with either the new or old format +* Create two datasets in the file: +* one dataset: fixed dimension, chunked layout, w/o filters +* one dataset: fixed dimension, chunked layout, w/ filters +* +*------------------------------------------------------------------------- +*/ +static +int gen_dataset_idx(const char *file, int format) +{ + hid_t fid; /* file id */ + hid_t did, did2; /* dataset id */ + hid_t sid; /* space id */ + hid_t fapl; /* file access property id */ + hid_t dcpl; /* dataset creation property id */ + hsize_t dims[1] = {10}; /* dataset dimension */ + hsize_t c_dims[1] = {2}; /* chunk dimension */ + herr_t status; /* return status */ + int buf[10]; /* data buffer */ + int i; /* local index variable */ + + /* Get a copy of the file aaccess property */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + + /* Set the "use the latest format" bounds for creating objects in the file */ + if(format) { + status = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + assert(status >= 0); + } + + /* Create a file */ + if((fid = H5Fcreate(file, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + return -1; + + /* Create data */ + for(i = 0; i < 10; i++) + buf[i] = i; + + /* Set chunk */ + dcpl = H5Pcreate(H5P_DATASET_CREATE); + status = H5Pset_chunk(dcpl, 1, c_dims); + assert(status >= 0); + + /* Create a 1D dataset */ + sid = H5Screate_simple(1, dims, NULL); + did = H5Dcreate2(fid, "dset", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); + + /* Write to the dataset */ + status = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); + assert(status >= 0); + +#if defined (H5_HAVE_FILTER_DEFLATE) + /* set deflate data */ + status = H5Pset_deflate(dcpl, 9); + assert(status >= 0); + + /* Create and write the dataset */ + did2 = H5Dcreate2(fid, "dset_filter", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); + status = H5Dwrite(did2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); + assert(status >= 0); + + /* Close the dataset */ + status = H5Dclose(did2); + assert(status >= 0); + +#endif + + /* closing: dataspace, dataset, file */ + status = H5Sclose(sid); + assert(status >= 0); + + status = H5Dclose(did); + assert(status >= 0); + + status = H5Fclose(fid); + assert(status >= 0); + + status = H5Pclose(dcpl); + assert(status >= 0); + + status = H5Pclose(fapl); + assert(status >= 0); + + return status; +} + +/*------------------------------------------------------------------------- * Function: write_attr_in * * Purpose: write attributes in LOC_ID (dataset, group, named datatype) diff --git a/tools/h5diff/testfiles/h5diff_dset_idx1.h5 b/tools/h5diff/testfiles/h5diff_dset_idx1.h5 Binary files differnew file mode 100644 index 0000000..684925d --- /dev/null +++ b/tools/h5diff/testfiles/h5diff_dset_idx1.h5 diff --git a/tools/h5diff/testfiles/h5diff_dset_idx2.h5 b/tools/h5diff/testfiles/h5diff_dset_idx2.h5 Binary files differnew file mode 100644 index 0000000..ea721d1 --- /dev/null +++ b/tools/h5diff/testfiles/h5diff_dset_idx2.h5 diff --git a/tools/h5diff/testfiles/h5diff_idx.txt b/tools/h5diff/testfiles/h5diff_idx.txt new file mode 100644 index 0000000..3b37720 --- /dev/null +++ b/tools/h5diff/testfiles/h5diff_idx.txt @@ -0,0 +1,19 @@ + +file1 file2 +--------------------------------------- + x x / + x x /dset + x x /dset_filter + +group : </> and </> +0 differences found +dataset: </dset> and </dset> +Not comparable: </dset> or </dset> is an empty dataset +0 differences found +dataset: </dset_filter> and </dset_filter> +Not comparable: </dset_filter> or </dset_filter> is an empty dataset +0 differences found +-------------------------------- +Some objects are not comparable +-------------------------------- +Use -c for a list of objects. diff --git a/tools/h5diff/testh5diff.sh b/tools/h5diff/testh5diff.sh index b8e743d..22c6f3f 100755 --- a/tools/h5diff/testh5diff.sh +++ b/tools/h5diff/testh5diff.sh @@ -42,6 +42,8 @@ SRCFILE8=h5diff_dset2.h5 SRCFILE9=h5diff_hyper1.h5 SRCFILE10=h5diff_hyper2.h5 SRCFILE11=h5diff_empty.h5 +SRCFILE12=h5diff_dset_idx1.h5 +SRCFILE13=h5diff_dset_idx2.h5 FILE1="$INDIR/$SRCFILE1" FILE2="$INDIR/$SRCFILE2" @@ -54,6 +56,8 @@ FILE8="$INDIR/$SRCFILE8" FILE9="$INDIR/$SRCFILE9" FILE10="$INDIR/$SRCFILE10" FILE11="$INDIR/$SRCFILE11" +FILE12="$INDIR/$SRCFILE12" +FILE13="$INDIR/$SRCFILE13" H5DIFF=h5diff # The tool name @@ -615,6 +619,13 @@ TESTING $H5DIFF -c $SRCFILE2 $SRCFILE2 g2/dset8 g2/dset9 TOOLTEST h5diff_207.txt -c $FILE2 $FILE2 g2/dset8 g2/dset9 +# ############################################################################## +# 12. The comparision for the two datasets between the 2 files should be the same +# SRCFILE12: B-tree indexing is used for the datasets +# SRCFILE13: Fixed Array indexing is used for the datasets +# ############################################################################## +TESTING $H5DIFF -v $SRCFILE12 $SRCFILE13 +TOOLTEST h5diff_idx.txt -v $FILE12 $FILE13 # ############################################################################## diff --git a/tools/h5dump/h5dumpgentest.c b/tools/h5dump/h5dumpgentest.c index b8ed41d..dcb491e 100644 --- a/tools/h5dump/h5dumpgentest.c +++ b/tools/h5dump/h5dumpgentest.c @@ -93,6 +93,7 @@ #define FILE63 "textlinkfar.h5" #define FILE64 "tarray8.h5" #define FILE65 "tattrreg.h5" +#define FILE66 "tdset_idx.h5" @@ -104,6 +105,8 @@ /* utility functions */ static int +make_dset(hid_t loc_id, const char *name, hid_t sid, hid_t tid, hid_t dcpl, void *buf); +static int write_attr(hid_t loc_id, int rank, hsize_t *dims, const char *attr_name, hid_t tid, void *buf); static int @@ -244,6 +247,16 @@ typedef struct s1_t { #define F64_ARRAY_BUF_LEN (4*1024) #define F64_DIM1 (F64_ARRAY_BUF_LEN / sizeof(int) + 1) +/* Declarations for gent_dataset_idx() for "FILE66" */ +#define DSET_FIXED "dset_fixed" +#define DSET_FIXED_FILTER "dset_filter" +#define DSET_BTREE "dset_btree" +#define DIM200 200 +#define DIM100 100 +#define DIM20 20 +#define DIM10 10 +#define CHUNK 5 + static void gent_group(void) { @@ -377,6 +390,84 @@ gent_dataset2(void) H5Fclose(fid); } +/* + * Create a file with new format. + * Create one dataset with (set_chunk, fixed dimension) + * so that Fixed Array indexing will be used. + * Create one dataset with (set_chunk, fixed dimension, filter) + * so that Fixed Array indexing will be used. + * Create one dataset with (set_chunk, non-fixed dimension) + * so that B-tree indexing will be used. + */ +static void +gent_dataset_idx(void) +{ + hid_t fid, space, dcpl, fapl; + hsize_t dims[2]; + hsize_t maxdims[2]; + int buf[20][10]; + int i, j, ret; + + /* Get a copy of the file aaccess property */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + + /* Set the "use the latest version of the format" bounds for creating objects in the file */ + ret = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + assert(ret >= 0); + + fid = H5Fcreate(FILE66, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + dcpl = H5Pcreate(H5P_DATASET_CREATE); + + dims[0] = CHUNK; + dims[1] = CHUNK; + + /* set chunk */ + ret = H5Pset_chunk(dcpl, RANK, dims); + assert(ret >= 0); + + /* dataset with fixed dimensions */ + dims[0] = DIM20; + dims[1] = DIM10; + space = H5Screate_simple(RANK, dims, NULL); + + for(i = 0; i < DIM20; i++) + for(j = 0; j < DIM10; j++) + buf[i][j] = j; + + ret = make_dset(fid, DSET_FIXED, space, H5T_NATIVE_INT, dcpl, buf); + assert(ret >= 0); + H5Sclose(space); + + /* dataset with non-fixed dimensions */ + maxdims[0] = DIM200; + maxdims[1] = DIM100; + space = H5Screate_simple(RANK, dims, maxdims); + + ret = make_dset(fid, DSET_BTREE, space, H5T_NATIVE_INT, dcpl, buf); + assert(ret >= 0); + H5Sclose(space); + +#if defined (H5_HAVE_FILTER_DEFLATE) + + /* dataset with fixed dimensions and filters */ + /* remove the filters from the dcpl */ + ret = H5Premove_filter(dcpl, H5Z_FILTER_ALL); + assert(ret >= 0); + + /* set deflate data */ + ret = H5Pset_deflate(dcpl, 9); + assert(ret >= 0); + + space = H5Screate_simple(RANK, dims, NULL); + ret = make_dset(fid, DSET_FIXED_FILTER, space, H5T_NATIVE_INT, dcpl, buf); + assert(ret >= 0); + + H5Sclose(space); +#endif + + H5Pclose(dcpl); + H5Fclose(fid); +} static void gent_attribute(void) @@ -6517,6 +6608,7 @@ int main(void) gent_fpformat(); gent_extlinks(); + gent_dataset_idx(); return 0; } diff --git a/tools/h5dump/testh5dump.sh.in b/tools/h5dump/testh5dump.sh.in index 3ffff50..6d2f273 100644 --- a/tools/h5dump/testh5dump.sh.in +++ b/tools/h5dump/testh5dump.sh.in @@ -515,8 +515,8 @@ TOOLTEST tfpformat.ddl -m %.7f tfpformat.h5 TOOLTEST textlinksrc.ddl textlinksrc.h5 TOOLTEST textlinkfar.ddl textlinkfar.h5 - - +# tests for Fixed Array chunk indices +TOOLTEST tdset_idx.ddl -p -H tdset_idx.h5 if test $nerrors -eq 0 ; then echo "All $DUMPER tests passed." diff --git a/tools/h5ls/testh5ls.sh.in b/tools/h5ls/testh5ls.sh.in index ddb09b5..0e6909a 100644 --- a/tools/h5ls/testh5ls.sh.in +++ b/tools/h5ls/testh5ls.sh.in @@ -190,6 +190,8 @@ else TOOLTEST tdataregbe.ls 0 -v tdatareg.h5 fi +# test for file with datasets that use Fixed Array chunk indices +TOOLTEST tdset_idx.ls 0 -w80 -d tdset_idx.h5 if test $nerrors -eq 0 ; then echo "All h5ls tests passed." diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index d8a503c..d0a7a7d 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -90,13 +90,13 @@ int h5repack(const char* infile, *------------------------------------------------------------------------- */ -int h5repack_init (pack_opt_t *options, - int verbose) +int h5repack_init (pack_opt_t *options, int verbose, int latest) { int k, n; memset(options,0,sizeof(pack_opt_t)); options->min_comp = 1024; - options->verbose = verbose; + options->verbose = verbose; + options->latest = latest ; for ( n = 0; n < H5_REPACK_MAX_NFILTERS; n++) { diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index cb3d75a..feabeaa 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -128,7 +128,7 @@ extern "C" { int h5repack (const char* infile, const char* outfile, pack_opt_t *options); int h5repack_addfilter (const char* str, pack_opt_t *options); int h5repack_addlayout (const char* str, pack_opt_t *options); -int h5repack_init (pack_opt_t *options, int verbose); +int h5repack_init (pack_opt_t *options, int verbose, int latest); int h5repack_end (pack_opt_t *options); int h5repack_verify (const char *fname,pack_opt_t *options); int h5repack_cmp_pl (const char *fname1, diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index d9ec7ad..d2aed2b 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -104,7 +104,7 @@ int main(int argc, const char **argv) int ret=-1; /* initialize options */ - h5repack_init (&options,0); + h5repack_init (&options,0, 0); parse_command_line(argc, argv, &options); diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c index d28750c..59652e0 100644 --- a/tools/h5repack/h5repacktst.c +++ b/tools/h5repack/h5repacktst.c @@ -184,7 +184,7 @@ int main (void) */ TESTING(" copy of datasets (fill values)"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME0,FNAME0OUT,&pack_options) < 0) GOERROR; @@ -204,7 +204,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of datasets (all datatypes)"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME1,FNAME1OUT,&pack_options) < 0) GOERROR; @@ -224,7 +224,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of datasets (attributes)"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME2,FNAME2OUT,&pack_options) < 0) GOERROR; @@ -243,7 +243,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of datasets (hardlinks)"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME3,FNAME3OUT,&pack_options) < 0) GOERROR; @@ -263,7 +263,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" copy of allocation early file"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME5,FNAME5OUT,&pack_options) < 0) GOERROR; @@ -284,7 +284,7 @@ int main (void) * deflate *------------------------------------------------------------------------- */ - TESTING(" adding deflate filter"); + TESTING(" adding deflate filter (old_format)"); #ifdef H5_HAVE_FILTER_DEFLATE @@ -293,7 +293,34 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) + GOERROR; + if (h5repack_addfilter("dset1:GZIP=9",&pack_options) < 0) + GOERROR; + if (h5repack_addlayout("dset1:CHUNK=20x10",&pack_options) < 0) + GOERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options) < 0) + GOERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) + GOERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + GOERROR; + if (h5repack_end (&pack_options) < 0) + GOERROR; + PASSED(); +#else + SKIPPED(); +#endif + + TESTING(" adding deflate filter (new format)"); +#ifdef H5_HAVE_FILTER_DEFLATE + /*------------------------------------------------------------------------- + * test an individual object option + * For new format, "dset1" should be using Fixed Array indexing + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0, 1) < 0) GOERROR; if (h5repack_addfilter("dset1:GZIP=9",&pack_options) < 0) GOERROR; @@ -321,7 +348,7 @@ int main (void) #ifdef H5_HAVE_FILTER_DEFLATE - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("GZIP=1",&pack_options) < 0) GOERROR; @@ -359,7 +386,7 @@ int main (void) */ if (szip_can_encode) { - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset2:SZIP=8,EC",&pack_options) < 0) GOERROR; @@ -391,7 +418,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("SZIP=8,NN",&pack_options) < 0) GOERROR; @@ -422,7 +449,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset1:SHUF",&pack_options) < 0) GOERROR; @@ -451,7 +478,7 @@ int main (void) #ifdef H5_HAVE_FILTER_SHUFFLE - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("SHUF",&pack_options) < 0) GOERROR; @@ -480,7 +507,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset1:FLET",&pack_options) < 0) GOERROR; @@ -510,7 +537,7 @@ int main (void) #ifdef H5_HAVE_FILTER_FLETCHER32 - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("FLET",&pack_options) < 0) GOERROR; @@ -538,7 +565,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("dset1:CHUNK 20x10",&pack_options) < 0) GOERROR; @@ -577,14 +604,36 @@ int main (void) PASSED(); - TESTING(" adding layout chunked"); + TESTING(" adding layout chunked (old format)"); + + /*------------------------------------------------------------------------- + * test an individual object option + *------------------------------------------------------------------------- + */ + + if (h5repack_init (&pack_options, 0, 0) < 0) + GOERROR; + if (h5repack_addlayout("dset1:CHUNK=20x10",&pack_options) < 0) + GOERROR; + if (h5repack(FNAME4,FNAME4OUT,&pack_options) < 0) + GOERROR; + if (h5diff(FNAME4,FNAME4OUT,NULL,NULL,&diff_options) >0) + GOERROR; + if (h5repack_verify(FNAME4OUT,&pack_options)<=0) + GOERROR; + if (h5repack_end (&pack_options) < 0) + GOERROR; + PASSED(); + + TESTING(" adding layout chunked (new format)"); /*------------------------------------------------------------------------- * test an individual object option + * For new format, "dset1" should be using Fixed Array indexing *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 1) < 0) GOERROR; if (h5repack_addlayout("dset1:CHUNK=20x10",&pack_options) < 0) GOERROR; @@ -604,7 +653,7 @@ int main (void) */ TESTING(" adding layout chunked to all"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("CHUNK=20x10",&pack_options) < 0) GOERROR; @@ -625,7 +674,7 @@ int main (void) * test an individual object option *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("dset1:CONTI",&pack_options) < 0) GOERROR; @@ -646,7 +695,7 @@ int main (void) * test all objects option *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("CONTI",&pack_options) < 0) GOERROR; @@ -663,7 +712,7 @@ int main (void) * do the same test for a file with filters (chunked) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("CONTI",&pack_options) < 0) GOERROR; @@ -685,7 +734,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("dset1:COMPA",&pack_options) < 0) GOERROR; @@ -706,7 +755,7 @@ int main (void) *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("COMPA",&pack_options) < 0) GOERROR; @@ -728,7 +777,7 @@ int main (void) * layout compact to contiguous conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("dset_compact:CONTI",&pack_options) < 0) GOERROR; @@ -748,7 +797,7 @@ int main (void) * layout compact to chunk conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("dset_compact:CHUNK=2x5",&pack_options) < 0) GOERROR; @@ -768,7 +817,7 @@ int main (void) * layout compact to compact conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("dset_compact:COMPA",&pack_options) < 0) GOERROR; @@ -787,7 +836,7 @@ int main (void) * layout contiguous to compact conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("dset_contiguous:COMPA",&pack_options) < 0) GOERROR; @@ -806,7 +855,7 @@ int main (void) * layout contiguous to chunk conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("dset_contiguous:CHUNK=3x6",&pack_options) < 0) GOERROR; @@ -826,7 +875,7 @@ int main (void) * layout contiguous to contiguous conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("dset_contiguous:CONTI",&pack_options) < 0) GOERROR; @@ -845,7 +894,7 @@ int main (void) * layout chunked to compact conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("dset_chunk:COMPA",&pack_options) < 0) GOERROR; @@ -865,7 +914,7 @@ int main (void) * layout chunked to contiguous conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("dset_chunk:CONTI",&pack_options) < 0) GOERROR; @@ -884,7 +933,7 @@ int main (void) * layout chunked to chunked conversion *------------------------------------------------------------------------- */ - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addlayout("dset_chunk:CHUNK=18x13",&pack_options) < 0) GOERROR; @@ -914,7 +963,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME7,FNAME7OUT,&pack_options) < 0) GOERROR; @@ -939,7 +988,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset_szip:NONE",&pack_options) < 0) GOERROR; @@ -964,7 +1013,7 @@ int main (void) TESTING(" copy of deflate filter"); #ifdef H5_HAVE_FILTER_DEFLATE - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME8,FNAME8OUT,&pack_options) < 0) GOERROR; @@ -984,7 +1033,7 @@ int main (void) TESTING(" removing deflate filter"); #ifdef H5_HAVE_FILTER_DEFLATE - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset_deflate:NONE",&pack_options) < 0) GOERROR; @@ -1007,7 +1056,7 @@ int main (void) TESTING(" copy of shuffle filter"); #ifdef H5_HAVE_FILTER_SHUFFLE - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME9,FNAME9OUT,&pack_options) < 0) GOERROR; @@ -1026,7 +1075,7 @@ int main (void) TESTING(" removing shuffle filter"); #ifdef H5_HAVE_FILTER_SHUFFLE - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset_shuffle:NONE",&pack_options) < 0) GOERROR; @@ -1047,7 +1096,7 @@ int main (void) TESTING(" copy of fletcher filter"); #ifdef H5_HAVE_FILTER_FLETCHER32 - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME10,FNAME10OUT,&pack_options) < 0) GOERROR; @@ -1066,7 +1115,7 @@ int main (void) TESTING(" removing fletcher filter"); #ifdef H5_HAVE_FILTER_FLETCHER32 - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset_fletcher32:NONE",&pack_options) < 0) GOERROR; @@ -1088,7 +1137,7 @@ int main (void) TESTING(" copy of nbit filter"); #ifdef H5_HAVE_FILTER_NBIT - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME12,FNAME12OUT,&pack_options) < 0) GOERROR; @@ -1107,7 +1156,7 @@ int main (void) TESTING(" removing nbit filter"); #ifdef H5_HAVE_FILTER_NBIT - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset_nbit:NONE",&pack_options) < 0) GOERROR; @@ -1129,7 +1178,7 @@ int main (void) TESTING(" adding nbit filter"); #ifdef H5_HAVE_FILTER_NBIT - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset_int31:NBIT",&pack_options) < 0) GOERROR; @@ -1151,7 +1200,7 @@ int main (void) TESTING(" copy of scaleoffset filter"); #ifdef H5_HAVE_FILTER_SCALEOFFSET - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME13,FNAME13OUT,&pack_options) < 0) GOERROR; @@ -1170,7 +1219,7 @@ int main (void) TESTING(" removing scaleoffset filter"); #ifdef H5_HAVE_FILTER_SCALEOFFSET - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset_scaleoffset:NONE",&pack_options) < 0) GOERROR; @@ -1192,7 +1241,7 @@ int main (void) TESTING(" adding scaleoffset filter"); #ifdef H5_HAVE_FILTER_SCALEOFFSET - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset_none:SOFF=31,IN",&pack_options) < 0) GOERROR; @@ -1230,7 +1279,7 @@ int main (void) && defined (H5_HAVE_FILTER_FLETCHER32) && defined (H5_HAVE_FILTER_SHUFFLE) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset_deflate:SZIP=8,NN",&pack_options) < 0) GOERROR; @@ -1258,7 +1307,7 @@ int main (void) && defined (H5_HAVE_FILTER_FLETCHER32) && defined (H5_HAVE_FILTER_SHUFFLE) if (szip_can_encode) { - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("dset_szip:GZIP=1",&pack_options) < 0) GOERROR; @@ -1290,7 +1339,7 @@ int main (void) #if defined (H5_HAVE_FILTER_SZIP) && defined (H5_HAVE_FILTER_DEFLATE) \ && defined (H5_HAVE_FILTER_FLETCHER32) && defined (H5_HAVE_FILTER_SHUFFLE) - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("NONE",&pack_options) < 0) GOERROR; @@ -1313,7 +1362,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" big file"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME14,FNAME14OUT,&pack_options) < 0) GOERROR; @@ -1330,7 +1379,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" external datasets"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack(FNAME15,FNAME15OUT,&pack_options) < 0) GOERROR; @@ -1347,7 +1396,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" file with userblock"); - if(h5repack_init(&pack_options, 0) < 0) + if(h5repack_init(&pack_options, 0, 0) < 0) GOERROR; if(h5repack(FNAME16, FNAME16OUT, &pack_options) < 0) GOERROR; @@ -1366,7 +1415,7 @@ int main (void) *------------------------------------------------------------------------- */ TESTING(" latest file format options"); - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; pack_options.latest=1; pack_options.grp_compact=10; @@ -1396,7 +1445,7 @@ int main (void) #if defined (H5_HAVE_FILTER_DEFLATE) && defined (H5_HAVE_FILTER_SHUFFLE) - if (h5repack_init (&pack_options, 0) < 0) + if (h5repack_init (&pack_options, 0, 0) < 0) GOERROR; if (h5repack_addfilter("GZIP=1",&pack_options) < 0) GOERROR; @@ -1425,7 +1474,7 @@ int main (void) #ifdef H5_HAVE_FILTER_DEFLATE - if(h5repack_init(&pack_options, 0) < 0) + if(h5repack_init(&pack_options, 0, 0) < 0) GOERROR; /* add the options for a user block size and user block filename */ @@ -1458,7 +1507,7 @@ int main (void) #ifdef H5_HAVE_FILTER_DEFLATE - if(h5repack_init(&pack_options, 0) < 0) + if(h5repack_init(&pack_options, 0, 0) < 0) GOERROR; /* add the options for aligment */ @@ -1513,7 +1562,7 @@ int main (void) */ TESTING(" file with committed datatypes"); - if(h5repack_init(&pack_options, 0) < 0) + if(h5repack_init(&pack_options, 0, 0) < 0) GOERROR; if(h5repack(FNAME17, FNAME17OUT, &pack_options) < 0) diff --git a/tools/h5stat/h5stat.c b/tools/h5stat/h5stat.c index 78fb4f5..657cf32 100644 --- a/tools/h5stat/h5stat.c +++ b/tools/h5stat/h5stat.c @@ -88,7 +88,7 @@ typedef struct iter_t { hsize_t SM_index_storage_size; /* index (btree & list) size for SOHM table (1.8) */ hsize_t SM_heap_storage_size; /* fractal heap size for SOHM table (1.8) */ hsize_t super_ext_size; /* superblock extension size */ - hsize_t datasets_btree_storage_size; /* btree size for chunked dataset */ + hsize_t datasets_index_storage_size; /* meta size for chunked dataset's indexing type */ unsigned long nexternal; /* Number of external files for a dataset */ int local; /* Flag to indicate iteration over the object*/ } iter_t; @@ -426,7 +426,7 @@ dataset_stats(iter_t *iter, const char *name, const H5O_info_t *oi) assert(did > 0); /* Update dataset metadata info */ - iter->datasets_btree_storage_size += oi->meta_size.obj.index_size; + iter->datasets_index_storage_size += oi->meta_size.obj.index_size; /* Update attribute metadata info */ ret = attribute_stats(iter, oi); @@ -860,7 +860,7 @@ print_file_metadata(const iter_t *iter) HDfprintf(stdout, "\t\tHeap: %Hu\n", iter->attrs_heap_storage_size); HDfprintf(stdout, "\tChunked datasets:\n"); - HDfprintf(stdout, "\t\tB-tree: %Hu\n", iter->datasets_btree_storage_size); + HDfprintf(stdout, "\t\tIndex: %Hu\n", iter->datasets_index_storage_size); HDfprintf(stdout, "\tShared Messages:\n"); HDfprintf(stdout, "\t\tHeader: %Hu\n", iter->SM_hdr_storage_size); diff --git a/tools/h5stat/h5stat_gentest.c b/tools/h5stat/h5stat_gentest.c index 3f933f2..2a7f210 100644 --- a/tools/h5stat/h5stat_gentest.c +++ b/tools/h5stat/h5stat_gentest.c @@ -33,6 +33,11 @@ #define NUM_GRPS 35000 #define NUM_ATTRS 100 +/* Declarations for gen_idx_file() */ +#define FILE_IDX "h5stat_idx.h5" +#define DSET "dset" +#define DSET_FILTER "dset_filter" + /* * Generate 1.8 HDF5 file * with NUM_GRPS groups @@ -81,12 +86,91 @@ static void gen_file(void) assert(ret >= 0); ret = H5Fclose(file); assert(ret >= 0); -} + +} /* gen_file() */ + +/* + * Function: gen_idx_file + * + * Purpose: Create a file with datasets that use Fixed Array indexing: + * one dataset: fixed dimension, chunked layout, w/o filters + * one dataset: fixed dimension, chunked layout, w/ filters + * + */ +static void gen_idx_file(void) +{ + hid_t fapl; /* file access property id */ + hid_t fid; /* file id */ + hid_t sid; /* space id */ + hid_t dcpl; /* dataset creation property id */ + hid_t did, did2; /* dataset id */ + hsize_t dims[1] = {10}; /* dataset dimension */ + hsize_t c_dims[1] = {2}; /* chunk dimension */ + herr_t status; /* return status */ + int i; /* local index variable */ + int buf[10]; /* data buffer */ + + + /* Get a copy of the file aaccess property */ + fapl = H5Pcreate(H5P_FILE_ACCESS); + + /* Set the "use the latest format" bounds for creating objects in the file */ + status = H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + assert(status >= 0); + + /* Create dataset */ + fid = H5Fcreate(FILE_IDX, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + + /* Create data */ + for(i = 0; i < 10; i++) + buf[i] = i; + + /* Set chunk */ + dcpl = H5Pcreate(H5P_DATASET_CREATE); + status = H5Pset_chunk(dcpl, 1, c_dims); + assert(status >= 0); + + /* Create a 1D dataset */ + sid = H5Screate_simple(1, dims, NULL); + did = H5Dcreate2(fid, DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); + + /* Write to the dataset */ + status = H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); + assert(status >= 0); + +#if defined (H5_HAVE_FILTER_DEFLATE) + /* set deflate data */ + status = H5Pset_deflate(dcpl, 9); + assert(status >= 0); + + /* Create and write the dataset */ + did2 = H5Dcreate2(fid, DSET_FILTER, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT); + status = H5Dwrite(did2, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf); + assert(status >= 0); + + /* Close the dataset */ + status = H5Dclose(did2); + assert(status >= 0); + +#endif + + /* closing: dataspace, dataset, file */ + status = H5Sclose(sid); + assert(status >= 0); + status = H5Dclose(did); + assert(status >= 0); + status = H5Fclose(fid); + assert(status >= 0); + +} /* gen_idx_file() */ int main(void) { gen_file(); + /* Generate an HDF file to test for datasets with Fixed Array indexing */ + gen_idx_file(); + return 0; } diff --git a/tools/h5stat/testfiles/h5stat_filters-F.ddl b/tools/h5stat/testfiles/h5stat_filters-F.ddl index 6bc6cc2..8f65c6d 100644 --- a/tools/h5stat/testfiles/h5stat_filters-F.ddl +++ b/tools/h5stat/testfiles/h5stat_filters-F.ddl @@ -14,7 +14,7 @@ Storage information: B-tree/List: 0 Heap: 0 Chunked datasets: - B-tree: 31392 + Index: 31392 Shared Messages: Header: 0 B-tree/List: 0 diff --git a/tools/h5stat/testfiles/h5stat_filters.ddl b/tools/h5stat/testfiles/h5stat_filters.ddl index 8243a23..d070dba 100644 --- a/tools/h5stat/testfiles/h5stat_filters.ddl +++ b/tools/h5stat/testfiles/h5stat_filters.ddl @@ -22,7 +22,7 @@ Storage information: B-tree/List: 0 Heap: 0 Chunked datasets: - B-tree: 31392 + Index: 31392 Shared Messages: Header: 0 B-tree/List: 0 diff --git a/tools/h5stat/testfiles/h5stat_idx.ddl b/tools/h5stat/testfiles/h5stat_idx.ddl new file mode 100644 index 0000000..f7e6031 --- /dev/null +++ b/tools/h5stat/testfiles/h5stat_idx.ddl @@ -0,0 +1,75 @@ +############################# +Expected output for 'h5stat h5stat_idx.h5' +############################# +Filename: h5stat_idx.h5 +File information + # of unique groups: 1 + # of unique datasets: 2 + # of unique named dataypes: 0 + # of unique links: 0 + # of unique other: 0 + Max. # of links to object: 1 + Max. # of objects in group: 2 +Object header size: (total/unused) + Groups: 147/47 + Datasets: 568/362 + Datatypes: 0/0 +Storage information: + Groups: + B-tree/List: 0 + Heap: 0 + Attributes: + B-tree/List: 0 + Heap: 0 + Chunked datasets: + Index: 202 + Shared Messages: + Header: 0 + B-tree/List: 0 + Heap: 0 + Superblock extension: 0 +Small groups: + # of groups of size 2: 1 + Total # of small groups: 1 +Group bins: + # of groups of size 1 - 9: 1 + Total # of groups: 1 +Dataset dimension information: + Max. rank of datasets: 1 + Dataset ranks: + # of dataset with rank 1: 2 +1-D Dataset information: + Max. dimension size of 1-D datasets: 10 + Small 1-D datasets: + Total small datasets: 0 + 1-D Dataset dimension bins: + # of datasets of size 10 - 99: 2 + Total # of datasets: 2 +Dataset storage information: + Total raw data size: 110 +Dataset layout information: + Dataset layout counts[COMPACT]: 0 + Dataset layout counts[CONTIG]: 0 + Dataset layout counts[CHUNKED]: 2 + Number of external files : 0 +Dataset filters information: + Number of datasets with: + NO filter: 1 + GZIP filter: 1 + SHUFFLE filter: 0 + FLETCHER32 filter: 0 + SZIP filter: 0 + NBIT filter: 0 + SCALEOFFSET filter: 0 + USER-DEFINED filter: 0 +Dataset datatype information: + # of unique datatypes used by datasets: 1 + Dataset datatype #0: + Count (total/named) = (2/0) + Size (desc./elmt) = (14/4) + Total dataset datatype count: 2 +Small # of attributes: + Total # of objects with small # of attributes: 0 +Attribute bins: + Total # of objects with attributes: 0 + Max. # of attributes to objects: 0 diff --git a/tools/h5stat/testfiles/h5stat_idx.h5 b/tools/h5stat/testfiles/h5stat_idx.h5 Binary files differnew file mode 100644 index 0000000..6a25272 --- /dev/null +++ b/tools/h5stat/testfiles/h5stat_idx.h5 diff --git a/tools/h5stat/testfiles/h5stat_newgrat.ddl b/tools/h5stat/testfiles/h5stat_newgrat.ddl index f87f297..ce93b9c 100644 --- a/tools/h5stat/testfiles/h5stat_newgrat.ddl +++ b/tools/h5stat/testfiles/h5stat_newgrat.ddl @@ -22,7 +22,7 @@ Storage information: B-tree/List: 2598 Heap: 4442 Chunked datasets: - B-tree: 0 + Index: 0 Shared Messages: Header: 0 B-tree/List: 0 diff --git a/tools/h5stat/testfiles/h5stat_tsohm.ddl b/tools/h5stat/testfiles/h5stat_tsohm.ddl index 577dfdd..25e1784 100644 --- a/tools/h5stat/testfiles/h5stat_tsohm.ddl +++ b/tools/h5stat/testfiles/h5stat_tsohm.ddl @@ -22,7 +22,7 @@ Storage information: B-tree/List: 0 Heap: 0 Chunked datasets: - B-tree: 0 + Index: 0 Shared Messages: Header: 38 B-tree/List: 550 diff --git a/tools/h5stat/testh5stat.sh.in b/tools/h5stat/testh5stat.sh.in index 66d05be..28837ec 100644 --- a/tools/h5stat/testh5stat.sh.in +++ b/tools/h5stat/testh5stat.sh.in @@ -124,6 +124,8 @@ TOOLTEST h5stat_filters-dT.ddl -dT h5stat_filters.h5 TOOLTEST h5stat_tsohm.ddl h5stat_tsohm.h5 # h5stat_newgrat.h5 is generated by h5stat_gentest.c TOOLTEST h5stat_newgrat.ddl h5stat_newgrat.h5 +# h5stat_idx.h5 is generated by h5stat_gentest.c +TOOLTEST h5stat_idx.ddl h5stat_idx.h5 echo diff --git a/tools/testfiles/tdset_idx.ddl b/tools/testfiles/tdset_idx.ddl new file mode 100644 index 0000000..65d9f44 --- /dev/null +++ b/tools/testfiles/tdset_idx.ddl @@ -0,0 +1,61 @@ +############################# +Expected output for 'h5dump -p -H tdset_idx.h5' +############################# +HDF5 "tdset_idx.h5" { +GROUP "/" { + DATASET "dset_btree" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 20, 10 ) / ( 200, 100 ) } + STORAGE_LAYOUT { + CHUNKED ( 5, 5 ) + SIZE 800 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE 0 + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset_filter" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 20, 10 ) / ( 20, 10 ) } + STORAGE_LAYOUT { + CHUNKED ( 5, 5 ) + SIZE 200 (4.000:1 COMPRESSION) + } + FILTERS { + COMPRESSION DEFLATE { LEVEL 9 } + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE 0 + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } + DATASET "dset_fixed" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 20, 10 ) / ( 20, 10 ) } + STORAGE_LAYOUT { + CHUNKED ( 5, 5 ) + SIZE 800 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE 0 + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_INCR + } + } +} +} diff --git a/tools/testfiles/tdset_idx.h5 b/tools/testfiles/tdset_idx.h5 Binary files differnew file mode 100644 index 0000000..920f59e --- /dev/null +++ b/tools/testfiles/tdset_idx.h5 diff --git a/tools/testfiles/tdset_idx.ls b/tools/testfiles/tdset_idx.ls new file mode 100644 index 0000000..4e6f3b5 --- /dev/null +++ b/tools/testfiles/tdset_idx.ls @@ -0,0 +1,39 @@ +############################# + output for 'h5ls -w80 -d tdset_idx.h5' +############################# +dset_btree Dataset {20/200, 10/100} + Data: + (0,0) 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, + (2,2) 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, + (4,4) 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, + (6,6) 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, + (8,8) 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + (11,0) 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, + (13,2) 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, + (15,4) 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, + (17,6) 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, + (19,8) 8, 9 +dset_filter Dataset {20, 10} + Data: + (0,0) 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, + (2,2) 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, + (4,4) 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, + (6,6) 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, + (8,8) 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + (11,0) 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, + (13,2) 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, + (15,4) 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, + (17,6) 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, + (19,8) 8, 9 +dset_fixed Dataset {20, 10} + Data: + (0,0) 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, + (2,2) 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, + (4,4) 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, + (6,6) 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, + (8,8) 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, + (11,0) 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, + (13,2) 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, + (15,4) 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, + (17,6) 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, 6, 7, + (19,8) 8, 9 |