diff options
author | Vailin Choi <vchoi@hdfgroup.org> | 2015-10-23 23:02:54 (GMT) |
---|---|---|
committer | Vailin Choi <vchoi@hdfgroup.org> | 2015-10-23 23:02:54 (GMT) |
commit | 66e1eeffa751dd58bcc00e6efc6f83cac9f203b4 (patch) | |
tree | 8c6383b6bb1d563a67c005f3222f21f47606612e | |
parent | bea706880a135b6d25aea6f211b7448b03326a6b (diff) | |
download | hdf5-66e1eeffa751dd58bcc00e6efc6f83cac9f203b4.zip hdf5-66e1eeffa751dd58bcc00e6efc6f83cac9f203b4.tar.gz hdf5-66e1eeffa751dd58bcc00e6efc6f83cac9f203b4.tar.bz2 |
[svn-r28206] 1) Implementation for Single Chunk indexing type.
2) Fix for changing h5format_convert testfiles for in-place build.
33 files changed, 1611 insertions, 348 deletions
@@ -497,6 +497,7 @@ ./src/H5Dpublic.h ./src/H5Dscatgath.c ./src/H5Dselect.c +./src/H5Dsingle.c ./src/H5Dtest.c ./src/H5Dvirtual.c ./src/H5E.c diff --git a/hl/test/test_ld.h5 b/hl/test/test_ld.h5 Binary files differindex bdf1182..bd5730c 100644 --- a/hl/test/test_ld.h5 +++ b/hl/test/test_ld.h5 diff --git a/src/H5Dbtree.c b/src/H5Dbtree.c index 0402cb8..20d1682 100644 --- a/src/H5Dbtree.c +++ b/src/H5Dbtree.c @@ -127,7 +127,7 @@ static herr_t H5D__btree_idx_init(const H5D_chk_idx_info_t *idx_info, static herr_t H5D__btree_idx_create(const H5D_chk_idx_info_t *idx_info); static hbool_t H5D__btree_idx_is_space_alloc(const H5O_storage_chunk_t *storage); static herr_t H5D__btree_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, - H5D_chunk_ud_t *udata); + H5D_chunk_ud_t *udata, H5D_t *dset); static herr_t H5D__btree_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); static int H5D__btree_idx_iterate(const H5D_chk_idx_info_t *idx_info, @@ -998,7 +998,7 @@ H5D__btree_idx_is_space_alloc(const H5O_storage_chunk_t *storage) *------------------------------------------------------------------------- */ static herr_t -H5D__btree_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) +H5D__btree_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, H5D_t H5_ATTR_UNUSED *dset) { herr_t ret_value = SUCCEED; /* Return value */ diff --git a/src/H5Dbtree2.c b/src/H5Dbtree2.c index 76c7e69..2f06e54 100644 --- a/src/H5Dbtree2.c +++ b/src/H5Dbtree2.c @@ -155,7 +155,7 @@ static herr_t H5D_bt2_idx_init(const H5D_chk_idx_info_t *idx_info, static herr_t H5D_bt2_idx_create(const H5D_chk_idx_info_t *idx_info); static hbool_t H5D_bt2_idx_is_space_alloc(const H5O_storage_chunk_t *storage); static herr_t H5D_bt2_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, - H5D_chunk_ud_t *udata); + H5D_chunk_ud_t *udata, H5D_t *dset); static herr_t H5D_bt2_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); static int H5D_bt2_idx_iterate(const H5D_chk_idx_info_t *idx_info, @@ -1047,7 +1047,7 @@ H5D_bt2_mod_filt_cb(void *_record, void *_op_data, hbool_t *changed) *------------------------------------------------------------------------- */ static herr_t -H5D_bt2_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) +H5D_bt2_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, H5D_t H5_ATTR_UNUSED *dset) { H5B2_t *bt2; /* v2 B-tree handle for indexing chunks */ H5D_bt2_find_ud_t bt2_udata; /* User data for v2 B-tree calls */ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 1bb4d67..caea0b9 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -72,6 +72,15 @@ #define H5D_CHUNK_GET_NODE_INFO(map, node) (map->use_single ? map->single_chunk_info : (H5D_chunk_info_t *)H5SL_item(node)) #define H5D_CHUNK_GET_NEXT_NODE(map, node) (map->use_single ? (H5SL_node_t *)NULL : H5SL_next(node)) +/* Sanity check on chunk index types: commonly used by a lot of routines in this file */ +#define H5D_CHUNK_STORAGE_INDEX_CHK(storage) \ + HDassert((H5D_CHUNK_IDX_EARRAY == storage->idx_type && H5D_COPS_EARRAY == storage->ops) || \ + (H5D_CHUNK_IDX_FARRAY == storage->idx_type && H5D_COPS_FARRAY == storage->ops) || \ + (H5D_CHUNK_IDX_BT2 == storage->idx_type && H5D_COPS_BT2 == storage->ops) || \ + (H5D_CHUNK_IDX_BTREE == storage->idx_type && H5D_COPS_BTREE == storage->ops) || \ + (H5D_CHUNK_IDX_SINGLE == storage->idx_type && H5D_COPS_SINGLE == storage->ops) || \ + (H5D_CHUNK_IDX_NONE == storage->idx_type && H5D_COPS_NONE == storage->ops)); + /* * Feature: If this constant is defined then every cache preemption and load * causes a character to be printed on the standard error stream: @@ -420,7 +429,7 @@ H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, uint32_t filters, /* Set the chunk's filter mask to the new settings */ udata.filter_mask = filters; - if((layout->storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0) + if((layout->storage.u.chunk.ops->insert_addr)(&idx_info, &udata, NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index") } /* end if */ @@ -613,6 +622,7 @@ H5D__chunk_init(H5F_t *f, hid_t dxpl_id, const H5D_t *dset, hid_t dapl_id) H5D_chk_idx_info_t idx_info; /* Chunked index info */ H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /* Convenience pointer to dataset's chunk cache */ H5P_genplist_t *dapl; /* Data access property list object pointer */ + H5O_storage_chunk_t *sc = &(dset->shared->layout.storage.u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -620,16 +630,7 @@ H5D__chunk_init(H5F_t *f, hid_t dxpl_id, const H5D_t *dset, hid_t dapl_id) /* Sanity check */ HDassert(f); HDassert(dset); - HDassert((H5D_CHUNK_IDX_EARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_EARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_FARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BT2 == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BTREE == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_NONE == dset->shared->layout.storage.u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); if(NULL == (dapl = (H5P_genplist_t *)H5I_object(dapl_id))) HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for fapl ID") @@ -713,22 +714,14 @@ done: hbool_t H5D__chunk_is_space_alloc(const H5O_storage_t *storage) { + const H5O_storage_chunk_t *sc = &(storage->u.chunk); hbool_t ret_value = FALSE; /* Return value */ FUNC_ENTER_PACKAGE_NOERR /* Sanity checks */ HDassert(storage); - HDassert((H5D_CHUNK_IDX_EARRAY == storage->u.chunk.idx_type && - H5D_COPS_EARRAY == storage->u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == storage->u.chunk.idx_type && - H5D_COPS_FARRAY == storage->u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == storage->u.chunk.idx_type && - H5D_COPS_BT2 == storage->u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == storage->u.chunk.idx_type && - H5D_COPS_BTREE == storage->u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == storage->u.chunk.idx_type && - H5D_COPS_NONE == storage->u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); /* Query index layer */ ret_value = (storage->u.chunk.ops->is_space_alloc)(&storage->u.chunk); @@ -2145,7 +2138,7 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, } /* end if */ else { if((need_insert || udata.need_modify) && io_info->dset->shared->layout.storage.u.chunk.ops->insert_addr) - if((io_info->dset->shared->layout.storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0) + if((io_info->dset->shared->layout.storage.u.chunk.ops->insert_addr)(&idx_info, &udata, NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index") } /* end else */ @@ -2277,22 +2270,14 @@ H5D__chunk_dest(H5D_t *dset, hid_t dxpl_id) H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /* Dataset's chunk cache */ H5D_rdcc_ent_t *ent = NULL, *next = NULL; /* Pointer to current & next cache entries */ int nerrors = 0; /* Accumulated count of errors */ + H5O_storage_chunk_t *sc = &(dset->shared->layout.storage.u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC_TAG(dxpl_id, dset->oloc.addr, FAIL) /* Sanity checks */ HDassert(dset); - HDassert((H5D_CHUNK_IDX_EARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_EARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_FARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BT2 == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BTREE == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_NONE == dset->shared->layout.storage.u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); /* Fill the DXPL cache values for later use */ if(H5D__get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) @@ -2353,16 +2338,7 @@ H5D_chunk_idx_reset(H5O_storage_chunk_t *storage, hbool_t reset_addr) /* Sanity checks */ HDassert(storage); HDassert(storage->ops); - HDassert((H5D_CHUNK_IDX_EARRAY == storage->idx_type && - H5D_COPS_EARRAY == storage->ops) || - (H5D_CHUNK_IDX_FARRAY == storage->idx_type && - H5D_COPS_FARRAY == storage->ops) || - (H5D_CHUNK_IDX_BT2 == storage->idx_type && - H5D_COPS_BT2 == storage->ops) || - (H5D_CHUNK_IDX_BTREE == storage->idx_type && - H5D_COPS_BTREE == storage->ops) || - (H5D_CHUNK_IDX_NONE == storage->idx_type && - H5D_COPS_NONE == storage->ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(storage); /* Reset index structures */ if((storage->ops->reset)(storage, reset_addr) < 0) @@ -2505,6 +2481,7 @@ herr_t H5D__chunk_create(const H5D_t *dset /*in,out*/, hid_t dxpl_id) { H5D_chk_idx_info_t idx_info; /* Chunked index info */ + H5O_storage_chunk_t *sc = &(dset->shared->layout.storage.u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -2513,16 +2490,8 @@ H5D__chunk_create(const H5D_t *dset /*in,out*/, hid_t dxpl_id) HDassert(dset); HDassert(H5D_CHUNKED == dset->shared->layout.type); HDassert(dset->shared->layout.u.chunk.ndims > 0 && dset->shared->layout.u.chunk.ndims <= H5O_LAYOUT_NDIMS); - HDassert((H5D_CHUNK_IDX_EARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_EARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_FARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BT2 == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BTREE == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_NONE == dset->shared->layout.storage.u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); + #ifndef NDEBUG { unsigned u; /* Local index variable */ @@ -2616,22 +2585,14 @@ H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id, const hsize_t *scaled, H5D_rdcc_ent_t *ent = NULL; /* Cache entry */ hbool_t found = FALSE; /* In cache? */ unsigned u; /* Counter */ + H5O_storage_chunk_t *sc = &(dset->shared->layout.storage.u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE HDassert(dset); HDassert(dset->shared->layout.u.chunk.ndims > 0); - HDassert((H5D_CHUNK_IDX_EARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_EARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_FARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BT2 == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BTREE == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_NONE == dset->shared->layout.storage.u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); HDassert(scaled); HDassert(udata); @@ -2679,7 +2640,7 @@ H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id, const hsize_t *scaled, idx_info.dxpl_id = dxpl_id; idx_info.pline = &dset->shared->dcpl_cache.pline; idx_info.layout = &dset->shared->layout.u.chunk; - idx_info.storage = &dset->shared->layout.storage.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; /* Go get the chunk information */ if((dset->shared->layout.storage.u.chunk.ops->get_addr)(&idx_info, udata) < 0) @@ -2716,22 +2677,14 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t { void *buf = NULL; /* Temporary buffer */ hbool_t point_of_no_return = FALSE; + H5O_storage_chunk_t *sc = &(dset->shared->layout.storage.u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC_TAG(dxpl_id, dset->oloc.addr, FAIL) HDassert(dset); HDassert(dset->shared); - HDassert((H5D_CHUNK_IDX_EARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_EARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_FARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BT2 == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BTREE == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_NONE == dset->shared->layout.storage.u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); HDassert(dxpl_cache); HDassert(ent); HDassert(!ent->locked); @@ -2845,7 +2798,7 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t /* Insert the chunk record into the index */ if((need_insert || udata.need_modify) && dset->shared->layout.storage.u.chunk.ops->insert_addr) - if((dset->shared->layout.storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0) + if((dset->shared->layout.storage.u.chunk.ops->insert_addr)(&idx_info, &udata, dset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index") /* Cache the chunk's info, in case it's accessed again shortly */ @@ -3644,22 +3597,14 @@ H5D__chunk_allocated(H5D_t *dset, hid_t dxpl_id, hsize_t *nbytes) H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ hsize_t chunk_bytes = 0; /* Number of bytes allocated for chunks */ + H5O_storage_chunk_t *sc = &(dset->shared->layout.storage.u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE HDassert(dset); HDassert(dset->shared); - HDassert((H5D_CHUNK_IDX_EARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_EARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_FARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BT2 == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BTREE == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_NONE == dset->shared->layout.storage.u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); /* Fill the DXPL cache values for later use */ if(H5D__get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) @@ -3743,6 +3688,7 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, hbool_t unfilt_edge_chunk_dim[H5O_LAYOUT_NDIMS]; /* Whether there are unfiltered edge chunks at the edge of each dimension */ hsize_t edge_chunk_scaled[H5O_LAYOUT_NDIMS]; /* Offset of the unfiltered edge chunks at the edge of each dimension */ unsigned nunfilt_edge_chunk_dims = 0; /* Number of dimensions on an edge */ + const H5O_storage_chunk_t *sc = &(layout->storage.u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_TAG(dxpl_id, dset->oloc.addr, FAIL) @@ -3750,16 +3696,7 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, /* Check args */ HDassert(dset && H5D_CHUNKED == layout->type); HDassert(layout->u.chunk.ndims > 0 && layout->u.chunk.ndims <= H5O_LAYOUT_NDIMS); - HDassert((H5D_CHUNK_IDX_EARRAY == layout->storage.u.chunk.idx_type && - H5D_COPS_EARRAY == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == layout->storage.u.chunk.idx_type && - H5D_COPS_FARRAY == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == layout->storage.u.chunk.idx_type && - H5D_COPS_BT2 == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == layout->storage.u.chunk.idx_type && - H5D_COPS_BTREE == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == layout->storage.u.chunk.idx_type && - H5D_COPS_NONE == layout->storage.u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); HDassert(TRUE == H5P_isa_class(dxpl_id, H5P_DATASET_XFER)); /* Retrieve the dataset dimensions */ @@ -4072,7 +4009,7 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, * serial operation. -QAK */ if((need_insert || udata.need_modify) && ops->insert_addr) - if((ops->insert_addr)(&idx_info, &udata) < 0) + if((ops->insert_addr)(&idx_info, &udata, dset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index") /* Increment indices and adjust the edge chunk state */ @@ -4192,6 +4129,7 @@ H5D__chunk_update_old_edge_chunks(H5D_t *dset, hid_t dxpl_id, hsize_t old_dim[]) H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ void *chunk; /* The file chunk */ hbool_t carry; /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */ + const H5O_storage_chunk_t *sc = &(layout->storage.u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -4199,16 +4137,7 @@ H5D__chunk_update_old_edge_chunks(H5D_t *dset, hid_t dxpl_id, hsize_t old_dim[]) /* Check args */ HDassert(dset && H5D_CHUNKED == layout->type); HDassert(layout->u.chunk.ndims > 0 && layout->u.chunk.ndims <= H5O_LAYOUT_NDIMS); - HDassert((H5D_CHUNK_IDX_EARRAY == layout->storage.u.chunk.idx_type && - H5D_COPS_EARRAY == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == layout->storage.u.chunk.idx_type && - H5D_COPS_FARRAY == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == layout->storage.u.chunk.idx_type && - H5D_COPS_BT2 == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == layout->storage.u.chunk.idx_type && - H5D_COPS_BTREE == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == layout->storage.u.chunk.idx_type && - H5D_COPS_NONE == layout->storage.u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); HDassert(TRUE == H5P_isa_class(dxpl_id, H5P_DATASET_XFER)); HDassert(pline->nused > 0); HDassert(layout->u.chunk.flags @@ -4750,6 +4679,7 @@ H5D__chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, const hsize_t *old_dim) hbool_t disable_edge_filters = FALSE; /* Whether to disable filters on partial edge chunks */ hbool_t new_unfilt_chunk = FALSE; /* Whether the chunk is newly unfiltered */ unsigned u; /* Local index variable */ + const H5O_storage_chunk_t *sc = &(layout->storage.u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -4757,16 +4687,7 @@ H5D__chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, const hsize_t *old_dim) /* Check args */ HDassert(dset && H5D_CHUNKED == layout->type); HDassert(layout->u.chunk.ndims > 0 && layout->u.chunk.ndims <= H5O_LAYOUT_NDIMS); - HDassert((H5D_CHUNK_IDX_EARRAY == layout->storage.u.chunk.idx_type && - H5D_COPS_EARRAY == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == layout->storage.u.chunk.idx_type && - H5D_COPS_FARRAY == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == layout->storage.u.chunk.idx_type && - H5D_COPS_BT2 == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == layout->storage.u.chunk.idx_type && - H5D_COPS_BTREE == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == layout->storage.u.chunk.idx_type && - H5D_COPS_NONE == layout->storage.u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); HDassert(dxpl_cache); /* Fill the DXPL cache values for later use */ @@ -5114,22 +5035,14 @@ H5D__chunk_addrmap(const H5D_io_info_t *io_info, haddr_t chunk_addr[]) H5D_chk_idx_info_t idx_info; /* Chunked index info */ const H5D_t *dset = io_info->dset; /* Local pointer to dataset info */ H5D_chunk_it_ud2_t udata; /* User data for iteration callback */ + H5O_storage_chunk_t *sc = &(dset->shared->layout.storage.u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE HDassert(dset); HDassert(dset->shared); - HDassert((H5D_CHUNK_IDX_EARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_EARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_FARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BT2 == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BTREE == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_NONE == dset->shared->layout.storage.u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); HDassert(chunk_addr); /* Set up user data for B-tree callback */ @@ -5177,6 +5090,7 @@ H5D__chunk_delete(H5F_t *f, hid_t dxpl_id, H5O_t *oh, H5O_storage_t *storage) H5O_pline_t pline; /* I/O pipeline message */ hbool_t pline_read = FALSE; /* Whether the I/O pipeline message was read from the file */ htri_t exists; /* Flag if header message of interest exists */ + H5O_storage_chunk_t *sc = &(storage->u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -5185,16 +5099,7 @@ H5D__chunk_delete(H5F_t *f, hid_t dxpl_id, H5O_t *oh, H5O_storage_t *storage) HDassert(f); HDassert(oh); HDassert(storage); - HDassert((H5D_CHUNK_IDX_EARRAY == storage->u.chunk.idx_type && - H5D_COPS_EARRAY == storage->u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == storage->u.chunk.idx_type && - H5D_COPS_FARRAY == storage->u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == storage->u.chunk.idx_type && - H5D_COPS_BT2 == storage->u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == storage->u.chunk.idx_type && - H5D_COPS_BTREE == storage->u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == storage->u.chunk.idx_type && - H5D_COPS_NONE == storage->u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); /* Check for I/O pipeline message */ if((exists = H5O_msg_exists_oh(oh, H5O_PLINE_ID)) < 0) @@ -5547,7 +5452,7 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) /* Insert chunk record into index */ if((need_insert || udata_dst.need_modify) && udata->idx_info_dst->storage->ops->insert_addr) - if((udata->idx_info_dst->storage->ops->insert_addr)(udata->idx_info_dst, &udata_dst) < 0) + if((udata->idx_info_dst->storage->ops->insert_addr)(udata->idx_info_dst, &udata_dst, NULL) < 0) HGOTO_ERROR_TAG(H5E_DATASET, H5E_CANTINSERT, H5_ITER_ERROR, "unable to insert chunk addr into index") /* Reset metadata tag in dxpl_id */ @@ -5609,29 +5514,11 @@ H5D__chunk_copy(H5F_t *f_src, H5O_storage_chunk_t *storage_src, /* Check args */ HDassert(f_src); HDassert(storage_src); - HDassert((H5D_CHUNK_IDX_EARRAY == storage_src->idx_type && - H5D_COPS_EARRAY == storage_src->ops) || - (H5D_CHUNK_IDX_FARRAY == storage_src->idx_type && - H5D_COPS_FARRAY == storage_src->ops) || - (H5D_CHUNK_IDX_BT2 == storage_src->idx_type && - H5D_COPS_BT2 == storage_src->ops) || - (H5D_CHUNK_IDX_BTREE == storage_src->idx_type && - H5D_COPS_BTREE == storage_src->ops) || - (H5D_CHUNK_IDX_NONE == storage_src->idx_type && - H5D_COPS_NONE == storage_src->ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(storage_src); HDassert(layout_src); HDassert(f_dst); HDassert(storage_dst); - HDassert((H5D_CHUNK_IDX_EARRAY == storage_dst->idx_type && - H5D_COPS_EARRAY == storage_dst->ops) || - (H5D_CHUNK_IDX_FARRAY == storage_dst->idx_type && - H5D_COPS_FARRAY == storage_dst->ops) || - (H5D_CHUNK_IDX_BT2 == storage_dst->idx_type && - H5D_COPS_BT2 == storage_dst->ops) || - (H5D_CHUNK_IDX_BTREE == storage_dst->idx_type && - H5D_COPS_BTREE == storage_dst->ops) || - (H5D_CHUNK_IDX_NONE == storage_dst->idx_type && - H5D_COPS_NONE == storage_dst->ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(storage_dst); HDassert(ds_extent_src); HDassert(dt_src); @@ -5859,6 +5746,7 @@ H5D__chunk_bh_info(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout, const H5O_pline_t *pline, hsize_t *index_size) { H5D_chk_idx_info_t idx_info; /* Chunked index info */ + H5O_storage_chunk_t *sc = &(layout->storage.u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE @@ -5866,16 +5754,7 @@ H5D__chunk_bh_info(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout, /* Check args */ HDassert(f); HDassert(layout); - HDassert((H5D_CHUNK_IDX_EARRAY == layout->storage.u.chunk.idx_type && - H5D_COPS_EARRAY == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == layout->storage.u.chunk.idx_type && - H5D_COPS_FARRAY == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == layout->storage.u.chunk.idx_type && - H5D_COPS_BT2 == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == layout->storage.u.chunk.idx_type && - H5D_COPS_BTREE == layout->storage.u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == layout->storage.u.chunk.idx_type && - H5D_COPS_NONE == layout->storage.u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); HDassert(pline); HDassert(index_size); @@ -5958,22 +5837,14 @@ H5D__chunk_dump_index_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) herr_t H5D__chunk_dump_index(H5D_t *dset, hid_t dxpl_id, FILE *stream) { + H5O_storage_chunk_t *sc = &(dset->shared->layout.storage.u.chunk); herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE /* Sanity check */ HDassert(dset); - HDassert((H5D_CHUNK_IDX_EARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_EARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_FARRAY == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_FARRAY == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BT2 == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BT2 == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_BTREE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_BTREE == dset->shared->layout.storage.u.chunk.ops) || - (H5D_CHUNK_IDX_NONE == dset->shared->layout.storage.u.chunk.idx_type && - H5D_COPS_NONE == dset->shared->layout.storage.u.chunk.ops)); + H5D_CHUNK_STORAGE_INDEX_CHK(sc); /* Only display info if stream is defined */ if(stream) { @@ -6243,7 +6114,6 @@ H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(idx_info->storage); - HDassert(H5F_addr_defined(idx_info->storage->idx_addr)); HDassert(new_chunk); HDassert(need_insert); HDassert(need_modify); @@ -6331,6 +6201,7 @@ H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old case H5D_CHUNK_IDX_FARRAY: case H5D_CHUNK_IDX_BT2: case H5D_CHUNK_IDX_BTREE: + case H5D_CHUNK_IDX_SINGLE: HDassert(new_chunk->length > 0); H5_CHECK_OVERFLOW(new_chunk->length, /*From: */uint32_t, /*To: */hsize_t); new_chunk->offset = H5MF_alloc(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, (hsize_t)new_chunk->length); @@ -6442,7 +6313,7 @@ H5D__chunk_format_convert_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) insert_udata.common.storage = new_idx_info->storage; /* Insert chunk into the v1 B-tree chunk index */ - if((new_idx_info->storage->ops->insert_addr)(new_idx_info, &insert_udata) < 0) + if((new_idx_info->storage->ops->insert_addr)(new_idx_info, &insert_udata, NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, H5_ITER_ERROR, "unable to insert chunk addr into index") done: diff --git a/src/H5Dearray.c b/src/H5Dearray.c index 6c77333..b42177e 100644 --- a/src/H5Dearray.c +++ b/src/H5Dearray.c @@ -126,7 +126,7 @@ static herr_t H5D_earray_idx_init(const H5D_chk_idx_info_t *idx_info, static herr_t H5D_earray_idx_create(const H5D_chk_idx_info_t *idx_info); static hbool_t H5D_earray_idx_is_space_alloc(const H5O_storage_chunk_t *storage); static herr_t H5D_earray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, - H5D_chunk_ud_t *udata); + H5D_chunk_ud_t *udata, H5D_t *dset); static herr_t H5D_earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); static herr_t H5D_earray_idx_resize(H5O_layout_chunk_t *layout); @@ -1078,7 +1078,7 @@ H5D_earray_idx_is_space_alloc(const H5O_storage_chunk_t *storage) *------------------------------------------------------------------------- */ static herr_t -H5D_earray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) +H5D_earray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, H5D_t H5_ATTR_UNUSED *dset) { H5EA_t *ea; /* Pointer to extensible array structure */ herr_t ret_value = SUCCEED; /* Return value */ diff --git a/src/H5Dfarray.c b/src/H5Dfarray.c index a031232..b3893c6 100644 --- a/src/H5Dfarray.c +++ b/src/H5Dfarray.c @@ -129,7 +129,7 @@ static herr_t H5D_farray_idx_init(const H5D_chk_idx_info_t *idx_info, static herr_t H5D_farray_idx_create(const H5D_chk_idx_info_t *idx_info); static hbool_t H5D_farray_idx_is_space_alloc(const H5O_storage_chunk_t *storage); static herr_t H5D_farray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, - H5D_chunk_ud_t *udata); + H5D_chunk_ud_t *udata, H5D_t *dset); static herr_t H5D_farray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); static int H5D_farray_idx_iterate(const H5D_chk_idx_info_t *idx_info, @@ -1033,7 +1033,7 @@ H5D_farray_idx_is_space_alloc(const H5O_storage_chunk_t *storage) *------------------------------------------------------------------------- */ static herr_t -H5D_farray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) +H5D_farray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, H5D_t H5_ATTR_UNUSED *dset) { H5FA_t *fa; /* Pointer to fixed array structure */ herr_t ret_value = SUCCEED; /* Return value */ diff --git a/src/H5Dint.c b/src/H5Dint.c index d1dd349..1bfe204 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -2808,8 +2808,18 @@ H5D__flush_real(H5D_t *dataset, hid_t dxpl_id) HDassert(dataset); HDassert(dataset->shared); + /* Avoid flushing the dataset (again) if it's closing */ if(!dataset->shared->closing) { + + /* Flush cached raw data for each kind of dataset layout */ + /* Need to flush first before the "layout" message because the single chunk address + is stored in the message */ + /* Continue flushing even if it fails */ + if(dataset->shared->layout.ops->flush && + (dataset->shared->layout.ops->flush)(dataset, dxpl_id) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "unable to flush raw data") + /* Check for metadata changes that will require updating the object's modification time */ if(dataset->shared->layout_dirty || dataset->shared->space_dirty) { unsigned update_flags = H5O_UPDATE_TIME; /* Modification time flag */ @@ -2842,10 +2852,6 @@ H5D__flush_real(H5D_t *dataset, hid_t dxpl_id) HDassert(update_flags == 0); } /* end if */ - /* Flush cached raw data for each kind of dataset layout */ - if(dataset->shared->layout.ops->flush && - (dataset->shared->layout.ops->flush)(dataset, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "unable to flush raw data") } /* end if */ done: diff --git a/src/H5Dlayout.c b/src/H5Dlayout.c index 05a7ae9..f4e5b15 100644 --- a/src/H5Dlayout.c +++ b/src/H5Dlayout.c @@ -105,6 +105,10 @@ H5D__layout_set_io_ops(const H5D_t *dataset) dataset->shared->layout.storage.u.chunk.ops = H5D_COPS_NONE; break; + case H5D_CHUNK_IDX_SINGLE: + dataset->shared->layout.storage.u.chunk.ops = H5D_COPS_SINGLE; + break; + case H5D_CHUNK_IDX_FARRAY: dataset->shared->layout.storage.u.chunk.ops = H5D_COPS_FARRAY; break; @@ -219,6 +223,14 @@ H5D__layout_meta_size(const H5F_t *f, const H5O_layout_t *layout, hbool_t includ switch(layout->u.chunk.idx_type) { case H5D_CHUNK_IDX_NONE: /* nothing */ + break; + + case H5D_CHUNK_IDX_SINGLE: + /* Possible filter information */ + if(layout->u.chunk.flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER) { + ret_value += H5F_SIZEOF_SIZE(f); /* Size of chunk (in file) */ + ret_value += 4; /* Filter mask for chunk */ + } break; case H5D_CHUNK_IDX_FARRAY: @@ -336,20 +348,25 @@ H5D__layout_set_latest_indexing(H5O_layout_t *layout, const H5S_t *space, HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "invalid dataspace rank") ndims = (unsigned)sndims; - /* Avoid scalar/null dataspace */ - if(ndims > 0) { - hsize_t max_dims[H5O_LAYOUT_NDIMS]; /* Maximum dimension sizes */ - unsigned unlim_count = 0; /* Count of unlimited max. dimensions */ - unsigned u; /* Local index variable */ - - /* Query the dataspace's dimensions */ - if(H5S_get_simple_extent_dims(space, NULL, max_dims) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataspace max. dimensions") - - /* Spin through the max. dimensions, looking for unlimited dimensions */ - for(u = 0; u < ndims; u++) - if(max_dims[u] == H5S_UNLIMITED) - unlim_count++; + /* Avoid scalar/null dataspace */ + if(ndims > 0) { + hsize_t max_dims[H5O_LAYOUT_NDIMS]; /* Maximum dimension sizes */ + hsize_t cur_dims[H5O_LAYOUT_NDIMS]; /* Current dimension sizes */ + unsigned unlim_count = 0; /* Count of unlimited max. dimensions */ + hbool_t single = TRUE; /* Fulfill single chunk indexing */ + unsigned u; /* Local index variable */ + + /* Query the dataspace's dimensions */ + if(H5S_get_simple_extent_dims(space, cur_dims, max_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataspace max. dimensions") + + /* Spin through the max. dimensions, looking for unlimited dimensions */ + for(u = 0; u < ndims; u++) { + if(max_dims[u] == H5S_UNLIMITED) + unlim_count++; + if(cur_dims[u] != max_dims[u] || cur_dims[u] != layout->u.chunk.dim[u]) + single = FALSE; + } /* Chunked datasets with unlimited dimension(s) */ if(unlim_count) { /* dataset with unlimited dimension(s) must be chunked */ @@ -382,16 +399,19 @@ H5D__layout_set_latest_indexing(H5O_layout_t *layout, const H5S_t *space, } /* end else */ } /* end if */ else { /* Chunked dataset with fixed dimensions */ - /* Check for correct condition for using "implicit" chunk index */ - if(!dcpl_cache->pline.nused && + /* Check for correct condition for using "single chunk" chunk index */ + if(single) { + layout->u.chunk.idx_type = H5D_CHUNK_IDX_SINGLE; + layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_SINGLE; + layout->storage.u.chunk.ops = H5D_COPS_SINGLE; + } else if(!dcpl_cache->pline.nused && dcpl_cache->fill.alloc_time == H5D_ALLOC_TIME_EARLY) { /* Set the chunk index type to Non Index */ layout->u.chunk.idx_type = H5D_CHUNK_IDX_NONE; layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_NONE; layout->storage.u.chunk.ops = H5D_COPS_NONE; - } /* end if */ - else { /* Used Fixed Array */ + } else { /* Used Fixed Array */ /* Set the chunk index type to Fixed Array */ layout->u.chunk.idx_type = H5D_CHUNK_IDX_FARRAY; layout->storage.u.chunk.idx_type = H5D_CHUNK_IDX_FARRAY; @@ -519,11 +539,10 @@ H5D__layout_oh_create(H5F_t *file, hid_t dxpl_id, H5O_t *oh, H5D_t *dset, } /* end if */ /* Create layout message */ - /* (Don't make layout message constant unless allocation time is early, since space may not be allocated) */ + /* (Don't make layout message constant unless allocation time is early and non-filtered, since space may not be allocated) */ /* (Note: this is relying on H5D_alloc_storage not calling H5O_msg_write during dataset creation) */ - if(H5O_msg_append_oh(file, dxpl_id, oh, H5O_LAYOUT_ID, ((fill_prop->alloc_time == H5D_ALLOC_TIME_EARLY && H5D_COMPACT != layout->type) ? H5O_MSG_FLAG_CONSTANT : 0), 0, layout) < 0) + if(H5O_msg_append_oh(file, dxpl_id, oh, H5O_LAYOUT_ID, ((fill_prop->alloc_time == H5D_ALLOC_TIME_EARLY && H5D_COMPACT != layout->type && !dset->shared->dcpl_cache.pline.nused) ? H5O_MSG_FLAG_CONSTANT : 0), 0, layout) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update layout") - done: /* Error cleanup */ if(ret_value < 0) diff --git a/src/H5Dnone.c b/src/H5Dnone.c index 3d761fb..d5e12ef 100644 --- a/src/H5Dnone.c +++ b/src/H5Dnone.c @@ -16,7 +16,7 @@ /* Programmer: Vailin Choi <vchoi@hdfgroup.org> * September 2010 * - * Purpose: Non Index chunked I/O functions. + * Purpose: Implicit (Non Index) chunked I/O functions. * This is used when the dataset is: * extendible but with fixed max. dims * with early allocation @@ -57,21 +57,21 @@ /********************/ /* Non Index chunking I/O ops */ -static herr_t H5D_none_create(const H5D_chk_idx_info_t *idx_info); -static hbool_t H5D_none_is_space_alloc(const H5O_storage_chunk_t *storage); -static herr_t H5D_none_get_addr(const H5D_chk_idx_info_t *idx_info, +static herr_t H5D_none_idx_create(const H5D_chk_idx_info_t *idx_info); +static hbool_t H5D_none_idx_is_space_alloc(const H5O_storage_chunk_t *storage); +static herr_t H5D_none_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); -static int H5D_none_iterate(const H5D_chk_idx_info_t *idx_info, +static int H5D_none_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb, void *chunk_udata); -static herr_t H5D_none_remove(const H5D_chk_idx_info_t *idx_info, +static herr_t H5D_none_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata); -static herr_t H5D_none_delete(const H5D_chk_idx_info_t *idx_info); -static herr_t H5D_none_copy_setup(const H5D_chk_idx_info_t *idx_info_src, +static herr_t H5D_none_idx_delete(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D_none_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk_idx_info_t *idx_info_dst); -static herr_t H5D_none_size(const H5D_chk_idx_info_t *idx_info, +static herr_t H5D_none_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *size); -static herr_t H5D_none_reset(H5O_storage_chunk_t *storage, hbool_t reset_addr); -static herr_t H5D_none_dump(const H5O_storage_chunk_t *storage, FILE *stream); +static herr_t H5D_none_idx_reset(H5O_storage_chunk_t *storage, hbool_t reset_addr); +static herr_t H5D_none_idx_dump(const H5O_storage_chunk_t *storage, FILE *stream); /*********************/ /* Package Variables */ @@ -79,22 +79,22 @@ static herr_t H5D_none_dump(const H5O_storage_chunk_t *storage, FILE *stream); /* Non Index chunk I/O ops */ const H5D_chunk_ops_t H5D_COPS_NONE[1] = {{ - TRUE, /* Non-indexed chunking don't current support SWMR access */ - NULL, /* init */ - H5D_none_create, /* create */ - H5D_none_is_space_alloc, /* is_space_alloc */ - NULL, /* insert */ - H5D_none_get_addr, /* get_addr */ - NULL, /* resize */ - H5D_none_iterate, /* iterate */ - H5D_none_remove, /* remove */ - H5D_none_delete, /* delete */ - H5D_none_copy_setup, /* copy_setup */ - NULL, /* copy_shutdown */ - H5D_none_size, /* size */ - H5D_none_reset, /* reset */ - H5D_none_dump, /* dump */ - NULL /* dest */ + FALSE, /* Non-indexed chunking don't current support SWMR access */ + NULL, /* init */ + H5D_none_idx_create, /* create */ + H5D_none_idx_is_space_alloc, /* is_space_alloc */ + NULL, /* insert */ + H5D_none_idx_get_addr, /* get_addr */ + NULL, /* resize */ + H5D_none_idx_iterate, /* iterate */ + H5D_none_idx_remove, /* remove */ + H5D_none_idx_delete, /* delete */ + H5D_none_idx_copy_setup, /* copy_setup */ + NULL, /* copy_shutdown */ + H5D_none_idx_size, /* size */ + H5D_none_idx_reset, /* reset */ + H5D_none_idx_dump, /* dump */ + NULL /* dest */ }}; /*****************************/ @@ -108,7 +108,7 @@ const H5D_chunk_ops_t H5D_COPS_NONE[1] = {{ /*------------------------------------------------------------------------- - * Function: H5D_none_create + * Function: H5D_none_idx_create * * Purpose: Allocate memory for the maximum # of chunks in the dataset. * @@ -120,7 +120,7 @@ const H5D_chunk_ops_t H5D_COPS_NONE[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5D_none_create(const H5D_chk_idx_info_t *idx_info) +H5D_none_idx_create(const H5D_chk_idx_info_t *idx_info) { hsize_t nbytes; /* Total size of dataset chunks */ haddr_t addr; /* The address of dataset chunks */ @@ -151,11 +151,11 @@ H5D_none_create(const H5D_chk_idx_info_t *idx_info) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_none_create() */ +} /* end H5D_none_idx_create() */ /*------------------------------------------------------------------------- - * Function: H5D_none_is_space_alloc + * Function: H5D_none_idx_is_space_alloc * * Purpose: Query if space for the dataset chunks is allocated * @@ -166,7 +166,7 @@ done: *------------------------------------------------------------------------- */ static hbool_t -H5D_none_is_space_alloc(const H5O_storage_chunk_t *storage) +H5D_none_idx_is_space_alloc(const H5O_storage_chunk_t *storage) { FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -174,11 +174,11 @@ H5D_none_is_space_alloc(const H5O_storage_chunk_t *storage) HDassert(storage); FUNC_LEAVE_NOAPI((hbool_t)H5F_addr_defined(storage->idx_addr)) -} /* end H5D_none_is_space_alloc() */ +} /* end H5D_none_idx_is_space_alloc() */ /*------------------------------------------------------------------------- - * Function: H5D_none_get_addr + * Function: H5D_none_idx_get_addr * * Purpose: Get the file address of a chunk. * Save the retrieved information in the udata supplied. @@ -190,7 +190,7 @@ H5D_none_is_space_alloc(const H5O_storage_chunk_t *storage) *------------------------------------------------------------------------- */ static herr_t -H5D_none_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) +H5D_none_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) { FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -214,11 +214,11 @@ H5D_none_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) udata->filter_mask = 0; FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5D_none_get_addr() */ +} /* H5D_none_idx_get_addr() */ /*------------------------------------------------------------------------- - * Function: H5D_none_iterate + * Function: H5D_none_idx_iterate * * Purpose: Iterate over the chunks in an index, making a callback * for each one. @@ -230,7 +230,7 @@ H5D_none_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) *------------------------------------------------------------------------- */ static int -H5D_none_iterate(const H5D_chk_idx_info_t *idx_info, +H5D_none_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_cb_func_t chunk_cb, void *chunk_udata) { H5D_chunk_rec_t chunk_rec; /* generic chunk record */ @@ -290,11 +290,11 @@ H5D_none_iterate(const H5D_chk_idx_info_t *idx_info, } /* end for */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_none_iterate() */ +} /* end H5D_none_idx_iterate() */ /*------------------------------------------------------------------------- - * Function: H5D_none_remove + * Function: H5D_none_idx_remove * * Purpose: Remove chunk from index. * @@ -309,18 +309,18 @@ H5D_none_iterate(const H5D_chk_idx_info_t *idx_info, *------------------------------------------------------------------------- */ static herr_t -H5D_none_remove(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, H5D_chunk_common_ud_t H5_ATTR_UNUSED *udata) +H5D_none_idx_remove(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, H5D_chunk_common_ud_t H5_ATTR_UNUSED *udata) { FUNC_ENTER_NOAPI_NOINIT_NOERR /* NO OP */ FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5D_none_remove() */ +} /* H5D_none_idx_remove() */ /*------------------------------------------------------------------------- - * Function: H5D_none_delete + * Function: H5D_none_idx_delete * * Purpose: Delete raw data storage for entire dataset (i.e. all chunks) * @@ -332,7 +332,7 @@ H5D_none_remove(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, H5D_chunk_com *------------------------------------------------------------------------- */ static herr_t -H5D_none_delete(const H5D_chk_idx_info_t *idx_info) +H5D_none_idx_delete(const H5D_chk_idx_info_t *idx_info) { hsize_t nbytes; /* Size of all chunks */ herr_t ret_value = SUCCEED; /* Return value */ @@ -356,11 +356,11 @@ H5D_none_delete(const H5D_chk_idx_info_t *idx_info) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_none_delete() */ +} /* end H5D_none_idx_delete() */ /*------------------------------------------------------------------------- - * Function: H5D_none_copy_setup + * Function: H5D_none_idx_copy_setup * * Purpose: Set up any necessary information for copying chunks * @@ -371,7 +371,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D_none_copy_setup(const H5D_chk_idx_info_t *idx_info_src, +H5D_none_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk_idx_info_t *idx_info_dst) { herr_t ret_value = SUCCEED; /* Return value */ @@ -398,7 +398,7 @@ H5D_none_copy_setup(const H5D_chk_idx_info_t *idx_info_src, H5_BEGIN_TAG(idx_info_dst->dxpl_id, H5AC__COPIED_TAG, FAIL); /* Allocate dataset chunks in the dest. file */ - if(H5D_none_create(idx_info_dst) < 0) + if(H5D_none_idx_create(idx_info_dst) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize chunked storage") /* Reset metadata tag */ @@ -406,11 +406,11 @@ H5D_none_copy_setup(const H5D_chk_idx_info_t *idx_info_src, done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_none_copy_setup() */ +} /* end H5D_none_idx_copy_setup() */ /*------------------------------------------------------------------------- - * Function: H5D_none_size + * Function: H5D_none_idx_size * * Purpose: Retrieve the amount of index storage for chunked dataset * @@ -422,7 +422,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D_none_size(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, hsize_t *index_size) +H5D_none_idx_size(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, hsize_t *index_size) { FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -432,11 +432,11 @@ H5D_none_size(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, hsize_t *index_ *index_size = 0; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5D_none_size() */ +} /* end H5D_none_idx_size() */ /*------------------------------------------------------------------------- - * Function: H5D_none_reset + * Function: H5D_none_idx_reset * * Purpose: Reset indexing information. * @@ -447,7 +447,7 @@ H5D_none_size(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, hsize_t *index_ *------------------------------------------------------------------------- */ static herr_t -H5D_none_reset(H5O_storage_chunk_t *storage, hbool_t reset_addr) +H5D_none_idx_reset(H5O_storage_chunk_t *storage, hbool_t reset_addr) { FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -459,11 +459,11 @@ H5D_none_reset(H5O_storage_chunk_t *storage, hbool_t reset_addr) storage->idx_addr = HADDR_UNDEF; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5D_none_reset() */ +} /* end H5D_none_idx_reset() */ /*------------------------------------------------------------------------- - * Function: H5D_none_dump + * Function: H5D_none_idx_dump * * Purpose: Dump * @@ -474,7 +474,7 @@ H5D_none_reset(H5O_storage_chunk_t *storage, hbool_t reset_addr) *------------------------------------------------------------------------- */ static herr_t -H5D_none_dump(const H5O_storage_chunk_t *storage, FILE *stream) +H5D_none_idx_dump(const H5O_storage_chunk_t *storage, FILE *stream) { FUNC_ENTER_NOAPI_NOINIT_NOERR @@ -485,5 +485,5 @@ H5D_none_dump(const H5O_storage_chunk_t *storage, FILE *stream) HDfprintf(stream, " Address: %a\n", storage->idx_addr); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5D_none_dump() */ +} /* end H5D_none_idx_dump() */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 5e006b5..48591e6 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -304,7 +304,7 @@ typedef herr_t (*H5D_chunk_init_func_t)(const H5D_chk_idx_info_t *idx_info, typedef herr_t (*H5D_chunk_create_func_t)(const H5D_chk_idx_info_t *idx_info); typedef hbool_t (*H5D_chunk_is_space_alloc_func_t)(const H5O_storage_chunk_t *storage); typedef herr_t (*H5D_chunk_insert_func_t)(const H5D_chk_idx_info_t *idx_info, - H5D_chunk_ud_t *udata); + H5D_chunk_ud_t *udata, H5D_t *dset); typedef herr_t (*H5D_chunk_get_addr_func_t)(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); typedef herr_t (*H5D_chunk_resize_func_t)(H5O_layout_chunk_t *layout); @@ -570,6 +570,7 @@ H5_DLLVAR const H5D_layout_ops_t H5D_LOPS_VIRTUAL[1]; /* Chunked layout operations */ H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_BTREE[1]; H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_NONE[1]; +H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_SINGLE[1]; H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_EARRAY[1]; H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_FARRAY[1]; H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_BT2[1]; diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index b442d63..0bb232f 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -61,10 +61,11 @@ typedef enum H5D_layout_t { /* Types of chunk index data structures */ typedef enum H5D_chunk_index_t { H5D_CHUNK_IDX_BTREE = 0, /* v1 B-tree index (default) */ - H5D_CHUNK_IDX_NONE = 1, /* No Index (H5D_ALLOC_TIME_EARLY, non-filtered, fixed dims) */ - H5D_CHUNK_IDX_FARRAY = 2, /* Fixed array (for 0 unlimited dims) */ - H5D_CHUNK_IDX_EARRAY = 3, /* Extensible array (for 1 unlimited dim) */ - H5D_CHUNK_IDX_BT2 = 4, /* v2 B-tree index (for >1 unlimited dims) */ + H5D_CHUNK_IDX_SINGLE = 1, /* Single Chunk index (cur dims[]=max dims[]=chunk dims[]; filtered & non-filtered) */ + H5D_CHUNK_IDX_NONE = 2, /* Implicit: No Index (H5D_ALLOC_TIME_EARLY, non-filtered, fixed dims) */ + H5D_CHUNK_IDX_FARRAY = 3, /* Fixed array (for 0 unlimited dims) */ + H5D_CHUNK_IDX_EARRAY = 4, /* Extensible array (for 1 unlimited dim) */ + H5D_CHUNK_IDX_BT2 = 5, /* v2 B-tree index (for >1 unlimited dims) */ H5D_CHUNK_IDX_NTYPES /*this one must be last! */ } H5D_chunk_index_t; diff --git a/src/H5Dsingle.c b/src/H5Dsingle.c new file mode 100644 index 0000000..a7c348e --- /dev/null +++ b/src/H5Dsingle.c @@ -0,0 +1,536 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Programmer: Vailin Choi <vchoi@hdfgroup.org> + * May 2011; updated 10/2015 + * + * Purpose: Single Chunk I/O functions. + * This is used when the dataset has only 1 chunk (with or without filter): + * cur_dims[] is equal to max_dims[] is equal to the chunk dims[] + * non-filter chunk record: [address of the chunk] + * filtered chunk record: [address of the chunk, chunk size, filter mask] + * + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5Dmodule.h" /* This source code file is part of the H5D module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Dpkg.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5MFprivate.h" /* File space management */ +#include "H5VMprivate.h" /* Vector functions */ + + +/****************/ +/* Local Macros */ +/****************/ + +/******************/ +/* Local Typedefs */ +/******************/ + +/********************/ +/* Local Prototypes */ +/********************/ + +/* Single Chunk Index chunking I/O ops */ +static herr_t H5D_single_idx_init(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D_single_idx_create(const H5D_chk_idx_info_t *idx_info); +static hbool_t H5D_single_idx_is_space_alloc(const H5O_storage_chunk_t *storage); +static herr_t H5D_single_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_ud_t *udata, H5D_t *dset); +static herr_t H5D_single_idx_get_addr(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_ud_t *udata); +static int H5D_single_idx_iterate(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_cb_func_t chunk_cb, void *chunk_udata); +static herr_t H5D_single_idx_remove(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_common_ud_t *udata); +static herr_t H5D_single_idx_delete(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D_single_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, + const H5D_chk_idx_info_t *idx_info_dst); +static herr_t H5D_single_idx_size(const H5D_chk_idx_info_t *idx_info, + hsize_t *size); +static herr_t H5D_single_idx_reset(H5O_storage_chunk_t *storage, hbool_t reset_addr); +static herr_t H5D_single_idx_dump(const H5O_storage_chunk_t *storage, FILE *stream); + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Non Index chunk I/O ops */ +const H5D_chunk_ops_t H5D_COPS_SINGLE[1] = {{ + FALSE, /* Single Chunk indexing don't current support SWMR access */ + H5D_single_idx_init, /* init */ + H5D_single_idx_create, /* create */ + H5D_single_idx_is_space_alloc, /* is_space_alloc */ + H5D_single_idx_insert_addr, /* insert */ + H5D_single_idx_get_addr, /* get_addr */ + NULL, /* resize */ + H5D_single_idx_iterate, /* iterate */ + H5D_single_idx_remove, /* remove */ + H5D_single_idx_delete, /* delete */ + H5D_single_idx_copy_setup, /* copy_setup */ + NULL, /* copy_shutdown */ + H5D_single_idx_size, /* size */ + H5D_single_idx_reset, /* reset */ + H5D_single_idx_dump, /* dump */ + NULL /* dest */ +}}; + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +static herr_t +H5D_single_idx_init(const H5D_chk_idx_info_t *idx_info) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check args */ + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->pline); + HDassert(idx_info->layout); + HDassert(idx_info->storage); + + if(idx_info->pline->nused) + idx_info->layout->flags |= H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER; + else + idx_info->layout->flags = 0; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_single_idx_init() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_single_idx_create + * + * Purpose: Set up Single Chunk Index: filtered or non-filtered + * + * Return: Non-negative on success + * Negative on failure. + * + * Programmer: Vailin Choi; July 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_single_idx_create(const H5D_chk_idx_info_t *idx_info) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check args */ + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->pline); + HDassert(idx_info->layout); + HDassert(idx_info->storage); + HDassert(idx_info->layout->max_nchunks == idx_info->layout->nchunks); + HDassert(idx_info->layout->nchunks == 1); + HDassert(!H5F_addr_defined(idx_info->storage->idx_addr)); + + if(idx_info->pline->nused) + HDassert(idx_info->layout->flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER); + else + HDassert(!(idx_info->layout->flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER)); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_single_idx_create() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_single_idx_is_space_alloc + * + * Purpose: Query if space is allocated for the single chunk + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; July 2011 + * + *------------------------------------------------------------------------- + */ +static hbool_t +H5D_single_idx_is_space_alloc(const H5O_storage_chunk_t *storage) +{ + hbool_t ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check args */ + HDassert(storage); + + /* Set return value */ + ret_value = (hbool_t)H5F_addr_defined(storage->idx_addr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_single_idx_is_space_alloc() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_single_idx_insert_addr + * + * Purpose: Allocate space for the single chunk + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; July 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_single_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata, H5D_t *dset) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->pline); + HDassert(idx_info->layout); + HDassert(idx_info->storage); + HDassert(idx_info->layout->nchunks == 1); + HDassert(idx_info->layout->max_nchunks == 1); + HDassert(udata); + + /* Set the address for the chunk */ + idx_info->storage->idx_addr = udata->chunk_block.offset; + HDassert(H5F_addr_defined(udata->chunk_block.offset)); + + if(idx_info->pline->nused > 0) { + idx_info->storage->u.single.nbytes = udata->chunk_block.length; + idx_info->storage->u.single.filter_mask = udata->filter_mask; + } + + if(dset) { + + if(dset->shared->dcpl_cache.fill.alloc_time != H5D_ALLOC_TIME_EARLY || idx_info->pline->nused > 0) { + + /* Mark the layout/storage dirty so that the address of the single chunk will be flushed later */ + if(H5D__mark(dset, idx_info->dxpl_id, H5D_MARK_LAYOUT) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to mark layout as dirty") + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D_single_idx_insert_addr() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_single_idx_get_addr + * + * Purpose: Get the file address of a chunk. + * Save the retrieved information in the udata supplied. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; July 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_single_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->pline); + HDassert(idx_info->layout); + HDassert(idx_info->storage); + HDassert(idx_info->layout->nchunks == 1); + HDassert(idx_info->layout->max_nchunks == 1); + HDassert(udata); + + udata->chunk_block.offset = idx_info->storage->idx_addr; + if(idx_info->layout->flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER) { + udata->chunk_block.length = idx_info->storage->u.single.nbytes; + udata->filter_mask = idx_info->storage->u.single.filter_mask; + } else { + udata->chunk_block.length = idx_info->layout->size; + udata->filter_mask = 0; + } + if(!H5F_addr_defined(udata->chunk_block.offset)) + udata->chunk_block.length = 0; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5D_single_idx_get_addr() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_single_idx_iterate + * + * Purpose: Make callback for the single chunk + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; July 2010 + * + *------------------------------------------------------------------------- + */ +static int +H5D_single_idx_iterate(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_cb_func_t chunk_cb, void *chunk_udata) +{ + H5D_chunk_rec_t chunk_rec; /* generic chunk record */ + int ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->pline); + HDassert(idx_info->layout); + HDassert(idx_info->storage); + HDassert(chunk_cb); + HDassert(chunk_udata); + HDassert(H5F_addr_defined(idx_info->storage->idx_addr)); + + /* Initialize generic chunk record */ + HDmemset(&chunk_rec, 0, sizeof(chunk_rec)); + chunk_rec.chunk_addr = idx_info->storage->idx_addr; + + if(idx_info->layout->flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER) { + chunk_rec.nbytes = idx_info->storage->u.single.nbytes; + chunk_rec.filter_mask = idx_info->storage->u.single.filter_mask; + } else { + chunk_rec.nbytes = idx_info->layout->size; + chunk_rec.filter_mask = 0; + } + + /* Make "generic chunk" callback */ + if((ret_value = (*chunk_cb)(&chunk_rec, chunk_udata)) < 0) + HERROR(H5E_DATASET, H5E_CALLBACK, "failure in generic chunk iterator callback"); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_single_idx_iterate() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_single_idx_remove + * + * Purpose: Remove the single chunk + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; July 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_single_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t H5_ATTR_UNUSED *udata) +{ + hsize_t nbytes; /* Size of all chunks */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->pline); + HDassert(idx_info->layout); + HDassert(idx_info->storage); + HDassert(H5F_addr_defined(idx_info->storage->idx_addr)); + + if(idx_info->layout->flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER) + nbytes = idx_info->storage->u.single.nbytes; + else + nbytes = idx_info->layout->size; + + if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, idx_info->storage->idx_addr, nbytes) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, H5_ITER_ERROR, "unable to free dataset chunks") + + idx_info->storage->idx_addr = HADDR_UNDEF; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D_single_idx_remove() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_single_idx_delete + * + * Purpose: Delete raw data storage for entire dataset (i.e. the only chunk) + * + * Return: Success: Non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Sept 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_single_idx_delete(const H5D_chk_idx_info_t *idx_info) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->pline); + HDassert(idx_info->layout); + HDassert(idx_info->storage); + + if(H5F_addr_defined(idx_info->storage->idx_addr)) { + ret_value = H5D_single_idx_remove(idx_info, NULL); + } else + HDassert(!H5F_addr_defined(idx_info->storage->idx_addr)); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_single_idx_delete() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_single_idx_copy_setup + * + * Purpose: Set up any necessary information for copying the single chunk + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Sept 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_single_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, + const H5D_chk_idx_info_t *idx_info_dst) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check args */ + HDassert(idx_info_src); + HDassert(idx_info_src->f); + HDassert(idx_info_src->pline); + HDassert(idx_info_src->layout); + HDassert(idx_info_src->storage); + HDassert(H5F_addr_defined(idx_info_src->storage->idx_addr)); + + HDassert(idx_info_dst); + HDassert(idx_info_dst->f); + HDassert(idx_info_dst->pline); + HDassert(idx_info_dst->layout); + HDassert(idx_info_dst->storage); + + /* Set copied metadata tag */ + H5_BEGIN_TAG(idx_info_dst->dxpl_id, H5AC__COPIED_TAG, FAIL); + + /* Set up information at the destination file */ + if(H5D_single_idx_create(idx_info_dst) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize chunked storage") + + /* Reset metadata tag */ + H5_END_TAG(FAIL); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_single_idx_copy_setup() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_single_idx_size + * + * Purpose: Retrieve the amount of index storage for the chunked dataset + * + * Return: Success: Non-negative + * Failure: negative + * + * Programmer: Vailin Choi; Sept 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_single_idx_size(const H5D_chk_idx_info_t H5_ATTR_UNUSED *idx_info, hsize_t *index_size) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check args */ + HDassert(index_size); + + *index_size = 0; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_single_idx_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_single_idx_reset + * + * Purpose: Reset indexing information. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Sept 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_single_idx_reset(H5O_storage_chunk_t *storage, hbool_t reset_addr) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check args */ + HDassert(storage); + + /* Reset index info */ + if(reset_addr) + storage->idx_addr = HADDR_UNDEF; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_single_idx_reset() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_single_idx_dump + * + * Purpose: Dump the address of the single chunk + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; September 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_single_idx_dump(const H5O_storage_chunk_t *storage, FILE *stream) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check args */ + HDassert(storage); + HDassert(stream); + + HDfprintf(stream, " Address: %a\n", storage->idx_addr); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_single_idx_dump() */ + diff --git a/src/H5Olayout.c b/src/H5Olayout.c index 0e44f1c..3a94df3 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -299,11 +299,20 @@ H5O__layout_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "v1 B-tree index type should never be in a v4 layout message") break; - case H5D_CHUNK_IDX_NONE: /* Non Index */ - /* Set the chunk operations */ + case H5D_CHUNK_IDX_NONE: /* Implicit Index */ mesg->storage.u.chunk.ops = H5D_COPS_NONE; break; + case H5D_CHUNK_IDX_SINGLE: /* Single Chunk Index */ + if(mesg->u.chunk.flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER) { + H5F_DECODE_LENGTH(f, p, mesg->storage.u.chunk.u.single.nbytes); + UINT32DECODE(p, mesg->storage.u.chunk.u.single.filter_mask); + } + + /* Set the chunk operations */ + mesg->storage.u.chunk.ops = H5D_COPS_SINGLE; + break; + case H5D_CHUNK_IDX_FARRAY: /* Fixed array creation parameters */ mesg->u.chunk.u.farray.cparam.max_dblk_page_nelmts_bits = *p++; @@ -624,7 +633,15 @@ H5O__layout_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "v1 B-tree index type should never be in a v4 layout message") break; - case H5D_CHUNK_IDX_NONE: /* Non Index */ + case H5D_CHUNK_IDX_NONE: /* Implicit */ + break; + + case H5D_CHUNK_IDX_SINGLE: /* Single Chunk */ + /* Filter information */ + if(mesg->u.chunk.flags & H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER) { + H5F_ENCODE_LENGTH(f, p, mesg->storage.u.chunk.u.single.nbytes); + UINT32ENCODE(p, mesg->storage.u.chunk.u.single.filter_mask); + } break; case H5D_CHUNK_IDX_FARRAY: @@ -652,7 +669,11 @@ H5O__layout_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "Invalid chunk index type") } /* end switch */ - /* Chunk index address */ + /* + * Implicit index: Address of the chunks + * Single chunk index: address of the single chunk + * Other indexes: chunk index address + */ H5F_addr_encode(f, &p, mesg->storage.u.chunk.idx_addr); } /* end else */ break; @@ -1181,7 +1202,12 @@ H5O__layout_debug(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, const v case H5D_CHUNK_IDX_NONE: HDfprintf(stream, "%*s%-*s %s\n", indent, "", fwidth, - "Index Type:", "None"); + "Index Type:", "Implicit"); + break; + + case H5D_CHUNK_IDX_SINGLE: + HDfprintf(stream, "%*s%-*s %s\n", indent, "", fwidth, + "Index Type:", "Single Chunk"); break; case H5D_CHUNK_IDX_FARRAY: diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 0fce28a..b1259d7 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -369,6 +369,7 @@ typedef struct H5O_efl_t { #define H5O_LAYOUT_CHUNK_ABBREVIATE_PARTIAL_BOUND_CHUNKS 0x08 #endif /* NOT_YET */ #define H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS 0x10 +#define H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER 0x20 #ifdef NOT_YET #define H5O_LAYOUT_ALL_CHUNK_FLAGS ( \ H5O_LAYOUT_CHUNK_STORE_ELEM_PHASE_CHANGE \ @@ -378,8 +379,8 @@ typedef struct H5O_efl_t { | H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS \ ) #else /* NOT_YET */ -#define H5O_LAYOUT_ALL_CHUNK_FLAGS ( \ - H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS \ +#define H5O_LAYOUT_ALL_CHUNK_FLAGS ( \ + H5O_LAYOUT_CHUNK_DONT_FILTER_PARTIAL_BOUND_CHUNKS|H5O_LAYOUT_CHUNK_SINGLE_INDEX_WITH_FILTER \ ) #endif /* NOT_YET */ @@ -412,7 +413,6 @@ typedef struct H5O_efl_t { * and 'size' callbacks for places to change when updating this. */ #define H5O_LAYOUT_VERSION_LATEST H5O_LAYOUT_VERSION_4 - /* Forward declaration of structs used below */ struct H5D_layout_ops_t; /* Defined in H5Dpkg.h */ struct H5D_chunk_ops_t; /* Defined in H5Dpkg.h */ @@ -443,6 +443,12 @@ typedef struct H5O_storage_chunk_earray_t { struct H5EA_t *ea; /* Pointer to extensible index array struct */ } H5O_storage_chunk_earray_t; +/* Filtered info for single chunk index */ +typedef struct H5O_storage_chunk_single_filt_t { + uint32_t nbytes; /* Size of chunk (in file) */ + uint32_t filter_mask; /* Excluded filters for chunk */ +} H5O_storage_chunk_single_filt_t; + /* Forward declaration of structs used below */ struct H5B2_t; /* Defined in H5B2pkg.h */ @@ -456,10 +462,11 @@ typedef struct H5O_storage_chunk_t { haddr_t idx_addr; /* File address of chunk index */ const struct H5D_chunk_ops_t *ops; /* Pointer to chunked storage operations */ union { - H5O_storage_chunk_btree_t btree; /* Information for v1 B-tree index */ + H5O_storage_chunk_btree_t btree; /* Information for v1 B-tree index */ H5O_storage_chunk_farray_t farray; /* Information for fixed array index */ H5O_storage_chunk_earray_t earray; /* Information for extensible array index */ - H5O_storage_chunk_bt2_t btree2; /* Information for v2 B-tree index */ + H5O_storage_chunk_bt2_t btree2; /* Information for v2 B-tree index */ + H5O_storage_chunk_single_filt_t single; /* Information for single chunk w/ filters index */ } u; } H5O_storage_chunk_t; diff --git a/src/Makefile.am b/src/Makefile.am index 1cdddfc..7f8babd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -48,7 +48,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5C.c \ H5CS.c \ H5D.c H5Dbtree.c H5Dbtree2.c H5Dchunk.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \ - H5Ddeprec.c H5Dearray.c H5Defl.c H5Dfarray.c H5Dfill.c H5Dint.c \ + H5Ddeprec.c H5Dearray.c H5Defl.c H5Dfarray.c H5Dsingle.c H5Dfill.c H5Dint.c \ H5Dio.c H5Dlayout.c H5Dnone.c H5Doh.c H5Dscatgath.c \ H5Dselect.c H5Dtest.c H5Dvirtual.c \ H5E.c H5Edeprec.c H5Eint.c \ diff --git a/test/dsets.c b/test/dsets.c index 7323c0d..10db417 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -61,7 +61,8 @@ const char *FILENAME[] = { "partial_chunks", /* 14 */ "layout_extend", /* 15 */ "swmr_fail", /* 16 */ - "zero_chunk", + "zero_chunk", /* 17 */ + "chunk_single", /* 18 */ NULL }; #define FILENAME_BUF_SIZE 1024 @@ -135,6 +136,10 @@ const char *FILENAME[] = { #define POINTS 72 #define POINTS_BIG 2500 +/* Dataset names for testing Implicit Indexing */ +#define DSET_SINGLE_MAX "DSET_SINGLE_MAX" +#define DSET_SINGLE_NOMAX "DSET_SINGLE_NOMAX" + #define USER_BLOCK 1024 #define SIXTY_FOUR_KB 65536 @@ -9085,7 +9090,7 @@ error: /*------------------------------------------------------------------------- * Function: test_fixed_array * - * Purpose: Tests support for Fixed Array and Non-Index Indexing + * Purpose: Tests support for Fixed Array and Implicit Indexing * * Create the following 3 datasets: * 1) extendible chunked dataset with fixed max. dims @@ -9100,10 +9105,10 @@ error: * verify that v1 btree indexing type is used for * all 3 datasets with all settings * For the new format: - * Verify that Non-Index type is used for - * all 3 datasets when ALLOC_TIME_EARLY and compression are true + * Verify that Implicit Index type is used for + * #1, #2, #3 datasets when ALLOC_TIME_EARLY and compression are true * Verify Fixed Array indexing type is used for - * all 3 datasets with the other settings + * #1, #2, #3 datasets with all other settings * * Return: Success: 0 * Failure: -1 @@ -9120,7 +9125,7 @@ test_fixed_array(hid_t fapl) hid_t dcpl = -1; /* Dataset creation property list ID */ hid_t sid = -1; /* Dataspace ID for dataset with fixed dimensions */ - hid_t sid_big = -1; /* Dataspace ID for big dataset */ + hid_t sid_big = -1; /* Dataspate ID for big dataset */ hid_t sid_max = -1; /* Dataspace ID for dataset with maximum dimensions set */ hid_t dsid = -1; /* Dataset ID for dataset with fixed dimensions */ @@ -9488,6 +9493,245 @@ error: /*------------------------------------------------------------------------- + * Function: test_single_chunk + * + * Purpose: Tests support for Single Chunk indexing type + * + * Create the following 2 datasets: + * 1) chunked dataset with NULL max dims and cur_dims = chunk_dims + * 2) chunked dataset with cur_dims = max_dims = chunk_dims + * + * Repeat the following test with/without compression filter + * Repeat the following test with H5D_ALLOC_TIME_EARLY/H5D_ALLOC_TIME_LATE/H5D_ALLOC_TIME_INCR + * For the old format, + * verify that v1 btree indexing type is used for + * all datasets with all settings + * For the new format: + * Verify that Single Chunk indexing type is used for + * all datasets with all settings + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Vailin Choi; July 2011 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_single_chunk(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t fid = -1; /* File ID */ + hid_t dcpl = -1; /* Dataset creation property list ID */ + hid_t t_dcpl = -1; /* Dataset creation property list ID */ + + hid_t sid = -1, sid_max = -1; /* Dataspace ID for dataset with fixed dimensions */ + hid_t did = -1, did_max = -1; /* Dataset ID for dataset with fixed dimensions */ + hsize_t dim2[2] = {DSET_DIM1, DSET_DIM2}; /* Dataset dimensions */ + hsize_t t_dim2[2] = {50, 100}; /* Dataset dimensions */ + int wbuf[DSET_DIM1*DSET_DIM2]; /* write buffer */ + int t_wbuf[50*100]; /* write buffer */ + int rbuf[DSET_DIM1*DSET_DIM2]; /* read buffer */ + int t_rbuf[50*100]; /* read buffer */ + + H5D_chunk_index_t idx_type; /* Dataset chunk index type */ + H5F_libver_t low, high; /* File format bounds */ + H5D_alloc_time_t alloc_time; /* Storage allocation time */ + +#ifdef H5_HAVE_FILTER_DEFLATE + hbool_t compress; /* Whether chunks should be compressed */ +#endif /* H5_HAVE_FILTER_DEFLATE */ + + size_t n, i; /* local index variables */ + herr_t ret; /* Generic return value */ + h5_stat_size_t empty_size; /* Size of an empty file */ + h5_stat_size_t file_size; /* Size of each file created */ + + TESTING("datasets w/Single Chunk indexing"); + + h5_fixname(FILENAME[18], fapl, filename, sizeof filename); + + /* Check if we are using the latest version of the format */ + if(H5Pget_libver_bounds(fapl, &low, &high) < 0) FAIL_STACK_ERROR + + /* Create and close the file to get the file size */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + STACK_ERROR + if(H5Fclose(fid) < 0) + STACK_ERROR + + /* Get the size of the empty file */ + if((empty_size = h5_get_file_size(filename, fapl)) < 0) + TEST_ERROR + + for(i = n = 0; i < (DSET_DIM1 * DSET_DIM2); i++) + wbuf[i] = n++; + + for(i = n = 0; i < (50* 100); i++) + t_wbuf[i] = n++; + +#ifdef H5_HAVE_FILTER_DEFLATE + /* Loop over compressing chunks */ + for(compress = FALSE; compress <= TRUE; compress++) { +#endif /* H5_HAVE_FILTER_DEFLATE */ + + /* Loop over storage allocation time */ + for(alloc_time = H5D_ALLOC_TIME_EARLY; alloc_time <= H5D_ALLOC_TIME_INCR; alloc_time++) { + /* Create file */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) FAIL_STACK_ERROR + if((t_dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) FAIL_STACK_ERROR + + /* Set chunking */ + if((ret = H5Pset_chunk(dcpl, 2, dim2)) < 0) + FAIL_PUTS_ERROR(" Problem with setting chunk.") + + if((ret = H5Pset_chunk(t_dcpl, 2, t_dim2)) < 0) + FAIL_PUTS_ERROR(" Problem with setting chunk.") + +#ifdef H5_HAVE_FILTER_DEFLATE + /* Check if we should compress the chunks */ + if(compress) { + if(H5Pset_deflate(dcpl, 9) < 0) FAIL_STACK_ERROR + if(H5Pset_deflate(t_dcpl, 9) < 0) FAIL_STACK_ERROR + } +#endif /* H5_HAVE_FILTER_DEFLATE */ + + /* Set fill time */ + if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) FAIL_STACK_ERROR + if(H5Pset_fill_time(t_dcpl, H5D_FILL_TIME_ALLOC) < 0) FAIL_STACK_ERROR + + /* Set allocation time */ + if(H5Pset_alloc_time(dcpl, alloc_time) < 0) FAIL_STACK_ERROR + if(H5Pset_alloc_time(t_dcpl, alloc_time) < 0) FAIL_STACK_ERROR + + /* Create first dataset with cur and max dimensions */ + if((sid_max = H5Screate_simple(2, dim2, dim2)) < 0) FAIL_STACK_ERROR + did_max = H5Dcreate2(fid, DSET_SINGLE_MAX, H5T_NATIVE_INT, sid_max, H5P_DEFAULT, dcpl, H5P_DEFAULT); + if(did_max < 0) + FAIL_PUTS_ERROR(" Creating Chunked Dataset with maximum dimensions.") + + /* Get the chunk index type */ + if(H5D__layout_idx_type_test(did_max, &idx_type) < 0) FAIL_STACK_ERROR + + /* Chunk index type depends on whether we are using the latest version of the format */ + if(low == H5F_LIBVER_LATEST) { + if(idx_type != H5D_CHUNK_IDX_SINGLE) + FAIL_PUTS_ERROR("should be using Single Chunk indexing"); + } /* end if */ + else { + if(idx_type != H5D_CHUNK_IDX_BTREE) + FAIL_PUTS_ERROR("should be using v1 B-tree as index"); + } /* end else */ + + /* Write into dataset */ + if(H5Dwrite(did_max, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) TEST_ERROR; + + /* Closing */ + if(H5Dclose(did_max) < 0) FAIL_STACK_ERROR + if(H5Sclose(sid_max) < 0) FAIL_STACK_ERROR + + /* Create second dataset with curr dim but NULL max dim */ + if((sid = H5Screate_simple(2, t_dim2, NULL)) < 0) FAIL_STACK_ERROR + did = H5Dcreate2(fid, DSET_SINGLE_NOMAX, H5T_NATIVE_INT, sid, H5P_DEFAULT, t_dcpl, H5P_DEFAULT); + if(did < 0) + FAIL_PUTS_ERROR(" Creating Chunked Dataset.") + + /* Get the chunk index type */ + if(H5D__layout_idx_type_test(did, &idx_type) < 0) FAIL_STACK_ERROR + + /* Chunk index type depends on whether we are using the latest version of the format */ + if(low == H5F_LIBVER_LATEST) { + if(idx_type != H5D_CHUNK_IDX_SINGLE) + FAIL_PUTS_ERROR("should be using Single Chunk indexing"); + } else { + if(idx_type != H5D_CHUNK_IDX_BTREE) + FAIL_PUTS_ERROR("should be using v1 B-tree as index"); + } /* end else */ + + /* Write into dataset */ + if(H5Dwrite(did, H5T_NATIVE_INT, H5S_ALL, sid, H5P_DEFAULT, t_wbuf) < 0) TEST_ERROR; + + /* Closing */ + if(H5Dclose(did) < 0) FAIL_STACK_ERROR + if(H5Sclose(sid) < 0) FAIL_STACK_ERROR + + /* Open the first dataset */ + if((did_max = H5Dopen2(fid, DSET_SINGLE_MAX, H5P_DEFAULT)) < 0) TEST_ERROR; + + /* Read from dataset */ + if(H5Dread(did_max, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf) < 0) TEST_ERROR; + + /* Verify that written and read data are the same */ + for(i = 0; i < (DSET_DIM1 * DSET_DIM2); i++) + if(rbuf[i] != wbuf[i]){ + printf(" Line %d: Incorrect value, wbuf[%u]=%d, rbuf[%u]=%d\n", + __LINE__,(unsigned)i,wbuf[i],(unsigned)i,rbuf[i]); + TEST_ERROR; + } /* end if */ + + /* Closing */ + if(H5Dclose(did_max) < 0) FAIL_STACK_ERROR + + /* Open the second dataset */ + if((did = H5Dopen2(fid, DSET_SINGLE_NOMAX, H5P_DEFAULT)) < 0) TEST_ERROR; + + HDmemset(rbuf, 0, sizeof(rbuf)); + + /* Read from dataset */ + if(H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, t_rbuf) < 0) TEST_ERROR; + + /* Verify that written and read data are the same */ + for(i = 0; i < (50* 100); i++) + if(t_rbuf[i] != t_wbuf[i]){ + printf(" Line %d: Incorrect value, t_wbuf[%u]=%d, t_rbuf[%u]=%d\n", + __LINE__,(unsigned)i,t_wbuf[i],(unsigned)i,t_rbuf[i]); + TEST_ERROR; + } /* end if */ + + /* Closing */ + if(H5Dclose(did) < 0) FAIL_STACK_ERROR + + /* Delete datasets */ + if(H5Ldelete(fid, DSET_SINGLE_NOMAX, H5P_DEFAULT) < 0) FAIL_STACK_ERROR + if(H5Ldelete(fid, DSET_SINGLE_MAX, H5P_DEFAULT) < 0) FAIL_STACK_ERROR + + /* Close everything */ + if(H5Fclose(fid) < 0) FAIL_STACK_ERROR + + /* Get the size of the file */ + if((file_size = h5_get_file_size(filename, fapl)) < 0) + TEST_ERROR + + /* Verify the file is correct size */ + if(file_size != empty_size) + TEST_ERROR + + } /* end for */ +#ifdef H5_HAVE_FILTER_DEFLATE + } /* end for */ +#endif /* H5_HAVE_FILTER_DEFLATE */ + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Pclose(t_dcpl); + H5Dclose(did); + H5Dclose(did_max); + H5Sclose(sid); + H5Sclose(sid_max); + H5Fclose(fid); + } H5E_END_TRY; + return -1; +} /* end test_single_chunk() */ + + +/*------------------------------------------------------------------------- * * test_idx_compatible(): * Verify that the library can read datasets created with @@ -11244,6 +11488,7 @@ main(void) nerrors += (test_fixed_array(my_fapl) < 0 ? 1 : 0); nerrors += (test_idx_compatible() < 0 ? 1 : 0); nerrors += (test_unfiltered_edge_chunks(my_fapl) < 0 ? 1 : 0); + nerrors += (test_single_chunk(my_fapl) < 0 ? 1 : 0); nerrors += (test_large_chunk_shrink(my_fapl) < 0 ? 1 : 0); nerrors += (test_zero_dim_dset(my_fapl) < 0 ? 1 : 0); diff --git a/test/objcopy.c b/test/objcopy.c index 215ec03..6ce8d6e 100644 --- a/test/objcopy.c +++ b/test/objcopy.c @@ -83,9 +83,13 @@ const char *FILENAME[] = { #define NAME_DATASET_SIMPLE3 "dataset_simple_another_copy" #define NAME_DATASET_COMPOUND "dataset_compound" #define NAME_DATASET_CHUNKED "dataset_chunked" +#define NAME_DATASET_CHUNKED_SINGLE "dataset_chunked_single" #define NAME_DATASET_CHUNKED2 "dataset_chunked2" +#define NAME_DATASET_CHUNKED2_SINGLE "dataset_chunked2_single" #define NAME_DATASET_CHUNKED3 "dataset_chunked3" +#define NAME_DATASET_CHUNKED3_SINGLE "dataset_chunked3_single" #define NAME_DATASET_CHUNKED4 "dataset_chunked4" +#define NAME_DATASET_CHUNKED4_SINGLE "dataset_chunked4_single" #define NAME_DATASET_COMPACT "dataset_compact" #define NAME_DATASET_EXTERNAL "dataset_ext" #define NAME_DATASET_NAMED_DTYPE "dataset_named_dtype" @@ -2563,6 +2567,35 @@ test_copy_dataset_chunked(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, hid_t /* close the dataset */ if(H5Dclose(did) < 0) TEST_ERROR + /* + * Create 1-D dataset: chunked, non-filterd, with data + * dims=max dims=chunk dims + * H5D_ALLOC_TIME_INC (default) + */ + /* create 1-D dataspace */ + if((sid = H5Screate_simple(1, dim1d, dim1d)) < 0) TEST_ERROR + + /* create and set chunk plist */ + if((pid = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR + if(H5Pset_chunk(pid, 1, dim1d) < 0) TEST_ERROR + + /* create dataset */ + if((did = H5Dcreate2(fid_src, NAME_DATASET_CHUNKED_SINGLE, H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, pid, H5P_DEFAULT)) < 0) TEST_ERROR + + /* close chunk plist */ + if(H5Pclose(pid) < 0) TEST_ERROR + + /* write data into file */ + if(H5Dwrite(did, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf1d) < 0) TEST_ERROR + + /* close dataspace */ + if(H5Sclose(sid) < 0) TEST_ERROR + + /* attach attributes to the dataset */ + if(test_copy_attach_attributes(did, H5T_NATIVE_INT) < 0) TEST_ERROR + + /* close the dataset */ + if(H5Dclose(did) < 0) TEST_ERROR /* Set 2-D dataspace dimensions */ dim2d[0] = DIM_SIZE_1; @@ -2597,18 +2630,96 @@ test_copy_dataset_chunked(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, hid_t /* close chunk plist */ if(H5Pclose(pid) < 0) TEST_ERROR + + /* write data into file */ + if(H5Dwrite(did, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf2d) < 0) TEST_ERROR + + /* close dataspace */ + if(H5Sclose(sid) < 0) TEST_ERROR + + /* attach attributes to the dataset */ + if(test_copy_attach_attributes(did, H5T_NATIVE_INT) < 0) TEST_ERROR + + /* close the dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + + /* + * Create 2-D dataset: chunked, non-filterd, with data, dims=chunk dims, + * H5D_ALLOC_TIME_INC (default) + */ + + /* create 2-D dataspace */ + if((sid = H5Screate_simple(2, dim2d, NULL)) < 0) TEST_ERROR + + /* create and set chunk plist to be the same as dims2d */ + if((pid = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR + if(H5Pset_chunk(pid, 2, dim2d) < 0) TEST_ERROR + + /* create dataset */ + if((did = H5Dcreate2(fid_src, NAME_DATASET_CHUNKED2_SINGLE, H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, pid, H5P_DEFAULT)) < 0) TEST_ERROR + + /* write data into file */ + if(H5Dwrite(did, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf2d) < 0) TEST_ERROR + + /* attach attributes to the dataset */ + if(test_copy_attach_attributes(did, H5T_NATIVE_INT) < 0) TEST_ERROR + + /* close the dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + + /* + * Create 2-D dataset: chunked, non-filterd, with data, dims=chunk dims, + * H5D_ALLOC_TIME_EARLY + */ + if(H5Pset_alloc_time(pid, H5D_ALLOC_TIME_EARLY) < 0) TEST_ERROR + + /* create dataset */ + if((did = H5Dcreate2(fid_src, NAME_DATASET_CHUNKED3_SINGLE, H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, pid, H5P_DEFAULT)) < 0) TEST_ERROR + /* write data into file */ if(H5Dwrite(did, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf2d) < 0) TEST_ERROR + /* attach attributes to the dataset */ + if(test_copy_attach_attributes(did, H5T_NATIVE_INT) < 0) TEST_ERROR + + /* close the dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + /* close dataspace */ if(H5Sclose(sid) < 0) TEST_ERROR + /* close chunk plist */ + if(H5Pclose(pid) < 0) TEST_ERROR + + /* + * Create 2-D dataset: chunked, non-filterd, with data, dims=max dims=chunk dims, + * H5D_ALLOC_TIME_LATE + */ + /* create 2-D dataspace */ + if((sid = H5Screate_simple(2, dim2d, dim2d)) < 0) TEST_ERROR + + /* create and set chunk plist to be the same as dims2d */ + if((pid = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR + if(H5Pset_chunk(pid, 2, dim2d) < 0) TEST_ERROR + if(H5Pset_alloc_time(pid, H5D_ALLOC_TIME_LATE) < 0) TEST_ERROR + + /* create dataset */ + if((did = H5Dcreate2(fid_src, NAME_DATASET_CHUNKED4_SINGLE, H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, pid, H5P_DEFAULT)) < 0) TEST_ERROR + + /* write data into file */ + if(H5Dwrite(did, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf2d) < 0) TEST_ERROR + /* attach attributes to the dataset */ if(test_copy_attach_attributes(did, H5T_NATIVE_INT) < 0) TEST_ERROR /* close the dataset */ if(H5Dclose(did) < 0) TEST_ERROR + /* close dataspace */ + if(H5Sclose(sid) < 0) TEST_ERROR + + /* close chunk plist */ + if(H5Pclose(pid) < 0) TEST_ERROR /* close the SRC file */ if(H5Fclose(fid_src) < 0) TEST_ERROR @@ -2628,6 +2739,11 @@ test_copy_dataset_chunked(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, hid_t if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED2, fid_dst, NAME_DATASET_CHUNKED2, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED3, fid_dst, NAME_DATASET_CHUNKED3, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED_SINGLE, fid_dst, NAME_DATASET_CHUNKED_SINGLE, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED2_SINGLE, fid_dst, NAME_DATASET_CHUNKED2_SINGLE, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED3_SINGLE, fid_dst, NAME_DATASET_CHUNKED3_SINGLE, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED4_SINGLE, fid_dst, NAME_DATASET_CHUNKED4_SINGLE, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + /* open the dataset for copy */ if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED, H5P_DEFAULT)) < 0) TEST_ERROR @@ -2648,6 +2764,25 @@ test_copy_dataset_chunked(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, hid_t if(H5Dclose(did) < 0) TEST_ERROR + /* open the dataset for copy */ + if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* open the 1-D destination dataset */ + if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* Check if the array index type is correct */ + if(compare_idx_type(src_fapl, did2, H5D_CHUNK_IDX_SINGLE, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + + /* Check if the datasets are equal */ + if(compare_datasets(did, did2, H5P_DEFAULT, buf1d) != TRUE) TEST_ERROR + + /* close the destination dataset */ + if(H5Dclose(did2) < 0) TEST_ERROR + + /* close the source dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + /* open the 2-D dataset for copy */ if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED2, H5P_DEFAULT)) < 0) TEST_ERROR @@ -2687,6 +2822,62 @@ test_copy_dataset_chunked(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, hid_t /* close the source dataset */ if(H5Dclose(did) < 0) TEST_ERROR + /* open the 2-D dataset for copy */ + if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED2_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* open the destination dataset */ + if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED2_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* Check if the array index type is correct */ + if(compare_idx_type(src_fapl, did2, H5D_CHUNK_IDX_SINGLE, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + + /* Check if the datasets are equal */ + if(compare_datasets(did, did2, H5P_DEFAULT, buf2d) != TRUE) TEST_ERROR + + /* close the destination dataset */ + if(H5Dclose(did2) < 0) TEST_ERROR + + /* close the source dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + + /* open the 2-D dataset for copy */ + if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED3_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* open the destination dataset */ + if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED3_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* Check if the array index type is correct */ + if(compare_idx_type(src_fapl, did2, H5D_CHUNK_IDX_SINGLE, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + + /* Check if the datasets are equal */ + if(compare_datasets(did, did2, H5P_DEFAULT, buf2d) != TRUE) TEST_ERROR + + /* close the destination dataset */ + if(H5Dclose(did2) < 0) TEST_ERROR + + /* close the source dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + + /* open the 2-D dataset for copy */ + if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED4_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* open the destination dataset */ + if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED4_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* Check if the array index type is correct */ + if(compare_idx_type(src_fapl, did2, H5D_CHUNK_IDX_SINGLE, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + + /* Check if the datasets are equal */ + if(compare_datasets(did, did2, H5P_DEFAULT, buf2d) != TRUE) TEST_ERROR + + /* close the destination dataset */ + if(H5Dclose(did2) < 0) TEST_ERROR + + /* close the source dataset */ + if(H5Dclose(did) < 0) TEST_ERROR /* close the SRC file */ if(H5Fclose(fid_src) < 0) TEST_ERROR @@ -2777,6 +2968,35 @@ test_copy_dataset_chunked_empty(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, /* close the dataset */ if(H5Dclose(did) < 0) TEST_ERROR + /* + * create 1-D dataset: chunked, empty, non-filtered, + * dims=max dims=chunk dims, H5D_ALLOC_TIME_INC(default) + */ + + /* Set 1-D dataspace dimensions */ + dim1d[0] = DIM_SIZE_1; + + /* create 1-D dataspace */ + if((sid = H5Screate_simple(1, dim1d, dim1d)) < 0) TEST_ERROR + + /* create and set chunk plist */ + if((pid = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR + if(H5Pset_chunk(pid, 1, dim1d) < 0) TEST_ERROR + + /* create dataset */ + if((did = H5Dcreate2(fid_src, NAME_DATASET_CHUNKED_SINGLE, H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, pid, H5P_DEFAULT)) < 0) TEST_ERROR + + /* close chunk plist */ + if(H5Pclose(pid) < 0) TEST_ERROR + + /* close dataspace */ + if(H5Sclose(sid) < 0) TEST_ERROR + + /* attach attributes to the dataset */ + if(test_copy_attach_attributes(did, H5T_NATIVE_INT) < 0) TEST_ERROR + + /* close the dataset */ + if(H5Dclose(did) < 0) TEST_ERROR /* Set 2-D dataspace dimensions */ dim2d[0] = DIM_SIZE_1; @@ -2817,6 +3037,88 @@ test_copy_dataset_chunked_empty(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, /* close the dataset */ if(H5Dclose(did) < 0) TEST_ERROR + + /* + * create 2-D dataset: chunked, empty, non-filtered, + * dims=chunk dims, H5D_ALLOC_TIME_INC (default) + */ + + /* Set 2-D dataspace dimensions */ + dim2d[0] = DIM_SIZE_1; + dim2d[1] = DIM_SIZE_2; + + /* create 2-D dataspace */ + if((sid = H5Screate_simple(2, dim2d, NULL)) < 0) TEST_ERROR + + /* create and set chunk plist */ + if((pid = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR + if(H5Pset_chunk(pid, 2, dim2d) < 0) TEST_ERROR + + /* create dataset */ + if((did = H5Dcreate2(fid_src, NAME_DATASET_CHUNKED2_SINGLE, H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, pid, H5P_DEFAULT)) < 0) TEST_ERROR + + /* attach attributes to the dataset */ + if(test_copy_attach_attributes(did, H5T_NATIVE_INT) < 0) TEST_ERROR + + /* close the dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + + /* + * create 2-D dataset: chunked, empty, non-filtered, dims=chunk dims + * H5D_ALLOC_TIME_EARLY + */ + /* Set allocation time to early */ + if(H5Pset_alloc_time(pid, H5D_ALLOC_TIME_EARLY) < 0) TEST_ERROR + + /* create dataset */ + if((did = H5Dcreate2(fid_src, NAME_DATASET_CHUNKED3_SINGLE, H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, pid, H5P_DEFAULT)) < 0) TEST_ERROR + + /* close chunk plist */ + if(H5Pclose(pid) < 0) TEST_ERROR + + /* close dataspace */ + if(H5Sclose(sid) < 0) TEST_ERROR + + /* attach attributes to the dataset */ + if(test_copy_attach_attributes(did, H5T_NATIVE_INT) < 0) TEST_ERROR + + /* close the dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + + /* + * create 2-D dataset: chunked, empty, non-filtered, + * dims=max dims=chunk dims, H5D_ALLOC_TIME_LATE + */ + + /* Set 2-D dataspace dimensions */ + dim2d[0] = DIM_SIZE_1; + dim2d[1] = DIM_SIZE_2; + + /* create 2-D dataspace */ + if((sid = H5Screate_simple(2, dim2d, dim2d)) < 0) TEST_ERROR + + /* create and set chunk plist */ + if((pid = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR + if(H5Pset_chunk(pid, 2, dim2d) < 0) TEST_ERROR + + /* Set allocation time to late */ + if(H5Pset_alloc_time(pid, H5D_ALLOC_TIME_LATE) < 0) TEST_ERROR + + /* create dataset */ + if((did = H5Dcreate2(fid_src, NAME_DATASET_CHUNKED4_SINGLE, H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, pid, H5P_DEFAULT)) < 0) TEST_ERROR + + /* attach attributes to the dataset */ + if(test_copy_attach_attributes(did, H5T_NATIVE_INT) < 0) TEST_ERROR + + /* close chunk plist */ + if(H5Pclose(pid) < 0) TEST_ERROR + + /* close dataspace */ + if(H5Sclose(sid) < 0) TEST_ERROR + + /* close the dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + /* close the SRC file */ if(H5Fclose(fid_src) < 0) TEST_ERROR @@ -2832,13 +3134,17 @@ test_copy_dataset_chunked_empty(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, /* copy the datasets from SRC to DST */ if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED, fid_dst, NAME_DATASET_CHUNKED, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED_SINGLE, fid_dst, NAME_DATASET_CHUNKED_SINGLE, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED2, fid_dst, NAME_DATASET_CHUNKED2, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED2_SINGLE, fid_dst, NAME_DATASET_CHUNKED2_SINGLE, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED3, fid_dst, NAME_DATASET_CHUNKED3, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED3_SINGLE, fid_dst, NAME_DATASET_CHUNKED3_SINGLE, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED4_SINGLE, fid_dst, NAME_DATASET_CHUNKED4_SINGLE, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR - /* open the dataset for copy */ + /* open the dataset NAME_DATASET_CHUNKED in SRC file */ if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED, H5P_DEFAULT)) < 0) TEST_ERROR - /* open the destination dataset */ + /* open the copied dataset NAME_DATASET_CHUNKED at destination */ if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED, H5P_DEFAULT)) < 0) TEST_ERROR /* Check if the array index type is correct */ @@ -2854,11 +3160,29 @@ test_copy_dataset_chunked_empty(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, /* close the source dataset */ if(H5Dclose(did) < 0) TEST_ERROR + /* open the dataset NAME_DATASET_CHUNKED_SINGLE in SRC file */ + if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR - /* open the dataset for copy */ + /* open the copied dataset NAME_DATASET_CHUNKED_SINGLE at destination */ + if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* Check if the array index type is correct */ + if(compare_idx_type(src_fapl, did2, H5D_CHUNK_IDX_SINGLE, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + + /* Check if the datasets are equal */ + if(compare_datasets(did, did2, H5P_DEFAULT, NULL) != TRUE) TEST_ERROR + + /* close the destination dataset */ + if(H5Dclose(did2) < 0) TEST_ERROR + + /* close the source dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + + /* open the dataset NAME_DATASET_CHUNKED2 in SRC file */ if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED2, H5P_DEFAULT)) < 0) TEST_ERROR - /* open the destination dataset */ + /* open the copied dataset NAME_DATASET_CHUNKED2 at destination */ if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED2, H5P_DEFAULT)) < 0) TEST_ERROR /* Check if the array index type is correct */ @@ -2874,11 +3198,30 @@ test_copy_dataset_chunked_empty(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, /* close the source dataset */ if(H5Dclose(did) < 0) TEST_ERROR + /* open the dataset "NAME_DATASET_CHUNKED2_SINGLE in SRC file */ + if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED2_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* open the copied dataset NAME_DATASET_CHUNKED2_SINGLE at destination */ + if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED2_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* Check if the array index type is correct */ + if(compare_idx_type(src_fapl, did2, H5D_CHUNK_IDX_SINGLE, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + + /* Check if the datasets are equal */ + if(compare_datasets(did, did2, H5P_DEFAULT, NULL) != TRUE) TEST_ERROR + + /* close the destination dataset */ + if(H5Dclose(did2) < 0) TEST_ERROR + + /* close the source dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + - /* open the third dataset for copy */ + /* open the dataset NAME_DATASET_CHUNKED3 in SRC file */ if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED3, H5P_DEFAULT)) < 0) TEST_ERROR - /* open the destination dataset */ + /* open the copied dataset NAME_DATASET_CHUNKED3 at destinaion */ if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED3, H5P_DEFAULT)) < 0) TEST_ERROR /* Check if the array index type is correct */ @@ -2895,6 +3238,44 @@ test_copy_dataset_chunked_empty(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, if(H5Dclose(did) < 0) TEST_ERROR + /* open the dataset NAME_DATASET_CHUNKED3_SINGLE in SRC file */ + if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED3_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* open the copied dataset NAME_DATASET_CHUNKED3_SINGLE at destination */ + if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED3_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* Check if the array index type is correct */ + if(compare_idx_type(src_fapl, did2, H5D_CHUNK_IDX_SINGLE, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + + /* Check if the datasets are equal */ + if(compare_datasets(did, did2, H5P_DEFAULT, NULL) != TRUE) TEST_ERROR + + /* close the destination dataset */ + if(H5Dclose(did2) < 0) TEST_ERROR + + /* close the source dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + + /* open the dataset NAME_DATASET_CHUNKED4_SINGLE in SRC file */ + if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED4_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* open the copied dataset NAME_DATASET_CHUNKED4_SINGLE at destination */ + if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED4_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* Check if the array index type is correct */ + if(compare_idx_type(src_fapl, did2, H5D_CHUNK_IDX_SINGLE, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + + /* Check if the datasets are equal */ + if(compare_datasets(did, did2, H5P_DEFAULT, NULL) != TRUE) TEST_ERROR + + /* close the destination dataset */ + if(H5Dclose(did2) < 0) TEST_ERROR + + /* close the source dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + /* close the SRC file */ if(H5Fclose(fid_src) < 0) TEST_ERROR @@ -3330,6 +3711,90 @@ test_copy_dataset_compressed(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, hid /* close the dataset */ if(H5Dclose(did) < 0) TEST_ERROR + /* + * create 2-D dataset: chunked, filtered, with data + * dims=max dims=chunk dims, H5D_ALLOC_TIME_INC(default) + */ + /* create dataspace */ + if((sid = H5Screate_simple(2, dim2d, dim2d)) < 0) TEST_ERROR + + /* create and set comp & chunk plist */ + if((pid = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR + if(H5Pset_chunk(pid, 2, dim2d) < 0) TEST_ERROR + if(H5Pset_deflate(pid, 9) < 0) TEST_ERROR + + /* create dataset */ + if((did = H5Dcreate2(fid_src, NAME_DATASET_CHUNKED2_SINGLE, H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, pid, H5P_DEFAULT)) < 0) TEST_ERROR + + /* write data into file */ + if(H5Dwrite(did, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) TEST_ERROR + + /* attach attributes to the dataset */ + if(test_copy_attach_attributes(did, H5T_NATIVE_INT) < 0) TEST_ERROR + + /* close the dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + + /* + * create 2-D dataset: chunked, filtered, with data + * dims=chunk dims, H5D_ALLOC_TIME_EARLY + */ + /* create dataspace */ + if((sid = H5Screate_simple(2, dim2d, NULL)) < 0) TEST_ERROR + + /* Set allocation time to early */ + if(H5Pset_alloc_time(pid, H5D_ALLOC_TIME_EARLY) < 0) TEST_ERROR + + /* create dataset */ + if((did = H5Dcreate2(fid_src, NAME_DATASET_CHUNKED3_SINGLE, H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, pid, H5P_DEFAULT)) < 0) TEST_ERROR + + /* close chunk plist */ + if(H5Pclose(pid) < 0) TEST_ERROR + + /* write data into file */ + if(H5Dwrite(did, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) TEST_ERROR + + /* close dataspace */ + if(H5Sclose(sid) < 0) TEST_ERROR + + /* attach attributes to the dataset */ + if(test_copy_attach_attributes(did, H5T_NATIVE_INT) < 0) TEST_ERROR + + /* close the dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + + /* + * create 2-D dataset: chunked, filtered, with data + * dims=chunk dims, H5D_ALLOC_TIME_LATE + */ + /* create dataspace */ + if((sid = H5Screate_simple(2, dim2d, NULL)) < 0) TEST_ERROR + + /* create and set comp & chunk plist */ + if((pid = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR + if(H5Pset_chunk(pid, 2, dim2d) < 0) TEST_ERROR + if(H5Pset_deflate(pid, 9) < 0) TEST_ERROR + + /* Set allocation time to late */ + if(H5Pset_alloc_time(pid, H5D_ALLOC_TIME_LATE) < 0) TEST_ERROR + + /* create dataset */ + if((did = H5Dcreate2(fid_src, NAME_DATASET_CHUNKED4_SINGLE, H5T_NATIVE_FLOAT, sid, H5P_DEFAULT, pid, H5P_DEFAULT)) < 0) TEST_ERROR + + /* close chunk plist */ + if(H5Pclose(pid) < 0) TEST_ERROR + + /* write data into file */ + if(H5Dwrite(did, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) TEST_ERROR + + /* close dataspace */ + if(H5Sclose(sid) < 0) TEST_ERROR + + /* attach attributes to the dataset */ + if(test_copy_attach_attributes(did, H5T_NATIVE_INT) < 0) TEST_ERROR + + /* close the dataset */ + if(H5Dclose(did) < 0) TEST_ERROR /* close the SRC file */ if(H5Fclose(fid_src) < 0) TEST_ERROR @@ -3347,6 +3812,16 @@ test_copy_dataset_compressed(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, hid /* copy the dataset from SRC to DST */ if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED, fid_dst, NAME_DATASET_CHUNKED, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED2, fid_dst, NAME_DATASET_CHUNKED2, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED2_SINGLE, fid_dst, NAME_DATASET_CHUNKED2_SINGLE, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED3_SINGLE, fid_dst, NAME_DATASET_CHUNKED3_SINGLE, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + if(H5Ocopy(fid_src, NAME_DATASET_CHUNKED4_SINGLE, fid_dst, NAME_DATASET_CHUNKED4_SINGLE, H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR + + if(H5Fclose(fid_dst) < 0) TEST_ERROR + if(H5Fclose(fid_src) < 0) TEST_ERROR + + /* Re-open the source and destination files for verification */ + if((fid_src = H5Fopen(src_filename, H5F_ACC_RDONLY, src_fapl)) < 0) TEST_ERROR + if((fid_dst = H5Fopen(dst_filename, H5F_ACC_RDONLY, dst_fapl)) < 0) TEST_ERROR /* open the dataset for copy */ if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED, H5P_DEFAULT)) < 0) TEST_ERROR @@ -3385,6 +3860,59 @@ test_copy_dataset_compressed(hid_t fcpl_src, hid_t fcpl_dst, hid_t src_fapl, hid /* close the source dataset */ if(H5Dclose(did) < 0) TEST_ERROR + /* open the dataset NAME_DATASET_CHUNKED2_SINGLE at source */ + if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED2_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* open the copied dataset NAME_DATASET_CHUNKED2_SINGLE at destination */ + if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED2_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + if(compare_idx_type(src_fapl, did2, H5D_CHUNK_IDX_SINGLE, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + + /* Check if the datasets are equal */ + if(compare_datasets(did, did2, H5P_DEFAULT, NULL) != TRUE) TEST_ERROR + + /* close the destination dataset */ + if(H5Dclose(did2) < 0) TEST_ERROR + + /* close the source dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + + /* open the dataset NAME_DATASET_CHUNKED3_SINGLE at source */ + if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED3_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* open the copied dataset NAME_DATASET_CHUNKED3_SINGLE at destination */ + if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED3_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + if(compare_idx_type(src_fapl, did2, H5D_CHUNK_IDX_SINGLE, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + + /* Check if the datasets are equal */ + if(compare_datasets(did, did2, H5P_DEFAULT, NULL) != TRUE) TEST_ERROR + + /* close the destination dataset */ + if(H5Dclose(did2) < 0) TEST_ERROR + + /* close the source dataset */ + if(H5Dclose(did) < 0) TEST_ERROR + + /* open the dataset NAME_DATASET_CHUNKED4_SINGLE at source */ + if((did = H5Dopen2(fid_src, NAME_DATASET_CHUNKED4_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + /* open the copied dataset NAME_DATASET_CHUNKED4_SINGLE at destination */ + if((did2 = H5Dopen2(fid_dst, NAME_DATASET_CHUNKED4_SINGLE, H5P_DEFAULT)) < 0) TEST_ERROR + + if(compare_idx_type(src_fapl, did2, H5D_CHUNK_IDX_SINGLE, H5D_CHUNK_IDX_BTREE) != TRUE) + TEST_ERROR + + /* Check if the datasets are equal */ + if(compare_datasets(did, did2, H5P_DEFAULT, NULL) != TRUE) TEST_ERROR + + /* close the destination dataset */ + if(H5Dclose(did2) < 0) TEST_ERROR + + /* close the source dataset */ + if(H5Dclose(did) < 0) TEST_ERROR /* close the SRC file */ if(H5Fclose(fid_src) < 0) TEST_ERROR diff --git a/tools/h5diff/testfiles/h5diff_dset_idx1.h5 b/tools/h5diff/testfiles/h5diff_dset_idx1.h5 Binary files differindex 9f71b42..3252303 100644 --- a/tools/h5diff/testfiles/h5diff_dset_idx1.h5 +++ b/tools/h5diff/testfiles/h5diff_dset_idx1.h5 diff --git a/tools/h5diff/testfiles/h5diff_dset_idx2.h5 b/tools/h5diff/testfiles/h5diff_dset_idx2.h5 Binary files differindex 3069fff..db7584d 100644 --- a/tools/h5diff/testfiles/h5diff_dset_idx2.h5 +++ b/tools/h5diff/testfiles/h5diff_dset_idx2.h5 diff --git a/tools/h5format_convert/testfiles/h5fc_all.h5 b/tools/h5format_convert/testfiles/h5fc_all.h5 Binary files differindex 23cc077..33a9166 100644 --- a/tools/h5format_convert/testfiles/h5fc_all.h5 +++ b/tools/h5format_convert/testfiles/h5fc_all.h5 diff --git a/tools/h5format_convert/testfiles/h5fc_edge.h5 b/tools/h5format_convert/testfiles/h5fc_edge.h5 Binary files differindex b3d4b37..abf6192 100644 --- a/tools/h5format_convert/testfiles/h5fc_edge.h5 +++ b/tools/h5format_convert/testfiles/h5fc_edge.h5 diff --git a/tools/h5format_convert/testfiles/h5fc_new.h5 b/tools/h5format_convert/testfiles/h5fc_new.h5 Binary files differindex aa9c7eb..fe4d858 100644 --- a/tools/h5format_convert/testfiles/h5fc_new.h5 +++ b/tools/h5format_convert/testfiles/h5fc_new.h5 diff --git a/tools/h5format_convert/testfiles/h5fc_old.h5 b/tools/h5format_convert/testfiles/h5fc_old.h5 Binary files differindex 13642a1..dc4e912 100644 --- a/tools/h5format_convert/testfiles/h5fc_old.h5 +++ b/tools/h5format_convert/testfiles/h5fc_old.h5 diff --git a/tools/h5format_convert/testfiles/h5fc_v_all.ddl b/tools/h5format_convert/testfiles/h5fc_v_all.ddl index 2d5b99c..3f474fe 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_all.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_all.ddl @@ -1,5 +1,5 @@ Process command line options -Open the file h5fc_old.h5 +Open the file tmp.h5 Processing all datasets in the file... Going to process dataset:/DSET_NDATA_BT1... Open the dataset diff --git a/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl b/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl index c63c1a0..abb0a89 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_bt1.ddl @@ -1,5 +1,5 @@ Process command line options -Open the file h5fc_old.h5 +Open the file tmp.h5 Going to process dataset: /GROUP/DSET_BT1... Open the dataset Retrieve the dataset's layout diff --git a/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl b/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl index 8cc6dec..a26dc66 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_n_1d.ddl @@ -1,6 +1,6 @@ Process command line options It is noop... -Open the file h5fc_new.h5 +Open the file tmp.h5 Going to process dataset: /DSET_EA... Open the dataset Retrieve the dataset's layout diff --git a/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl b/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl index e02a465..d4b7a2d 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_n_all.ddl @@ -1,6 +1,6 @@ Process command line options It is noop... -Open the file h5fc_all.h5 +Open the file tmp.h5 Processing all datasets in the file... Going to process dataset:/DSET_NDATA_BT1... Open the dataset diff --git a/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl b/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl index 32f04aa..86081f3 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_ndata_bt1.ddl @@ -1,6 +1,6 @@ Process command line options It is noop... -Open the file h5fc_old.h5 +Open the file tmp.h5 Going to process dataset: /DSET_NDATA_BT1... Open the dataset Retrieve the dataset's layout diff --git a/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl b/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl index d4f0f6c..baba0e4 100644 --- a/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl +++ b/tools/h5format_convert/testfiles/h5fc_v_non_chunked.ddl @@ -1,5 +1,5 @@ Process command line options -Open the file h5fc_old.h5 +Open the file tmp.h5 Going to process dataset: /DSET_NON_CHUNKED... Open the dataset Retrieve the dataset's layout diff --git a/tools/h5format_convert/testh5fc.sh.in b/tools/h5format_convert/testh5fc.sh.in index 7f41a21..ac51c08 100644 --- a/tools/h5format_convert/testh5fc.sh.in +++ b/tools/h5format_convert/testh5fc.sh.in @@ -66,6 +66,9 @@ SRC_H5FORMCONV_TESTFILES="$SRC_TOOLS/h5format_convert/testfiles" TESTDIR=./testfiles test -d $TESTDIR || mkdir $TESTDIR +# Copy the testfile to a temporary file for testing as h5format_convert is changing the file in place +TMPFILE=tmp.h5 + ###################################################################### # test files # -------------------------------------------------------------------- @@ -140,6 +143,8 @@ CLEAN_TESTFILES_AND_TESTDIR() INODE_DDIR=`$LS -i -d $TESTDIR | $AWK -F' ' '{print $1}'` if [ "$INODE_SDIR" != "$INODE_DDIR" ]; then $RM $TESTDIR + else + $RM $TESTDIR/$TMPFILE fi } @@ -160,27 +165,40 @@ TESTING() { # non-zero value. # # $1: expected output -# $2 to at most $6 : options to the tool and the input fname -# -d dname or --dname=dname -# -v or --verbose -# -n or --noop -# fname +# $2: the test file name +# --fname might be empty or fname does not exist +# --fname is copied to a temporary file for testing +# $3 to at most $6--options to the tool such as: +# -d dname or --dname=dname +# -v or --verbose +# -n or --noop TOOLTEST_OUT() { + # Prepare expected and actual output expect="$TESTDIR/$1" actual="$TESTDIR/`basename $1 .ddl`.out" actual_err="$TESTDIR/`basename $1 .ddl`.err" actual_sav=${actual}-sav actual_err_sav=${actual_err}-sav + + # Prepare the test file + $RM $TESTDIR/$TMPFILE + TFILE=$2 + if [ ! -z "$2" ] && [ -e $TESTDIR/$2 ] ; then + $CP $TESTDIR/$2 $TESTDIR/$TMPFILE + TFILE=$TMPFILE + fi + # Run test. - TESTING $FORMCONV $@ + TESTING $FORMCONV $3 $4 $5 $6 $2 ( cd $TESTDIR - $RUNSERIAL $FORMCONV_BIN $2 $3 $4 $5 $6 + $RUNSERIAL $FORMCONV_BIN $3 $4 $5 $6 $TFILE ) >$actual 2>$actual_err cp $actual $actual_sav cp $actual_err $actual_err_sav cat $actual_err >> $actual + # Compare output if $CMP $expect $actual; then echo " PASSED" else @@ -192,20 +210,24 @@ TOOLTEST_OUT() { # Clean up output file if test -z "$HDF5_NOCLEANUP"; then - rm -f $actual $actual_err - rm -f $actual $actual_err $actual_sav $actual_err_sav + $RM $actual $actual_err + $RM $actual $actual_err $actual_sav $actual_err_sav fi } # To check that the tool exits success, no output # Assume all short options -# Assume $1 is fname -# $2 to at most $4: options to the tool -# -d dname or -a -# -n +# $1 is the test file name +# --fname exists +# --fname is copied to a temporary file for testing +# $2 to at most $4--options to the tool such as: +# -d dname +# -n TOOLTEST() { - TESTING $FORMCONV $@ - $RUNSERIAL $FORMCONV_BIN $2 $3 $4 $TESTDIR/$1 + TESTING $FORMCONV $2 $3 $4 $1 + $RM $TESTDIR/$TMPFILE + $CP $TESTDIR/$1 $TESTDIR/$TMPFILE + $RUNSERIAL $FORMCONV_BIN $2 $3 $4 $TESTDIR/$TMPFILE exitcode=$? if [ $exitcode -ne 0 ]; then echo "*FAILED*" @@ -218,14 +240,14 @@ TOOLTEST() { CHECKING() { SPACES=" " - echo "Verifing $* $SPACES" | cut -c1-80 | tr -d '\012' + echo "Verifying $* $SPACES" | cut -c1-80 | tr -d '\012' } -# $1 file name -# $2 dataset name +# $1 dataset name +# Assume $TESTDIR/$TMPFILE is the converted test file IDX_CHECK() { - CHECKING $1 $2 - $RUNSERIAL $CHK_IDX_BIN $TESTDIR/$1 $2 + CHECKING $1 + $RUNSERIAL $CHK_IDX_BIN $TESTDIR/$TMPFILE $1 ret=$? if [ $ret -eq 0 ]; then echo " PASSED" @@ -257,8 +279,8 @@ COPY_TESTFILES_TO_TESTDIR # h5format_convert --help # h5format_convert (no options) # h5format_convert nonexist.h5 (no options, file does not exist) -TOOLTEST_OUT h5fc_help.ddl --help -TOOLTEST_OUT h5fc_nooption.ddl +TOOLTEST_OUT h5fc_help.ddl '' --help +TOOLTEST_OUT h5fc_nooption.ddl '' TOOLTEST_OUT h5fc_nonexistfile.ddl nonexist.h5 # # @@ -266,10 +288,10 @@ TOOLTEST_OUT h5fc_nonexistfile.ddl nonexist.h5 # h5format_convert --dname h5fc_old.h5 (just --dname option, file exists) # h5format_convert --dname (just --dname option) # h5format_convert --dname=nonexist h5fc_old.h5 (dataset does not exist, file exists) -TOOLTEST_OUT h5fc_d_file.ddl -d h5fc_old.h5 -TOOLTEST_OUT h5fc_d_file.ddl --dname h5fc_old.h5 -TOOLTEST_OUT h5fc_dname.ddl --dname -TOOLTEST_OUT h5fc_nonexistdset_file.ddl --dname=nonexist h5fc_old.h5 +TOOLTEST_OUT h5fc_d_file.ddl h5fc_old.h5 -d +TOOLTEST_OUT h5fc_d_file.ddl h5fc_old.h5 --dname +TOOLTEST_OUT h5fc_dname.ddl '' --dname +TOOLTEST_OUT h5fc_nonexistdset_file.ddl h5fc_old.h5 --dname=nonexist # # # @@ -277,17 +299,17 @@ TOOLTEST_OUT h5fc_nonexistdset_file.ddl --dname=nonexist h5fc_old.h5 # h5format_convert -d /GROUP/DSET_BT1 --verbose h5fc_old.h5 (verbose, bt1 dataset) # h5format_convert -d /DSET_NDATA_BT1 -v -n h5fc_old.h5 (verbose, noop, bt1+nodata dataset) # h5format_convert -v h5fc_old.h5 (verbose, all datasets) -TOOLTEST_OUT h5fc_v_non_chunked.ddl -d /DSET_NON_CHUNKED -v h5fc_old.h5 -TOOLTEST_OUT h5fc_v_bt1.ddl -d /GROUP/DSET_BT1 --verbose h5fc_old.h5 -TOOLTEST_OUT h5fc_v_ndata_bt1.ddl -d /DSET_NDATA_BT1 -v -n h5fc_old.h5 -TOOLTEST_OUT h5fc_v_all.ddl -v h5fc_old.h5 +TOOLTEST_OUT h5fc_v_non_chunked.ddl h5fc_old.h5 -d /DSET_NON_CHUNKED -v +TOOLTEST_OUT h5fc_v_bt1.ddl h5fc_old.h5 -d /GROUP/DSET_BT1 --verbose +TOOLTEST_OUT h5fc_v_ndata_bt1.ddl h5fc_old.h5 -d /DSET_NDATA_BT1 -v -n +TOOLTEST_OUT h5fc_v_all.ddl h5fc_old.h5 -v # # # # h5format_convert -d /DSET_EA -v -n h5fc_new.h5 (verbose, noop, one ea dataset) # h5format_convert -v -n h5fc_all.h5 (verbose, noop, all datasets) -TOOLTEST_OUT h5fc_v_n_1d.ddl -d /DSET_EA -v -n h5fc_new.h5 -TOOLTEST_OUT h5fc_v_n_all.ddl -v -n h5fc_all.h5 +TOOLTEST_OUT h5fc_v_n_1d.ddl h5fc_new.h5 -d /DSET_EA -v -n +TOOLTEST_OUT h5fc_v_n_all.ddl h5fc_all.h5 -v -n # # # @@ -303,28 +325,28 @@ TOOLTEST_OUT h5fc_v_n_all.ddl -v -n h5fc_all.h5 # h5format_convert -d /DSET_NONE h5fc_new.h5 # h5format_convert -d /GROUP/DSET_NONE h5fc_new.h5 TOOLTEST h5fc_new.h5 -d /DSET_EA -IDX_CHECK h5fc_new.h5 /DSET_EA +IDX_CHECK /DSET_EA # TOOLTEST h5fc_new.h5 -d /GROUP/DSET_NDATA_EA -IDX_CHECK h5fc_new.h5 /GROUP/DSET_NDATA_EA +IDX_CHECK /GROUP/DSET_NDATA_EA # TOOLTEST h5fc_new.h5 -d /GROUP/DSET_BT2 -IDX_CHECK h5fc_new.h5 /GROUP/DSET_BT2 +IDX_CHECK /GROUP/DSET_BT2 # TOOLTEST h5fc_new.h5 -d /DSET_NDATA_BT2 -IDX_CHECK h5fc_new.h5 /DSET_NDATA_BT2 +IDX_CHECK /DSET_NDATA_BT2 # TOOLTEST h5fc_new.h5 -d /DSET_FA -IDX_CHECK h5fc_new.h5 /DSET_FA +IDX_CHECK /DSET_FA # TOOLTEST h5fc_new.h5 -d /GROUP/DSET_NDATA_FA -IDX_CHECK h5fc_new.h5 /GROUP/DSET_NDATA_FA +IDX_CHECK /GROUP/DSET_NDATA_FA # TOOLTEST h5fc_new.h5 -d /DSET_NONE -IDX_CHECK h5fc_new.h5 /DSET_NONE +IDX_CHECK /DSET_NONE # TOOLTEST h5fc_new.h5 -d /GROUP/DSET_NDATA_NONE -IDX_CHECK h5fc_new.h5 /GROUP/DSET_NDATA_NONE +IDX_CHECK /GROUP/DSET_NDATA_NONE # # # @@ -349,10 +371,10 @@ TOOLTEST h5fc_all.h5 -n # 1) convert all datasets # 2) verify indexing types TOOLTEST h5fc_all.h5 -IDX_CHECK h5fc_all.h5 /DSET_NDATA_BT1 -IDX_CHECK h5fc_all.h5 /DSET_NDATA_EA -IDX_CHECK h5fc_all.h5 /GROUP/DSET_BT1 -IDX_CHECK h5fc_all.h5 /GROUP/DSET_BT2 +IDX_CHECK /DSET_NDATA_BT1 +IDX_CHECK /DSET_NDATA_EA +IDX_CHECK /GROUP/DSET_BT1 +IDX_CHECK /GROUP/DSET_BT2 # # # @@ -361,7 +383,7 @@ IDX_CHECK h5fc_all.h5 /GROUP/DSET_BT2 # 1) convert the chunked dataset (filter, no-filter-edge-chunk) # 2) verify the indexing type TOOLTEST h5fc_edge.h5 -IDX_CHECK h5fc_edge.h5 /DSET_EDGE +IDX_CHECK /DSET_EDGE # # # diff --git a/tools/h5stat/testfiles/h5stat_idx.h5 b/tools/h5stat/testfiles/h5stat_idx.h5 Binary files differindex 76e4ba3..303d1f8 100644 --- a/tools/h5stat/testfiles/h5stat_idx.h5 +++ b/tools/h5stat/testfiles/h5stat_idx.h5 diff --git a/tools/testfiles/tdset_idx.h5 b/tools/testfiles/tdset_idx.h5 Binary files differindex 3884789..314de9b 100644 --- a/tools/testfiles/tdset_idx.h5 +++ b/tools/testfiles/tdset_idx.h5 |