From 90c3d4e0cabb555a53ade75b9039465e39ca0070 Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Thu, 5 Feb 2009 19:08:38 -0500 Subject: [svn-r16447] Description: Add support for using I/O pipeline filters with extensible array chunked dataset indices. Tested on: FreeBSD/32 6.3 (duty) in debug mode FreeBSD/64 6.3 (liberty) w/C++ & FORTRAN, in debug mode Linux/32 2.6 (jam) w/PGI compilers, w/C++ & FORTRAN, w/threadsafe, in debug mode Linux/64-amd64 2.6 (smirom) w/Intel compilers w/default API=1.6.x, w/C++ & FORTRAN, in production mode Linux/64-ia64 2.6 (cobalt) w/Intel compilers, w/C++ & FORTRAN, in production mode Linux/64-ia64 2.4 (tg-login3) w/parallel, w/FORTRAN, in production mode Linux/64-amd64 2.6 (abe) w/parallel, w/FORTRAN, in production mode Mac OS X/32 10.5.6 (amazon) in debug mode Mac OS X/32 10.5.6 (amazon) w/C++ & FORTRAN, w/threadsafe, in production mode --- src/H5Dbtree.c | 12 ++ src/H5Dchunk.c | 69 ++++++-- src/H5Dearray.c | 465 ++++++++++++++++++++++++++++++++++++++++++++++-------- src/H5Doh.c | 20 ++- src/H5Dpkg.h | 4 +- src/H5Dprivate.h | 3 +- src/H5Dtest.c | 41 +++++ src/H5EAprivate.h | 3 +- src/H5Olayout.c | 5 +- test/Makefile.am | 3 +- test/dsets.c | 298 +++++++++++++++++++++++++++++++--- 11 files changed, 804 insertions(+), 119 deletions(-) diff --git a/src/H5Dbtree.c b/src/H5Dbtree.c index 86a86cb..7db99d0 100644 --- a/src/H5Dbtree.c +++ b/src/H5Dbtree.c @@ -851,6 +851,7 @@ H5D_btree_idx_init(const H5D_chk_idx_info_t *idx_info) /* Check args */ HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); /* Allocate the shared structure */ @@ -891,6 +892,7 @@ H5D_btree_idx_create(const H5D_chk_idx_info_t *idx_info) /* Check args */ HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(!H5F_addr_defined(idx_info->layout->u.chunk.u.btree.addr)); @@ -957,6 +959,7 @@ H5D_btree_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.btree.addr)); HDassert(udata); @@ -996,6 +999,7 @@ H5D_btree_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(idx_info->layout->u.chunk.ndims > 0); HDassert(udata); @@ -1081,6 +1085,7 @@ H5D_btree_idx_iterate(const H5D_chk_idx_info_t *idx_info, HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.btree.addr)); HDassert(chunk_cb); @@ -1121,6 +1126,7 @@ H5D_btree_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t * HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.btree.addr)); HDassert(udata); @@ -1160,6 +1166,7 @@ H5D_btree_idx_delete(const H5D_chk_idx_info_t *idx_info) /* Sanity checks */ HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); /* Check if the index data structure has been allocated */ @@ -1214,9 +1221,11 @@ H5D_btree_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, HDassert(idx_info_src); HDassert(idx_info_src->f); + HDassert(idx_info_src->pline); HDassert(idx_info_src->layout); HDassert(idx_info_dst); HDassert(idx_info_dst->f); + HDassert(idx_info_dst->pline); HDassert(idx_info_dst->layout); HDassert(!H5F_addr_defined(idx_info_dst->layout->u.chunk.u.btree.addr)); @@ -1296,6 +1305,7 @@ H5D_btree_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size) /* Check args */ HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(index_size); @@ -1373,6 +1383,7 @@ H5D_btree_idx_dump(const H5D_chk_idx_info_t *idx_info, FILE *stream) HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(stream); @@ -1403,6 +1414,7 @@ H5D_btree_idx_dest(const H5D_chk_idx_info_t *idx_info) HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); /* Free the raw B-tree node buffer */ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 9e04e7f..8318af1 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -1852,6 +1852,7 @@ H5D_chunk_init(H5F_t *f, hid_t dapl_id, hid_t dxpl_id, const H5D_t *dset) /* Compose chunked index info struct */ idx_info.f = f; idx_info.dxpl_id = dxpl_id; + idx_info.pline = &dset->shared->dcpl_cache.pline; idx_info.layout = &dset->shared->layout; /* Allocate any indexing structures */ @@ -2056,6 +2057,7 @@ H5D_chunk_create(H5D_t *dset /*in,out*/, hid_t dxpl_id) /* Compose chunked index info struct */ idx_info.f = dset->oloc.file; idx_info.dxpl_id = dxpl_id; + idx_info.pline = &dset->shared->dcpl_cache.pline; idx_info.layout = &dset->shared->layout; /* Create the index for the chunks */ @@ -2114,6 +2116,7 @@ H5D_chunk_get_info(const H5D_t *dset, hid_t dxpl_id, const hsize_t *chunk_offset /* Compose chunked index info struct */ idx_info.f = dset->oloc.file; idx_info.dxpl_id = dxpl_id; + idx_info.pline = &dset->shared->dcpl_cache.pline; idx_info.layout = &dset->shared->layout; /* Go get the chunk information */ @@ -2225,6 +2228,7 @@ H5D_chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t * /* Compose chunked index info struct */ idx_info.f = dset->oloc.file; idx_info.dxpl_id = dxpl_id; + idx_info.pline = &dset->shared->dcpl_cache.pline; idx_info.layout = &dset->shared->layout; /* Create the chunk it if it doesn't exist, or reallocate the chunk @@ -2460,7 +2464,7 @@ done: * directly into the chunk cache and should not be freed * by the caller but will be valid until it is unlocked. The * input value IDX_HINT is used to speed up cache lookups and - * it's output value should be given to H5F_chunk_unlock(). + * it's output value should be given to H5D_chunk_unlock(). * IDX_HINT is ignored if it is out of range, and if it points * to the wrong entry then we fall back to the normal search * method. @@ -2767,18 +2771,18 @@ H5D_chunk_unlock(const H5D_io_info_t *io_info, hbool_t dirty, unsigned idx_hint, * don't discard the `const' qualifier. */ if(dirty) { - H5D_rdcc_ent_t x; + H5D_rdcc_ent_t ent; /* "fake" chunk cache entry */ - HDmemset(&x, 0, sizeof(x)); - x.dirty = TRUE; - HDmemcpy(x.offset, io_info->store->chunk.offset, layout->u.chunk.ndims * sizeof(x.offset[0])); + HDmemset(&ent, 0, sizeof(ent)); + ent.dirty = TRUE; + HDmemcpy(ent.offset, io_info->store->chunk.offset, layout->u.chunk.ndims * sizeof(ent.offset[0])); HDassert(layout->u.chunk.size > 0); - x.chunk_addr = HADDR_UNDEF; - x.chunk_size = layout->u.chunk.size; - H5_ASSIGN_OVERFLOW(x.alloc_size, x.chunk_size, uint32_t, size_t); - x.chunk = (uint8_t *)chunk; + ent.chunk_addr = HADDR_UNDEF; + ent.chunk_size = layout->u.chunk.size; + H5_ASSIGN_OVERFLOW(ent.alloc_size, ent.chunk_size, uint32_t, size_t); + ent.chunk = (uint8_t *)chunk; - if(H5D_chunk_flush_entry(io_info->dset, io_info->dxpl_id, io_info->dxpl_cache, &x, TRUE) < 0) + if(H5D_chunk_flush_entry(io_info->dset, io_info->dxpl_id, io_info->dxpl_cache, &ent, TRUE) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") } /* end if */ else { @@ -2934,6 +2938,7 @@ H5D_chunk_allocated(H5D_t *dset, hid_t dxpl_id, hsize_t *nbytes) /* Compose chunked index info struct */ idx_info.f = dset->oloc.file; idx_info.dxpl_id = dxpl_id; + idx_info.pline = &dset->shared->dcpl_cache.pline; idx_info.layout = &dset->shared->layout; /* Iterate over the chunks */ @@ -3083,6 +3088,7 @@ H5D_chunk_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) /* Compose chunked index info struct */ idx_info.f = dset->oloc.file; idx_info.dxpl_id = dxpl_id; + idx_info.pline = &dset->shared->dcpl_cache.pline; idx_info.layout = &dset->shared->layout; /* Reset the chunk offset indices */ @@ -3440,7 +3446,7 @@ done: (void)H5FL_FREE(H5D_chunk_sl_ck_t, sl_node); FUNC_LEAVE_NOAPI(ret_value) -} /* H5D_chunk_prune_sl_rm_cb() */ +} /* H5D_chunk_prune_sl_rm_cb() */ /*------------------------------------------------------------------------- @@ -3638,6 +3644,7 @@ H5D_chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, const hsize_t *old_dims) /* Compose chunked index info struct */ idx_info.f = dset->oloc.file; idx_info.dxpl_id = dxpl_id; + idx_info.pline = &dset->shared->dcpl_cache.pline; idx_info.layout = &dset->shared->layout; /* Initialize the user data for the iteration */ @@ -3787,22 +3794,38 @@ done: *------------------------------------------------------------------------- */ herr_t -H5D_chunk_delete(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout) +H5D_chunk_delete(H5F_t *f, hid_t dxpl_id, H5O_t *oh, H5O_layout_t *layout) { H5D_chk_idx_info_t idx_info; /* Chunked index info */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_pline_t pline; /* I/O pipeline message */ + htri_t exists; /* Flag if header message of interest exists */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5D_chunk_delete, FAIL) /* Sanity check */ + HDassert(f); + HDassert(oh); + HDassert(layout); HDassert((H5D_CHUNK_EARRAY == layout->u.chunk.idx_type && H5D_COPS_EARRAY == layout->u.chunk.ops) || (H5D_CHUNK_BTREE == layout->u.chunk.idx_type && H5D_COPS_BTREE == layout->u.chunk.ops)); + /* Check for I/O pipeline message */ + if((exists = H5O_msg_exists_oh(oh, H5O_PLINE_ID)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to read object header") + else if(exists) { + if(NULL == H5O_msg_read_oh(f, dxpl_id, oh, H5O_PLINE_ID, &pline)) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't find I/O pipeline message") + } /* end else if */ + else + HDmemset(&pline, 0, sizeof(pline)); + /* Compose chunked index info struct */ idx_info.f = f; idx_info.dxpl_id = dxpl_id; + idx_info.pline = &pline; idx_info.layout = layout; /* Delete the chunked storage information in the file */ @@ -4100,11 +4123,13 @@ done: herr_t H5D_chunk_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, H5O_layout_t *layout_dst, H5T_t *dt_src, H5O_copy_t *cpy_info, - H5O_pline_t *pline, hid_t dxpl_id) + H5O_pline_t *pline_src, hid_t dxpl_id) { H5D_chunk_it_ud3_t udata; /* User data for iteration callback */ H5D_chk_idx_info_t idx_info_dst; /* Dest. chunked index info */ H5D_chk_idx_info_t idx_info_src; /* Source chunked index info */ + H5O_pline_t _pline; /* Temporary pipeline info */ + H5O_pline_t *pline; /* Pointer to pipeline info to use */ H5T_path_t *tpath_src_mem = NULL, *tpath_mem_dst = NULL; /* Datatype conversion paths */ hid_t tid_src = -1; /* Datatype ID for source datatype */ hid_t tid_dst = -1; /* Datatype ID for destination datatype */ @@ -4138,13 +4163,23 @@ H5D_chunk_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, H5D_COPS_BTREE == layout_dst->u.chunk.ops)); HDassert(dt_src); + /* Initialize the temporary pipeline info */ + if(NULL == pline_src) { + HDmemset(&_pline, 0, sizeof(_pline)); + pline = &_pline; + } /* end if */ + else + pline = pline_src; + /* Compose source & dest chunked index info structs */ idx_info_src.f = f_src; idx_info_src.dxpl_id = dxpl_id; + idx_info_src.pline = pline; idx_info_src.layout = layout_src; idx_info_dst.f = f_dst; idx_info_dst.dxpl_id = dxpl_id; + idx_info_dst.pline = pline; /* Use same I/O filter pipeline for dest. */ idx_info_dst.layout = layout_dst; /* Call the index-specific "copy setup" routine */ @@ -4323,7 +4358,7 @@ done: */ herr_t H5D_chunk_bh_info(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout, - hsize_t *index_size) + const H5O_pline_t *pline, hsize_t *index_size) { H5D_chk_idx_info_t idx_info; /* Chunked index info */ herr_t ret_value = SUCCEED; /* Return value */ @@ -4337,11 +4372,13 @@ H5D_chunk_bh_info(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout, H5D_COPS_EARRAY == layout->u.chunk.ops) || (H5D_CHUNK_BTREE == layout->u.chunk.idx_type && H5D_COPS_BTREE == layout->u.chunk.ops)); + HDassert(pline); HDassert(index_size); /* Compose chunked index info struct */ idx_info.f = f; idx_info.dxpl_id = dxpl_id; + idx_info.pline = pline; idx_info.layout = layout; /* Get size of index structure */ @@ -4435,6 +4472,7 @@ H5D_chunk_dump_index(H5D_t *dset, hid_t dxpl_id, FILE *stream) /* Compose chunked index info struct */ idx_info.f = dset->oloc.file; idx_info.dxpl_id = dxpl_id; + idx_info.pline = &dset->shared->dcpl_cache.pline; idx_info.layout = &dset->shared->layout; /* Display info for index */ @@ -4510,6 +4548,7 @@ H5D_chunk_dest(H5F_t *f, hid_t dxpl_id, H5D_t *dset) /* Compose chunked index info struct */ idx_info.f = f; idx_info.dxpl_id = dxpl_id; + idx_info.pline = &dset->shared->dcpl_cache.pline; idx_info.layout = &dset->shared->layout; /* Free any index structures */ diff --git a/src/H5Dearray.c b/src/H5Dearray.c index 02747c5..03e3434 100644 --- a/src/H5Dearray.c +++ b/src/H5Dearray.c @@ -48,6 +48,7 @@ /* Value to fill unset array elements with */ #define H5D_EARRAY_FILL HADDR_UNDEF +#define H5D_EARRAY_FILT_FILL {HADDR_UNDEF, 0, 0} /* Extensible array creation values */ #define H5D_EARRAY_MAX_NELMTS_BITS 32 /* i.e. 4 giga-elements */ @@ -72,12 +73,19 @@ typedef struct H5D_earray_ud_t { hid_t dxpl_id; /* DXPL ID for operation */ } H5D_earray_ud_t; +/* Native extensible array element for chunks w/filters */ +typedef struct H5D_earray_filt_elmt_t { + haddr_t addr; /* Address of chunk */ + uint32_t nbytes; /* Size of chunk (in file) */ + uint32_t filter_mask; /* Excluded filters for chunk */ +} H5D_earray_filt_elmt_t; + /********************/ /* Local Prototypes */ /********************/ -/* Extensible array class callbacks */ +/* Extensible array class callbacks for chunks w/o filters */ static void *H5D_earray_crt_context(const H5F_t *f); static herr_t H5D_earray_dst_context(void *ctx); static herr_t H5D_earray_fill(void *nat_blk, size_t nelmts); @@ -88,6 +96,16 @@ static herr_t H5D_earray_decode(const void *raw, void *elmt, size_t nelmts, static herr_t H5D_earray_debug(FILE *stream, int indent, int fwidth, hsize_t idx, const void *elmt); +/* Extensible array class callbacks for chunks w/filters */ +/* (some shared with callbacks for chunks w/o filters) */ +static herr_t H5D_earray_filt_fill(void *nat_blk, size_t nelmts); +static herr_t H5D_earray_filt_encode(void *raw, const void *elmt, size_t nelmts, + void *ctx); +static herr_t H5D_earray_filt_decode(const void *raw, void *elmt, size_t nelmts, + void *ctx); +static herr_t H5D_earray_filt_debug(FILE *stream, int indent, int fwidth, + hsize_t idx, const void *elmt); + /* Chunked layout indexing callbacks */ static herr_t H5D_earray_idx_create(const H5D_chk_idx_info_t *idx_info); static hbool_t H5D_earray_idx_is_space_alloc(const H5O_layout_t *layout); @@ -144,7 +162,7 @@ const H5D_chunk_ops_t H5D_COPS_EARRAY[1] = {{ /* Local Variables */ /*******************/ -/* Extensible array class callbacks for dataset chunks */ +/* Extensible array class callbacks for dataset chunks w/o filters */ const H5EA_class_t H5EA_CLS_CHUNK[1]={{ H5EA_CLS_CHUNK_ID, /* Type of extensible array */ sizeof(haddr_t), /* Size of native element */ @@ -156,6 +174,18 @@ const H5EA_class_t H5EA_CLS_CHUNK[1]={{ H5D_earray_debug /* Element debugging callback */ }}; +/* Extensible array class callbacks for dataset chunks w/filters */ +const H5EA_class_t H5EA_CLS_FILT_CHUNK[1]={{ + H5EA_CLS_FILT_CHUNK_ID, /* Type of extensible array */ + sizeof(H5D_earray_filt_elmt_t), /* Size of native element */ + H5D_earray_crt_context, /* Create context */ + H5D_earray_dst_context, /* Destroy context */ + H5D_earray_filt_fill, /* Fill block of missing elements callback */ + H5D_earray_filt_encode, /* Element encoding callback */ + H5D_earray_filt_decode, /* Element decoding callback */ + H5D_earray_filt_debug /* Element debugging callback */ +}}; + /* Declare a free list to manage the H5D_earray_ctx_t struct */ H5FL_DEFINE_STATIC(H5D_earray_ctx_t); @@ -383,6 +413,165 @@ H5D_earray_debug(FILE *stream, int indent, int fwidth, hsize_t idx, /*------------------------------------------------------------------------- + * Function: H5D_earray_filt_fill + * + * Purpose: Fill "missing elements" in block of elements + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Saturday, January 31, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_filt_fill(void *nat_blk, size_t nelmts) +{ + H5D_earray_filt_elmt_t fill_val = H5D_EARRAY_FILT_FILL; /* Value to fill elements with */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_earray_filt_fill) + + /* Sanity checks */ + HDassert(nat_blk); + HDassert(nelmts); + HDassert(sizeof(fill_val) == H5EA_CLS_FILT_CHUNK->nat_elmt_size); + + H5V_array_fill(nat_blk, &fill_val, H5EA_CLS_FILT_CHUNK->nat_elmt_size, nelmts); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_earray_filt_fill() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_filt_encode + * + * Purpose: Encode an element from "native" to "raw" form + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Saturday, January 31, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_filt_encode(void *_raw, const void *_elmt, size_t nelmts, void *_ctx) +{ + H5D_earray_ctx_t *ctx = (H5D_earray_ctx_t *)_ctx; /* Extensible array callback context */ + uint8_t *raw = (uint8_t *)_raw; /* Convenience pointer to raw elements */ + const H5D_earray_filt_elmt_t *elmt = (const H5D_earray_filt_elmt_t *)_elmt; /* Convenience pointer to native elements */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_earray_filt_encode) + + /* Sanity checks */ + HDassert(raw); + HDassert(elmt); + HDassert(nelmts); + HDassert(ctx); + + /* Encode native elements into raw elements */ + while(nelmts) { + /* Encode element */ + /* (advances 'raw' pointer */ + H5F_addr_encode_len(ctx->file_addr_len, &raw, elmt->addr); + UINT32ENCODE(raw, elmt->nbytes); + UINT32ENCODE(raw, elmt->filter_mask); + + /* Advance native element pointer */ + elmt++; + + /* Decrement # of elements to encode */ + nelmts--; + } /* end while */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_earray_filt_encode() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_filt_decode + * + * Purpose: Decode an element from "raw" to "native" form + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Saturday, January 31, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_filt_decode(const void *_raw, void *_elmt, size_t nelmts, void *_ctx) +{ + H5D_earray_ctx_t *ctx = (H5D_earray_ctx_t *)_ctx; /* Extensible array callback context */ + H5D_earray_filt_elmt_t *elmt = (H5D_earray_filt_elmt_t *)_elmt; /* Convenience pointer to native elements */ + const uint8_t *raw = (const uint8_t *)_raw; /* Convenience pointer to raw elements */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_earray_filt_decode) + + /* Sanity checks */ + HDassert(raw); + HDassert(elmt); + HDassert(nelmts); + + /* Decode raw elements into native elements */ + while(nelmts) { + /* Decode element */ + /* (advances 'raw' pointer */ + H5F_addr_decode_len(ctx->file_addr_len, &raw, &elmt->addr); + UINT32DECODE(raw, elmt->nbytes); + UINT32DECODE(raw, elmt->filter_mask); + + /* Advance native element pointer */ + elmt++; + + /* Decrement # of elements to decode */ + nelmts--; + } /* end while */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_earray_filt_decode() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_filt_debug + * + * Purpose: Display an element for debugging + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Saturday, January 31, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_filt_debug(FILE *stream, int indent, int fwidth, hsize_t idx, + const void *_elmt) +{ + const H5D_earray_filt_elmt_t *elmt = (const H5D_earray_filt_elmt_t *)_elmt; /* Convenience pointer to native elements */ + char temp_str[128]; /* Temporary string, for formatting */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_earray_filt_debug) + + /* Sanity checks */ + HDassert(stream); + HDassert(elmt); + + /* Print element */ + sprintf(temp_str, "Element #%llu:", (unsigned long_long)idx); + HDfprintf(stream, "%*s%-*s {%a, %u, %0x}\n", indent, "", fwidth, temp_str, + elmt->addr, elmt->nbytes, elmt->filter_mask); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_earray_filt_debug() */ + + +/*------------------------------------------------------------------------- * Function: H5D_earray_idx_open * * Purpose: Opens an existing extensible array and initializes @@ -399,6 +588,7 @@ H5D_earray_debug(FILE *stream, int indent, int fwidth, hsize_t idx, static herr_t H5D_earray_idx_open(const H5D_chk_idx_info_t *idx_info) { + const H5EA_class_t *cls; /* Extensible array class to use */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_open) @@ -406,13 +596,15 @@ H5D_earray_idx_open(const H5D_chk_idx_info_t *idx_info) /* Check args */ HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(H5D_CHUNK_EARRAY == idx_info->layout->u.chunk.idx_type); HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); HDassert(NULL == idx_info->layout->u.chunk.u.earray.ea); /* Open the extensible array for the chunk index */ - if(NULL == (idx_info->layout->u.chunk.u.earray.ea = H5EA_open(idx_info->f, idx_info->dxpl_id, idx_info->layout->u.chunk.u.earray.addr, H5EA_CLS_CHUNK))) + cls = (idx_info->pline->nused > 0) ? H5EA_CLS_FILT_CHUNK : H5EA_CLS_CHUNK; + if(NULL == (idx_info->layout->u.chunk.u.earray.ea = H5EA_open(idx_info->f, idx_info->dxpl_id, idx_info->layout->u.chunk.u.earray.addr, cls))) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open extensible array") done: @@ -449,13 +641,20 @@ H5D_earray_idx_create(const H5D_chk_idx_info_t *idx_info) /* Check args */ HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(!H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); HDassert(NULL == idx_info->layout->u.chunk.u.earray.ea); /* General parameters */ - cparam.cls = H5EA_CLS_CHUNK; - cparam.raw_elmt_size = (uint8_t)H5F_SIZEOF_ADDR(idx_info->f); + if(idx_info->pline->nused > 0) { + cparam.cls = H5EA_CLS_FILT_CHUNK; + cparam.raw_elmt_size = (uint8_t)(H5F_SIZEOF_ADDR(idx_info->f) + 4 + 4); + } /* end if */ + else { + cparam.cls = H5EA_CLS_CHUNK; + cparam.raw_elmt_size = (uint8_t)H5F_SIZEOF_ADDR(idx_info->f); + } /* end else */ cparam.max_nelmts_bits = H5D_EARRAY_MAX_NELMTS_BITS; cparam.idx_blk_elmts = H5D_EARRAY_IDX_BLK_ELMTS; cparam.sup_blk_min_data_ptrs = H5D_EARRAY_SUP_BLK_MIN_DATA_PTRS; @@ -510,9 +709,6 @@ H5D_earray_idx_is_space_alloc(const H5O_layout_t *layout) * Purpose: Create the chunk it if it doesn't exist, or reallocate the * chunk if its size changed. * - * Note: Doesn't support reallocating chunks that change size yet - * (as chunks w/filters aren't supported yet) - * * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol @@ -524,7 +720,6 @@ static herr_t H5D_earray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) { H5EA_t *ea; /* Pointer to extensible array structure */ - haddr_t addr = HADDR_UNDEF; /* Address of chunk in file */ hsize_t idx; /* Array index of chunk */ herr_t ret_value = SUCCEED; /* Return value */ @@ -532,6 +727,7 @@ H5D_earray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(idx_info->layout->u.chunk.ndims == 2); /* (for now) */ HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); @@ -550,33 +746,83 @@ H5D_earray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) /* Compute array index for chunk offset */ idx = udata->common.offset[0] / idx_info->layout->u.chunk.dim[0]; + /* Check for filters on chunks */ + if(idx_info->pline->nused > 0) { + H5D_earray_filt_elmt_t elmt; /* Extensible array element */ + hbool_t alloc_chunk = FALSE; /* Whether to allocate chunk */ + + /* Get the information for the chunk */ + if(H5EA_get(ea, idx_info->dxpl_id, idx, &elmt) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info") + + /* Check for previous chunk */ + if(H5F_addr_defined(elmt.addr)) { + /* Sanity check */ + HDassert(!H5F_addr_defined(udata->addr) || H5F_addr_eq(udata->addr, elmt.addr)); + + /* Check for chunk being same size */ + if(udata->nbytes != elmt.nbytes) { + /* Release previous chunk */ + H5_CHECK_OVERFLOW(elmt.nbytes, uint32_t, hsize_t); + if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, elmt.addr, (hsize_t)elmt.nbytes) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + elmt.addr = HADDR_UNDEF; + alloc_chunk = TRUE; + } /* end if */ + else { + /* Don't need to reallocate chunk, but send its address back up */ + if(!H5F_addr_defined(udata->addr)) + udata->addr = elmt.addr; + } /* end else */ + } /* end if */ + else + alloc_chunk = TRUE; + + /* Check if we need to allocate the chunk */ + if(alloc_chunk) { + H5_CHECK_OVERFLOW(udata->nbytes, uint32_t, hsize_t); + udata->addr = H5MF_alloc(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, (hsize_t)udata->nbytes); + if(!H5F_addr_defined(udata->addr)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk") + + /* Update the element information */ + elmt.addr = udata->addr; + elmt.nbytes = udata->nbytes; + elmt.filter_mask = udata->filter_mask; + + /* Set the info for the chunk */ + if(H5EA_set(ea, idx_info->dxpl_id, idx, &elmt) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set chunk info") + } /* end if */ + } /* end if */ + else { + HDassert(!H5F_addr_defined(udata->addr)); + HDassert(udata->nbytes == idx_info->layout->u.chunk.size); + #ifndef NDEBUG +{ + haddr_t addr = HADDR_UNDEF; /* Address of chunk in file */ + /* Get the address for the chunk */ if(H5EA_get(ea, idx_info->dxpl_id, idx, &addr) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") HDassert(!H5F_addr_defined(addr)); - addr = HADDR_UNDEF; +} #endif /* NDEBUG */ - /* - * Create the chunk it if it doesn't exist, or reallocate the chunk if - * its size changed. - */ - /* (not yet) */ - HDassert(!H5F_addr_defined(udata->addr)); - HDassert(udata->nbytes == idx_info->layout->u.chunk.size); - - /* - * Allocate storage for the new chunk - */ - H5_CHECK_OVERFLOW(udata->nbytes, /*From: */uint32_t, /*To: */hsize_t); - udata->addr = H5MF_alloc(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, (hsize_t)udata->nbytes); - if(!H5F_addr_defined(udata->addr)) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "file allocation failed") - - /* Set the address for the chunk */ - if(H5EA_set(ea, idx_info->dxpl_id, idx, &udata->addr) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + /* + * Allocate storage for the new chunk + */ + H5_CHECK_OVERFLOW(udata->nbytes, /*From: */uint32_t, /*To: */hsize_t); + udata->addr = H5MF_alloc(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, (hsize_t)udata->nbytes); + if(!H5F_addr_defined(udata->addr)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "file allocation failed") + + /* Set the address for the chunk */ + if(H5EA_set(ea, idx_info->dxpl_id, idx, &udata->addr) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set chunk address") + } /* end else */ + HDassert(H5F_addr_defined(udata->addr)); done: FUNC_LEAVE_NOAPI(ret_value) @@ -584,7 +830,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5D_earraye_idx_get_addr + * Function: H5D_earray_idx_get_addr * * Purpose: Get the file address of a chunk if file space has been * assigned. Save the retrieved information in the udata @@ -608,6 +854,7 @@ H5D_earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(idx_info->layout->u.chunk.ndims == 2); /* (for now) */ HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); @@ -626,9 +873,28 @@ H5D_earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat /* Compute array index for chunk offset */ idx = udata->common.offset[0] / idx_info->layout->u.chunk.dim[0]; - /* Get the address for the chunk */ - if(H5EA_get(ea, idx_info->dxpl_id, idx, &udata->addr) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + /* Check for filters on chunks */ + if(idx_info->pline->nused > 0) { + H5D_earray_filt_elmt_t elmt; /* Extensible array element */ + + /* Get the information for the chunk */ + if(H5EA_get(ea, idx_info->dxpl_id, idx, &elmt) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info") + + /* Set the info for the chunk */ + udata->addr = elmt.addr; + udata->nbytes = elmt.nbytes; + udata->filter_mask = elmt.filter_mask; + } /* end if */ + else { + /* Get the address for the chunk */ + if(H5EA_get(ea, idx_info->dxpl_id, idx, &udata->addr) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + + /* Update the other (constant) information for the chunk */ + udata->nbytes = idx_info->layout->u.chunk.size; + udata->filter_mask = 0; + } /* end else */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -664,6 +930,7 @@ H5D_earray_idx_iterate(const H5D_chk_idx_info_t *idx_info, HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); HDassert(chunk_cb); @@ -688,31 +955,59 @@ H5D_earray_idx_iterate(const H5D_chk_idx_info_t *idx_info, H5D_chunk_rec_t chunk_rec; /* Generic chunk record for callback */ hsize_t u; /* Local index variable */ - /* Prepare common fields of chunk record for callback */ - /* (Note: these are not common for chunks w/filters) */ - HDmemset(&chunk_rec, 0, sizeof(chunk_rec)); - chunk_rec.nbytes = idx_info->layout->u.chunk.size; - chunk_rec.filter_mask = 0; - - /* Loop over array elements */ - /* (Note: this is too simple for datasets with >1 dimension) */ - for(u = 0; u < ea_stat.max_idx_set; u++, chunk_rec.offset[0] += idx_info->layout->u.chunk.dim[0]) { - haddr_t addr; /* Chunk address */ - - /* Get the address of the chunk for the index */ - if(H5EA_get(ea, idx_info->dxpl_id, u, &addr) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") - - /* Check if chunk exists */ - if(H5F_addr_defined(addr)) { - /* Set chunk address for callback record */ - chunk_rec.chunk_addr = addr; - - /* Make chunk callback */ - if((ret_value = (*chunk_cb)(&chunk_rec, chunk_udata)) < 0) - HERROR(H5E_DATASET, H5E_CALLBACK, "failure in generic chunk iterator callback"); - } /* end if */ - } /* end for */ + /* Check for filters on chunks */ + if(idx_info->pline->nused > 0) { + /* Prepare common fields of chunk record for callback */ + HDmemset(&chunk_rec, 0, sizeof(chunk_rec)); + + /* Loop over array elements */ + /* (Note: this may be too simple for datasets with >1 dimension) */ + for(u = 0; u < ea_stat.max_idx_set; u++, chunk_rec.offset[0] += idx_info->layout->u.chunk.dim[0]) { + H5D_earray_filt_elmt_t elmt; /* Extensible array element */ + + /* Get the info about the chunk for the index */ + if(H5EA_get(ea, idx_info->dxpl_id, u, &elmt) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info") + + /* Check if chunk exists */ + if(H5F_addr_defined(elmt.addr)) { + /* Set chunk info for callback record */ + chunk_rec.chunk_addr = elmt.addr; + chunk_rec.nbytes = elmt.nbytes; + chunk_rec.filter_mask = elmt.filter_mask; + + /* Make chunk callback */ + if((ret_value = (*chunk_cb)(&chunk_rec, chunk_udata)) < 0) + HERROR(H5E_DATASET, H5E_CALLBACK, "failure in generic chunk iterator callback"); + } /* end if */ + } /* end for */ + } /* end if */ + else { + /* Prepare common fields of chunk record for callback */ + HDmemset(&chunk_rec, 0, sizeof(chunk_rec)); + chunk_rec.nbytes = idx_info->layout->u.chunk.size; + chunk_rec.filter_mask = 0; + + /* Loop over array elements */ + /* (Note: this may be too simple for datasets with >1 dimension) */ + for(u = 0; u < ea_stat.max_idx_set; u++, chunk_rec.offset[0] += idx_info->layout->u.chunk.dim[0]) { + haddr_t addr; /* Chunk address */ + + /* Get the address of the chunk for the index */ + if(H5EA_get(ea, idx_info->dxpl_id, u, &addr) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + + /* Check if chunk exists */ + if(H5F_addr_defined(addr)) { + /* Set chunk address for callback record */ + chunk_rec.chunk_addr = addr; + + /* Make chunk callback */ + if((ret_value = (*chunk_cb)(&chunk_rec, chunk_udata)) < 0) + HERROR(H5E_DATASET, H5E_CALLBACK, "failure in generic chunk iterator callback"); + } /* end if */ + } /* end for */ + } /* end else */ } /* end if */ done: @@ -736,7 +1031,6 @@ static herr_t H5D_earray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata) { H5EA_t *ea; /* Pointer to extensible array structure */ - haddr_t addr = HADDR_UNDEF; /* Chunk address */ hsize_t idx; /* Array index of chunk */ herr_t ret_value = SUCCEED; /* Return value */ @@ -744,6 +1038,7 @@ H5D_earray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); HDassert(udata); @@ -761,19 +1056,45 @@ H5D_earray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t /* Compute array index for chunk offset */ idx = udata->offset[0] / idx_info->layout->u.chunk.dim[0]; - /* Get the address of the chunk for the index */ - if(H5EA_get(ea, idx_info->dxpl_id, idx, &addr) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + /* Check for filters on chunks */ + if(idx_info->pline->nused > 0) { + H5D_earray_filt_elmt_t elmt; /* Extensible array element */ + + /* Get the info about the chunk for the index */ + if(H5EA_get(ea, idx_info->dxpl_id, idx, &elmt) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info") + + /* Remove raw data chunk from file */ + HDassert(H5F_addr_defined(elmt.addr)); + H5_CHECK_OVERFLOW(elmt.nbytes, /*From: */uint32_t, /*To: */hsize_t); + if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, elmt.addr, (hsize_t)elmt.nbytes) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + + /* Reset the info about the chunk for the index */ + elmt.addr = HADDR_UNDEF; + elmt.nbytes = 0; + elmt.filter_mask = 0; + if(H5EA_set(ea, idx_info->dxpl_id, idx, &elmt) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to reset chunk info") + } /* end if */ + else { + haddr_t addr = HADDR_UNDEF; /* Chunk address */ - /* Remove raw data chunk from file */ - H5_CHECK_OVERFLOW(idx_info->layout->u.chunk.size, /*From: */uint32_t, /*To: */hsize_t); - if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, addr, (hsize_t)idx_info->layout->u.chunk.size) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + /* Get the address of the chunk for the index */ + if(H5EA_get(ea, idx_info->dxpl_id, idx, &addr) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + + /* Remove raw data chunk from file */ + HDassert(H5F_addr_defined(addr)); + H5_CHECK_OVERFLOW(idx_info->layout->u.chunk.size, /*From: */uint32_t, /*To: */hsize_t); + if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, addr, (hsize_t)idx_info->layout->u.chunk.size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") - /* Reset the address of the chunk for the index */ - addr = HADDR_UNDEF; - if(H5EA_set(ea, idx_info->dxpl_id, idx, &addr) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to reset chunk address") + /* Reset the address of the chunk for the index */ + addr = HADDR_UNDEF; + if(H5EA_set(ea, idx_info->dxpl_id, idx, &addr) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to reset chunk address") + } /* end else */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -846,6 +1167,7 @@ H5D_earray_idx_delete(const H5D_chk_idx_info_t *idx_info) /* Sanity checks */ HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); /* Check if the index data structure has been allocated */ @@ -901,9 +1223,11 @@ H5D_earray_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, HDassert(idx_info_src); HDassert(idx_info_src->f); + HDassert(idx_info_src->pline); HDassert(idx_info_src->layout); HDassert(idx_info_dst); HDassert(idx_info_dst->f); + HDassert(idx_info_dst->pline); HDassert(idx_info_dst->layout); HDassert(!H5F_addr_defined(idx_info_dst->layout->u.chunk.u.earray.addr)); @@ -990,6 +1314,7 @@ H5D_earray_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size) /* Check args */ HDassert(idx_info); HDassert(idx_info->f); + HDassert(idx_info->pline); HDassert(idx_info->layout); HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); HDassert(index_size); diff --git a/src/H5Doh.c b/src/H5Doh.c index 039f055..93fd21a 100644 --- a/src/H5Doh.c +++ b/src/H5Doh.c @@ -373,12 +373,26 @@ H5O_dset_bh_info(H5F_t *f, hid_t dxpl_id, H5O_t *oh, H5_ih_info_t *bh_info) /* Get the layout message from the object header */ if(NULL == H5O_msg_read_oh(f, dxpl_id, oh, H5O_LAYOUT_ID, &layout)) - HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't find LAYOUT message") + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't find layout message") /* Check for chunked dataset storage */ - if(layout.type == H5D_CHUNKED && H5D_chunk_is_space_alloc(&layout)) - if(H5D_chunk_bh_info(f, dxpl_id, &layout, &(bh_info->index_size)) < 0) + if(layout.type == H5D_CHUNKED && H5D_chunk_is_space_alloc(&layout)) { + H5O_pline_t pline; /* I/O pipeline message */ + htri_t exists; /* Flag if header message of interest exists */ + + /* Check for I/O pipeline message */ + if((exists = H5O_msg_exists_oh(oh, H5O_PLINE_ID)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to read object header") + else if(exists) { + if(NULL == H5O_msg_read_oh(f, dxpl_id, oh, H5O_PLINE_ID, &pline)) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't find I/O pipeline message") + } /* end else if */ + else + HDmemset(&pline, 0, sizeof(pline)); + + if(H5D_chunk_bh_info(f, dxpl_id, &layout, &pline, &(bh_info->index_size)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine chunked dataset btree info") + } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 0d8ca8c..5cef2fe 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -215,6 +215,7 @@ typedef struct H5D_io_info_t { typedef struct H5D_chk_idx_info_t { H5F_t *f; /* File pointer for operation */ hid_t dxpl_id; /* DXPL ID for operation */ + const H5O_pline_t *pline; /* I/O pipeline info */ H5O_layout_t *layout; /* Layout info for chunks */ } H5D_chk_idx_info_t; @@ -597,7 +598,7 @@ H5_DLL herr_t H5D_chunk_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, H5O_layout_t *layout_dst, H5T_t *src_dtype, H5O_copy_t *cpy_info, H5O_pline_t *pline, hid_t dxpl_id); H5_DLL herr_t H5D_chunk_bh_info(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout, - hsize_t *btree_size); + const H5O_pline_t *pline, hsize_t *btree_size); H5_DLL herr_t H5D_chunk_dump_index(H5D_t *dset, hid_t dxpl_id, FILE *stream); H5_DLL herr_t H5D_chunk_dest(H5F_t *f, hid_t dxpl_id, H5D_t *dset); #ifdef H5D_CHUNK_DEBUG @@ -668,6 +669,7 @@ H5_DLL htri_t H5D_mpio_opt_possible(const H5D_io_info_t *io_info, #ifdef H5D_TESTING H5_DLL herr_t H5D_layout_version_test(hid_t did, unsigned *version); H5_DLL herr_t H5D_layout_contig_size_test(hid_t did, hsize_t *size); +H5_DLL herr_t H5D_layout_idx_type_test(hid_t did, H5D_chunk_index_t *idx_type); #endif /* H5D_TESTING */ #endif /*_H5Dpkg_H*/ diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index 7c78d71..7327d0d 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -167,7 +167,8 @@ H5_DLL herr_t H5D_contig_delete(H5F_t *f, hid_t dxpl_id, /* Functions that operate on chunked storage */ H5_DLL herr_t H5D_chunk_idx_reset(H5O_layout_t *layout); -H5_DLL herr_t H5D_chunk_delete(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout); +H5_DLL herr_t H5D_chunk_delete(H5F_t *f, hid_t dxpl_id, H5O_t *oh, + H5O_layout_t *layout); /* Functions that operate on indexed storage */ H5_DLL herr_t H5D_btree_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, diff --git a/src/H5Dtest.c b/src/H5Dtest.c index f7124f0..b71ba30 100644 --- a/src/H5Dtest.c +++ b/src/H5Dtest.c @@ -135,3 +135,44 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5D_layout_contig_size_test() */ + +/*-------------------------------------------------------------------------- + NAME + H5D_layout_idx_type_test + PURPOSE + Determine the storage layout index type for a dataset's layout information + USAGE + herr_t H5D_layout_idx_type_test(did, idx_type) + hid_t did; IN: Dataset to query + H5D_chunk_index_t *idx_type; OUT: Pointer to location to place index type info + RETURNS + Non-negative on success, negative on failure + DESCRIPTION + Checks the index type of the storage layout information for a dataset. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + DO NOT USE THIS FUNCTION FOR ANYTHING EXCEPT TESTING + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5D_layout_idx_type_test(hid_t did, H5D_chunk_index_t *idx_type) +{ + H5D_t *dset; /* Pointer to dataset to query */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_NOAPI(H5D_layout_idx_type_test, FAIL) + + /* Check args */ + if(NULL == (dset = (H5D_t *)H5I_object_verify(did, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") + if(dset->shared->layout.type != H5D_CHUNKED) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "dataset is not chunked") + + if(idx_type) + *idx_type = dset->shared->layout.u.chunk.idx_type; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D_layout_idx_type_test() */ + diff --git a/src/H5EAprivate.h b/src/H5EAprivate.h index fcfccd0..ca30f73 100644 --- a/src/H5EAprivate.h +++ b/src/H5EAprivate.h @@ -48,7 +48,8 @@ /* Extensible array class IDs */ typedef enum H5EA_cls_id_t { - H5EA_CLS_CHUNK_ID = 0, /* Extensible array is for indexing dataset chunks */ + H5EA_CLS_CHUNK_ID = 0, /* Extensible array is for indexing dataset chunks w/o filters */ + H5EA_CLS_FILT_CHUNK_ID, /* Extensible array is for indexing dataset chunks w/filters */ /* (keep these last) */ H5EA_CLS_TEST_ID, /* Extensible array is for testing (do not use for actual data) */ diff --git a/src/H5Olayout.c b/src/H5Olayout.c index 9ca6ba6..1975173 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -590,7 +590,7 @@ H5O_layout_free(void *_mesg) *------------------------------------------------------------------------- */ static herr_t -H5O_layout_delete(H5F_t *f, hid_t dxpl_id, H5O_t UNUSED *open_oh, void *_mesg) +H5O_layout_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, void *_mesg) { H5O_layout_t *mesg = (H5O_layout_t *) _mesg; herr_t ret_value = SUCCEED; /* Return value */ @@ -599,6 +599,7 @@ H5O_layout_delete(H5F_t *f, hid_t dxpl_id, H5O_t UNUSED *open_oh, void *_mesg) /* check args */ HDassert(f); + HDassert(open_oh); HDassert(mesg); /* Perform different actions, depending on the type of storage */ @@ -615,7 +616,7 @@ H5O_layout_delete(H5F_t *f, hid_t dxpl_id, H5O_t UNUSED *open_oh, void *_mesg) case H5D_CHUNKED: /* Chunked blocks on disk */ /* Free the file space for the index & chunk raw data */ - if(H5D_chunk_delete(f, dxpl_id, mesg) < 0) + if(H5D_chunk_delete(f, dxpl_id, open_oh, mesg) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free raw data") break; diff --git a/test/Makefile.am b/test/Makefile.am index 1709d44..3a729fd 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -104,7 +104,8 @@ flush2.chkexe_: flush1.chkexe_ # the temporary file name in ways that the makefile is not aware of. CHECK_CLEANFILES+=cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offset.h5 \ max_compact_dataset.h5 simple.h5 set_local.h5 random_chunks.h5 \ - huge_chunks.h5 extend.h5 istore.h5 extlinks*.h5 frspace.h5 links*.h5 \ + huge_chunks.h5 chunk_cache.h5 chunk_fast.h5 extend.h5 istore.h5 \ + extlinks*.h5 frspace.h5 links*.h5 \ tfile[1-4].h5 th5s[1-3].h5 lheap.h5 fheap.h5 ohdr.h5 stab.h5 \ extern_[1-3].h5 extern_[1-4][ab].raw gheap[0-4].h5 dt_arith[1-2] \ links.h5 links[0-6]*.h5 extlinks[0-15].h5 tmp \ diff --git a/test/dsets.c b/test/dsets.c index e3bac54..95ac4a3 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -29,6 +29,13 @@ #endif /* + * This file needs to access private datatypes from the H5D package. + */ +#define H5D_PACKAGE +#define H5D_TESTING +#include "H5Dpkg.h" + +/* * This file needs to access private datatypes from the H5Z package. */ #define H5Z_PACKAGE @@ -44,6 +51,7 @@ const char *FILENAME[] = { "random_chunks", "huge_chunks", "chunk_cache", + "chunk_fast", NULL }; #define FILENAME_BUF_SIZE 1024 @@ -132,10 +140,12 @@ const char *FILENAME[] = { #define FILTER_HS_SIZE2 50 /* Names for noencoder test */ +#ifdef H5_HAVE_FILTER_SZIP #define NOENCODER_FILENAME "noencoder.h5" #define NOENCODER_TEST_DATASET "noencoder_tdset.h5" #define NOENCODER_SZIP_DATASET "noencoder_szip_dset.h5" #define NOENCODER_SZIP_SHUFF_FLETCH_DATASET "noencoder_szip_shuffle_fletcher_dset.h5" +#endif /* H5_HAVE_FILTER_SZIP */ /* Names for zero-dim test */ #define ZERODIM_DATASET "zerodim" @@ -959,9 +969,9 @@ test_tconv(hid_t file) hsize_t dims[1]; hid_t space, dataset; - out = HDmalloc((size_t)(4 * 1000 * 1000)); + out = (char *)HDmalloc((size_t)(4 * 1000 * 1000)); HDassert(out); - in = HDmalloc((size_t)(4 * 1000 * 1000)); + in = (char *)HDmalloc((size_t)(4 * 1000 * 1000)); HDassert(in); TESTING("data type conversion"); @@ -1116,7 +1126,7 @@ set_local_bogus2(hid_t dcpl_id, hid_t type_id, hid_t UNUSED space_id) return(FAIL); /* Set "local" parameters for this dataset */ - cd_values[2]=(add_on>0); /* Flag to indicate data is modified */ + cd_values[2]=(unsigned)(add_on>0); /* Flag to indicate data is modified */ cd_values[3]=add_on; /* Amount the data was modified by */ /* Modify the filter's parameters for this dataset */ @@ -1170,12 +1180,12 @@ filter_bogus2(unsigned int flags, size_t cd_nelmts, /* "Compressing" */ else { unsigned add_on=cd_values[3]; /* Get "add on" value */ - int *int_ptr=*buf; /* Pointer to the data values */ + int *int_ptr=(int *)*buf; /* Pointer to the data values */ size_t buf_left=*buf_size; /* Amount of data buffer left to process */ /* Add the "add on" value to all the data values */ while(buf_left>0) { - *int_ptr++ += add_on; + *int_ptr++ += (int)add_on; buf_left -= sizeof(int); } /* end while */ } /* end else */ @@ -2646,7 +2656,7 @@ test_nbit_int(hid_t file) /* Check that the values read are the same as the values written * Use mask for checking the significant bits, ignoring the padding bits */ - mask = ~(~0 << (precision + offset)) & (~0 << offset); + mask = ~((unsigned)~0 << (precision + offset)) & ((unsigned)~0 << offset); for(i=0; i<(size_t)size[0]; i++) { for(j=0; j<(size_t)size[1]; j++) { if((new_data[i][j] & mask) != (orig_data[i][j] & mask)) { @@ -3205,7 +3215,7 @@ test_nbit_compound(hid_t file) /* some even-numbered integer values are negtive */ if((i*size[1]+j+1)%2 == 0) { orig_data[i][j].i = -orig_data[i][j].i; - orig_data[i][j].s = -orig_data[i][j].s; + orig_data[i][j].s = (short)-orig_data[i][j].s; } } @@ -3247,9 +3257,9 @@ test_nbit_compound(hid_t file) /* Check that the values read are the same as the values written * Use mask for checking the significant bits, ignoring the padding bits */ - i_mask = ~(~0 << (precision[0] + offset[0])) & (~0 << offset[0]); - c_mask = ~(~0 << (precision[1] + offset[1])) & (~0 << offset[1]); - s_mask = ~(~0 << (precision[2] + offset[2])) & (~0 << offset[2]); + i_mask = ~((unsigned)~0 << (precision[0] + offset[0])) & ((unsigned)~0 << offset[0]); + c_mask = ~((unsigned)~0 << (precision[1] + offset[1])) & ((unsigned)~0 << offset[1]); + s_mask = ~((unsigned)~0 << (precision[2] + offset[2])) & ((unsigned)~0 << offset[2]); for(i=0; i