From f368891ac839f5b6e20d717b586fc14afa194f3b Mon Sep 17 00:00:00 2001 From: Pedro Vicente Nunes Date: Thu, 11 Apr 2002 16:28:59 -0500 Subject: [svn-r5168] Purpose: H5Dset_extent API function Description: 1 new public function: H5Dset_extent, that allows shrinking a dataset several private functions that free the previously allocated chunks H5F_istore_prune_by_extent, H5B_prune_by_extent : removes the chunks H5F_istore_initialize_by_extent : initializes them with the fill value H5S_select_fill is used to fill the selection in the memory buffer Platforms tested: Windows 2000 SUN( arabica) LINUX (eirene) IRIX64 (modi4) --- src/H5B.c | 110 ++++++++++ src/H5Bprivate.h | 10 + src/H5D.c | 288 +++++++++++++------------ src/H5Distore.c | 638 ++++++++++++++++++++++++++++++++++++++++++++++++++----- src/H5Dprivate.h | 4 +- src/H5Dpublic.h | 4 +- src/H5Fistore.c | 638 ++++++++++++++++++++++++++++++++++++++++++++++++++----- src/H5Fprivate.h | 10 + src/H5S.c | 8 +- src/H5Sall.c | 4 +- src/H5Shyper.c | 3 - src/H5Sprivate.h | 10 +- 12 files changed, 1482 insertions(+), 245 deletions(-) diff --git a/src/H5B.c b/src/H5B.c index 0adbca0..04993a1 100644 --- a/src/H5B.c +++ b/src/H5B.c @@ -2367,3 +2367,113 @@ H5B_assert(H5F_t *f, haddr_t addr, const H5B_class_t *type, void *udata) FUNC_LEAVE(SUCCEED); } #endif /* H5B_DEBUG */ + + +/*------------------------------------------------------------------------- + * Function: H5B_prune_by_extent + * + * Purpose: Search for chunks that are no longer necessary in the B-tree. + * The function iterates trough the B-tree and calls an operator function prune_extent + * + * Return: Success: 0, Failure: -1 + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: March 26, 2002 + * + * Comments: Private function + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +herr_t H5B_prune_by_extent( H5F_t *f, const H5B_class_t *type, haddr_t addr, void *udata, + hsize_t *size ) +{ + H5B_t *bt = NULL; + haddr_t next_addr; + haddr_t cur_addr = HADDR_UNDEF; + uint8_t *key = NULL; + int i, nchildren; + herr_t ret_value = FAIL; + + FUNC_ENTER( H5B_prune_by_extent, FAIL); + +/* + * Check arguments. + */ + assert( f ); + assert( type ); + assert( type->prune_extent); + assert( H5F_addr_defined( addr ) ); + assert( udata ); + + if ( NULL == ( bt = H5AC_find( f, H5AC_BT, addr, type, udata ))) + { + HGOTO_ERROR(H5E_BTREE, H5E_CANTLOAD, FAIL, "unable to load B-tree node" ); + } + + if ( bt->level > 0 ) + { + /* Keep following the left-most child until we reach a leaf node. */ + if (( ret_value = H5B_prune_by_extent(f, type, bt->child[0], udata, size )) < 0 ) + { + HGOTO_ERROR(H5E_BTREE, H5E_CANTLIST, FAIL, "unable to list B-tree node"); + } + } + else + { + +/* + * We've reached the left-most leaf. Now follow the right-sibling + * pointer from leaf to leaf until we've processed all leaves. + */ + + if ( NULL== ( key = H5MM_malloc((2*H5B_Kvalue(f, type)+1)*type->sizeof_nkey ))) + { + HGOTO_ERROR( H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed" ); + } + + for ( cur_addr = addr, ret_value = 0; H5F_addr_defined(cur_addr) && !ret_value; cur_addr = next_addr) + { + + /* + * Save all the native keys since we can't leave the B-tree node protected during an application callback. + */ + + if ( NULL == ( bt = H5AC_find( f, H5AC_BT, cur_addr, type, udata ))) + { + HGOTO_ERROR( H5E_BTREE, H5E_CANTLOAD, FAIL, "B-tree node" ); + } + + for ( i = 0; i < bt->nchildren+1; i++ ) + { + if ( !bt->key[i].nkey ) + H5B_decode_key(f, bt, i); + HDmemcpy( key+i*type->sizeof_nkey, bt->key[i].nkey, type->sizeof_nkey ); + } + + next_addr = bt->right; + nchildren = bt->nchildren; + bt = NULL; + + /* Figure out what chunks are no longer in use for the specified extent and release them */ + + for ( i = 0, ret_value = 0; i < nchildren && !ret_value; i++ ) + { + ret_value = ( type->prune_extent )( f, key+i*type->sizeof_nkey, addr, udata, size ); + if ( ret_value < 0 ) + { + HGOTO_ERROR( H5E_BTREE, H5E_CANTINIT, FAIL, "iterator function failed" ); + } + } /*i*/ + } /*addr*/ + } /*level*/ + +done: + if( key != NULL ) + H5MM_xfree( key ); + FUNC_LEAVE( ret_value ); +} + diff --git a/src/H5Bprivate.h b/src/H5Bprivate.h index 8cab893..e6fc2ac 100644 --- a/src/H5Bprivate.h +++ b/src/H5Bprivate.h @@ -87,6 +87,13 @@ typedef struct H5B_class_t { herr_t (*decode)(H5F_t*, struct H5B_t*, uint8_t*, void*); herr_t (*encode)(H5F_t*, struct H5B_t*, uint8_t*, void*); herr_t (*debug_key)(FILE*, int, int, const void*, const void*); + + /*pvn */ + + /* iterate through the leaf nodes, removing chunks upon H5Dset_extend request */ + herr_t (*prune_extent)( H5F_t *f, void *_lt_key, haddr_t addr, + void *_udata, hsize_t *size ); + } H5B_class_t; /* @@ -132,5 +139,8 @@ __DLL__ herr_t H5B_remove(H5F_t *f, const H5B_class_t *type, haddr_t addr, __DLL__ herr_t H5B_iterate (H5F_t *f, const H5B_class_t *type, haddr_t addr, void *udata); __DLL__ int H5B_Kvalue(H5F_t *f, const H5B_class_t *type); +__DLL__ herr_t H5B_prune_by_extent( H5F_t *f, const H5B_class_t *type, haddr_t addr, + void *udata, hsize_t *size ); + #endif diff --git a/src/H5D.c b/src/H5D.c index dc591ba..c2980ae 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -27,6 +27,8 @@ #include "H5Vprivate.h" /* Vector and array functions */ #include "H5Zprivate.h" /* Data filters */ +/*#define H5D_DEBUG*/ + /* * The MPIO driver is needed because there are kludges in this file and * places where we check for things that aren't handled by this driver. @@ -40,10 +42,6 @@ #define PABLO_MASK H5D_mask - -/* forward declarations */ -static herr_t H5D_update_chunk( H5D_t *dataset ); - /* * A dataset is the following struct. */ @@ -3681,80 +3679,36 @@ done: FUNC_LEAVE (ret_value); } /* H5Dfill() */ - -/*------------------------------------------------------------------------- - * Function: H5Ddebug - * - * Purpose: Prints various information about a dataset. This function is - * not to be documented in the API at this time. - * - * Return: Success: Non-negative - * - * Failure: Negative - * - * Programmer: Robb Matzke - * Wednesday, April 28, 1999 - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -herr_t -H5Ddebug(hid_t dset_id, unsigned UNUSED flags) -{ - H5D_t *dset=NULL; - - FUNC_ENTER(H5Ddebug, FAIL); - H5TRACE2("e","iIu",dset_id,flags); - - /* Check args */ - if (H5I_DATASET!=H5I_get_type(dset_id) || - NULL==(dset=H5I_object(dset_id))) { - HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset"); - } - - /* Print B-tree information */ - if (H5D_CHUNKED==dset->layout.type) { - H5F_istore_dump_btree(dset->ent.file, stdout, dset->layout.ndims, - dset->layout.addr); - } else if (H5D_CONTIGUOUS==dset->layout.type) { - HDfprintf(stdout, " %-10s %a\n", "Address:", - dset->layout.addr); - } - - FUNC_LEAVE(SUCCEED); -} /*------------------------------------------------------------------------- - * Function: H5Dset_extend + * Function: H5Dset_extent * - * Purpose: Modifies the dimensions of a dataset, based on H5Dextend + * Purpose: Modifies the dimensions of a dataset, based on H5Dextend. + * Can change to a lower dimension. * * Return: Success: 0, Failure: -1 * * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * Robb Matzke * - * Date: March 13, 2002 + * Date: April 9, 2002 * - * Comments: Public function, calls private H5D_set_extend + * Comments: Public function, calls private H5D_set_extent * * Modifications: * - * *------------------------------------------------------------------------- */ -herr_t H5Dset_extend( hid_t dset_id, const hsize_t *size ) +herr_t H5Dset_extent( hid_t dset_id, const hsize_t *size ) { H5D_t *dset = NULL; - FUNC_ENTER( H5Dset_extend, FAIL ); - H5TRACE2( "e", "i*h", dset_id, size ); - + FUNC_ENTER( H5Dset_extent, FAIL ); + /* Check args */ if ( H5I_DATASET != H5I_get_type( dset_id ) || NULL == ( dset = H5I_object( dset_id ))) { @@ -3764,144 +3718,210 @@ herr_t H5Dset_extend( hid_t dset_id, const hsize_t *size ) { HRETURN_ERROR( H5E_ARGS, H5E_BADVALUE, FAIL, "no size specified" ); } - - /* Modify size */ - if ( H5D_set_extend (dset, size) < 0 ) + + /* Private function */ + if ( H5D_set_extent ( dset, size ) < 0 ) { HRETURN_ERROR( H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set extend dataset"); } - + FUNC_LEAVE (SUCCEED); } - - /*------------------------------------------------------------------------- - * Function: H5D_set_extend + * Function: H5D_set_extent * - * Purpose: Same as H5D_extend, allows change to a lower dimension, calls H5S_modify + * Purpose: Based in H5D_extend, allows change to a lower dimension, + * calls H5S_set_extent and H5F_istore_prune_by_extent instead * * Return: Success: 0, Failure: -1 * * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * Robb Matzke * - * Date: March 13, 2002 + * Date: April 9, 2002 * - * Comments: + * Comments: Private function * * Modifications: * - * *------------------------------------------------------------------------- */ -herr_t H5D_set_extend( H5D_t *dataset, const hsize_t *size ) +herr_t H5D_set_extent( H5D_t *dset, const hsize_t *size ) { + hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /* Current dimension sizes */ + int rank; /* Dataspace # of dimensions */ herr_t changed; herr_t ret_value = FAIL; H5S_t *space = NULL; - H5O_fill_t fill; H5P_genplist_t *plist; - - FUNC_ENTER( H5D_set_extend, FAIL ); - + H5O_fill_t fill; + H5O_pline_t pline; + unsigned u; + int shrink = 0; + + FUNC_ENTER( H5D_set_extent, FAIL ); + /* Check args */ - assert( dataset ); + assert( dset ); assert( size ); - -/* - * NOTE: Restrictions on extensions were checked when the dataset was - * created. All extensions are allowed here since none should be - * able to muck things up. - */ - - /* Increase the size of the data space */ - if ( NULL == ( space = H5S_read (&(dataset->ent)))) - HGOTO_ERROR( H5E_DATASET, H5E_CANTINIT, FAIL, "unable to read data space info from dataset header"); - if ( ( changed = H5S_set_extend( space, size )) < 0 ) + /*------------------------------------------------------------------------- + * Get the data space + *------------------------------------------------------------------------- + */ + + if ( NULL == ( space = H5S_read (&(dset->ent)))) + { + HGOTO_ERROR( H5E_DATASET, H5E_CANTINIT, FAIL, "unable to read data space info from dset header"); + } + + /*------------------------------------------------------------------------- + * Check if we are shrinking in any of the dimensions + *------------------------------------------------------------------------- + */ + if((rank=H5S_get_simple_extent_dims(space, curr_dims, NULL))<0) + HGOTO_ERROR( H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions"); + + for ( u = 0; u < rank; u++ ) + { + if ( size[u] < curr_dims[u] ) + { + shrink = 1; + break; + } + } + + /*------------------------------------------------------------------------- + * Modify the size of the data space + *------------------------------------------------------------------------- + */ + + if ( ( changed = H5S_set_extent( space, size )) < 0 ) + { HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space"); - + } + + /*------------------------------------------------------------------------- + * Modify the dataset storage if changed space + *------------------------------------------------------------------------- + */ + if ( changed > 0 ) { /* Save the new dataspace in the file if necessary */ - if ( H5S_modify ( &(dataset->ent ), space ) < 0 ) + if ( H5S_modify ( &(dset->ent ), space ) < 0 ) + { HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to update file with new dataspace"); - - /* Initialize the new parts of the dataset */ -#ifdef LATER - if ( H5S_select_all( space ) < 0 || H5S_select_hyperslab( space, H5S_SELECT_DIFF, zero, NULL, old_dims, NULL ) < 0 ) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to select new extents for fill value"); -#else - - /* - * We don't have the H5S_SELECT_DIFF operator yet. We really only - * need it for contiguous datasets because the chunked datasets will - * either fill on demand during I/O or attempt a fill of all chunks. - */ + } - if ( NULL == ( plist = H5I_object(dataset->dcpl_id ))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list"); + /* Initialize the new parts of the dset */ + if ( NULL == ( plist = H5I_object( dset->dcpl_id ))) + { + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dset creation property list"); + } if( H5P_get( plist, H5D_CRT_FILL_VALUE_NAME, &fill ) < 0 ) + { HGOTO_ERROR( H5E_DATASET, H5E_CANTGET, FAIL, "can't get fill value" ); - if( H5D_CONTIGUOUS == dataset->layout.type && fill.buf ) + } + if( H5D_CONTIGUOUS == dset->layout.type && fill.buf ) + { HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to select fill value region"); -#endif - - if ( H5D_init_storage( dataset, space ) < 0 ) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize dataset with fill value"); - - - /* Update chunk information */ - if ( H5D_CHUNKED == dataset->layout.type ) - { - - if ( H5D_update_chunk( dataset ) < 0 ) - goto done; - - - } - - + } + if ( H5D_init_storage( dset, space ) < 0 ) + { + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize dset with fill value"); + } + } /* end if changed */ + + + /*------------------------------------------------------------------------- + * Remove chunk information in the case of chunked datasets + * This removal takes place only in case we are shrinking the dateset + *------------------------------------------------------------------------- + */ + + if ( changed > 0 && shrink && H5D_CHUNKED == dset->layout.type ) + { + +#if defined (PVN) + H5F_istore_dump_btree(dset->ent.file, stdout, dset->layout.ndims, dset->layout.addr); +#endif + if ( H5F_istore_prune_by_extent( dset->ent.file, &dset->layout, space ) < 0 ) + { + HRETURN_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to remove chunks "); + } + +#if defined (PVN) + H5F_istore_dump_btree(dset->ent.file, stdout, dset->layout.ndims, dset->layout.addr); +#endif + + if( H5P_get( plist, H5D_CRT_DATA_PIPELINE_NAME, &pline) < 0) + { + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get pipeline"); + } + + if ( H5F_istore_initialize_by_extent( dset->ent.file, &dset->layout, &pline, &fill, space ) < 0 ) + { + HRETURN_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to initialize chunks "); + } + + } + ret_value = SUCCEED; - + done: - H5S_close( space ); + if(space) + H5S_close( space ); FUNC_LEAVE( ret_value ); } - + /*------------------------------------------------------------------------- - * Function: H5D_update_chunk - * - * Purpose: + * Function: H5Ddebug * - * Return: Success: 0, Failure: -1 + * Purpose: Prints various information about a dataset. This function is + * not to be documented in the API at this time. * - * Programmer: Implementation: Pedro Vicente, pvn@ncsa.uiuc.edu - * Algorithm: Robb Matzke + * Return: Success: Non-negative * - * Date: March 13, 2002 + * Failure: Negative * - * Comments: + * Programmer: Robb Matzke + * Wednesday, April 28, 1999 * * Modifications: * - * *------------------------------------------------------------------------- */ - -herr_t H5D_update_chunk( H5D_t *dset ) +herr_t +H5Ddebug(hid_t dset_id, unsigned UNUSED flags) { + H5D_t *dset=NULL; - H5F_istore_dump_btree( dset->ent.file, stdout, dset->layout.ndims, dset->layout.addr ); + FUNC_ENTER(H5Ddebug, FAIL); + H5TRACE2("e","iIu",dset_id,flags); - return 0; + /* Check args */ + if (H5I_DATASET!=H5I_get_type(dset_id) || + NULL==(dset=H5I_object(dset_id))) { + HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset"); + } + /* Print B-tree information */ + if (H5D_CHUNKED==dset->layout.type) { + H5F_istore_dump_btree(dset->ent.file, stdout, dset->layout.ndims, + dset->layout.addr); + } else if (H5D_CONTIGUOUS==dset->layout.type) { + HDfprintf(stdout, " %-10s %a\n", "Address:", + dset->layout.addr); + } + + FUNC_LEAVE(SUCCEED); } diff --git a/src/H5Distore.c b/src/H5Distore.c index df8b04a..1ed6679 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -42,6 +42,7 @@ #include "H5MMprivate.h" #include "H5Oprivate.h" #include "H5Pprivate.h" /* Property lists */ +#include "H5Sprivate.h" /* Dataspaces */ #include "H5Vprivate.h" /* MPIO driver needed for special checks */ @@ -70,7 +71,8 @@ * the disk block size, chunks are aligned on disk block boundaries, * and the operating system can also eliminate a read operation. */ -/* #define H5F_ISTORE_DEBUG */ + +/*#define H5F_ISTORE_DEBUG */ /* Interface initialization */ #define PABLO_MASK H5Fistore_mask @@ -106,7 +108,7 @@ typedef H5F_rdcc_ent_t *H5F_rdcc_ent_ptr_t; /* For free lists */ static size_t H5F_istore_sizeof_rkey(H5F_t *f, const void *_udata); static herr_t H5F_istore_new_node(H5F_t *f, H5B_ins_t, void *_lt_key, void *_udata, void *_rt_key, - haddr_t*/*out*/); + haddr_t *addr_p /*out*/); static int H5F_istore_cmp2(H5F_t *f, void *_lt_key, void *_udata, void *_rt_key); static int H5F_istore_cmp3(H5F_t *f, void *_lt_key, void *_udata, @@ -129,6 +131,15 @@ static herr_t H5F_istore_debug_key(FILE *stream, int indent, int fwidth, static haddr_t H5F_istore_get_addr(H5F_t *f, const H5O_layout_t *layout, const hssize_t offset[]); +static herr_t H5F_istore_prune_extent( H5F_t *f, void *left_key, haddr_t addr, + void *_udata, hsize_t *size ); +static H5B_ins_t H5F_istore_remove( H5F_t *f, haddr_t addr, + void *_lt_key /*in,out*/, + hbool_t *lt_key_changed /*out*/, + void UNUSED *_udata /*in,out*/, + void UNUSED *_rt_key /*in,out*/, + hbool_t *rt_key_changed /*out*/); + /* * B-tree key. A key contains the minimum logical N-dimensional address and * the logical size of the chunk to which this key refers. The @@ -169,11 +180,15 @@ H5B_class_t H5B_ISTORE[1] = {{ H5F_istore_insert, /*insert */ FALSE, /*follow min branch? */ FALSE, /*follow max branch? */ - NULL, /*remove */ + /*pvn*/ + /*NULL,*/ /*remove */ + H5F_istore_remove, /*remove */ H5F_istore_iterate, /*iterator */ H5F_istore_decode_key, /*decode */ H5F_istore_encode_key, /*encode */ H5F_istore_debug_key, /*debug */ + /*pvn*/ + H5F_istore_prune_extent, /*remove chunks, upon H5Dset_extend call */ }}; #define H5F_HASH_DIVISOR 8 /* Attempt to spread out the hashing */ @@ -1052,52 +1067,71 @@ done: * Programmer: Robb Matzke * Thursday, May 21, 1998 * - * Modifications: + * Modifications: Pedro Vicente, March 28, 2002 + * Added flush parameter that switches the call to H5F_istore_flush_entry + * The call with FALSE is used by the H5F_istore_prune_by_extent function * *------------------------------------------------------------------------- */ -static herr_t -H5F_istore_preempt (H5F_t *f, H5F_rdcc_ent_t *ent) -{ - H5F_rdcc_t *rdcc = &(f->shared->rdcc); - - FUNC_ENTER (H5F_istore_preempt, FAIL); - - assert(f); - assert(ent); - assert(!ent->locked); - assert(ent->idxnslots); - - /* Flush */ - if (H5F_istore_flush_entry(f, ent, TRUE)<0) { - HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, - "cannot flush indexed storage buffer"); - } - - /* Unlink from list */ - if (ent->prev) { - ent->prev->next = ent->next; - } else { - rdcc->head = ent->next; - } - if (ent->next) { - ent->next->prev = ent->prev; - } else { - rdcc->tail = ent->prev; - } - ent->prev = ent->next = NULL; - - /* Remove from cache */ - rdcc->slot[ent->idx] = NULL; - ent->idx = UINT_MAX; - rdcc->nbytes -= ent->chunk_size; - --rdcc->nused; + + static herr_t + H5F_istore_preempt (H5F_t *f, H5F_rdcc_ent_t *ent, hbool_t flush ) + { + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + + FUNC_ENTER (H5F_istore_preempt, FAIL); + + assert(f); + assert(ent); + assert(!ent->locked); + assert(ent->idxnslots); + + if ( flush ) + { + + /* Flush */ + if (H5F_istore_flush_entry(f, ent, TRUE)<0) { + HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, + "cannot flush indexed storage buffer"); + } + } + + else + { + + /* Reset, but do not free or remove from list */ + ent->layout = H5O_free(H5O_LAYOUT, ent->layout); + ent->pline = H5O_free(H5O_PLINE, ent->pline); + if(ent->chunk!=NULL) + ent->chunk = H5F_istore_chunk_free(ent->chunk); + + } + + /* Unlink from list */ + if (ent->prev) { + ent->prev->next = ent->next; + } else { + rdcc->head = ent->next; + } + if (ent->next) { + ent->next->prev = ent->prev; + } else { + rdcc->tail = ent->prev; + } + ent->prev = ent->next = NULL; + + /* Remove from cache */ + rdcc->slot[ent->idx] = NULL; + ent->idx = UINT_MAX; + rdcc->nbytes -= ent->chunk_size; + --rdcc->nused; + + /* Free */ + H5FL_FREE(H5F_rdcc_ent_t, ent); + + FUNC_LEAVE (SUCCEED); + } - /* Free */ - H5FL_FREE(H5F_rdcc_ent_t, ent); - - FUNC_LEAVE (SUCCEED); -} /*------------------------------------------------------------------------- @@ -1112,6 +1146,8 @@ H5F_istore_preempt (H5F_t *f, H5F_rdcc_ent_t *ent) * Thursday, May 21, 1998 * * Modifications: + * Pedro Vicente, March 28, 2002 + * Added TRUE parameter to the call to H5F_istore_preempt * *------------------------------------------------------------------------- */ @@ -1127,7 +1163,7 @@ H5F_istore_flush (H5F_t *f, hbool_t preempt) for (ent=rdcc->head; ent; ent=next) { next = ent->next; if (preempt) { - if (H5F_istore_preempt(f, ent)<0) { + if (H5F_istore_preempt(f, ent, TRUE )<0) { nerrors++; } } else { @@ -1157,6 +1193,8 @@ H5F_istore_flush (H5F_t *f, hbool_t preempt) * Thursday, May 21, 1998 * * Modifications: + * Pedro Vicente, March 28, 2002 + * Added TRUE parameter to the call to H5F_istore_preempt * *------------------------------------------------------------------------- */ @@ -1175,7 +1213,7 @@ H5F_istore_dest (H5F_t *f) HDfflush(stderr); #endif next = ent->next; - if (H5F_istore_preempt(f, ent)<0) { + if (H5F_istore_preempt(f, ent, TRUE )<0) { nerrors++; } } @@ -1203,6 +1241,8 @@ H5F_istore_dest (H5F_t *f) * Thursday, May 21, 1998 * * Modifications: + * Pedro Vicente, March 28, 2002 + * Added TRUE parameter to the call to H5F_istore_preempt * *------------------------------------------------------------------------- */ @@ -1280,7 +1320,7 @@ H5F_istore_prune (H5F_t *f, size_t size) if (p[j]==cur) p[j] = NULL; if (n[j]==cur) n[j] = cur->next; } - if (H5F_istore_preempt(f, cur)<0) nerrors++; + if (H5F_istore_preempt(f, cur, TRUE)<0) nerrors++; } } @@ -1326,6 +1366,9 @@ H5F_istore_prune (H5F_t *f, size_t size) * Robb Matzke, 1999-08-02 * The split ratios are passed in as part of the data transfer * property list. + * + * Pedro Vicente, March 28, 2002 + * Added TRUE parameter to the call to H5F_istore_preempt *------------------------------------------------------------------------- */ static void * @@ -1477,7 +1520,7 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, } fprintf(stderr, "}\n"); #endif - if (H5F_istore_preempt(f, ent)<0) { + if (H5F_istore_preempt(f, ent, TRUE)<0) { HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk from cache"); } @@ -2457,3 +2500,500 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, FUNC_LEAVE(SUCCEED); } + + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_prune_by_extent + * + * Purpose: This function searches for chunks that are no longer necessary both in the + * raw data cache and in the B-tree. + * + * Return: Success: 0, Failure: -1 + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * Algorithm: Robb Matzke + * + * Date: March 27, 2002 + * + * The algorithm is: + * + * For chunks that are no longer necessary: + * + * 1. Search in the raw data cache for each chunk + * 2. If found then preempt it from the cache + * 3. Search in the B-tree for each chunk + * 4. If found then remove it from the B-tree and deallocate file storage for the chunk + * + * This example shows a 2d dataset of 90x90 with a chunk size of 20x20. + * + * + * 0 20 40 60 80 90 100 + * 0 +---------+---------+---------+---------+-----+...+ + * |:::::X:::::::::::::: : : | : + * |:::::::X:::::::::::: : : | : Key + * |::::::::::X::::::::: : : | : -------- + * |::::::::::::X::::::: : : | : +-+ Dataset + * 20+::::::::::::::::::::.........:.........:.....+...: | | Extent + * | :::::X::::: : : | : +-+ + * | ::::::::::: : : | : + * | ::::::::::: : : | : ... Chunk + * | :::::::X::: : : | : : : Boundary + * 40+.........:::::::::::.........:.........:.....+...: :.: + * | : : : : | : + * | : : : : | : ... Allocated + * | : : : : | : ::: & Filled + * | : : : : | : ::: Chunk + * 60+.........:.........:.........:.........:.....+...: + * | : :::::::X::: : | : X Element + * | : ::::::::::: : | : Written + * | : ::::::::::: : | : + * | : ::::::::::: : | : + * 80+.........:.........:::::::::::.........:.....+...: O Fill Val + * | : : ::::::::::: | : Explicitly + * | : : ::::::X:::: | : Written + * 90+---------+---------+---------+---------+-----+ : + * : : : ::::::::::: : + * 100:.........:.........:.........:::::::::::.........: + * + * + * We have 25 total chunks for this dataset, 5 of which have space + * allocated in the file because they were written to one or more + * elements. These five chunks (and only these five) also have entries in + * the storage B-tree for this dataset. + * + * Now lets say we want to shrink the dataset down to 70x70: + * + * + * 0 20 40 60 70 80 90 100 + * 0 +---------+---------+---------+----+----+-----+...+ + * |:::::X:::::::::::::: : | : | : + * |:::::::X:::::::::::: : | : | : Key + * |::::::::::X::::::::: : | : | : -------- + * |::::::::::::X::::::: : | : | : +-+ Dataset + * 20+::::::::::::::::::::.........:....+....:.....|...: | | Extent + * | :::::X::::: : | : | : +-+ + * | ::::::::::: : | : | : + * | ::::::::::: : | : | : ... Chunk + * | :::::::X::: : | : | : : : Boundary + * 40+.........:::::::::::.........:....+....:.....|...: :.: + * | : : : | : | : + * | : : : | : | : ... Allocated + * | : : : | : | : ::: & Filled + * | : : : | : | : ::: Chunk + * 60+.........:.........:.........:....+....:.....|...: + * | : :::::::X::: | : | : X Element + * | : ::::::::::: | : | : Written + * +---------+---------+---------+----+ : | : + * | : ::::::::::: : | : + * 80+.........:.........:::::::::X:.........:.....|...: O Fill Val + * | : : ::::::::::: | : Explicitly + * | : : ::::::X:::: | : Written + * 90+---------+---------+---------+---------+-----+ : + * : : : ::::::::::: : + * 100:.........:.........:.........:::::::::::.........: + * + * + * That means that the nine chunks along the bottom and right side should + * no longer exist. Of those nine chunks, (0,80), (20,80), (40,80), + * (60,80), (80,80), (80,60), (80,40), (80,20), and (80,0), one is actually allocated + * that needs to be released. + * To release the chunks, we traverse the B-tree to obtain a list of unused + * allocated chunks, and then call H5B_remove() for each chunk. + * + *------------------------------------------------------------------------- + */ + +herr_t H5F_istore_prune_by_extent( H5F_t *f, H5O_layout_t *layout, H5S_t *space ) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); /*raw data chunk cache*/ + H5F_rdcc_ent_t *ent = NULL, *next = NULL; /*cache entry */ + unsigned u; /*counters */ + int found = 0; /*remove this entry */ + H5F_istore_ud1_t udata; /*B-tree pass-through */ + hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /*current dataspace dimensions */ + + FUNC_ENTER( H5F_istore_prune_by_extent, FAIL ); + + /* Check args */ + assert(f); + assert(layout && H5D_CHUNKED==layout->type); + assert(layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS); + assert(H5F_addr_defined(layout->addr)); + assert(space); + + /* Go get the rank & dimensions */ + if(H5S_get_simple_extent_dims(space, curr_dims, NULL)<0) + HRETURN_ERROR( H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions"); + + + /*------------------------------------------------------------------------- + * Figure out what chunks are no longer in use for the specified extent + * and release them from the linked list raw data cache + *------------------------------------------------------------------------- + */ + + for ( ent = rdcc->head; ent; ent=next ) + { + next = ent->next; + + found = 0; + for ( u = 0; u < ent->layout->ndims-1; u++ ) + { + if ( (hsize_t)ent->offset[u] > curr_dims[u] ) + { + found = 1; + break; + } + } + + if ( found ) + { +#if defined (H5F_ISTORE_DEBUG) + HDfputs("cache:remove:[", stdout); + for ( u = 0; u < ent->layout->ndims-1; u++) + { + HDfprintf( stdout, "%s%Hd", u?", ":"", ent->offset[u]); + } + HDfputs("]\n", stdout ); +#endif + + /* Preempt the entry from the cache, but do not flush it to disk */ + if ( H5F_istore_preempt( f, ent, FALSE ) < 0 ) + { + HRETURN_ERROR(H5E_IO, H5E_CANTINIT, 0, "unable to preempt chunk"); + } + + } + + } + +/*------------------------------------------------------------------------- + * Check if there are any chunks on the B-tree + *------------------------------------------------------------------------- + */ + + HDmemset(&udata, 0, sizeof udata); + udata.stream = stdout; + udata.mesg.addr = layout->addr; + udata.mesg.ndims = layout->ndims; + for ( u = 0; u < udata.mesg.ndims; u++ ) + { + udata.mesg.dim[u] = layout->dim[u]; + } + + if ( H5B_prune_by_extent( f, H5B_ISTORE, layout->addr, &udata, curr_dims ) < 0 ) + { + HRETURN_ERROR( H5E_IO, H5E_CANTINIT, 0, "unable to iterate over B-tree" ); + } + + FUNC_LEAVE( SUCCEED ); + +} + +/*------------------------------------------------------------------------- + * Function: H5F_istore_prune_extent + * + * Purpose: Search for chunks that are no longer necessary in the B-tree. + * + * Return: Success: 0, Failure: -1 + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: March 26, 2002 + * + * Comments: Called by H5B_prune_by_extent, part of H5B_ISTORE + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +static herr_t H5F_istore_prune_extent( H5F_t *f, void *_lt_key, haddr_t addr, void *_udata, + hsize_t *size ) +{ + H5F_istore_ud1_t *bt_udata = (H5F_istore_ud1_t *)_udata; + H5F_istore_key_t *lt_key = (H5F_istore_key_t *)_lt_key; + unsigned u; + int found = 0; + H5F_istore_ud1_t udata; + + /* The LT_KEY is the left key (the onethat describes the chunk). It points to a chunk of + storage that contains the beginning of the logical address space represented by UDATA. + */ + + FUNC_ENTER( H5F_istore_prune_extent, FAIL ); + + /* Figure out what chunks are no longer in use for the specified extent and release them */ + + for ( u = 0; u < bt_udata->mesg.ndims-1; u++ ) + { + if ( (hsize_t)lt_key->offset[u] > size[u] ) + { + found = 1; + break; + } + } + + if ( found ) + { + +#if defined (H5F_ISTORE_DEBUG) + HDfputs("b-tree:remove:[", bt_udata->stream); + for ( u = 0; u < bt_udata->mesg.ndims-1; u++) + { + HDfprintf(bt_udata->stream, "%s%Hd", u?", ":"", lt_key->offset[u]); + } + HDfputs("]\n", bt_udata->stream); +#endif + + HDmemset( &udata, 0, sizeof udata ); + udata.key = *lt_key; + udata.mesg = bt_udata->mesg; + + /* Remove */ + if ( H5B_remove( f, H5B_ISTORE, addr, &udata ) < 0 ) + { + HRETURN_ERROR( H5E_SYM, H5E_CANTINIT, FAIL, "unable to remove entry" ); + } + } + + FUNC_LEAVE( SUCCEED ); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_remove + * + * Purpose: Removes chunks that are no longer necessary in the B-tree. + * + * Return: Success: 0, Failure: -1 + * + * Programmer: Robb Matzke + * Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: March 28, 2002 + * + * Comments: Part of H5B_ISTORE + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +static H5B_ins_t H5F_istore_remove( H5F_t *f, + haddr_t addr, + void *_lt_key /*in,out*/, + hbool_t *lt_key_changed /*out*/, + void UNUSED *_udata /*in,out*/, + void UNUSED *_rt_key /*in,out*/, + hbool_t *rt_key_changed /*out*/) +{ + H5F_istore_key_t *lt_key = (H5F_istore_key_t*)_lt_key; + H5FD_free(f->shared->lf, H5FD_MEM_DRAW, addr, lt_key->nbytes); + *lt_key_changed = FALSE; + *rt_key_changed = FALSE; + return H5B_INS_REMOVE; +} + + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_initialize_by_extent + * + * Purpose: This function searches for chunks that have to be initialized with the fill + * value both in the raw data cache and in the B-tree. + * + * Return: Success: 0, Failure: -1 + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: April 4, 2002 + * + * Comments: + * + * (See the example of H5F_istore_prune_by_extent) + * Next, there are seven chunks where the database extent boundary is + * within the chunk. We find those seven just like we did with the previous nine. + * Fot the ones that are allocated we initialize the part that lies outside the boundary + * with the fill value. + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + + +herr_t H5F_istore_initialize_by_extent( H5F_t *f, H5O_layout_t *layout, H5O_pline_t *pline, + H5O_fill_t *fill, H5S_t *space ) +{ + hid_t dxpl_id; /*dataset transfer property list*/ + uint8_t *chunk = NULL; /*the file chunk */ + unsigned idx_hint = 0; /*input value for H5F_istore_lock*/ + hssize_t chunk_offset[H5O_LAYOUT_NDIMS]; /*logical location of the chunks */ + hsize_t idx_cur[H5O_LAYOUT_NDIMS]; /*multi-dimensional counters */ + hsize_t idx_min[H5O_LAYOUT_NDIMS]; + hsize_t idx_max[H5O_LAYOUT_NDIMS]; + hsize_t sub_size[H5O_LAYOUT_NDIMS]; + hsize_t naccessed; /*bytes accessed in chunk */ + hsize_t elm_size; /*size of an element in bytes */ + hsize_t end_chunk; /*chunk position counter */ + hssize_t start[H5O_LAYOUT_NDIMS]; /*starting location of hyperslab */ + hsize_t count[H5O_LAYOUT_NDIMS]; /*element count of hyperslab */ + hsize_t size[H5O_LAYOUT_NDIMS]; /*current size of dimensions */ + H5S_t *space_chunk=NULL; /*dataspace for a chunk */ + hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /*current dataspace dimensions */ + int rank; /*current # of dimensions */ + int i, carry; /*counters */ + unsigned u; + int found = 0; /*initialize this entry */ + + FUNC_ENTER( H5F_istore_initialize_by_extent, FAIL ); + + /* Check args */ + assert(f); + assert(layout && H5D_CHUNKED==layout->type); + assert(layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS); + assert(H5F_addr_defined(layout->addr)); + assert(space); + assert(pline); + assert(fill); + + HDmemset( start, 0, sizeof(start) ); + HDmemset( count, 0, sizeof(count) ); + + /* Go get the rank & dimensions */ + if((rank=H5S_get_simple_extent_dims(space, curr_dims, NULL))<0) + HRETURN_ERROR( H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions"); + + for ( i = 0; i < rank; i++ ) + { + size[i] = curr_dims[i]; + } + size[i] = layout->dim[i]; + elm_size = size[i]; + + /* Default dataset transfer property list */ + dxpl_id = H5P_DATASET_XFER_DEFAULT; + + /* Create a data space for a chunk & set the extent */ + if(NULL==(space_chunk=H5S_create(H5S_SIMPLE))) { + HRETURN_ERROR (H5E_DATASPACE, H5E_CANTCREATE, FAIL, + "can't create simple dataspace"); + } + if(H5S_set_extent_simple(space_chunk,(unsigned)rank,layout->dim,NULL)<0) { + HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, + "can't set dimensions"); + } + +/* + * Set up multi-dimensional counters (idx_min, idx_max, and idx_cur) and + * loop through the chunks copying each chunk from the application to the + * chunk cache. + */ + for ( u = 0; u < layout->ndims; u++) + { + idx_min[u] = 0; + idx_max[u] = (size[u]-1) / layout->dim[u] + 1; + idx_cur[u] = idx_min[u]; + } + + /* Loop over all chunks */ + while ( 1 ) + { + + for ( u = 0, naccessed=1; u < layout->ndims; u++ ) + { + /* The location and size of the chunk being accessed */ + chunk_offset[u] = idx_cur[u] * (hssize_t)(layout->dim[u]); + sub_size[u] = MIN((idx_cur[u]+1)*layout->dim[u],size[u]) - chunk_offset[u]; + naccessed *= sub_size[u]; + } + + /* + Figure out what chunks have to be initialized. These are the chunks where the database + extent boundary is within the chunk + */ + + for ( u = 0, found = 0; u < layout->ndims-1; u++ ) + { + end_chunk = chunk_offset[u] + layout->dim[u]; + if ( end_chunk > size[u] ) + { + found = 1; + break; + } + } + + if ( found ) + { + + if (NULL==(chunk=H5F_istore_lock( f, dxpl_id, layout, pline, fill, chunk_offset, FALSE, &idx_hint))) + { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to read raw data chunk"); + } + + if ( H5S_select_all( space_chunk ) < 0 ) + { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to select space"); + } + + for ( i = 0; i < rank; i++ ) + { + count[i] = MIN( (idx_cur[i]+1)*layout->dim[i], size[i]-chunk_offset[i] ); + } + +#if defined (H5F_ISTORE_DEBUG) + HDfputs("cache:initialize:offset:[", stdout); + for ( u = 0; u < layout->ndims-1; u++) + { + HDfprintf( stdout, "%s%Hd", u?", ":"", chunk_offset[u]); + } + HDfputs("]", stdout ); + HDfputs(":count:[", stdout); + for ( u = 0; u < layout->ndims-1; u++) + { + HDfprintf( stdout, "%s%Hd", u?", ":"", count[u]); + } + HDfputs("]\n", stdout ); +#endif + + if ( H5S_select_hyperslab( space_chunk, H5S_SELECT_NOTB, start, NULL, count, NULL)<0) + { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to select hyperslab"); + } + + /* Fill the selection in the memory buffer */ + if ( H5S_select_fill( fill->buf, fill->size, space_chunk, chunk ) < 0 ) + { + HRETURN_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "filling selection failed"); + } + + if (H5F_istore_unlock(f, dxpl_id, layout, pline, TRUE, chunk_offset, &idx_hint, chunk, (size_t)naccessed)<0) + { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to unlock raw data chunk"); + } + + } /*found*/ + + + /* Increment indices */ + for ( i = layout->ndims-1, carry=1; i >= 0 && carry; --i) + { + if (++idx_cur[i] >= idx_max[i]) + idx_cur[i] = idx_min[i]; + else + carry = 0; + } + if (carry) + break; + + } + + + if(space_chunk) + H5S_close( space_chunk); + + FUNC_LEAVE( SUCCEED ); +} + diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index 1c7d940..33fa930 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -155,7 +155,6 @@ __DLL__ herr_t H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, const H5S_t *file_space, hid_t dset_xfer_plist, const void *buf); __DLL__ herr_t H5D_extend(H5D_t *dataset, const hsize_t *size); -__DLL__ herr_t H5D_set_extend(H5D_t *dataset, const hsize_t *size); __DLL__ H5G_entry_t *H5D_entof(H5D_t *dataset); __DLL__ H5T_t *H5D_typeof(H5D_t *dset); __DLL__ H5S_t *H5D_get_space(H5D_t *dset); @@ -172,5 +171,8 @@ __DLL__ herr_t H5D_xfer_create(hid_t dxpl_id, void *create_data); __DLL__ herr_t H5D_xfer_copy(hid_t new_plist_id, hid_t old_plist_id, void *copy_data); __DLL__ herr_t H5D_xfer_close(hid_t dxpl_id, void *close_data); +/*pvn*/ +__DLL__ herr_t H5D_set_extent(H5D_t *dataset, const hsize_t *size); + #endif diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 41e479d..66b6b2f 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -52,7 +52,6 @@ __DLL__ herr_t H5Dread (hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, __DLL__ herr_t H5Dwrite (hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, const void *buf); __DLL__ herr_t H5Dextend (hid_t dset_id, const hsize_t *size); -__DLL__ herr_t H5Dset_extend (hid_t dset_id, const hsize_t *size); __DLL__ herr_t H5Diterate(void *buf, hid_t type_id, hid_t space_id, H5D_operator_t op, void *operator_data); __DLL__ herr_t H5Dvlen_reclaim(hid_t type_id, hid_t space_id, hid_t plist_id, void *buf); @@ -60,6 +59,9 @@ __DLL__ herr_t H5Dvlen_get_buf_size(hid_t dataset_id, hid_t type_id, hid_t space __DLL__ herr_t H5Dfill(const void *fill, hid_t fill_type, void *buf, hid_t buf_type, hid_t space); __DLL__ herr_t H5Ddebug(hid_t dset_id, unsigned int flags); +/*pvn*/ +__DLL__ herr_t H5Dset_extent (hid_t dset_id, const hsize_t *size); + #ifdef __cplusplus } diff --git a/src/H5Fistore.c b/src/H5Fistore.c index df8b04a..1ed6679 100644 --- a/src/H5Fistore.c +++ b/src/H5Fistore.c @@ -42,6 +42,7 @@ #include "H5MMprivate.h" #include "H5Oprivate.h" #include "H5Pprivate.h" /* Property lists */ +#include "H5Sprivate.h" /* Dataspaces */ #include "H5Vprivate.h" /* MPIO driver needed for special checks */ @@ -70,7 +71,8 @@ * the disk block size, chunks are aligned on disk block boundaries, * and the operating system can also eliminate a read operation. */ -/* #define H5F_ISTORE_DEBUG */ + +/*#define H5F_ISTORE_DEBUG */ /* Interface initialization */ #define PABLO_MASK H5Fistore_mask @@ -106,7 +108,7 @@ typedef H5F_rdcc_ent_t *H5F_rdcc_ent_ptr_t; /* For free lists */ static size_t H5F_istore_sizeof_rkey(H5F_t *f, const void *_udata); static herr_t H5F_istore_new_node(H5F_t *f, H5B_ins_t, void *_lt_key, void *_udata, void *_rt_key, - haddr_t*/*out*/); + haddr_t *addr_p /*out*/); static int H5F_istore_cmp2(H5F_t *f, void *_lt_key, void *_udata, void *_rt_key); static int H5F_istore_cmp3(H5F_t *f, void *_lt_key, void *_udata, @@ -129,6 +131,15 @@ static herr_t H5F_istore_debug_key(FILE *stream, int indent, int fwidth, static haddr_t H5F_istore_get_addr(H5F_t *f, const H5O_layout_t *layout, const hssize_t offset[]); +static herr_t H5F_istore_prune_extent( H5F_t *f, void *left_key, haddr_t addr, + void *_udata, hsize_t *size ); +static H5B_ins_t H5F_istore_remove( H5F_t *f, haddr_t addr, + void *_lt_key /*in,out*/, + hbool_t *lt_key_changed /*out*/, + void UNUSED *_udata /*in,out*/, + void UNUSED *_rt_key /*in,out*/, + hbool_t *rt_key_changed /*out*/); + /* * B-tree key. A key contains the minimum logical N-dimensional address and * the logical size of the chunk to which this key refers. The @@ -169,11 +180,15 @@ H5B_class_t H5B_ISTORE[1] = {{ H5F_istore_insert, /*insert */ FALSE, /*follow min branch? */ FALSE, /*follow max branch? */ - NULL, /*remove */ + /*pvn*/ + /*NULL,*/ /*remove */ + H5F_istore_remove, /*remove */ H5F_istore_iterate, /*iterator */ H5F_istore_decode_key, /*decode */ H5F_istore_encode_key, /*encode */ H5F_istore_debug_key, /*debug */ + /*pvn*/ + H5F_istore_prune_extent, /*remove chunks, upon H5Dset_extend call */ }}; #define H5F_HASH_DIVISOR 8 /* Attempt to spread out the hashing */ @@ -1052,52 +1067,71 @@ done: * Programmer: Robb Matzke * Thursday, May 21, 1998 * - * Modifications: + * Modifications: Pedro Vicente, March 28, 2002 + * Added flush parameter that switches the call to H5F_istore_flush_entry + * The call with FALSE is used by the H5F_istore_prune_by_extent function * *------------------------------------------------------------------------- */ -static herr_t -H5F_istore_preempt (H5F_t *f, H5F_rdcc_ent_t *ent) -{ - H5F_rdcc_t *rdcc = &(f->shared->rdcc); - - FUNC_ENTER (H5F_istore_preempt, FAIL); - - assert(f); - assert(ent); - assert(!ent->locked); - assert(ent->idxnslots); - - /* Flush */ - if (H5F_istore_flush_entry(f, ent, TRUE)<0) { - HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, - "cannot flush indexed storage buffer"); - } - - /* Unlink from list */ - if (ent->prev) { - ent->prev->next = ent->next; - } else { - rdcc->head = ent->next; - } - if (ent->next) { - ent->next->prev = ent->prev; - } else { - rdcc->tail = ent->prev; - } - ent->prev = ent->next = NULL; - - /* Remove from cache */ - rdcc->slot[ent->idx] = NULL; - ent->idx = UINT_MAX; - rdcc->nbytes -= ent->chunk_size; - --rdcc->nused; + + static herr_t + H5F_istore_preempt (H5F_t *f, H5F_rdcc_ent_t *ent, hbool_t flush ) + { + H5F_rdcc_t *rdcc = &(f->shared->rdcc); + + FUNC_ENTER (H5F_istore_preempt, FAIL); + + assert(f); + assert(ent); + assert(!ent->locked); + assert(ent->idxnslots); + + if ( flush ) + { + + /* Flush */ + if (H5F_istore_flush_entry(f, ent, TRUE)<0) { + HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, + "cannot flush indexed storage buffer"); + } + } + + else + { + + /* Reset, but do not free or remove from list */ + ent->layout = H5O_free(H5O_LAYOUT, ent->layout); + ent->pline = H5O_free(H5O_PLINE, ent->pline); + if(ent->chunk!=NULL) + ent->chunk = H5F_istore_chunk_free(ent->chunk); + + } + + /* Unlink from list */ + if (ent->prev) { + ent->prev->next = ent->next; + } else { + rdcc->head = ent->next; + } + if (ent->next) { + ent->next->prev = ent->prev; + } else { + rdcc->tail = ent->prev; + } + ent->prev = ent->next = NULL; + + /* Remove from cache */ + rdcc->slot[ent->idx] = NULL; + ent->idx = UINT_MAX; + rdcc->nbytes -= ent->chunk_size; + --rdcc->nused; + + /* Free */ + H5FL_FREE(H5F_rdcc_ent_t, ent); + + FUNC_LEAVE (SUCCEED); + } - /* Free */ - H5FL_FREE(H5F_rdcc_ent_t, ent); - - FUNC_LEAVE (SUCCEED); -} /*------------------------------------------------------------------------- @@ -1112,6 +1146,8 @@ H5F_istore_preempt (H5F_t *f, H5F_rdcc_ent_t *ent) * Thursday, May 21, 1998 * * Modifications: + * Pedro Vicente, March 28, 2002 + * Added TRUE parameter to the call to H5F_istore_preempt * *------------------------------------------------------------------------- */ @@ -1127,7 +1163,7 @@ H5F_istore_flush (H5F_t *f, hbool_t preempt) for (ent=rdcc->head; ent; ent=next) { next = ent->next; if (preempt) { - if (H5F_istore_preempt(f, ent)<0) { + if (H5F_istore_preempt(f, ent, TRUE )<0) { nerrors++; } } else { @@ -1157,6 +1193,8 @@ H5F_istore_flush (H5F_t *f, hbool_t preempt) * Thursday, May 21, 1998 * * Modifications: + * Pedro Vicente, March 28, 2002 + * Added TRUE parameter to the call to H5F_istore_preempt * *------------------------------------------------------------------------- */ @@ -1175,7 +1213,7 @@ H5F_istore_dest (H5F_t *f) HDfflush(stderr); #endif next = ent->next; - if (H5F_istore_preempt(f, ent)<0) { + if (H5F_istore_preempt(f, ent, TRUE )<0) { nerrors++; } } @@ -1203,6 +1241,8 @@ H5F_istore_dest (H5F_t *f) * Thursday, May 21, 1998 * * Modifications: + * Pedro Vicente, March 28, 2002 + * Added TRUE parameter to the call to H5F_istore_preempt * *------------------------------------------------------------------------- */ @@ -1280,7 +1320,7 @@ H5F_istore_prune (H5F_t *f, size_t size) if (p[j]==cur) p[j] = NULL; if (n[j]==cur) n[j] = cur->next; } - if (H5F_istore_preempt(f, cur)<0) nerrors++; + if (H5F_istore_preempt(f, cur, TRUE)<0) nerrors++; } } @@ -1326,6 +1366,9 @@ H5F_istore_prune (H5F_t *f, size_t size) * Robb Matzke, 1999-08-02 * The split ratios are passed in as part of the data transfer * property list. + * + * Pedro Vicente, March 28, 2002 + * Added TRUE parameter to the call to H5F_istore_preempt *------------------------------------------------------------------------- */ static void * @@ -1477,7 +1520,7 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, } fprintf(stderr, "}\n"); #endif - if (H5F_istore_preempt(f, ent)<0) { + if (H5F_istore_preempt(f, ent, TRUE)<0) { HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk from cache"); } @@ -2457,3 +2500,500 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout, FUNC_LEAVE(SUCCEED); } + + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_prune_by_extent + * + * Purpose: This function searches for chunks that are no longer necessary both in the + * raw data cache and in the B-tree. + * + * Return: Success: 0, Failure: -1 + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * Algorithm: Robb Matzke + * + * Date: March 27, 2002 + * + * The algorithm is: + * + * For chunks that are no longer necessary: + * + * 1. Search in the raw data cache for each chunk + * 2. If found then preempt it from the cache + * 3. Search in the B-tree for each chunk + * 4. If found then remove it from the B-tree and deallocate file storage for the chunk + * + * This example shows a 2d dataset of 90x90 with a chunk size of 20x20. + * + * + * 0 20 40 60 80 90 100 + * 0 +---------+---------+---------+---------+-----+...+ + * |:::::X:::::::::::::: : : | : + * |:::::::X:::::::::::: : : | : Key + * |::::::::::X::::::::: : : | : -------- + * |::::::::::::X::::::: : : | : +-+ Dataset + * 20+::::::::::::::::::::.........:.........:.....+...: | | Extent + * | :::::X::::: : : | : +-+ + * | ::::::::::: : : | : + * | ::::::::::: : : | : ... Chunk + * | :::::::X::: : : | : : : Boundary + * 40+.........:::::::::::.........:.........:.....+...: :.: + * | : : : : | : + * | : : : : | : ... Allocated + * | : : : : | : ::: & Filled + * | : : : : | : ::: Chunk + * 60+.........:.........:.........:.........:.....+...: + * | : :::::::X::: : | : X Element + * | : ::::::::::: : | : Written + * | : ::::::::::: : | : + * | : ::::::::::: : | : + * 80+.........:.........:::::::::::.........:.....+...: O Fill Val + * | : : ::::::::::: | : Explicitly + * | : : ::::::X:::: | : Written + * 90+---------+---------+---------+---------+-----+ : + * : : : ::::::::::: : + * 100:.........:.........:.........:::::::::::.........: + * + * + * We have 25 total chunks for this dataset, 5 of which have space + * allocated in the file because they were written to one or more + * elements. These five chunks (and only these five) also have entries in + * the storage B-tree for this dataset. + * + * Now lets say we want to shrink the dataset down to 70x70: + * + * + * 0 20 40 60 70 80 90 100 + * 0 +---------+---------+---------+----+----+-----+...+ + * |:::::X:::::::::::::: : | : | : + * |:::::::X:::::::::::: : | : | : Key + * |::::::::::X::::::::: : | : | : -------- + * |::::::::::::X::::::: : | : | : +-+ Dataset + * 20+::::::::::::::::::::.........:....+....:.....|...: | | Extent + * | :::::X::::: : | : | : +-+ + * | ::::::::::: : | : | : + * | ::::::::::: : | : | : ... Chunk + * | :::::::X::: : | : | : : : Boundary + * 40+.........:::::::::::.........:....+....:.....|...: :.: + * | : : : | : | : + * | : : : | : | : ... Allocated + * | : : : | : | : ::: & Filled + * | : : : | : | : ::: Chunk + * 60+.........:.........:.........:....+....:.....|...: + * | : :::::::X::: | : | : X Element + * | : ::::::::::: | : | : Written + * +---------+---------+---------+----+ : | : + * | : ::::::::::: : | : + * 80+.........:.........:::::::::X:.........:.....|...: O Fill Val + * | : : ::::::::::: | : Explicitly + * | : : ::::::X:::: | : Written + * 90+---------+---------+---------+---------+-----+ : + * : : : ::::::::::: : + * 100:.........:.........:.........:::::::::::.........: + * + * + * That means that the nine chunks along the bottom and right side should + * no longer exist. Of those nine chunks, (0,80), (20,80), (40,80), + * (60,80), (80,80), (80,60), (80,40), (80,20), and (80,0), one is actually allocated + * that needs to be released. + * To release the chunks, we traverse the B-tree to obtain a list of unused + * allocated chunks, and then call H5B_remove() for each chunk. + * + *------------------------------------------------------------------------- + */ + +herr_t H5F_istore_prune_by_extent( H5F_t *f, H5O_layout_t *layout, H5S_t *space ) +{ + H5F_rdcc_t *rdcc = &(f->shared->rdcc); /*raw data chunk cache*/ + H5F_rdcc_ent_t *ent = NULL, *next = NULL; /*cache entry */ + unsigned u; /*counters */ + int found = 0; /*remove this entry */ + H5F_istore_ud1_t udata; /*B-tree pass-through */ + hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /*current dataspace dimensions */ + + FUNC_ENTER( H5F_istore_prune_by_extent, FAIL ); + + /* Check args */ + assert(f); + assert(layout && H5D_CHUNKED==layout->type); + assert(layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS); + assert(H5F_addr_defined(layout->addr)); + assert(space); + + /* Go get the rank & dimensions */ + if(H5S_get_simple_extent_dims(space, curr_dims, NULL)<0) + HRETURN_ERROR( H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions"); + + + /*------------------------------------------------------------------------- + * Figure out what chunks are no longer in use for the specified extent + * and release them from the linked list raw data cache + *------------------------------------------------------------------------- + */ + + for ( ent = rdcc->head; ent; ent=next ) + { + next = ent->next; + + found = 0; + for ( u = 0; u < ent->layout->ndims-1; u++ ) + { + if ( (hsize_t)ent->offset[u] > curr_dims[u] ) + { + found = 1; + break; + } + } + + if ( found ) + { +#if defined (H5F_ISTORE_DEBUG) + HDfputs("cache:remove:[", stdout); + for ( u = 0; u < ent->layout->ndims-1; u++) + { + HDfprintf( stdout, "%s%Hd", u?", ":"", ent->offset[u]); + } + HDfputs("]\n", stdout ); +#endif + + /* Preempt the entry from the cache, but do not flush it to disk */ + if ( H5F_istore_preempt( f, ent, FALSE ) < 0 ) + { + HRETURN_ERROR(H5E_IO, H5E_CANTINIT, 0, "unable to preempt chunk"); + } + + } + + } + +/*------------------------------------------------------------------------- + * Check if there are any chunks on the B-tree + *------------------------------------------------------------------------- + */ + + HDmemset(&udata, 0, sizeof udata); + udata.stream = stdout; + udata.mesg.addr = layout->addr; + udata.mesg.ndims = layout->ndims; + for ( u = 0; u < udata.mesg.ndims; u++ ) + { + udata.mesg.dim[u] = layout->dim[u]; + } + + if ( H5B_prune_by_extent( f, H5B_ISTORE, layout->addr, &udata, curr_dims ) < 0 ) + { + HRETURN_ERROR( H5E_IO, H5E_CANTINIT, 0, "unable to iterate over B-tree" ); + } + + FUNC_LEAVE( SUCCEED ); + +} + +/*------------------------------------------------------------------------- + * Function: H5F_istore_prune_extent + * + * Purpose: Search for chunks that are no longer necessary in the B-tree. + * + * Return: Success: 0, Failure: -1 + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: March 26, 2002 + * + * Comments: Called by H5B_prune_by_extent, part of H5B_ISTORE + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +static herr_t H5F_istore_prune_extent( H5F_t *f, void *_lt_key, haddr_t addr, void *_udata, + hsize_t *size ) +{ + H5F_istore_ud1_t *bt_udata = (H5F_istore_ud1_t *)_udata; + H5F_istore_key_t *lt_key = (H5F_istore_key_t *)_lt_key; + unsigned u; + int found = 0; + H5F_istore_ud1_t udata; + + /* The LT_KEY is the left key (the onethat describes the chunk). It points to a chunk of + storage that contains the beginning of the logical address space represented by UDATA. + */ + + FUNC_ENTER( H5F_istore_prune_extent, FAIL ); + + /* Figure out what chunks are no longer in use for the specified extent and release them */ + + for ( u = 0; u < bt_udata->mesg.ndims-1; u++ ) + { + if ( (hsize_t)lt_key->offset[u] > size[u] ) + { + found = 1; + break; + } + } + + if ( found ) + { + +#if defined (H5F_ISTORE_DEBUG) + HDfputs("b-tree:remove:[", bt_udata->stream); + for ( u = 0; u < bt_udata->mesg.ndims-1; u++) + { + HDfprintf(bt_udata->stream, "%s%Hd", u?", ":"", lt_key->offset[u]); + } + HDfputs("]\n", bt_udata->stream); +#endif + + HDmemset( &udata, 0, sizeof udata ); + udata.key = *lt_key; + udata.mesg = bt_udata->mesg; + + /* Remove */ + if ( H5B_remove( f, H5B_ISTORE, addr, &udata ) < 0 ) + { + HRETURN_ERROR( H5E_SYM, H5E_CANTINIT, FAIL, "unable to remove entry" ); + } + } + + FUNC_LEAVE( SUCCEED ); +} + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_remove + * + * Purpose: Removes chunks that are no longer necessary in the B-tree. + * + * Return: Success: 0, Failure: -1 + * + * Programmer: Robb Matzke + * Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: March 28, 2002 + * + * Comments: Part of H5B_ISTORE + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +static H5B_ins_t H5F_istore_remove( H5F_t *f, + haddr_t addr, + void *_lt_key /*in,out*/, + hbool_t *lt_key_changed /*out*/, + void UNUSED *_udata /*in,out*/, + void UNUSED *_rt_key /*in,out*/, + hbool_t *rt_key_changed /*out*/) +{ + H5F_istore_key_t *lt_key = (H5F_istore_key_t*)_lt_key; + H5FD_free(f->shared->lf, H5FD_MEM_DRAW, addr, lt_key->nbytes); + *lt_key_changed = FALSE; + *rt_key_changed = FALSE; + return H5B_INS_REMOVE; +} + + + +/*------------------------------------------------------------------------- + * Function: H5F_istore_initialize_by_extent + * + * Purpose: This function searches for chunks that have to be initialized with the fill + * value both in the raw data cache and in the B-tree. + * + * Return: Success: 0, Failure: -1 + * + * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu + * + * Date: April 4, 2002 + * + * Comments: + * + * (See the example of H5F_istore_prune_by_extent) + * Next, there are seven chunks where the database extent boundary is + * within the chunk. We find those seven just like we did with the previous nine. + * Fot the ones that are allocated we initialize the part that lies outside the boundary + * with the fill value. + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + + +herr_t H5F_istore_initialize_by_extent( H5F_t *f, H5O_layout_t *layout, H5O_pline_t *pline, + H5O_fill_t *fill, H5S_t *space ) +{ + hid_t dxpl_id; /*dataset transfer property list*/ + uint8_t *chunk = NULL; /*the file chunk */ + unsigned idx_hint = 0; /*input value for H5F_istore_lock*/ + hssize_t chunk_offset[H5O_LAYOUT_NDIMS]; /*logical location of the chunks */ + hsize_t idx_cur[H5O_LAYOUT_NDIMS]; /*multi-dimensional counters */ + hsize_t idx_min[H5O_LAYOUT_NDIMS]; + hsize_t idx_max[H5O_LAYOUT_NDIMS]; + hsize_t sub_size[H5O_LAYOUT_NDIMS]; + hsize_t naccessed; /*bytes accessed in chunk */ + hsize_t elm_size; /*size of an element in bytes */ + hsize_t end_chunk; /*chunk position counter */ + hssize_t start[H5O_LAYOUT_NDIMS]; /*starting location of hyperslab */ + hsize_t count[H5O_LAYOUT_NDIMS]; /*element count of hyperslab */ + hsize_t size[H5O_LAYOUT_NDIMS]; /*current size of dimensions */ + H5S_t *space_chunk=NULL; /*dataspace for a chunk */ + hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /*current dataspace dimensions */ + int rank; /*current # of dimensions */ + int i, carry; /*counters */ + unsigned u; + int found = 0; /*initialize this entry */ + + FUNC_ENTER( H5F_istore_initialize_by_extent, FAIL ); + + /* Check args */ + assert(f); + assert(layout && H5D_CHUNKED==layout->type); + assert(layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS); + assert(H5F_addr_defined(layout->addr)); + assert(space); + assert(pline); + assert(fill); + + HDmemset( start, 0, sizeof(start) ); + HDmemset( count, 0, sizeof(count) ); + + /* Go get the rank & dimensions */ + if((rank=H5S_get_simple_extent_dims(space, curr_dims, NULL))<0) + HRETURN_ERROR( H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions"); + + for ( i = 0; i < rank; i++ ) + { + size[i] = curr_dims[i]; + } + size[i] = layout->dim[i]; + elm_size = size[i]; + + /* Default dataset transfer property list */ + dxpl_id = H5P_DATASET_XFER_DEFAULT; + + /* Create a data space for a chunk & set the extent */ + if(NULL==(space_chunk=H5S_create(H5S_SIMPLE))) { + HRETURN_ERROR (H5E_DATASPACE, H5E_CANTCREATE, FAIL, + "can't create simple dataspace"); + } + if(H5S_set_extent_simple(space_chunk,(unsigned)rank,layout->dim,NULL)<0) { + HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, + "can't set dimensions"); + } + +/* + * Set up multi-dimensional counters (idx_min, idx_max, and idx_cur) and + * loop through the chunks copying each chunk from the application to the + * chunk cache. + */ + for ( u = 0; u < layout->ndims; u++) + { + idx_min[u] = 0; + idx_max[u] = (size[u]-1) / layout->dim[u] + 1; + idx_cur[u] = idx_min[u]; + } + + /* Loop over all chunks */ + while ( 1 ) + { + + for ( u = 0, naccessed=1; u < layout->ndims; u++ ) + { + /* The location and size of the chunk being accessed */ + chunk_offset[u] = idx_cur[u] * (hssize_t)(layout->dim[u]); + sub_size[u] = MIN((idx_cur[u]+1)*layout->dim[u],size[u]) - chunk_offset[u]; + naccessed *= sub_size[u]; + } + + /* + Figure out what chunks have to be initialized. These are the chunks where the database + extent boundary is within the chunk + */ + + for ( u = 0, found = 0; u < layout->ndims-1; u++ ) + { + end_chunk = chunk_offset[u] + layout->dim[u]; + if ( end_chunk > size[u] ) + { + found = 1; + break; + } + } + + if ( found ) + { + + if (NULL==(chunk=H5F_istore_lock( f, dxpl_id, layout, pline, fill, chunk_offset, FALSE, &idx_hint))) + { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to read raw data chunk"); + } + + if ( H5S_select_all( space_chunk ) < 0 ) + { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to select space"); + } + + for ( i = 0; i < rank; i++ ) + { + count[i] = MIN( (idx_cur[i]+1)*layout->dim[i], size[i]-chunk_offset[i] ); + } + +#if defined (H5F_ISTORE_DEBUG) + HDfputs("cache:initialize:offset:[", stdout); + for ( u = 0; u < layout->ndims-1; u++) + { + HDfprintf( stdout, "%s%Hd", u?", ":"", chunk_offset[u]); + } + HDfputs("]", stdout ); + HDfputs(":count:[", stdout); + for ( u = 0; u < layout->ndims-1; u++) + { + HDfprintf( stdout, "%s%Hd", u?", ":"", count[u]); + } + HDfputs("]\n", stdout ); +#endif + + if ( H5S_select_hyperslab( space_chunk, H5S_SELECT_NOTB, start, NULL, count, NULL)<0) + { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to select hyperslab"); + } + + /* Fill the selection in the memory buffer */ + if ( H5S_select_fill( fill->buf, fill->size, space_chunk, chunk ) < 0 ) + { + HRETURN_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "filling selection failed"); + } + + if (H5F_istore_unlock(f, dxpl_id, layout, pline, TRUE, chunk_offset, &idx_hint, chunk, (size_t)naccessed)<0) + { + HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to unlock raw data chunk"); + } + + } /*found*/ + + + /* Increment indices */ + for ( i = layout->ndims-1, carry=1; i >= 0 && carry; --i) + { + if (++idx_cur[i] >= idx_max[i]) + idx_cur[i] = idx_min[i]; + else + carry = 0; + } + if (carry) + break; + + } + + + if(space_chunk) + H5S_close( space_chunk); + + FUNC_LEAVE( SUCCEED ); +} + diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 8a77e7a..898da84 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -26,6 +26,7 @@ typedef struct H5F_t H5F_t; + /* * Encode and decode macros for file meta-data. * Currently, all file meta-data is little-endian. @@ -391,6 +392,15 @@ __DLL__ hsize_t H5F_istore_allocated(H5F_t *f, unsigned ndims, haddr_t addr); __DLL__ herr_t H5F_istore_dump_btree(H5F_t *f, FILE *stream, unsigned ndims, haddr_t addr); +__DLL__ herr_t H5F_istore_prune_by_extent( H5F_t *f, + struct H5O_layout_t *layout, + struct H5S_t *space ); +__DLL__ herr_t H5F_istore_initialize_by_extent( H5F_t *f, + struct H5O_layout_t *layout, + struct H5O_pline_t *pline, + struct H5O_fill_t *fill, + struct H5S_t *space ); + /* Functions for allocation/releasing chunks */ __DLL__ void * H5F_istore_chunk_alloc(size_t chunk_size); __DLL__ void * H5F_istore_chunk_realloc(void *chunk, size_t new_size); diff --git a/src/H5S.c b/src/H5S.c index 383c851..f681615 100644 --- a/src/H5S.c +++ b/src/H5S.c @@ -24,8 +24,6 @@ #include "H5Spkg.h" /* Data-space functions */ /* Local static function prototypes */ -herr_t H5S_set_extent_simple (H5S_t *space, unsigned rank, const hsize_t *dims, - const hsize_t *max); /* Interface initialization */ #define PABLO_MASK H5S_mask @@ -1979,7 +1977,7 @@ H5S_debug(H5F_t *f, const void *_mesg, FILE *stream, int indent, int fwidth) /*------------------------------------------------------------------------- - * Function: H5S_set_extend + * Function: H5S_set_extent * * Purpose: Modify the dimensions of a data space. Based on H5S_extend * @@ -1994,12 +1992,12 @@ H5S_debug(H5F_t *f, const void *_mesg, FILE *stream, int indent, int fwidth) *------------------------------------------------------------------------- */ -int H5S_set_extend( H5S_t *space, const hsize_t *size ) +int H5S_set_extent( H5S_t *space, const hsize_t *size ) { int ret_value = 0; unsigned u; - FUNC_ENTER( H5S_set_extend, FAIL ); + FUNC_ENTER( H5S_set_extent, FAIL ); /* Check args */ assert( space && H5S_SIMPLE==space->extent.type ); diff --git a/src/H5Sall.c b/src/H5Sall.c index d988de2..36ad983 100644 --- a/src/H5Sall.c +++ b/src/H5Sall.c @@ -46,7 +46,6 @@ static hsize_t H5S_all_mgath (const void *_buf, size_t elmt_size, static herr_t H5S_all_mscat (const void *_tconv_buf, size_t elmt_size, const H5S_t *mem_space, H5S_sel_iter_t *mem_iter, hsize_t nelmts, void *_buf/*out*/); -static herr_t H5S_select_all(H5S_t *space); const H5S_fconv_t H5S_ALL_FCONV[1] = {{ "all", /*name */ @@ -904,7 +903,8 @@ H5S_all_bounds(H5S_t *space, hsize_t *start, hsize_t *end) EXAMPLES REVISION LOG --------------------------------------------------------------------------*/ -static herr_t H5S_select_all (H5S_t *space) +herr_t +H5S_select_all (H5S_t *space) { herr_t ret_value=SUCCEED; /* return value */ diff --git a/src/H5Shyper.c b/src/H5Shyper.c index 029b7d7..0857aeb 100644 --- a/src/H5Shyper.c +++ b/src/H5Shyper.c @@ -66,9 +66,6 @@ static herr_t H5S_hyper_free_span (H5S_hyper_span_t *span); static H5S_hyper_span_info_t *H5S_hyper_copy_span (H5S_hyper_span_info_t *spans); static herr_t H5S_hyper_span_scratch (H5S_hyper_span_info_t *spans, void *scr_value); static herr_t H5S_hyper_span_precompute (H5S_hyper_span_info_t *spans, size_t elmt_size); -static herr_t H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, const hssize_t start[], - const hsize_t stride[], const hsize_t count[], - const hsize_t block[]); const H5S_fconv_t H5S_HYPER_FCONV[1] = {{ "hslab", /*name */ diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index 0380d36..3fd1903 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -196,6 +196,8 @@ __DLL__ hsize_t H5S_get_npoints_max(const H5S_t *ds); __DLL__ int H5S_get_simple_extent_ndims(const H5S_t *ds); __DLL__ int H5S_get_simple_extent_dims(const H5S_t *ds, hsize_t dims[]/*out*/, hsize_t max_dims[]/*out*/); +__DLL__ herr_t H5S_set_extent_simple (H5S_t *space, unsigned rank, const hsize_t *dims, + const hsize_t *max); __DLL__ herr_t H5S_modify(struct H5G_entry_t *ent, const H5S_t *space); __DLL__ H5S_t *H5S_read(struct H5G_entry_t *ent); __DLL__ int H5S_cmp(const H5S_t *ds1, const H5S_t *ds2); @@ -208,7 +210,7 @@ __DLL__ herr_t H5S_extent_release(H5S_t *space); __DLL__ herr_t H5S_select_release(H5S_t *space); __DLL__ hssize_t H5S_get_select_npoints(const H5S_t *space); __DLL__ int H5S_extend(H5S_t *space, const hsize_t *size); -__DLL__ int H5S_set_extend(H5S_t *space, const hsize_t *size); +__DLL__ int H5S_set_extent(H5S_t *space, const hsize_t *size); __DLL__ htri_t H5S_select_valid(const H5S_t *space); __DLL__ herr_t H5S_debug(H5F_t *f, const void *_mesg, FILE *stream, int indent, int fwidth); @@ -226,6 +228,12 @@ __DLL__ herr_t H5S_sel_iter_release(const H5S_t *space, __DLL__ herr_t H5S_select_fill(const void *fill, size_t fill_size, H5S_t *space, void *buf); +/* Needed for internal use of selections in H5Fistore code */ +__DLL__ herr_t H5S_select_all(H5S_t *space); +__DLL__ herr_t H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, const hssize_t start[], + const hsize_t stride[], const hsize_t count[], + const hsize_t block[]); + #ifdef H5_HAVE_PARALLEL /* MPI-IO function to check whether its possible to transfer directly from app buffer to file */ -- cgit v0.12