summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/H5B.c110
-rw-r--r--src/H5Bprivate.h10
-rw-r--r--src/H5D.c288
-rw-r--r--src/H5Distore.c638
-rw-r--r--src/H5Dprivate.h4
-rw-r--r--src/H5Dpublic.h4
-rw-r--r--src/H5Fistore.c638
-rw-r--r--src/H5Fprivate.h10
-rw-r--r--src/H5S.c8
-rw-r--r--src/H5Sall.c4
-rw-r--r--src/H5Shyper.c3
-rw-r--r--src/H5Sprivate.h10
12 files changed, 1482 insertions, 245 deletions
diff --git a/src/H5B.c b/src/H5B.c
index 0adbca0..04993a1 100644
--- a/src/H5B.c
+++ b/src/H5B.c
@@ -2367,3 +2367,113 @@ H5B_assert(H5F_t *f, haddr_t addr, const H5B_class_t *type, void *udata)
FUNC_LEAVE(SUCCEED);
}
#endif /* H5B_DEBUG */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5B_prune_by_extent
+ *
+ * Purpose: Search for chunks that are no longer necessary in the B-tree.
+ * The function iterates trough the B-tree and calls an operator function prune_extent
+ *
+ * Return: Success: 0, Failure: -1
+ *
+ * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
+ *
+ * Date: March 26, 2002
+ *
+ * Comments: Private function
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+
+herr_t H5B_prune_by_extent( H5F_t *f, const H5B_class_t *type, haddr_t addr, void *udata,
+ hsize_t *size )
+{
+ H5B_t *bt = NULL;
+ haddr_t next_addr;
+ haddr_t cur_addr = HADDR_UNDEF;
+ uint8_t *key = NULL;
+ int i, nchildren;
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER( H5B_prune_by_extent, FAIL);
+
+/*
+ * Check arguments.
+ */
+ assert( f );
+ assert( type );
+ assert( type->prune_extent);
+ assert( H5F_addr_defined( addr ) );
+ assert( udata );
+
+ if ( NULL == ( bt = H5AC_find( f, H5AC_BT, addr, type, udata )))
+ {
+ HGOTO_ERROR(H5E_BTREE, H5E_CANTLOAD, FAIL, "unable to load B-tree node" );
+ }
+
+ if ( bt->level > 0 )
+ {
+ /* Keep following the left-most child until we reach a leaf node. */
+ if (( ret_value = H5B_prune_by_extent(f, type, bt->child[0], udata, size )) < 0 )
+ {
+ HGOTO_ERROR(H5E_BTREE, H5E_CANTLIST, FAIL, "unable to list B-tree node");
+ }
+ }
+ else
+ {
+
+/*
+ * We've reached the left-most leaf. Now follow the right-sibling
+ * pointer from leaf to leaf until we've processed all leaves.
+ */
+
+ if ( NULL== ( key = H5MM_malloc((2*H5B_Kvalue(f, type)+1)*type->sizeof_nkey )))
+ {
+ HGOTO_ERROR( H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed" );
+ }
+
+ for ( cur_addr = addr, ret_value = 0; H5F_addr_defined(cur_addr) && !ret_value; cur_addr = next_addr)
+ {
+
+ /*
+ * Save all the native keys since we can't leave the B-tree node protected during an application callback.
+ */
+
+ if ( NULL == ( bt = H5AC_find( f, H5AC_BT, cur_addr, type, udata )))
+ {
+ HGOTO_ERROR( H5E_BTREE, H5E_CANTLOAD, FAIL, "B-tree node" );
+ }
+
+ for ( i = 0; i < bt->nchildren+1; i++ )
+ {
+ if ( !bt->key[i].nkey )
+ H5B_decode_key(f, bt, i);
+ HDmemcpy( key+i*type->sizeof_nkey, bt->key[i].nkey, type->sizeof_nkey );
+ }
+
+ next_addr = bt->right;
+ nchildren = bt->nchildren;
+ bt = NULL;
+
+ /* Figure out what chunks are no longer in use for the specified extent and release them */
+
+ for ( i = 0, ret_value = 0; i < nchildren && !ret_value; i++ )
+ {
+ ret_value = ( type->prune_extent )( f, key+i*type->sizeof_nkey, addr, udata, size );
+ if ( ret_value < 0 )
+ {
+ HGOTO_ERROR( H5E_BTREE, H5E_CANTINIT, FAIL, "iterator function failed" );
+ }
+ } /*i*/
+ } /*addr*/
+ } /*level*/
+
+done:
+ if( key != NULL )
+ H5MM_xfree( key );
+ FUNC_LEAVE( ret_value );
+}
+
diff --git a/src/H5Bprivate.h b/src/H5Bprivate.h
index 8cab893..e6fc2ac 100644
--- a/src/H5Bprivate.h
+++ b/src/H5Bprivate.h
@@ -87,6 +87,13 @@ typedef struct H5B_class_t {
herr_t (*decode)(H5F_t*, struct H5B_t*, uint8_t*, void*);
herr_t (*encode)(H5F_t*, struct H5B_t*, uint8_t*, void*);
herr_t (*debug_key)(FILE*, int, int, const void*, const void*);
+
+ /*pvn */
+
+ /* iterate through the leaf nodes, removing chunks upon H5Dset_extend request */
+ herr_t (*prune_extent)( H5F_t *f, void *_lt_key, haddr_t addr,
+ void *_udata, hsize_t *size );
+
} H5B_class_t;
/*
@@ -132,5 +139,8 @@ __DLL__ herr_t H5B_remove(H5F_t *f, const H5B_class_t *type, haddr_t addr,
__DLL__ herr_t H5B_iterate (H5F_t *f, const H5B_class_t *type, haddr_t addr,
void *udata);
__DLL__ int H5B_Kvalue(H5F_t *f, const H5B_class_t *type);
+__DLL__ herr_t H5B_prune_by_extent( H5F_t *f, const H5B_class_t *type, haddr_t addr,
+ void *udata, hsize_t *size );
+
#endif
diff --git a/src/H5D.c b/src/H5D.c
index dc591ba..c2980ae 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -27,6 +27,8 @@
#include "H5Vprivate.h" /* Vector and array functions */
#include "H5Zprivate.h" /* Data filters */
+/*#define H5D_DEBUG*/
+
/*
* The MPIO driver is needed because there are kludges in this file and
* places where we check for things that aren't handled by this driver.
@@ -40,10 +42,6 @@
#define PABLO_MASK H5D_mask
-
-/* forward declarations */
-static herr_t H5D_update_chunk( H5D_t *dataset );
-
/*
* A dataset is the following struct.
*/
@@ -3681,80 +3679,36 @@ done:
FUNC_LEAVE (ret_value);
} /* H5Dfill() */
-
-/*-------------------------------------------------------------------------
- * Function: H5Ddebug
- *
- * Purpose: Prints various information about a dataset. This function is
- * not to be documented in the API at this time.
- *
- * Return: Success: Non-negative
- *
- * Failure: Negative
- *
- * Programmer: Robb Matzke
- * Wednesday, April 28, 1999
- *
- * Modifications:
- *
- *-------------------------------------------------------------------------
- */
-herr_t
-H5Ddebug(hid_t dset_id, unsigned UNUSED flags)
-{
- H5D_t *dset=NULL;
-
- FUNC_ENTER(H5Ddebug, FAIL);
- H5TRACE2("e","iIu",dset_id,flags);
-
- /* Check args */
- if (H5I_DATASET!=H5I_get_type(dset_id) ||
- NULL==(dset=H5I_object(dset_id))) {
- HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset");
- }
-
- /* Print B-tree information */
- if (H5D_CHUNKED==dset->layout.type) {
- H5F_istore_dump_btree(dset->ent.file, stdout, dset->layout.ndims,
- dset->layout.addr);
- } else if (H5D_CONTIGUOUS==dset->layout.type) {
- HDfprintf(stdout, " %-10s %a\n", "Address:",
- dset->layout.addr);
- }
-
- FUNC_LEAVE(SUCCEED);
-}
/*-------------------------------------------------------------------------
- * Function: H5Dset_extend
+ * Function: H5Dset_extent
*
- * Purpose: Modifies the dimensions of a dataset, based on H5Dextend
+ * Purpose: Modifies the dimensions of a dataset, based on H5Dextend.
+ * Can change to a lower dimension.
*
* Return: Success: 0, Failure: -1
*
* Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
* Robb Matzke
*
- * Date: March 13, 2002
+ * Date: April 9, 2002
*
- * Comments: Public function, calls private H5D_set_extend
+ * Comments: Public function, calls private H5D_set_extent
*
* Modifications:
*
- *
*-------------------------------------------------------------------------
*/
-herr_t H5Dset_extend( hid_t dset_id, const hsize_t *size )
+herr_t H5Dset_extent( hid_t dset_id, const hsize_t *size )
{
H5D_t *dset = NULL;
- FUNC_ENTER( H5Dset_extend, FAIL );
- H5TRACE2( "e", "i*h", dset_id, size );
-
+ FUNC_ENTER( H5Dset_extent, FAIL );
+
/* Check args */
if ( H5I_DATASET != H5I_get_type( dset_id ) || NULL == ( dset = H5I_object( dset_id )))
{
@@ -3764,144 +3718,210 @@ herr_t H5Dset_extend( hid_t dset_id, const hsize_t *size )
{
HRETURN_ERROR( H5E_ARGS, H5E_BADVALUE, FAIL, "no size specified" );
}
-
- /* Modify size */
- if ( H5D_set_extend (dset, size) < 0 )
+
+ /* Private function */
+ if ( H5D_set_extent ( dset, size ) < 0 )
{
HRETURN_ERROR( H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set extend dataset");
}
-
+
FUNC_LEAVE (SUCCEED);
}
-
-
/*-------------------------------------------------------------------------
- * Function: H5D_set_extend
+ * Function: H5D_set_extent
*
- * Purpose: Same as H5D_extend, allows change to a lower dimension, calls H5S_modify
+ * Purpose: Based in H5D_extend, allows change to a lower dimension,
+ * calls H5S_set_extent and H5F_istore_prune_by_extent instead
*
* Return: Success: 0, Failure: -1
*
* Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
* Robb Matzke
*
- * Date: March 13, 2002
+ * Date: April 9, 2002
*
- * Comments:
+ * Comments: Private function
*
* Modifications:
*
- *
*-------------------------------------------------------------------------
*/
-herr_t H5D_set_extend( H5D_t *dataset, const hsize_t *size )
+herr_t H5D_set_extent( H5D_t *dset, const hsize_t *size )
{
+ hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /* Current dimension sizes */
+ int rank; /* Dataspace # of dimensions */
herr_t changed;
herr_t ret_value = FAIL;
H5S_t *space = NULL;
- H5O_fill_t fill;
H5P_genplist_t *plist;
-
- FUNC_ENTER( H5D_set_extend, FAIL );
-
+ H5O_fill_t fill;
+ H5O_pline_t pline;
+ unsigned u;
+ int shrink = 0;
+
+ FUNC_ENTER( H5D_set_extent, FAIL );
+
/* Check args */
- assert( dataset );
+ assert( dset );
assert( size );
-
-/*
- * NOTE: Restrictions on extensions were checked when the dataset was
- * created. All extensions are allowed here since none should be
- * able to muck things up.
- */
-
- /* Increase the size of the data space */
- if ( NULL == ( space = H5S_read (&(dataset->ent))))
- HGOTO_ERROR( H5E_DATASET, H5E_CANTINIT, FAIL, "unable to read data space info from dataset header");
- if ( ( changed = H5S_set_extend( space, size )) < 0 )
+ /*-------------------------------------------------------------------------
+ * Get the data space
+ *-------------------------------------------------------------------------
+ */
+
+ if ( NULL == ( space = H5S_read (&(dset->ent))))
+ {
+ HGOTO_ERROR( H5E_DATASET, H5E_CANTINIT, FAIL, "unable to read data space info from dset header");
+ }
+
+ /*-------------------------------------------------------------------------
+ * Check if we are shrinking in any of the dimensions
+ *-------------------------------------------------------------------------
+ */
+ if((rank=H5S_get_simple_extent_dims(space, curr_dims, NULL))<0)
+ HGOTO_ERROR( H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions");
+
+ for ( u = 0; u < rank; u++ )
+ {
+ if ( size[u] < curr_dims[u] )
+ {
+ shrink = 1;
+ break;
+ }
+ }
+
+ /*-------------------------------------------------------------------------
+ * Modify the size of the data space
+ *-------------------------------------------------------------------------
+ */
+
+ if ( ( changed = H5S_set_extent( space, size )) < 0 )
+ {
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space");
-
+ }
+
+ /*-------------------------------------------------------------------------
+ * Modify the dataset storage if changed space
+ *-------------------------------------------------------------------------
+ */
+
if ( changed > 0 )
{
/* Save the new dataspace in the file if necessary */
- if ( H5S_modify ( &(dataset->ent ), space ) < 0 )
+ if ( H5S_modify ( &(dset->ent ), space ) < 0 )
+ {
HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to update file with new dataspace");
-
- /* Initialize the new parts of the dataset */
-#ifdef LATER
- if ( H5S_select_all( space ) < 0 || H5S_select_hyperslab( space, H5S_SELECT_DIFF, zero, NULL, old_dims, NULL ) < 0 )
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to select new extents for fill value");
-#else
-
- /*
- * We don't have the H5S_SELECT_DIFF operator yet. We really only
- * need it for contiguous datasets because the chunked datasets will
- * either fill on demand during I/O or attempt a fill of all chunks.
- */
+ }
- if ( NULL == ( plist = H5I_object(dataset->dcpl_id )))
- HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list");
+ /* Initialize the new parts of the dset */
+ if ( NULL == ( plist = H5I_object( dset->dcpl_id )))
+ {
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dset creation property list");
+ }
if( H5P_get( plist, H5D_CRT_FILL_VALUE_NAME, &fill ) < 0 )
+ {
HGOTO_ERROR( H5E_DATASET, H5E_CANTGET, FAIL, "can't get fill value" );
- if( H5D_CONTIGUOUS == dataset->layout.type && fill.buf )
+ }
+ if( H5D_CONTIGUOUS == dset->layout.type && fill.buf )
+ {
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to select fill value region");
-#endif
-
- if ( H5D_init_storage( dataset, space ) < 0 )
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize dataset with fill value");
-
-
- /* Update chunk information */
- if ( H5D_CHUNKED == dataset->layout.type )
- {
-
- if ( H5D_update_chunk( dataset ) < 0 )
- goto done;
-
-
- }
-
-
+ }
+ if ( H5D_init_storage( dset, space ) < 0 )
+ {
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize dset with fill value");
+ }
+
} /* end if changed */
+
+
+ /*-------------------------------------------------------------------------
+ * Remove chunk information in the case of chunked datasets
+ * This removal takes place only in case we are shrinking the dateset
+ *-------------------------------------------------------------------------
+ */
+
+ if ( changed > 0 && shrink && H5D_CHUNKED == dset->layout.type )
+ {
+
+#if defined (PVN)
+ H5F_istore_dump_btree(dset->ent.file, stdout, dset->layout.ndims, dset->layout.addr);
+#endif
+ if ( H5F_istore_prune_by_extent( dset->ent.file, &dset->layout, space ) < 0 )
+ {
+ HRETURN_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to remove chunks ");
+ }
+
+#if defined (PVN)
+ H5F_istore_dump_btree(dset->ent.file, stdout, dset->layout.ndims, dset->layout.addr);
+#endif
+
+ if( H5P_get( plist, H5D_CRT_DATA_PIPELINE_NAME, &pline) < 0)
+ {
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get pipeline");
+ }
+
+ if ( H5F_istore_initialize_by_extent( dset->ent.file, &dset->layout, &pline, &fill, space ) < 0 )
+ {
+ HRETURN_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to initialize chunks ");
+ }
+
+ }
+
ret_value = SUCCEED;
-
+
done:
- H5S_close( space );
+ if(space)
+ H5S_close( space );
FUNC_LEAVE( ret_value );
}
-
+
/*-------------------------------------------------------------------------
- * Function: H5D_update_chunk
- *
- * Purpose:
+ * Function: H5Ddebug
*
- * Return: Success: 0, Failure: -1
+ * Purpose: Prints various information about a dataset. This function is
+ * not to be documented in the API at this time.
*
- * Programmer: Implementation: Pedro Vicente, pvn@ncsa.uiuc.edu
- * Algorithm: Robb Matzke
+ * Return: Success: Non-negative
*
- * Date: March 13, 2002
+ * Failure: Negative
*
- * Comments:
+ * Programmer: Robb Matzke
+ * Wednesday, April 28, 1999
*
* Modifications:
*
- *
*-------------------------------------------------------------------------
*/
-
-herr_t H5D_update_chunk( H5D_t *dset )
+herr_t
+H5Ddebug(hid_t dset_id, unsigned UNUSED flags)
{
+ H5D_t *dset=NULL;
- H5F_istore_dump_btree( dset->ent.file, stdout, dset->layout.ndims, dset->layout.addr );
+ FUNC_ENTER(H5Ddebug, FAIL);
+ H5TRACE2("e","iIu",dset_id,flags);
- return 0;
+ /* Check args */
+ if (H5I_DATASET!=H5I_get_type(dset_id) ||
+ NULL==(dset=H5I_object(dset_id))) {
+ HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset");
+ }
+ /* Print B-tree information */
+ if (H5D_CHUNKED==dset->layout.type) {
+ H5F_istore_dump_btree(dset->ent.file, stdout, dset->layout.ndims,
+ dset->layout.addr);
+ } else if (H5D_CONTIGUOUS==dset->layout.type) {
+ HDfprintf(stdout, " %-10s %a\n", "Address:",
+ dset->layout.addr);
+ }
+
+ FUNC_LEAVE(SUCCEED);
}
diff --git a/src/H5Distore.c b/src/H5Distore.c
index df8b04a..1ed6679 100644
--- a/src/H5Distore.c
+++ b/src/H5Distore.c
@@ -42,6 +42,7 @@
#include "H5MMprivate.h"
#include "H5Oprivate.h"
#include "H5Pprivate.h" /* Property lists */
+#include "H5Sprivate.h" /* Dataspaces */
#include "H5Vprivate.h"
/* MPIO driver needed for special checks */
@@ -70,7 +71,8 @@
* the disk block size, chunks are aligned on disk block boundaries,
* and the operating system can also eliminate a read operation.
*/
-/* #define H5F_ISTORE_DEBUG */
+
+/*#define H5F_ISTORE_DEBUG */
/* Interface initialization */
#define PABLO_MASK H5Fistore_mask
@@ -106,7 +108,7 @@ typedef H5F_rdcc_ent_t *H5F_rdcc_ent_ptr_t; /* For free lists */
static size_t H5F_istore_sizeof_rkey(H5F_t *f, const void *_udata);
static herr_t H5F_istore_new_node(H5F_t *f, H5B_ins_t, void *_lt_key,
void *_udata, void *_rt_key,
- haddr_t*/*out*/);
+ haddr_t *addr_p /*out*/);
static int H5F_istore_cmp2(H5F_t *f, void *_lt_key, void *_udata,
void *_rt_key);
static int H5F_istore_cmp3(H5F_t *f, void *_lt_key, void *_udata,
@@ -129,6 +131,15 @@ static herr_t H5F_istore_debug_key(FILE *stream, int indent, int fwidth,
static haddr_t H5F_istore_get_addr(H5F_t *f, const H5O_layout_t *layout,
const hssize_t offset[]);
+static herr_t H5F_istore_prune_extent( H5F_t *f, void *left_key, haddr_t addr,
+ void *_udata, hsize_t *size );
+static H5B_ins_t H5F_istore_remove( H5F_t *f, haddr_t addr,
+ void *_lt_key /*in,out*/,
+ hbool_t *lt_key_changed /*out*/,
+ void UNUSED *_udata /*in,out*/,
+ void UNUSED *_rt_key /*in,out*/,
+ hbool_t *rt_key_changed /*out*/);
+
/*
* B-tree key. A key contains the minimum logical N-dimensional address and
* the logical size of the chunk to which this key refers. The
@@ -169,11 +180,15 @@ H5B_class_t H5B_ISTORE[1] = {{
H5F_istore_insert, /*insert */
FALSE, /*follow min branch? */
FALSE, /*follow max branch? */
- NULL, /*remove */
+ /*pvn*/
+ /*NULL,*/ /*remove */
+ H5F_istore_remove, /*remove */
H5F_istore_iterate, /*iterator */
H5F_istore_decode_key, /*decode */
H5F_istore_encode_key, /*encode */
H5F_istore_debug_key, /*debug */
+ /*pvn*/
+ H5F_istore_prune_extent, /*remove chunks, upon H5Dset_extend call */
}};
#define H5F_HASH_DIVISOR 8 /* Attempt to spread out the hashing */
@@ -1052,52 +1067,71 @@ done:
* Programmer: Robb Matzke
* Thursday, May 21, 1998
*
- * Modifications:
+ * Modifications: Pedro Vicente, March 28, 2002
+ * Added flush parameter that switches the call to H5F_istore_flush_entry
+ * The call with FALSE is used by the H5F_istore_prune_by_extent function
*
*-------------------------------------------------------------------------
*/
-static herr_t
-H5F_istore_preempt (H5F_t *f, H5F_rdcc_ent_t *ent)
-{
- H5F_rdcc_t *rdcc = &(f->shared->rdcc);
-
- FUNC_ENTER (H5F_istore_preempt, FAIL);
-
- assert(f);
- assert(ent);
- assert(!ent->locked);
- assert(ent->idx<rdcc->nslots);
-
- /* Flush */
- if (H5F_istore_flush_entry(f, ent, TRUE)<0) {
- HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
- "cannot flush indexed storage buffer");
- }
-
- /* Unlink from list */
- if (ent->prev) {
- ent->prev->next = ent->next;
- } else {
- rdcc->head = ent->next;
- }
- if (ent->next) {
- ent->next->prev = ent->prev;
- } else {
- rdcc->tail = ent->prev;
- }
- ent->prev = ent->next = NULL;
-
- /* Remove from cache */
- rdcc->slot[ent->idx] = NULL;
- ent->idx = UINT_MAX;
- rdcc->nbytes -= ent->chunk_size;
- --rdcc->nused;
+
+ static herr_t
+ H5F_istore_preempt (H5F_t *f, H5F_rdcc_ent_t *ent, hbool_t flush )
+ {
+ H5F_rdcc_t *rdcc = &(f->shared->rdcc);
+
+ FUNC_ENTER (H5F_istore_preempt, FAIL);
+
+ assert(f);
+ assert(ent);
+ assert(!ent->locked);
+ assert(ent->idx<rdcc->nslots);
+
+ if ( flush )
+ {
+
+ /* Flush */
+ if (H5F_istore_flush_entry(f, ent, TRUE)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
+ "cannot flush indexed storage buffer");
+ }
+ }
+
+ else
+ {
+
+ /* Reset, but do not free or remove from list */
+ ent->layout = H5O_free(H5O_LAYOUT, ent->layout);
+ ent->pline = H5O_free(H5O_PLINE, ent->pline);
+ if(ent->chunk!=NULL)
+ ent->chunk = H5F_istore_chunk_free(ent->chunk);
+
+ }
+
+ /* Unlink from list */
+ if (ent->prev) {
+ ent->prev->next = ent->next;
+ } else {
+ rdcc->head = ent->next;
+ }
+ if (ent->next) {
+ ent->next->prev = ent->prev;
+ } else {
+ rdcc->tail = ent->prev;
+ }
+ ent->prev = ent->next = NULL;
+
+ /* Remove from cache */
+ rdcc->slot[ent->idx] = NULL;
+ ent->idx = UINT_MAX;
+ rdcc->nbytes -= ent->chunk_size;
+ --rdcc->nused;
+
+ /* Free */
+ H5FL_FREE(H5F_rdcc_ent_t, ent);
+
+ FUNC_LEAVE (SUCCEED);
+ }
- /* Free */
- H5FL_FREE(H5F_rdcc_ent_t, ent);
-
- FUNC_LEAVE (SUCCEED);
-}
/*-------------------------------------------------------------------------
@@ -1112,6 +1146,8 @@ H5F_istore_preempt (H5F_t *f, H5F_rdcc_ent_t *ent)
* Thursday, May 21, 1998
*
* Modifications:
+ * Pedro Vicente, March 28, 2002
+ * Added TRUE parameter to the call to H5F_istore_preempt
*
*-------------------------------------------------------------------------
*/
@@ -1127,7 +1163,7 @@ H5F_istore_flush (H5F_t *f, hbool_t preempt)
for (ent=rdcc->head; ent; ent=next) {
next = ent->next;
if (preempt) {
- if (H5F_istore_preempt(f, ent)<0) {
+ if (H5F_istore_preempt(f, ent, TRUE )<0) {
nerrors++;
}
} else {
@@ -1157,6 +1193,8 @@ H5F_istore_flush (H5F_t *f, hbool_t preempt)
* Thursday, May 21, 1998
*
* Modifications:
+ * Pedro Vicente, March 28, 2002
+ * Added TRUE parameter to the call to H5F_istore_preempt
*
*-------------------------------------------------------------------------
*/
@@ -1175,7 +1213,7 @@ H5F_istore_dest (H5F_t *f)
HDfflush(stderr);
#endif
next = ent->next;
- if (H5F_istore_preempt(f, ent)<0) {
+ if (H5F_istore_preempt(f, ent, TRUE )<0) {
nerrors++;
}
}
@@ -1203,6 +1241,8 @@ H5F_istore_dest (H5F_t *f)
* Thursday, May 21, 1998
*
* Modifications:
+ * Pedro Vicente, March 28, 2002
+ * Added TRUE parameter to the call to H5F_istore_preempt
*
*-------------------------------------------------------------------------
*/
@@ -1280,7 +1320,7 @@ H5F_istore_prune (H5F_t *f, size_t size)
if (p[j]==cur) p[j] = NULL;
if (n[j]==cur) n[j] = cur->next;
}
- if (H5F_istore_preempt(f, cur)<0) nerrors++;
+ if (H5F_istore_preempt(f, cur, TRUE)<0) nerrors++;
}
}
@@ -1326,6 +1366,9 @@ H5F_istore_prune (H5F_t *f, size_t size)
* Robb Matzke, 1999-08-02
* The split ratios are passed in as part of the data transfer
* property list.
+ *
+ * Pedro Vicente, March 28, 2002
+ * Added TRUE parameter to the call to H5F_istore_preempt
*-------------------------------------------------------------------------
*/
static void *
@@ -1477,7 +1520,7 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
}
fprintf(stderr, "}\n");
#endif
- if (H5F_istore_preempt(f, ent)<0) {
+ if (H5F_istore_preempt(f, ent, TRUE)<0) {
HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL,
"unable to preempt chunk from cache");
}
@@ -2457,3 +2500,500 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
FUNC_LEAVE(SUCCEED);
}
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_istore_prune_by_extent
+ *
+ * Purpose: This function searches for chunks that are no longer necessary both in the
+ * raw data cache and in the B-tree.
+ *
+ * Return: Success: 0, Failure: -1
+ *
+ * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
+ * Algorithm: Robb Matzke
+ *
+ * Date: March 27, 2002
+ *
+ * The algorithm is:
+ *
+ * For chunks that are no longer necessary:
+ *
+ * 1. Search in the raw data cache for each chunk
+ * 2. If found then preempt it from the cache
+ * 3. Search in the B-tree for each chunk
+ * 4. If found then remove it from the B-tree and deallocate file storage for the chunk
+ *
+ * This example shows a 2d dataset of 90x90 with a chunk size of 20x20.
+ *
+ *
+ * 0 20 40 60 80 90 100
+ * 0 +---------+---------+---------+---------+-----+...+
+ * |:::::X:::::::::::::: : : | :
+ * |:::::::X:::::::::::: : : | : Key
+ * |::::::::::X::::::::: : : | : --------
+ * |::::::::::::X::::::: : : | : +-+ Dataset
+ * 20+::::::::::::::::::::.........:.........:.....+...: | | Extent
+ * | :::::X::::: : : | : +-+
+ * | ::::::::::: : : | :
+ * | ::::::::::: : : | : ... Chunk
+ * | :::::::X::: : : | : : : Boundary
+ * 40+.........:::::::::::.........:.........:.....+...: :.:
+ * | : : : : | :
+ * | : : : : | : ... Allocated
+ * | : : : : | : ::: & Filled
+ * | : : : : | : ::: Chunk
+ * 60+.........:.........:.........:.........:.....+...:
+ * | : :::::::X::: : | : X Element
+ * | : ::::::::::: : | : Written
+ * | : ::::::::::: : | :
+ * | : ::::::::::: : | :
+ * 80+.........:.........:::::::::::.........:.....+...: O Fill Val
+ * | : : ::::::::::: | : Explicitly
+ * | : : ::::::X:::: | : Written
+ * 90+---------+---------+---------+---------+-----+ :
+ * : : : ::::::::::: :
+ * 100:.........:.........:.........:::::::::::.........:
+ *
+ *
+ * We have 25 total chunks for this dataset, 5 of which have space
+ * allocated in the file because they were written to one or more
+ * elements. These five chunks (and only these five) also have entries in
+ * the storage B-tree for this dataset.
+ *
+ * Now lets say we want to shrink the dataset down to 70x70:
+ *
+ *
+ * 0 20 40 60 70 80 90 100
+ * 0 +---------+---------+---------+----+----+-----+...+
+ * |:::::X:::::::::::::: : | : | :
+ * |:::::::X:::::::::::: : | : | : Key
+ * |::::::::::X::::::::: : | : | : --------
+ * |::::::::::::X::::::: : | : | : +-+ Dataset
+ * 20+::::::::::::::::::::.........:....+....:.....|...: | | Extent
+ * | :::::X::::: : | : | : +-+
+ * | ::::::::::: : | : | :
+ * | ::::::::::: : | : | : ... Chunk
+ * | :::::::X::: : | : | : : : Boundary
+ * 40+.........:::::::::::.........:....+....:.....|...: :.:
+ * | : : : | : | :
+ * | : : : | : | : ... Allocated
+ * | : : : | : | : ::: & Filled
+ * | : : : | : | : ::: Chunk
+ * 60+.........:.........:.........:....+....:.....|...:
+ * | : :::::::X::: | : | : X Element
+ * | : ::::::::::: | : | : Written
+ * +---------+---------+---------+----+ : | :
+ * | : ::::::::::: : | :
+ * 80+.........:.........:::::::::X:.........:.....|...: O Fill Val
+ * | : : ::::::::::: | : Explicitly
+ * | : : ::::::X:::: | : Written
+ * 90+---------+---------+---------+---------+-----+ :
+ * : : : ::::::::::: :
+ * 100:.........:.........:.........:::::::::::.........:
+ *
+ *
+ * That means that the nine chunks along the bottom and right side should
+ * no longer exist. Of those nine chunks, (0,80), (20,80), (40,80),
+ * (60,80), (80,80), (80,60), (80,40), (80,20), and (80,0), one is actually allocated
+ * that needs to be released.
+ * To release the chunks, we traverse the B-tree to obtain a list of unused
+ * allocated chunks, and then call H5B_remove() for each chunk.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+herr_t H5F_istore_prune_by_extent( H5F_t *f, H5O_layout_t *layout, H5S_t *space )
+{
+ H5F_rdcc_t *rdcc = &(f->shared->rdcc); /*raw data chunk cache*/
+ H5F_rdcc_ent_t *ent = NULL, *next = NULL; /*cache entry */
+ unsigned u; /*counters */
+ int found = 0; /*remove this entry */
+ H5F_istore_ud1_t udata; /*B-tree pass-through */
+ hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /*current dataspace dimensions */
+
+ FUNC_ENTER( H5F_istore_prune_by_extent, FAIL );
+
+ /* Check args */
+ assert(f);
+ assert(layout && H5D_CHUNKED==layout->type);
+ assert(layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS);
+ assert(H5F_addr_defined(layout->addr));
+ assert(space);
+
+ /* Go get the rank & dimensions */
+ if(H5S_get_simple_extent_dims(space, curr_dims, NULL)<0)
+ HRETURN_ERROR( H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions");
+
+
+ /*-------------------------------------------------------------------------
+ * Figure out what chunks are no longer in use for the specified extent
+ * and release them from the linked list raw data cache
+ *-------------------------------------------------------------------------
+ */
+
+ for ( ent = rdcc->head; ent; ent=next )
+ {
+ next = ent->next;
+
+ found = 0;
+ for ( u = 0; u < ent->layout->ndims-1; u++ )
+ {
+ if ( (hsize_t)ent->offset[u] > curr_dims[u] )
+ {
+ found = 1;
+ break;
+ }
+ }
+
+ if ( found )
+ {
+#if defined (H5F_ISTORE_DEBUG)
+ HDfputs("cache:remove:[", stdout);
+ for ( u = 0; u < ent->layout->ndims-1; u++)
+ {
+ HDfprintf( stdout, "%s%Hd", u?", ":"", ent->offset[u]);
+ }
+ HDfputs("]\n", stdout );
+#endif
+
+ /* Preempt the entry from the cache, but do not flush it to disk */
+ if ( H5F_istore_preempt( f, ent, FALSE ) < 0 )
+ {
+ HRETURN_ERROR(H5E_IO, H5E_CANTINIT, 0, "unable to preempt chunk");
+ }
+
+ }
+
+ }
+
+/*-------------------------------------------------------------------------
+ * Check if there are any chunks on the B-tree
+ *-------------------------------------------------------------------------
+ */
+
+ HDmemset(&udata, 0, sizeof udata);
+ udata.stream = stdout;
+ udata.mesg.addr = layout->addr;
+ udata.mesg.ndims = layout->ndims;
+ for ( u = 0; u < udata.mesg.ndims; u++ )
+ {
+ udata.mesg.dim[u] = layout->dim[u];
+ }
+
+ if ( H5B_prune_by_extent( f, H5B_ISTORE, layout->addr, &udata, curr_dims ) < 0 )
+ {
+ HRETURN_ERROR( H5E_IO, H5E_CANTINIT, 0, "unable to iterate over B-tree" );
+ }
+
+ FUNC_LEAVE( SUCCEED );
+
+}
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_istore_prune_extent
+ *
+ * Purpose: Search for chunks that are no longer necessary in the B-tree.
+ *
+ * Return: Success: 0, Failure: -1
+ *
+ * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
+ *
+ * Date: March 26, 2002
+ *
+ * Comments: Called by H5B_prune_by_extent, part of H5B_ISTORE
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static herr_t H5F_istore_prune_extent( H5F_t *f, void *_lt_key, haddr_t addr, void *_udata,
+ hsize_t *size )
+{
+ H5F_istore_ud1_t *bt_udata = (H5F_istore_ud1_t *)_udata;
+ H5F_istore_key_t *lt_key = (H5F_istore_key_t *)_lt_key;
+ unsigned u;
+ int found = 0;
+ H5F_istore_ud1_t udata;
+
+ /* The LT_KEY is the left key (the onethat describes the chunk). It points to a chunk of
+ storage that contains the beginning of the logical address space represented by UDATA.
+ */
+
+ FUNC_ENTER( H5F_istore_prune_extent, FAIL );
+
+ /* Figure out what chunks are no longer in use for the specified extent and release them */
+
+ for ( u = 0; u < bt_udata->mesg.ndims-1; u++ )
+ {
+ if ( (hsize_t)lt_key->offset[u] > size[u] )
+ {
+ found = 1;
+ break;
+ }
+ }
+
+ if ( found )
+ {
+
+#if defined (H5F_ISTORE_DEBUG)
+ HDfputs("b-tree:remove:[", bt_udata->stream);
+ for ( u = 0; u < bt_udata->mesg.ndims-1; u++)
+ {
+ HDfprintf(bt_udata->stream, "%s%Hd", u?", ":"", lt_key->offset[u]);
+ }
+ HDfputs("]\n", bt_udata->stream);
+#endif
+
+ HDmemset( &udata, 0, sizeof udata );
+ udata.key = *lt_key;
+ udata.mesg = bt_udata->mesg;
+
+ /* Remove */
+ if ( H5B_remove( f, H5B_ISTORE, addr, &udata ) < 0 )
+ {
+ HRETURN_ERROR( H5E_SYM, H5E_CANTINIT, FAIL, "unable to remove entry" );
+ }
+ }
+
+ FUNC_LEAVE( SUCCEED );
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_istore_remove
+ *
+ * Purpose: Removes chunks that are no longer necessary in the B-tree.
+ *
+ * Return: Success: 0, Failure: -1
+ *
+ * Programmer: Robb Matzke
+ * Pedro Vicente, pvn@ncsa.uiuc.edu
+ *
+ * Date: March 28, 2002
+ *
+ * Comments: Part of H5B_ISTORE
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static H5B_ins_t H5F_istore_remove( H5F_t *f,
+ haddr_t addr,
+ void *_lt_key /*in,out*/,
+ hbool_t *lt_key_changed /*out*/,
+ void UNUSED *_udata /*in,out*/,
+ void UNUSED *_rt_key /*in,out*/,
+ hbool_t *rt_key_changed /*out*/)
+{
+ H5F_istore_key_t *lt_key = (H5F_istore_key_t*)_lt_key;
+ H5FD_free(f->shared->lf, H5FD_MEM_DRAW, addr, lt_key->nbytes);
+ *lt_key_changed = FALSE;
+ *rt_key_changed = FALSE;
+ return H5B_INS_REMOVE;
+}
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_istore_initialize_by_extent
+ *
+ * Purpose: This function searches for chunks that have to be initialized with the fill
+ * value both in the raw data cache and in the B-tree.
+ *
+ * Return: Success: 0, Failure: -1
+ *
+ * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
+ *
+ * Date: April 4, 2002
+ *
+ * Comments:
+ *
+ * (See the example of H5F_istore_prune_by_extent)
+ * Next, there are seven chunks where the database extent boundary is
+ * within the chunk. We find those seven just like we did with the previous nine.
+ * Fot the ones that are allocated we initialize the part that lies outside the boundary
+ * with the fill value.
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+
+
+herr_t H5F_istore_initialize_by_extent( H5F_t *f, H5O_layout_t *layout, H5O_pline_t *pline,
+ H5O_fill_t *fill, H5S_t *space )
+{
+ hid_t dxpl_id; /*dataset transfer property list*/
+ uint8_t *chunk = NULL; /*the file chunk */
+ unsigned idx_hint = 0; /*input value for H5F_istore_lock*/
+ hssize_t chunk_offset[H5O_LAYOUT_NDIMS]; /*logical location of the chunks */
+ hsize_t idx_cur[H5O_LAYOUT_NDIMS]; /*multi-dimensional counters */
+ hsize_t idx_min[H5O_LAYOUT_NDIMS];
+ hsize_t idx_max[H5O_LAYOUT_NDIMS];
+ hsize_t sub_size[H5O_LAYOUT_NDIMS];
+ hsize_t naccessed; /*bytes accessed in chunk */
+ hsize_t elm_size; /*size of an element in bytes */
+ hsize_t end_chunk; /*chunk position counter */
+ hssize_t start[H5O_LAYOUT_NDIMS]; /*starting location of hyperslab */
+ hsize_t count[H5O_LAYOUT_NDIMS]; /*element count of hyperslab */
+ hsize_t size[H5O_LAYOUT_NDIMS]; /*current size of dimensions */
+ H5S_t *space_chunk=NULL; /*dataspace for a chunk */
+ hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /*current dataspace dimensions */
+ int rank; /*current # of dimensions */
+ int i, carry; /*counters */
+ unsigned u;
+ int found = 0; /*initialize this entry */
+
+ FUNC_ENTER( H5F_istore_initialize_by_extent, FAIL );
+
+ /* Check args */
+ assert(f);
+ assert(layout && H5D_CHUNKED==layout->type);
+ assert(layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS);
+ assert(H5F_addr_defined(layout->addr));
+ assert(space);
+ assert(pline);
+ assert(fill);
+
+ HDmemset( start, 0, sizeof(start) );
+ HDmemset( count, 0, sizeof(count) );
+
+ /* Go get the rank & dimensions */
+ if((rank=H5S_get_simple_extent_dims(space, curr_dims, NULL))<0)
+ HRETURN_ERROR( H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions");
+
+ for ( i = 0; i < rank; i++ )
+ {
+ size[i] = curr_dims[i];
+ }
+ size[i] = layout->dim[i];
+ elm_size = size[i];
+
+ /* Default dataset transfer property list */
+ dxpl_id = H5P_DATASET_XFER_DEFAULT;
+
+ /* Create a data space for a chunk & set the extent */
+ if(NULL==(space_chunk=H5S_create(H5S_SIMPLE))) {
+ HRETURN_ERROR (H5E_DATASPACE, H5E_CANTCREATE, FAIL,
+ "can't create simple dataspace");
+ }
+ if(H5S_set_extent_simple(space_chunk,(unsigned)rank,layout->dim,NULL)<0) {
+ HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
+ "can't set dimensions");
+ }
+
+/*
+ * Set up multi-dimensional counters (idx_min, idx_max, and idx_cur) and
+ * loop through the chunks copying each chunk from the application to the
+ * chunk cache.
+ */
+ for ( u = 0; u < layout->ndims; u++)
+ {
+ idx_min[u] = 0;
+ idx_max[u] = (size[u]-1) / layout->dim[u] + 1;
+ idx_cur[u] = idx_min[u];
+ }
+
+ /* Loop over all chunks */
+ while ( 1 )
+ {
+
+ for ( u = 0, naccessed=1; u < layout->ndims; u++ )
+ {
+ /* The location and size of the chunk being accessed */
+ chunk_offset[u] = idx_cur[u] * (hssize_t)(layout->dim[u]);
+ sub_size[u] = MIN((idx_cur[u]+1)*layout->dim[u],size[u]) - chunk_offset[u];
+ naccessed *= sub_size[u];
+ }
+
+ /*
+ Figure out what chunks have to be initialized. These are the chunks where the database
+ extent boundary is within the chunk
+ */
+
+ for ( u = 0, found = 0; u < layout->ndims-1; u++ )
+ {
+ end_chunk = chunk_offset[u] + layout->dim[u];
+ if ( end_chunk > size[u] )
+ {
+ found = 1;
+ break;
+ }
+ }
+
+ if ( found )
+ {
+
+ if (NULL==(chunk=H5F_istore_lock( f, dxpl_id, layout, pline, fill, chunk_offset, FALSE, &idx_hint)))
+ {
+ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to read raw data chunk");
+ }
+
+ if ( H5S_select_all( space_chunk ) < 0 )
+ {
+ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to select space");
+ }
+
+ for ( i = 0; i < rank; i++ )
+ {
+ count[i] = MIN( (idx_cur[i]+1)*layout->dim[i], size[i]-chunk_offset[i] );
+ }
+
+#if defined (H5F_ISTORE_DEBUG)
+ HDfputs("cache:initialize:offset:[", stdout);
+ for ( u = 0; u < layout->ndims-1; u++)
+ {
+ HDfprintf( stdout, "%s%Hd", u?", ":"", chunk_offset[u]);
+ }
+ HDfputs("]", stdout );
+ HDfputs(":count:[", stdout);
+ for ( u = 0; u < layout->ndims-1; u++)
+ {
+ HDfprintf( stdout, "%s%Hd", u?", ":"", count[u]);
+ }
+ HDfputs("]\n", stdout );
+#endif
+
+ if ( H5S_select_hyperslab( space_chunk, H5S_SELECT_NOTB, start, NULL, count, NULL)<0)
+ {
+ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to select hyperslab");
+ }
+
+ /* Fill the selection in the memory buffer */
+ if ( H5S_select_fill( fill->buf, fill->size, space_chunk, chunk ) < 0 )
+ {
+ HRETURN_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "filling selection failed");
+ }
+
+ if (H5F_istore_unlock(f, dxpl_id, layout, pline, TRUE, chunk_offset, &idx_hint, chunk, (size_t)naccessed)<0)
+ {
+ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to unlock raw data chunk");
+ }
+
+ } /*found*/
+
+
+ /* Increment indices */
+ for ( i = layout->ndims-1, carry=1; i >= 0 && carry; --i)
+ {
+ if (++idx_cur[i] >= idx_max[i])
+ idx_cur[i] = idx_min[i];
+ else
+ carry = 0;
+ }
+ if (carry)
+ break;
+
+ }
+
+
+ if(space_chunk)
+ H5S_close( space_chunk);
+
+ FUNC_LEAVE( SUCCEED );
+}
+
diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h
index 1c7d940..33fa930 100644
--- a/src/H5Dprivate.h
+++ b/src/H5Dprivate.h
@@ -155,7 +155,6 @@ __DLL__ herr_t H5D_write(H5D_t *dataset, const H5T_t *mem_type,
const H5S_t *mem_space, const H5S_t *file_space,
hid_t dset_xfer_plist, const void *buf);
__DLL__ herr_t H5D_extend(H5D_t *dataset, const hsize_t *size);
-__DLL__ herr_t H5D_set_extend(H5D_t *dataset, const hsize_t *size);
__DLL__ H5G_entry_t *H5D_entof(H5D_t *dataset);
__DLL__ H5T_t *H5D_typeof(H5D_t *dset);
__DLL__ H5S_t *H5D_get_space(H5D_t *dset);
@@ -172,5 +171,8 @@ __DLL__ herr_t H5D_xfer_create(hid_t dxpl_id, void *create_data);
__DLL__ herr_t H5D_xfer_copy(hid_t new_plist_id, hid_t old_plist_id,
void *copy_data);
__DLL__ herr_t H5D_xfer_close(hid_t dxpl_id, void *close_data);
+/*pvn*/
+__DLL__ herr_t H5D_set_extent(H5D_t *dataset, const hsize_t *size);
+
#endif
diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h
index 41e479d..66b6b2f 100644
--- a/src/H5Dpublic.h
+++ b/src/H5Dpublic.h
@@ -52,7 +52,6 @@ __DLL__ herr_t H5Dread (hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id,
__DLL__ herr_t H5Dwrite (hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id,
hid_t file_space_id, hid_t plist_id, const void *buf);
__DLL__ herr_t H5Dextend (hid_t dset_id, const hsize_t *size);
-__DLL__ herr_t H5Dset_extend (hid_t dset_id, const hsize_t *size);
__DLL__ herr_t H5Diterate(void *buf, hid_t type_id, hid_t space_id,
H5D_operator_t op, void *operator_data);
__DLL__ herr_t H5Dvlen_reclaim(hid_t type_id, hid_t space_id, hid_t plist_id, void *buf);
@@ -60,6 +59,9 @@ __DLL__ herr_t H5Dvlen_get_buf_size(hid_t dataset_id, hid_t type_id, hid_t space
__DLL__ herr_t H5Dfill(const void *fill, hid_t fill_type, void *buf,
hid_t buf_type, hid_t space);
__DLL__ herr_t H5Ddebug(hid_t dset_id, unsigned int flags);
+/*pvn*/
+__DLL__ herr_t H5Dset_extent (hid_t dset_id, const hsize_t *size);
+
#ifdef __cplusplus
}
diff --git a/src/H5Fistore.c b/src/H5Fistore.c
index df8b04a..1ed6679 100644
--- a/src/H5Fistore.c
+++ b/src/H5Fistore.c
@@ -42,6 +42,7 @@
#include "H5MMprivate.h"
#include "H5Oprivate.h"
#include "H5Pprivate.h" /* Property lists */
+#include "H5Sprivate.h" /* Dataspaces */
#include "H5Vprivate.h"
/* MPIO driver needed for special checks */
@@ -70,7 +71,8 @@
* the disk block size, chunks are aligned on disk block boundaries,
* and the operating system can also eliminate a read operation.
*/
-/* #define H5F_ISTORE_DEBUG */
+
+/*#define H5F_ISTORE_DEBUG */
/* Interface initialization */
#define PABLO_MASK H5Fistore_mask
@@ -106,7 +108,7 @@ typedef H5F_rdcc_ent_t *H5F_rdcc_ent_ptr_t; /* For free lists */
static size_t H5F_istore_sizeof_rkey(H5F_t *f, const void *_udata);
static herr_t H5F_istore_new_node(H5F_t *f, H5B_ins_t, void *_lt_key,
void *_udata, void *_rt_key,
- haddr_t*/*out*/);
+ haddr_t *addr_p /*out*/);
static int H5F_istore_cmp2(H5F_t *f, void *_lt_key, void *_udata,
void *_rt_key);
static int H5F_istore_cmp3(H5F_t *f, void *_lt_key, void *_udata,
@@ -129,6 +131,15 @@ static herr_t H5F_istore_debug_key(FILE *stream, int indent, int fwidth,
static haddr_t H5F_istore_get_addr(H5F_t *f, const H5O_layout_t *layout,
const hssize_t offset[]);
+static herr_t H5F_istore_prune_extent( H5F_t *f, void *left_key, haddr_t addr,
+ void *_udata, hsize_t *size );
+static H5B_ins_t H5F_istore_remove( H5F_t *f, haddr_t addr,
+ void *_lt_key /*in,out*/,
+ hbool_t *lt_key_changed /*out*/,
+ void UNUSED *_udata /*in,out*/,
+ void UNUSED *_rt_key /*in,out*/,
+ hbool_t *rt_key_changed /*out*/);
+
/*
* B-tree key. A key contains the minimum logical N-dimensional address and
* the logical size of the chunk to which this key refers. The
@@ -169,11 +180,15 @@ H5B_class_t H5B_ISTORE[1] = {{
H5F_istore_insert, /*insert */
FALSE, /*follow min branch? */
FALSE, /*follow max branch? */
- NULL, /*remove */
+ /*pvn*/
+ /*NULL,*/ /*remove */
+ H5F_istore_remove, /*remove */
H5F_istore_iterate, /*iterator */
H5F_istore_decode_key, /*decode */
H5F_istore_encode_key, /*encode */
H5F_istore_debug_key, /*debug */
+ /*pvn*/
+ H5F_istore_prune_extent, /*remove chunks, upon H5Dset_extend call */
}};
#define H5F_HASH_DIVISOR 8 /* Attempt to spread out the hashing */
@@ -1052,52 +1067,71 @@ done:
* Programmer: Robb Matzke
* Thursday, May 21, 1998
*
- * Modifications:
+ * Modifications: Pedro Vicente, March 28, 2002
+ * Added flush parameter that switches the call to H5F_istore_flush_entry
+ * The call with FALSE is used by the H5F_istore_prune_by_extent function
*
*-------------------------------------------------------------------------
*/
-static herr_t
-H5F_istore_preempt (H5F_t *f, H5F_rdcc_ent_t *ent)
-{
- H5F_rdcc_t *rdcc = &(f->shared->rdcc);
-
- FUNC_ENTER (H5F_istore_preempt, FAIL);
-
- assert(f);
- assert(ent);
- assert(!ent->locked);
- assert(ent->idx<rdcc->nslots);
-
- /* Flush */
- if (H5F_istore_flush_entry(f, ent, TRUE)<0) {
- HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
- "cannot flush indexed storage buffer");
- }
-
- /* Unlink from list */
- if (ent->prev) {
- ent->prev->next = ent->next;
- } else {
- rdcc->head = ent->next;
- }
- if (ent->next) {
- ent->next->prev = ent->prev;
- } else {
- rdcc->tail = ent->prev;
- }
- ent->prev = ent->next = NULL;
-
- /* Remove from cache */
- rdcc->slot[ent->idx] = NULL;
- ent->idx = UINT_MAX;
- rdcc->nbytes -= ent->chunk_size;
- --rdcc->nused;
+
+ static herr_t
+ H5F_istore_preempt (H5F_t *f, H5F_rdcc_ent_t *ent, hbool_t flush )
+ {
+ H5F_rdcc_t *rdcc = &(f->shared->rdcc);
+
+ FUNC_ENTER (H5F_istore_preempt, FAIL);
+
+ assert(f);
+ assert(ent);
+ assert(!ent->locked);
+ assert(ent->idx<rdcc->nslots);
+
+ if ( flush )
+ {
+
+ /* Flush */
+ if (H5F_istore_flush_entry(f, ent, TRUE)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
+ "cannot flush indexed storage buffer");
+ }
+ }
+
+ else
+ {
+
+ /* Reset, but do not free or remove from list */
+ ent->layout = H5O_free(H5O_LAYOUT, ent->layout);
+ ent->pline = H5O_free(H5O_PLINE, ent->pline);
+ if(ent->chunk!=NULL)
+ ent->chunk = H5F_istore_chunk_free(ent->chunk);
+
+ }
+
+ /* Unlink from list */
+ if (ent->prev) {
+ ent->prev->next = ent->next;
+ } else {
+ rdcc->head = ent->next;
+ }
+ if (ent->next) {
+ ent->next->prev = ent->prev;
+ } else {
+ rdcc->tail = ent->prev;
+ }
+ ent->prev = ent->next = NULL;
+
+ /* Remove from cache */
+ rdcc->slot[ent->idx] = NULL;
+ ent->idx = UINT_MAX;
+ rdcc->nbytes -= ent->chunk_size;
+ --rdcc->nused;
+
+ /* Free */
+ H5FL_FREE(H5F_rdcc_ent_t, ent);
+
+ FUNC_LEAVE (SUCCEED);
+ }
- /* Free */
- H5FL_FREE(H5F_rdcc_ent_t, ent);
-
- FUNC_LEAVE (SUCCEED);
-}
/*-------------------------------------------------------------------------
@@ -1112,6 +1146,8 @@ H5F_istore_preempt (H5F_t *f, H5F_rdcc_ent_t *ent)
* Thursday, May 21, 1998
*
* Modifications:
+ * Pedro Vicente, March 28, 2002
+ * Added TRUE parameter to the call to H5F_istore_preempt
*
*-------------------------------------------------------------------------
*/
@@ -1127,7 +1163,7 @@ H5F_istore_flush (H5F_t *f, hbool_t preempt)
for (ent=rdcc->head; ent; ent=next) {
next = ent->next;
if (preempt) {
- if (H5F_istore_preempt(f, ent)<0) {
+ if (H5F_istore_preempt(f, ent, TRUE )<0) {
nerrors++;
}
} else {
@@ -1157,6 +1193,8 @@ H5F_istore_flush (H5F_t *f, hbool_t preempt)
* Thursday, May 21, 1998
*
* Modifications:
+ * Pedro Vicente, March 28, 2002
+ * Added TRUE parameter to the call to H5F_istore_preempt
*
*-------------------------------------------------------------------------
*/
@@ -1175,7 +1213,7 @@ H5F_istore_dest (H5F_t *f)
HDfflush(stderr);
#endif
next = ent->next;
- if (H5F_istore_preempt(f, ent)<0) {
+ if (H5F_istore_preempt(f, ent, TRUE )<0) {
nerrors++;
}
}
@@ -1203,6 +1241,8 @@ H5F_istore_dest (H5F_t *f)
* Thursday, May 21, 1998
*
* Modifications:
+ * Pedro Vicente, March 28, 2002
+ * Added TRUE parameter to the call to H5F_istore_preempt
*
*-------------------------------------------------------------------------
*/
@@ -1280,7 +1320,7 @@ H5F_istore_prune (H5F_t *f, size_t size)
if (p[j]==cur) p[j] = NULL;
if (n[j]==cur) n[j] = cur->next;
}
- if (H5F_istore_preempt(f, cur)<0) nerrors++;
+ if (H5F_istore_preempt(f, cur, TRUE)<0) nerrors++;
}
}
@@ -1326,6 +1366,9 @@ H5F_istore_prune (H5F_t *f, size_t size)
* Robb Matzke, 1999-08-02
* The split ratios are passed in as part of the data transfer
* property list.
+ *
+ * Pedro Vicente, March 28, 2002
+ * Added TRUE parameter to the call to H5F_istore_preempt
*-------------------------------------------------------------------------
*/
static void *
@@ -1477,7 +1520,7 @@ H5F_istore_lock(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
}
fprintf(stderr, "}\n");
#endif
- if (H5F_istore_preempt(f, ent)<0) {
+ if (H5F_istore_preempt(f, ent, TRUE)<0) {
HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL,
"unable to preempt chunk from cache");
}
@@ -2457,3 +2500,500 @@ H5F_istore_allocate(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout,
FUNC_LEAVE(SUCCEED);
}
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_istore_prune_by_extent
+ *
+ * Purpose: This function searches for chunks that are no longer necessary both in the
+ * raw data cache and in the B-tree.
+ *
+ * Return: Success: 0, Failure: -1
+ *
+ * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
+ * Algorithm: Robb Matzke
+ *
+ * Date: March 27, 2002
+ *
+ * The algorithm is:
+ *
+ * For chunks that are no longer necessary:
+ *
+ * 1. Search in the raw data cache for each chunk
+ * 2. If found then preempt it from the cache
+ * 3. Search in the B-tree for each chunk
+ * 4. If found then remove it from the B-tree and deallocate file storage for the chunk
+ *
+ * This example shows a 2d dataset of 90x90 with a chunk size of 20x20.
+ *
+ *
+ * 0 20 40 60 80 90 100
+ * 0 +---------+---------+---------+---------+-----+...+
+ * |:::::X:::::::::::::: : : | :
+ * |:::::::X:::::::::::: : : | : Key
+ * |::::::::::X::::::::: : : | : --------
+ * |::::::::::::X::::::: : : | : +-+ Dataset
+ * 20+::::::::::::::::::::.........:.........:.....+...: | | Extent
+ * | :::::X::::: : : | : +-+
+ * | ::::::::::: : : | :
+ * | ::::::::::: : : | : ... Chunk
+ * | :::::::X::: : : | : : : Boundary
+ * 40+.........:::::::::::.........:.........:.....+...: :.:
+ * | : : : : | :
+ * | : : : : | : ... Allocated
+ * | : : : : | : ::: & Filled
+ * | : : : : | : ::: Chunk
+ * 60+.........:.........:.........:.........:.....+...:
+ * | : :::::::X::: : | : X Element
+ * | : ::::::::::: : | : Written
+ * | : ::::::::::: : | :
+ * | : ::::::::::: : | :
+ * 80+.........:.........:::::::::::.........:.....+...: O Fill Val
+ * | : : ::::::::::: | : Explicitly
+ * | : : ::::::X:::: | : Written
+ * 90+---------+---------+---------+---------+-----+ :
+ * : : : ::::::::::: :
+ * 100:.........:.........:.........:::::::::::.........:
+ *
+ *
+ * We have 25 total chunks for this dataset, 5 of which have space
+ * allocated in the file because they were written to one or more
+ * elements. These five chunks (and only these five) also have entries in
+ * the storage B-tree for this dataset.
+ *
+ * Now lets say we want to shrink the dataset down to 70x70:
+ *
+ *
+ * 0 20 40 60 70 80 90 100
+ * 0 +---------+---------+---------+----+----+-----+...+
+ * |:::::X:::::::::::::: : | : | :
+ * |:::::::X:::::::::::: : | : | : Key
+ * |::::::::::X::::::::: : | : | : --------
+ * |::::::::::::X::::::: : | : | : +-+ Dataset
+ * 20+::::::::::::::::::::.........:....+....:.....|...: | | Extent
+ * | :::::X::::: : | : | : +-+
+ * | ::::::::::: : | : | :
+ * | ::::::::::: : | : | : ... Chunk
+ * | :::::::X::: : | : | : : : Boundary
+ * 40+.........:::::::::::.........:....+....:.....|...: :.:
+ * | : : : | : | :
+ * | : : : | : | : ... Allocated
+ * | : : : | : | : ::: & Filled
+ * | : : : | : | : ::: Chunk
+ * 60+.........:.........:.........:....+....:.....|...:
+ * | : :::::::X::: | : | : X Element
+ * | : ::::::::::: | : | : Written
+ * +---------+---------+---------+----+ : | :
+ * | : ::::::::::: : | :
+ * 80+.........:.........:::::::::X:.........:.....|...: O Fill Val
+ * | : : ::::::::::: | : Explicitly
+ * | : : ::::::X:::: | : Written
+ * 90+---------+---------+---------+---------+-----+ :
+ * : : : ::::::::::: :
+ * 100:.........:.........:.........:::::::::::.........:
+ *
+ *
+ * That means that the nine chunks along the bottom and right side should
+ * no longer exist. Of those nine chunks, (0,80), (20,80), (40,80),
+ * (60,80), (80,80), (80,60), (80,40), (80,20), and (80,0), one is actually allocated
+ * that needs to be released.
+ * To release the chunks, we traverse the B-tree to obtain a list of unused
+ * allocated chunks, and then call H5B_remove() for each chunk.
+ *
+ *-------------------------------------------------------------------------
+ */
+
+herr_t H5F_istore_prune_by_extent( H5F_t *f, H5O_layout_t *layout, H5S_t *space )
+{
+ H5F_rdcc_t *rdcc = &(f->shared->rdcc); /*raw data chunk cache*/
+ H5F_rdcc_ent_t *ent = NULL, *next = NULL; /*cache entry */
+ unsigned u; /*counters */
+ int found = 0; /*remove this entry */
+ H5F_istore_ud1_t udata; /*B-tree pass-through */
+ hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /*current dataspace dimensions */
+
+ FUNC_ENTER( H5F_istore_prune_by_extent, FAIL );
+
+ /* Check args */
+ assert(f);
+ assert(layout && H5D_CHUNKED==layout->type);
+ assert(layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS);
+ assert(H5F_addr_defined(layout->addr));
+ assert(space);
+
+ /* Go get the rank & dimensions */
+ if(H5S_get_simple_extent_dims(space, curr_dims, NULL)<0)
+ HRETURN_ERROR( H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions");
+
+
+ /*-------------------------------------------------------------------------
+ * Figure out what chunks are no longer in use for the specified extent
+ * and release them from the linked list raw data cache
+ *-------------------------------------------------------------------------
+ */
+
+ for ( ent = rdcc->head; ent; ent=next )
+ {
+ next = ent->next;
+
+ found = 0;
+ for ( u = 0; u < ent->layout->ndims-1; u++ )
+ {
+ if ( (hsize_t)ent->offset[u] > curr_dims[u] )
+ {
+ found = 1;
+ break;
+ }
+ }
+
+ if ( found )
+ {
+#if defined (H5F_ISTORE_DEBUG)
+ HDfputs("cache:remove:[", stdout);
+ for ( u = 0; u < ent->layout->ndims-1; u++)
+ {
+ HDfprintf( stdout, "%s%Hd", u?", ":"", ent->offset[u]);
+ }
+ HDfputs("]\n", stdout );
+#endif
+
+ /* Preempt the entry from the cache, but do not flush it to disk */
+ if ( H5F_istore_preempt( f, ent, FALSE ) < 0 )
+ {
+ HRETURN_ERROR(H5E_IO, H5E_CANTINIT, 0, "unable to preempt chunk");
+ }
+
+ }
+
+ }
+
+/*-------------------------------------------------------------------------
+ * Check if there are any chunks on the B-tree
+ *-------------------------------------------------------------------------
+ */
+
+ HDmemset(&udata, 0, sizeof udata);
+ udata.stream = stdout;
+ udata.mesg.addr = layout->addr;
+ udata.mesg.ndims = layout->ndims;
+ for ( u = 0; u < udata.mesg.ndims; u++ )
+ {
+ udata.mesg.dim[u] = layout->dim[u];
+ }
+
+ if ( H5B_prune_by_extent( f, H5B_ISTORE, layout->addr, &udata, curr_dims ) < 0 )
+ {
+ HRETURN_ERROR( H5E_IO, H5E_CANTINIT, 0, "unable to iterate over B-tree" );
+ }
+
+ FUNC_LEAVE( SUCCEED );
+
+}
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_istore_prune_extent
+ *
+ * Purpose: Search for chunks that are no longer necessary in the B-tree.
+ *
+ * Return: Success: 0, Failure: -1
+ *
+ * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
+ *
+ * Date: March 26, 2002
+ *
+ * Comments: Called by H5B_prune_by_extent, part of H5B_ISTORE
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static herr_t H5F_istore_prune_extent( H5F_t *f, void *_lt_key, haddr_t addr, void *_udata,
+ hsize_t *size )
+{
+ H5F_istore_ud1_t *bt_udata = (H5F_istore_ud1_t *)_udata;
+ H5F_istore_key_t *lt_key = (H5F_istore_key_t *)_lt_key;
+ unsigned u;
+ int found = 0;
+ H5F_istore_ud1_t udata;
+
+ /* The LT_KEY is the left key (the onethat describes the chunk). It points to a chunk of
+ storage that contains the beginning of the logical address space represented by UDATA.
+ */
+
+ FUNC_ENTER( H5F_istore_prune_extent, FAIL );
+
+ /* Figure out what chunks are no longer in use for the specified extent and release them */
+
+ for ( u = 0; u < bt_udata->mesg.ndims-1; u++ )
+ {
+ if ( (hsize_t)lt_key->offset[u] > size[u] )
+ {
+ found = 1;
+ break;
+ }
+ }
+
+ if ( found )
+ {
+
+#if defined (H5F_ISTORE_DEBUG)
+ HDfputs("b-tree:remove:[", bt_udata->stream);
+ for ( u = 0; u < bt_udata->mesg.ndims-1; u++)
+ {
+ HDfprintf(bt_udata->stream, "%s%Hd", u?", ":"", lt_key->offset[u]);
+ }
+ HDfputs("]\n", bt_udata->stream);
+#endif
+
+ HDmemset( &udata, 0, sizeof udata );
+ udata.key = *lt_key;
+ udata.mesg = bt_udata->mesg;
+
+ /* Remove */
+ if ( H5B_remove( f, H5B_ISTORE, addr, &udata ) < 0 )
+ {
+ HRETURN_ERROR( H5E_SYM, H5E_CANTINIT, FAIL, "unable to remove entry" );
+ }
+ }
+
+ FUNC_LEAVE( SUCCEED );
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_istore_remove
+ *
+ * Purpose: Removes chunks that are no longer necessary in the B-tree.
+ *
+ * Return: Success: 0, Failure: -1
+ *
+ * Programmer: Robb Matzke
+ * Pedro Vicente, pvn@ncsa.uiuc.edu
+ *
+ * Date: March 28, 2002
+ *
+ * Comments: Part of H5B_ISTORE
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+
+static H5B_ins_t H5F_istore_remove( H5F_t *f,
+ haddr_t addr,
+ void *_lt_key /*in,out*/,
+ hbool_t *lt_key_changed /*out*/,
+ void UNUSED *_udata /*in,out*/,
+ void UNUSED *_rt_key /*in,out*/,
+ hbool_t *rt_key_changed /*out*/)
+{
+ H5F_istore_key_t *lt_key = (H5F_istore_key_t*)_lt_key;
+ H5FD_free(f->shared->lf, H5FD_MEM_DRAW, addr, lt_key->nbytes);
+ *lt_key_changed = FALSE;
+ *rt_key_changed = FALSE;
+ return H5B_INS_REMOVE;
+}
+
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5F_istore_initialize_by_extent
+ *
+ * Purpose: This function searches for chunks that have to be initialized with the fill
+ * value both in the raw data cache and in the B-tree.
+ *
+ * Return: Success: 0, Failure: -1
+ *
+ * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu
+ *
+ * Date: April 4, 2002
+ *
+ * Comments:
+ *
+ * (See the example of H5F_istore_prune_by_extent)
+ * Next, there are seven chunks where the database extent boundary is
+ * within the chunk. We find those seven just like we did with the previous nine.
+ * Fot the ones that are allocated we initialize the part that lies outside the boundary
+ * with the fill value.
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+
+
+herr_t H5F_istore_initialize_by_extent( H5F_t *f, H5O_layout_t *layout, H5O_pline_t *pline,
+ H5O_fill_t *fill, H5S_t *space )
+{
+ hid_t dxpl_id; /*dataset transfer property list*/
+ uint8_t *chunk = NULL; /*the file chunk */
+ unsigned idx_hint = 0; /*input value for H5F_istore_lock*/
+ hssize_t chunk_offset[H5O_LAYOUT_NDIMS]; /*logical location of the chunks */
+ hsize_t idx_cur[H5O_LAYOUT_NDIMS]; /*multi-dimensional counters */
+ hsize_t idx_min[H5O_LAYOUT_NDIMS];
+ hsize_t idx_max[H5O_LAYOUT_NDIMS];
+ hsize_t sub_size[H5O_LAYOUT_NDIMS];
+ hsize_t naccessed; /*bytes accessed in chunk */
+ hsize_t elm_size; /*size of an element in bytes */
+ hsize_t end_chunk; /*chunk position counter */
+ hssize_t start[H5O_LAYOUT_NDIMS]; /*starting location of hyperslab */
+ hsize_t count[H5O_LAYOUT_NDIMS]; /*element count of hyperslab */
+ hsize_t size[H5O_LAYOUT_NDIMS]; /*current size of dimensions */
+ H5S_t *space_chunk=NULL; /*dataspace for a chunk */
+ hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /*current dataspace dimensions */
+ int rank; /*current # of dimensions */
+ int i, carry; /*counters */
+ unsigned u;
+ int found = 0; /*initialize this entry */
+
+ FUNC_ENTER( H5F_istore_initialize_by_extent, FAIL );
+
+ /* Check args */
+ assert(f);
+ assert(layout && H5D_CHUNKED==layout->type);
+ assert(layout->ndims>0 && layout->ndims<=H5O_LAYOUT_NDIMS);
+ assert(H5F_addr_defined(layout->addr));
+ assert(space);
+ assert(pline);
+ assert(fill);
+
+ HDmemset( start, 0, sizeof(start) );
+ HDmemset( count, 0, sizeof(count) );
+
+ /* Go get the rank & dimensions */
+ if((rank=H5S_get_simple_extent_dims(space, curr_dims, NULL))<0)
+ HRETURN_ERROR( H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions");
+
+ for ( i = 0; i < rank; i++ )
+ {
+ size[i] = curr_dims[i];
+ }
+ size[i] = layout->dim[i];
+ elm_size = size[i];
+
+ /* Default dataset transfer property list */
+ dxpl_id = H5P_DATASET_XFER_DEFAULT;
+
+ /* Create a data space for a chunk & set the extent */
+ if(NULL==(space_chunk=H5S_create(H5S_SIMPLE))) {
+ HRETURN_ERROR (H5E_DATASPACE, H5E_CANTCREATE, FAIL,
+ "can't create simple dataspace");
+ }
+ if(H5S_set_extent_simple(space_chunk,(unsigned)rank,layout->dim,NULL)<0) {
+ HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
+ "can't set dimensions");
+ }
+
+/*
+ * Set up multi-dimensional counters (idx_min, idx_max, and idx_cur) and
+ * loop through the chunks copying each chunk from the application to the
+ * chunk cache.
+ */
+ for ( u = 0; u < layout->ndims; u++)
+ {
+ idx_min[u] = 0;
+ idx_max[u] = (size[u]-1) / layout->dim[u] + 1;
+ idx_cur[u] = idx_min[u];
+ }
+
+ /* Loop over all chunks */
+ while ( 1 )
+ {
+
+ for ( u = 0, naccessed=1; u < layout->ndims; u++ )
+ {
+ /* The location and size of the chunk being accessed */
+ chunk_offset[u] = idx_cur[u] * (hssize_t)(layout->dim[u]);
+ sub_size[u] = MIN((idx_cur[u]+1)*layout->dim[u],size[u]) - chunk_offset[u];
+ naccessed *= sub_size[u];
+ }
+
+ /*
+ Figure out what chunks have to be initialized. These are the chunks where the database
+ extent boundary is within the chunk
+ */
+
+ for ( u = 0, found = 0; u < layout->ndims-1; u++ )
+ {
+ end_chunk = chunk_offset[u] + layout->dim[u];
+ if ( end_chunk > size[u] )
+ {
+ found = 1;
+ break;
+ }
+ }
+
+ if ( found )
+ {
+
+ if (NULL==(chunk=H5F_istore_lock( f, dxpl_id, layout, pline, fill, chunk_offset, FALSE, &idx_hint)))
+ {
+ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to read raw data chunk");
+ }
+
+ if ( H5S_select_all( space_chunk ) < 0 )
+ {
+ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to select space");
+ }
+
+ for ( i = 0; i < rank; i++ )
+ {
+ count[i] = MIN( (idx_cur[i]+1)*layout->dim[i], size[i]-chunk_offset[i] );
+ }
+
+#if defined (H5F_ISTORE_DEBUG)
+ HDfputs("cache:initialize:offset:[", stdout);
+ for ( u = 0; u < layout->ndims-1; u++)
+ {
+ HDfprintf( stdout, "%s%Hd", u?", ":"", chunk_offset[u]);
+ }
+ HDfputs("]", stdout );
+ HDfputs(":count:[", stdout);
+ for ( u = 0; u < layout->ndims-1; u++)
+ {
+ HDfprintf( stdout, "%s%Hd", u?", ":"", count[u]);
+ }
+ HDfputs("]\n", stdout );
+#endif
+
+ if ( H5S_select_hyperslab( space_chunk, H5S_SELECT_NOTB, start, NULL, count, NULL)<0)
+ {
+ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to select hyperslab");
+ }
+
+ /* Fill the selection in the memory buffer */
+ if ( H5S_select_fill( fill->buf, fill->size, space_chunk, chunk ) < 0 )
+ {
+ HRETURN_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "filling selection failed");
+ }
+
+ if (H5F_istore_unlock(f, dxpl_id, layout, pline, TRUE, chunk_offset, &idx_hint, chunk, (size_t)naccessed)<0)
+ {
+ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to unlock raw data chunk");
+ }
+
+ } /*found*/
+
+
+ /* Increment indices */
+ for ( i = layout->ndims-1, carry=1; i >= 0 && carry; --i)
+ {
+ if (++idx_cur[i] >= idx_max[i])
+ idx_cur[i] = idx_min[i];
+ else
+ carry = 0;
+ }
+ if (carry)
+ break;
+
+ }
+
+
+ if(space_chunk)
+ H5S_close( space_chunk);
+
+ FUNC_LEAVE( SUCCEED );
+}
+
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 8a77e7a..898da84 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -26,6 +26,7 @@
typedef struct H5F_t H5F_t;
+
/*
* Encode and decode macros for file meta-data.
* Currently, all file meta-data is little-endian.
@@ -391,6 +392,15 @@ __DLL__ hsize_t H5F_istore_allocated(H5F_t *f, unsigned ndims, haddr_t addr);
__DLL__ herr_t H5F_istore_dump_btree(H5F_t *f, FILE *stream, unsigned ndims,
haddr_t addr);
+__DLL__ herr_t H5F_istore_prune_by_extent( H5F_t *f,
+ struct H5O_layout_t *layout,
+ struct H5S_t *space );
+__DLL__ herr_t H5F_istore_initialize_by_extent( H5F_t *f,
+ struct H5O_layout_t *layout,
+ struct H5O_pline_t *pline,
+ struct H5O_fill_t *fill,
+ struct H5S_t *space );
+
/* Functions for allocation/releasing chunks */
__DLL__ void * H5F_istore_chunk_alloc(size_t chunk_size);
__DLL__ void * H5F_istore_chunk_realloc(void *chunk, size_t new_size);
diff --git a/src/H5S.c b/src/H5S.c
index 383c851..f681615 100644
--- a/src/H5S.c
+++ b/src/H5S.c
@@ -24,8 +24,6 @@
#include "H5Spkg.h" /* Data-space functions */
/* Local static function prototypes */
-herr_t H5S_set_extent_simple (H5S_t *space, unsigned rank, const hsize_t *dims,
- const hsize_t *max);
/* Interface initialization */
#define PABLO_MASK H5S_mask
@@ -1979,7 +1977,7 @@ H5S_debug(H5F_t *f, const void *_mesg, FILE *stream, int indent, int fwidth)
/*-------------------------------------------------------------------------
- * Function: H5S_set_extend
+ * Function: H5S_set_extent
*
* Purpose: Modify the dimensions of a data space. Based on H5S_extend
*
@@ -1994,12 +1992,12 @@ H5S_debug(H5F_t *f, const void *_mesg, FILE *stream, int indent, int fwidth)
*-------------------------------------------------------------------------
*/
-int H5S_set_extend( H5S_t *space, const hsize_t *size )
+int H5S_set_extent( H5S_t *space, const hsize_t *size )
{
int ret_value = 0;
unsigned u;
- FUNC_ENTER( H5S_set_extend, FAIL );
+ FUNC_ENTER( H5S_set_extent, FAIL );
/* Check args */
assert( space && H5S_SIMPLE==space->extent.type );
diff --git a/src/H5Sall.c b/src/H5Sall.c
index d988de2..36ad983 100644
--- a/src/H5Sall.c
+++ b/src/H5Sall.c
@@ -46,7 +46,6 @@ static hsize_t H5S_all_mgath (const void *_buf, size_t elmt_size,
static herr_t H5S_all_mscat (const void *_tconv_buf, size_t elmt_size,
const H5S_t *mem_space, H5S_sel_iter_t *mem_iter,
hsize_t nelmts, void *_buf/*out*/);
-static herr_t H5S_select_all(H5S_t *space);
const H5S_fconv_t H5S_ALL_FCONV[1] = {{
"all", /*name */
@@ -904,7 +903,8 @@ H5S_all_bounds(H5S_t *space, hsize_t *start, hsize_t *end)
EXAMPLES
REVISION LOG
--------------------------------------------------------------------------*/
-static herr_t H5S_select_all (H5S_t *space)
+herr_t
+H5S_select_all (H5S_t *space)
{
herr_t ret_value=SUCCEED; /* return value */
diff --git a/src/H5Shyper.c b/src/H5Shyper.c
index 029b7d7..0857aeb 100644
--- a/src/H5Shyper.c
+++ b/src/H5Shyper.c
@@ -66,9 +66,6 @@ static herr_t H5S_hyper_free_span (H5S_hyper_span_t *span);
static H5S_hyper_span_info_t *H5S_hyper_copy_span (H5S_hyper_span_info_t *spans);
static herr_t H5S_hyper_span_scratch (H5S_hyper_span_info_t *spans, void *scr_value);
static herr_t H5S_hyper_span_precompute (H5S_hyper_span_info_t *spans, size_t elmt_size);
-static herr_t H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, const hssize_t start[],
- const hsize_t stride[], const hsize_t count[],
- const hsize_t block[]);
const H5S_fconv_t H5S_HYPER_FCONV[1] = {{
"hslab", /*name */
diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h
index 0380d36..3fd1903 100644
--- a/src/H5Sprivate.h
+++ b/src/H5Sprivate.h
@@ -196,6 +196,8 @@ __DLL__ hsize_t H5S_get_npoints_max(const H5S_t *ds);
__DLL__ int H5S_get_simple_extent_ndims(const H5S_t *ds);
__DLL__ int H5S_get_simple_extent_dims(const H5S_t *ds, hsize_t dims[]/*out*/,
hsize_t max_dims[]/*out*/);
+__DLL__ herr_t H5S_set_extent_simple (H5S_t *space, unsigned rank, const hsize_t *dims,
+ const hsize_t *max);
__DLL__ herr_t H5S_modify(struct H5G_entry_t *ent, const H5S_t *space);
__DLL__ H5S_t *H5S_read(struct H5G_entry_t *ent);
__DLL__ int H5S_cmp(const H5S_t *ds1, const H5S_t *ds2);
@@ -208,7 +210,7 @@ __DLL__ herr_t H5S_extent_release(H5S_t *space);
__DLL__ herr_t H5S_select_release(H5S_t *space);
__DLL__ hssize_t H5S_get_select_npoints(const H5S_t *space);
__DLL__ int H5S_extend(H5S_t *space, const hsize_t *size);
-__DLL__ int H5S_set_extend(H5S_t *space, const hsize_t *size);
+__DLL__ int H5S_set_extent(H5S_t *space, const hsize_t *size);
__DLL__ htri_t H5S_select_valid(const H5S_t *space);
__DLL__ herr_t H5S_debug(H5F_t *f, const void *_mesg, FILE *stream,
int indent, int fwidth);
@@ -226,6 +228,12 @@ __DLL__ herr_t H5S_sel_iter_release(const H5S_t *space,
__DLL__ herr_t H5S_select_fill(const void *fill, size_t fill_size, H5S_t *space,
void *buf);
+/* Needed for internal use of selections in H5Fistore code */
+__DLL__ herr_t H5S_select_all(H5S_t *space);
+__DLL__ herr_t H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, const hssize_t start[],
+ const hsize_t stride[], const hsize_t count[],
+ const hsize_t block[]);
+
#ifdef H5_HAVE_PARALLEL
/* MPI-IO function to check whether its possible to transfer directly from app buffer to file */
generated by cgit v0.12 at 2024-11-01 14:11:00 (GMT)