diff options
| author | Quincey Koziol <koziol@hdfgroup.org> | 2008-06-03 19:44:12 (GMT) |
|---|---|---|
| committer | Quincey Koziol <koziol@hdfgroup.org> | 2008-06-03 19:44:12 (GMT) |
| commit | 771bae88881f4ae8102c9553fd10e0938aa3094c (patch) | |
| tree | 62ac7bce1e109e1b0b8899ea14da603a2b6e6ffc /src/H5Distore.c | |
| parent | d36f67c0e27a39a54f870c4f917ab2c1754e80d6 (diff) | |
| download | hdf5-771bae88881f4ae8102c9553fd10e0938aa3094c.zip hdf5-771bae88881f4ae8102c9553fd10e0938aa3094c.tar.gz hdf5-771bae88881f4ae8102c9553fd10e0938aa3094c.tar.bz2 | |
[svn-r15131] Description:
Finish omnibus chunked dataset I/O refactoring, to separate general
actions on chunked datasets from actions that are specific to using the v1
B-tree index.
Cleaned up a few bugs and added some additional tests also.
Tested on:
FreeBSD/32 6.2 (duty) in debug mode
FreeBSD/64 6.2 (liberty) w/C++ & FORTRAN, in debug mode
Linux/32 2.6 (kagiso) w/PGI compilers, w/C++ & FORTRAN, w/threadsafe,
in debug mode
Linux/64-amd64 2.6 (smirom) w/default API=1.6.x, w/C++ & FORTRAN,
in production mode
Linux/64-ia64 2.6 (cobalt) w/Intel compilers, w/C++ & FORTRAN,
in production mode
Solaris/32 2.10 (linew) w/deprecated symbols disabled, w/C++ & FORTRAN,
w/szip filter, in production mode
Mac OS X/32 10.5.2 (amazon) in debug mode
Linux/64-ia64 2.4 (tg-login3) w/parallel, w/FORTRAN, in production mode
Diffstat (limited to 'src/H5Distore.c')
| -rw-r--r-- | src/H5Distore.c | 3210 |
1 files changed, 384 insertions, 2826 deletions
diff --git a/src/H5Distore.c b/src/H5Distore.c index aed5add..b6005db 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -73,44 +73,36 @@ /****************/ /* - * Feature: If this constant is defined then every cache preemption and load - * causes a character to be printed on the standard error stream: - * - * `.': Entry was preempted because it has been completely read or - * completely written but not partially read and not partially - * written. This is often a good reason for preemption because such - * a chunk will be unlikely to be referenced in the near future. - * - * `:': Entry was preempted because it hasn't been used recently. - * - * `#': Entry was preempted because another chunk collided with it. This - * is usually a relatively bad thing. If there are too many of - * these then the number of entries in the cache can be increased. - * - * c: Entry was preempted because the file is closing. - * - * w: A chunk read operation was eliminated because the library is - * about to write new values to the entire chunk. This is a good - * thing, especially on files where the chunk size is the same as - * the disk block size, chunks are aligned on disk block boundaries, - * and the operating system can also eliminate a read operation. - */ - -/*#define H5D_ISTORE_DEBUG */ - -/* * Given a B-tree node return the dimensionality of the chunks pointed to by * that node. */ #define H5D_ISTORE_NDIMS(X) (((X)->sizeof_rkey-8)/8) -#define H5D_ISTORE_DEFAULT_SKIPLIST_HEIGHT 8 - /******************/ /* Local Typedefs */ /******************/ /* + * B-tree key. A key contains the minimum logical N-dimensional coordinates and + * the logical size of the chunk to which this key refers. The + * fastest-varying dimension is assumed to reference individual bytes of the + * array, so a 100-element 1-d array of 4-byte integers would really be a 2-d + * array with the slow varying dimension of size 100 and the fast varying + * dimension of size 4 (the storage dimensionality has very little to do with + * the real dimensionality). + * + * Only the first few values of the OFFSET and SIZE fields are actually + * stored on disk, depending on the dimensionality. + * + * The chunk's file address is part of the B-tree and not part of the key. + */ +typedef struct H5D_istore_key_t { + size_t nbytes; /*size of stored data */ + hsize_t offset[H5O_LAYOUT_NDIMS]; /*logical offset to start*/ + unsigned filter_mask; /*excluded filters */ +} H5D_istore_key_t; + +/* * Data exchange structure for indexed storage nodes. This structure is * passed through the B-link tree layer to the methods for the objects * to which the B-link tree points for operations which require no @@ -118,99 +110,25 @@ * * (Just an alias for the "common" info). */ -typedef H5D_istore_bt_ud_common_t H5D_istore_ud0_t; - -/* B-tree callback info for iteration to total allocated space */ -typedef struct H5D_istore_it_ud1_t { - H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ - hsize_t total_storage; /*output from iterator */ -} H5D_istore_it_ud1_t; - -/* B-tree callback info for iteration to dump node's info */ -typedef struct H5D_istore_it_ud2_t { - H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ - FILE *stream; /*debug output stream */ - hbool_t header_displayed; /* Node's header is displayed? */ -} H5D_istore_it_ud2_t; - -/* B-tree callback info for iteration to prune chunks */ -typedef struct H5D_istore_it_ud3_t { - H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ - const hsize_t *dims; /* New dataset dimensions */ - const hsize_t *down_chunks; /* "down" size of number of chunks in each dimension */ - H5SL_t *outside; /* Skip list to hold chunks outside the new dimensions */ -} H5D_istore_it_ud3_t; - -/* B-tree callback info for iteration to copy data */ -typedef struct H5D_istore_it_ud4_t { - H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ - H5F_t *file_src; /* Source file for copy */ - haddr_t addr_dst; /* Address of dest. B-tree */ - void *buf; /* Buffer to hold chunk data for read/write */ - void *bkg; /* Buffer for background information during type conversion */ - size_t buf_size; /* Buffer size */ - hbool_t do_convert; /* Whether to perform type conversions */ - - /* needed for converting variable-length data */ - hid_t tid_src; /* Datatype ID for source datatype */ - hid_t tid_dst; /* Datatype ID for destination datatype */ - hid_t tid_mem; /* Datatype ID for memory datatype */ - H5T_t *dt_src; /* Source datatype */ - H5T_path_t *tpath_src_mem; /* Datatype conversion path from source file to memory */ - H5T_path_t *tpath_mem_dst; /* Datatype conversion path from memory to dest. file */ - void *reclaim_buf; /* Buffer for reclaiming data */ - size_t reclaim_buf_size; /* Reclaim buffer size */ - uint32_t nelmts; /* Number of elements in buffer */ - H5S_t *buf_space; /* Dataspace describing buffer */ - - /* needed for compressed variable-length data */ - H5O_pline_t *pline; /* Filter pipeline */ - - /* needed for copy object pointed by refs */ - H5F_t *file_dst; /* Destination file for copy */ - H5O_copy_t *cpy_info; /* Copy options */ -} H5D_istore_it_ud4_t; - -/* B-tree callback info for iteration to obtain chunk address and the index of the chunk for all chunks in the B-tree. */ -typedef struct H5D_istore_it_ud5_t { - H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ - const hsize_t *down_chunks; - haddr_t *chunk_addr; -} H5D_istore_it_ud5_t; - -/* Skip list node for storing chunks to remove during an iteration */ -typedef struct H5D_istore_sl_ck_t { - hsize_t index; /* Index of chunk to remove (must be first) */ - H5D_istore_key_t key; /* Chunk key */ -} H5D_istore_sl_ck_t; - -/* Skip list callback info when destroying list & removing chunks */ -typedef struct H5D_istore_sl_rm_t { - H5F_t *f; /* Pointer to file for B-tree */ - hid_t dxpl_id; /* DXPL to use */ - const H5O_layout_t *mesg; /* Layout message */ -} H5D_istore_sl_rm_t; +typedef H5D_chunk_common_ud_t H5D_istore_ud0_t; + +/* B-tree callback info for iteration over chunks */ +typedef struct H5D_istore_it_ud_t { + H5D_chunk_common_ud_t common; /* Common info for B-tree user data (must be first) */ + H5D_chunk_cb_func_t cb; /* Chunk callback routine */ + void *udata; /* User data for chunk callback routine */ +} H5D_istore_it_ud_t; + /********************/ /* Local Prototypes */ /********************/ -static void *H5D_istore_chunk_alloc(size_t size, const H5O_pline_t *pline); -static void *H5D_istore_chunk_xfree(void *chk, const H5O_pline_t *pline); -static herr_t H5D_istore_shared_create (const H5F_t *f, H5O_layout_t *layout); -static herr_t H5D_istore_shared_free (void *page); +static herr_t H5D_istore_shared_create(const H5F_t *f, H5O_layout_t *layout); /* B-tree iterator callbacks */ -static int H5D_istore_iter_chunkmap(H5F_t *f, hid_t dxpl_id, const void *left_key, haddr_t addr, - const void *right_key, void *_udata); -static int H5D_istore_iter_allocated(H5F_t *f, hid_t dxpl_id, const void *left_key, haddr_t addr, - const void *right_key, void *_udata); -static int H5D_istore_iter_dump(H5F_t *f, hid_t dxpl_id, const void *left_key, haddr_t addr, - const void *right_key, void *_udata); -static int H5D_istore_prune_check(H5F_t *f, hid_t dxpl_id, const void *_lt_key, haddr_t addr, - const void *_rt_key, void *_udata); -static int H5D_istore_iter_copy(H5F_t *f, hid_t dxpl_id, const void *_lt_key, haddr_t addr, - const void *_rt_key, void *_udata); +static int H5D_istore_idx_iterate_cb(H5F_t *f, hid_t dxpl_id, const void *left_key, + haddr_t addr, const void *right_key, void *_udata); /* B-tree callbacks */ static H5RC_t *H5D_istore_get_shared(const H5F_t *f, const void *_udata); @@ -239,6 +157,51 @@ static herr_t H5D_istore_debug_key(FILE *stream, H5F_t *f, hid_t dxpl_id, int indent, int fwidth, const void *key, const void *udata); +/* Chunked layout indexing callbacks */ +static herr_t H5D_istore_idx_init(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D_istore_idx_create(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D_istore_idx_insert(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_ud_t *udata); +static haddr_t H5D_istore_idx_get_addr(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_ud_t *udata); +static int H5D_istore_idx_iterate(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_cb_func_t chunk_cb, void *chunk_udata); +static herr_t H5D_istore_idx_remove(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_common_ud_t *udata); +static herr_t H5D_istore_idx_delete(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D_istore_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, + const H5D_chk_idx_info_t *idx_info_dst); +static herr_t H5D_istore_idx_copy_shutdown(H5O_layout_t *layout_src, + H5O_layout_t *layout_dst); +static herr_t H5D_istore_idx_size(const H5D_chk_idx_info_t *idx_info, + hsize_t *size); +static herr_t H5D_istore_idx_dest(const H5D_chk_idx_info_t *idx_info); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* v1 B-tree indexed chunk I/O ops */ +const H5D_chunk_ops_t H5D_COPS_ISTORE[1] = {{ + H5D_istore_idx_init, + H5D_istore_idx_create, + H5D_istore_idx_insert, + H5D_istore_idx_get_addr, + H5D_istore_idx_iterate, + H5D_istore_idx_remove, + H5D_istore_idx_delete, + H5D_istore_idx_copy_setup, + H5D_istore_idx_copy_shutdown, + H5D_istore_idx_size, + H5D_istore_idx_dest +}}; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + /* inherits B-tree like properties from H5B */ H5B_class_t H5B_ISTORE[1] = {{ H5B_ISTORE_ID, /*id */ @@ -257,36 +220,11 @@ H5B_class_t H5B_ISTORE[1] = {{ H5D_istore_debug_key, /*debug */ }}; -/*********************/ -/* Package Variables */ -/*********************/ - -/*****************************/ -/* Library Private Variables */ -/*****************************/ /*******************/ /* Local Variables */ /*******************/ -/* Declare a free list to manage H5F_rdcc_ent_t objects */ -H5FL_DEFINE_STATIC(H5D_rdcc_ent_t); - -/* Declare a free list to manage the H5F_rdcc_ent_ptr_t sequence information */ -H5FL_SEQ_DEFINE_STATIC(H5D_rdcc_ent_ptr_t); - -/* Declare a free list to manage the chunk sequence information */ -H5FL_BLK_DEFINE_STATIC(chunk); - -/* Declare a free list to manage the native key offset sequence information */ -H5FL_SEQ_DEFINE_STATIC(size_t); - -/* Declare a free list to manage the raw page information */ -H5FL_BLK_DEFINE_STATIC(chunk_page); - -/* Declare a free list to manage H5D_istore_sl_ck_t objects */ -H5FL_DEFINE_STATIC(H5D_istore_sl_ck_t); - /*------------------------------------------------------------------------- * Function: H5D_istore_get_shared @@ -580,7 +518,7 @@ H5D_istore_new_node(H5F_t *f, hid_t dxpl_id, H5B_ins_t op, { H5D_istore_key_t *lt_key = (H5D_istore_key_t *) _lt_key; H5D_istore_key_t *rt_key = (H5D_istore_key_t *) _rt_key; - H5D_istore_ud1_t *udata = (H5D_istore_ud1_t *) _udata; + H5D_chunk_ud_t *udata = (H5D_chunk_ud_t *) _udata; unsigned u; herr_t ret_value = SUCCEED; /* Return value */ @@ -658,7 +596,7 @@ static herr_t H5D_istore_found(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, haddr_t addr, const void *_lt_key, void *_udata) { - H5D_istore_ud1_t *udata = (H5D_istore_ud1_t *) _udata; + H5D_chunk_ud_t *udata = (H5D_chunk_ud_t *) _udata; const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *) _lt_key; unsigned u; herr_t ret_value = SUCCEED; /* Return value */ @@ -726,7 +664,7 @@ H5D_istore_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key, H5D_istore_key_t *lt_key = (H5D_istore_key_t *) _lt_key; H5D_istore_key_t *md_key = (H5D_istore_key_t *) _md_key; H5D_istore_key_t *rt_key = (H5D_istore_key_t *) _rt_key; - H5D_istore_ud1_t *udata = (H5D_istore_ud1_t *) _udata; + H5D_chunk_ud_t *udata = (H5D_chunk_ud_t *) _udata; int cmp; unsigned u; H5B_ins_t ret_value; @@ -825,738 +763,43 @@ done: /*------------------------------------------------------------------------- - * Function: H5D_istore_iter_allocated - * - * Purpose: Simply counts the number of chunks for a dataset. - * - * Return: Success: Non-negative - * - * Failure: Negative - * - * Programmer: Robb Matzke - * Wednesday, April 21, 1999 - * - *------------------------------------------------------------------------- - */ -/* ARGSUSED */ -static int -H5D_istore_iter_allocated (H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const void *_lt_key, haddr_t UNUSED addr, - const void UNUSED *_rt_key, void *_udata) -{ - H5D_istore_it_ud1_t *udata = (H5D_istore_it_ud1_t *)_udata; - const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; - - FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_iter_allocated) - - udata->total_storage += lt_key->nbytes; - - FUNC_LEAVE_NOAPI(H5_ITER_CONT) -} /* H5D_istore_iter_allocated() */ - -/*------------------------------------------------------------------------- - * Function: H5D_istore_iter_chunkmap - * - * Purpose: obtain chunk address and the corresponding index - * - * Return: Success: Non-negative - * - * Failure: Negative - * - * Programmer: Kent Yang - * Tuesday, November 15, 2005 - * - *------------------------------------------------------------------------- - */ -/* ARGSUSED */ -static int -H5D_istore_iter_chunkmap (H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const void *_lt_key, haddr_t addr, - const void UNUSED *_rt_key, void *_udata) -{ - H5D_istore_it_ud5_t *udata = (H5D_istore_it_ud5_t *)_udata; - const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; - unsigned rank = udata->common.mesg->u.chunk.ndims - 1; - hsize_t chunk_index; - int ret_value = H5_ITER_CONT; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_iter_chunkmap) - - if(H5V_chunk_index(rank, lt_key->offset, udata->common.mesg->u.chunk.dim, udata->down_chunks, &chunk_index) < 0) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index") - - udata->chunk_addr[chunk_index] = addr; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5D_istore_iter_chunkmap() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_iter_dump - * - * Purpose: If the UDATA.STREAM member is non-null then debugging - * information is written to that stream. - * - * Return: Success: Non-negative - * - * Failure: Negative - * - * Programmer: Robb Matzke - * Wednesday, April 21, 1999 - * - *------------------------------------------------------------------------- - */ -/* ARGSUSED */ -static int -H5D_istore_iter_dump (H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const void *_lt_key, haddr_t UNUSED addr, - const void UNUSED *_rt_key, void *_udata) -{ - H5D_istore_it_ud2_t *udata = (H5D_istore_it_ud2_t *)_udata; - const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; - unsigned u; - - FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_iter_dump) - - if(udata->stream) { - if(!udata->header_displayed) { - HDfprintf(udata->stream, " Flags Bytes Address Logical Offset\n"); - HDfprintf(udata->stream, " ========== ======== ========== ==============================\n"); - - /* Set flag that the headers has been printed */ - udata->header_displayed = TRUE; - } /* end if */ - HDfprintf(udata->stream, " 0x%08x %8Zu %10a [", lt_key->filter_mask, lt_key->nbytes, addr); - for(u = 0; u < udata->common.mesg->u.chunk.ndims; u++) - HDfprintf(udata->stream, "%s%Hd", (u ? ", " : ""), lt_key->offset[u]); - HDfputs("]\n", udata->stream); - } /* end if */ - - FUNC_LEAVE_NOAPI(H5_ITER_CONT) -} /* H5D_istore_iter_dump() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_iter_copy - * - * Purpose: copy chunked raw data from source file and insert to the - * B-tree node in the destination file - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Peter Cao - * August 20, 2005 - * - *------------------------------------------------------------------------- - */ -static int -H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, - haddr_t addr_src, const void UNUSED *_rt_key, void *_udata) -{ - H5D_istore_it_ud4_t *udata = (H5D_istore_it_ud4_t *)_udata; - const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; - H5D_istore_ud1_t udata_dst; /* User data about new destination chunk */ - hbool_t is_vlen = FALSE; - hbool_t fix_ref = FALSE; - - /* General information about chunk copy */ - void *bkg = udata->bkg; - void *buf = udata->buf; - size_t buf_size = udata->buf_size; - H5O_pline_t *pline = udata->pline; - - /* needed for commpressed variable length data */ - hbool_t is_compressed = FALSE; - H5Z_EDC_t edc_read = H5Z_NO_EDC; - size_t nbytes; /* Size of chunk (in bytes) */ - H5Z_cb_t cb_struct; - - int ret_value = H5_ITER_CONT; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_iter_copy) - - /* Get 'size_t' local value for number of bytes in chunk */ - H5_ASSIGN_OVERFLOW(nbytes, lt_key->nbytes, uint32_t, size_t); - - /* Check parameter for type conversion */ - if(udata->do_convert) { - if(H5T_detect_class(udata->dt_src, H5T_VLEN) > 0) - is_vlen = TRUE; - else if((H5T_get_class(udata->dt_src, FALSE) == H5T_REFERENCE) && (udata->file_src != udata->file_dst)) - fix_ref = TRUE; - else - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy dataset elements") - } /* end if */ - - /* Check for filtered chunks */ - if(pline && pline->nused) { - is_compressed = TRUE; - cb_struct.func = NULL; /* no callback function when failed */ - } /* end if */ - - /* Resize the buf if it is too small to hold the data */ - if(nbytes > buf_size) { - void *new_buf; /* New buffer for data */ - - /* Re-allocate memory for copying the chunk */ - if(NULL == (new_buf = H5MM_realloc(udata->buf, nbytes))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5_ITER_ERROR, "memory allocation failed for raw data chunk") - udata->buf = new_buf; - if(udata->bkg) { - if(NULL == (new_buf = H5MM_realloc(udata->bkg, nbytes))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5_ITER_ERROR, "memory allocation failed for raw data chunk") - udata->bkg = new_buf; - if(!udata->cpy_info->expand_ref) - HDmemset((uint8_t *)udata->bkg + buf_size, 0, (size_t)(nbytes - buf_size)); - - bkg = udata->bkg; - } /* end if */ - - buf = udata->buf; - udata->buf_size = buf_size = nbytes; - } /* end if */ - - /* read chunk data from the source file */ - if(H5F_block_read(f_src, H5FD_MEM_DRAW, addr_src, nbytes, dxpl_id, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_READERROR, H5_ITER_ERROR, "unable to read raw data chunk") - - /* Need to uncompress variable-length & reference data elements */ - if(is_compressed && (is_vlen || fix_ref)) { - unsigned filter_mask = lt_key->filter_mask; - - if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, edc_read, cb_struct, &nbytes, &buf_size, &buf) < 0) - HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "data pipeline read failed") - } /* end if */ - - /* Perform datatype conversion, if necessary */ - if(is_vlen) { - H5T_path_t *tpath_src_mem = udata->tpath_src_mem; - H5T_path_t *tpath_mem_dst = udata->tpath_mem_dst; - H5S_t *buf_space = udata->buf_space; - hid_t tid_src = udata->tid_src; - hid_t tid_dst = udata->tid_dst; - hid_t tid_mem = udata->tid_mem; - void *reclaim_buf = udata->reclaim_buf; - size_t reclaim_buf_size = udata->reclaim_buf_size; - - /* Convert from source file to memory */ - H5_CHECK_OVERFLOW(udata->nelmts, uint32_t, size_t); - if(H5T_convert(tpath_src_mem, tid_src, tid_mem, (size_t)udata->nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, H5_ITER_ERROR, "datatype conversion failed") - - /* Copy into another buffer, to reclaim memory later */ - HDmemcpy(reclaim_buf, buf, reclaim_buf_size); - - /* Set background buffer to all zeros */ - HDmemset(bkg, 0, buf_size); - - /* Convert from memory to destination file */ - if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, udata->nelmts, (size_t)0, (size_t)0, buf, bkg, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, H5_ITER_ERROR, "datatype conversion failed") - - /* Reclaim space from variable length data */ - if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_BADITER, H5_ITER_ERROR, "unable to reclaim variable-length data") - } /* end if */ - else if(fix_ref) { - /* Check for expanding references */ - /* (background buffer has already been zeroed out, if not expanding) */ - if(udata->cpy_info->expand_ref) { - size_t ref_count; - - /* Determine # of reference elements to copy */ - ref_count = nbytes / H5T_get_size(udata->dt_src); - - /* Copy the reference elements */ - if(H5O_copy_expand_ref(f_src, buf, dxpl_id, udata->file_dst, bkg, ref_count, H5T_get_ref_type(udata->dt_src), udata->cpy_info) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy reference attribute") - } /* end if */ - - /* After fix ref, copy the new reference elements to the buffer to write out */ - HDmemcpy(buf, bkg, buf_size); - } /* end if */ - - /* Set up destination chunk callback information for insertion */ - udata_dst.common.mesg = udata->common.mesg; /* Share this pointer for a short while */ - udata_dst.common.offset = lt_key->offset; - udata_dst.nbytes = lt_key->nbytes; - udata_dst.filter_mask = lt_key->filter_mask; - udata_dst.addr = HADDR_UNDEF; - - /* Need to compress variable-length & reference data elements before writing to file */ - if(is_compressed && (is_vlen || fix_ref) ) { - if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), edc_read, - cb_struct, &nbytes, &buf_size, &buf) < 0) - HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed") - H5_ASSIGN_OVERFLOW(udata_dst.nbytes, nbytes, size_t, uint32_t); - udata->buf = buf; - udata->buf_size = buf_size; - } /* end if */ - - /* Insert chunk into the destination Btree */ - if(H5B_insert(udata->file_dst, dxpl_id, H5B_ISTORE, udata->addr_dst, &udata_dst) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, H5_ITER_ERROR, "unable to allocate chunk") - - /* Write chunk data to destination file */ - HDassert(H5F_addr_defined(udata_dst.addr)); - if(H5F_block_write(udata->file_dst, H5FD_MEM_DRAW, udata_dst.addr, nbytes, dxpl_id, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, H5_ITER_ERROR, "unable to write raw data to file") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_iter_copy() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_cinfo_cache_reset - * - * Purpose: Reset the cached chunk info - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * November 27, 2007 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D_istore_cinfo_cache_reset(H5D_chunk_cached_t *last) -{ - FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_cinfo_cache_reset) - - /* Sanity check */ - HDassert(last); - - /* Indicate that the cached info is not valid */ - last->valid = FALSE; - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5D_istore_cinfo_cache_reset() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_cinfo_cache_update - * - * Purpose: Update the cached chunk info - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * November 27, 2007 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D_istore_cinfo_cache_update(H5D_chunk_cached_t *last, const H5D_istore_ud1_t *udata) -{ - unsigned u; /* Local index variable */ - - FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_cinfo_cache_update) - - /* Sanity check */ - HDassert(last); - HDassert(udata); - HDassert(udata->common.mesg); - HDassert(udata->common.offset); - - /* Stored the information to cache */ - for(u = 0; u < udata->common.mesg->u.chunk.ndims; u++) - last->offset[u] = udata->common.offset[u]; - last->nbytes = udata->nbytes; - last->filter_mask = udata->filter_mask; - last->addr = udata->addr; - - /* Indicate that the cached info is valid */ - last->valid = TRUE; - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5D_istore_cinfo_cache_update() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_cinfo_cache_found - * - * Purpose: Look for chunk info in cache - * - * Return: TRUE/FALSE/FAIL - * - * Programmer: Quincey Koziol - * November 27, 2007 - * - *------------------------------------------------------------------------- - */ -static hbool_t -H5D_istore_cinfo_cache_found(const H5D_chunk_cached_t *last, H5D_istore_ud1_t *udata) -{ - hbool_t ret_value = FALSE; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_cinfo_cache_found) - - /* Sanity check */ - HDassert(last); - HDassert(udata); - HDassert(udata->common.mesg); - HDassert(udata->common.offset); - - /* Check if the cached information is what is desired */ - if(last->valid) { - unsigned u; /* Local index variable */ - - /* Check that the offset is the same */ - for(u = 0; u < udata->common.mesg->u.chunk.ndims; u++) - if(last->offset[u] != udata->common.offset[u]) - HGOTO_DONE(FALSE) - - /* Retrieve the information from the cache */ - udata->nbytes = last->nbytes; - udata->filter_mask = last->filter_mask; - udata->addr = last->addr; - - /* Indicate that the data was found */ - HGOTO_DONE(TRUE) - } /* end if */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5D_istore_cinfo_cache_found() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_init - * - * Purpose: Initialize the raw data chunk cache for a dataset. This is - * called when the dataset is initialized. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Robb Matzke - * Monday, May 18, 1998 - * - *------------------------------------------------------------------------- - */ -herr_t -H5D_istore_init(const H5F_t *f, const H5D_t *dset) -{ - H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5D_istore_init, FAIL) - - if(H5F_RDCC_NBYTES(f) > 0 && H5F_RDCC_NELMTS(f) > 0) { - rdcc->nbytes = H5F_RDCC_NBYTES(f); - rdcc->nslots = H5F_RDCC_NELMTS(f); - rdcc->slot = H5FL_SEQ_CALLOC (H5D_rdcc_ent_ptr_t,rdcc->nslots); - if(NULL==rdcc->slot) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - - /* Reset any cached chunk info for this dataset */ - H5D_istore_cinfo_cache_reset(&(rdcc->last)); - } /* end if */ - - /* Allocate the shared structure */ - if(H5D_istore_shared_create(f, &dset->shared->layout) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_init() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_flush_entry - * - * Purpose: Writes a chunk to disk. If RESET is non-zero then the - * entry is cleared -- it's slightly faster to flush a chunk if - * the RESET flag is turned on because it results in one fewer - * memory copy. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Robb Matzke - * Thursday, May 21, 1998 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_t reset) -{ - void *buf = NULL; /*temporary buffer */ - hbool_t point_of_no_return = FALSE; - herr_t ret_value = SUCCEED; /*return value */ - - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_flush_entry) - - assert(io_info); - assert(io_info->dset); - assert(ent); - assert(!ent->locked); - - buf = ent->chunk; - if(ent->dirty) { - H5D_istore_ud1_t udata; /*pass through B-tree */ - - /* Set up user data for B-tree callbacks */ - udata.common.mesg = &io_info->dset->shared->layout; - udata.common.offset = ent->offset; - udata.filter_mask = 0; - udata.nbytes = ent->chunk_size; - udata.addr = HADDR_UNDEF; - - /* Should the chunk be filtered before writing it to disk? */ - if(io_info->dset->shared->dcpl_cache.pline.nused) { - size_t alloc = ent->alloc_size; /* Bytes allocated for BUF */ - size_t nbytes; /* Chunk size (in bytes) */ - - if(!reset) { - /* - * Copy the chunk to a new buffer before running it through - * the pipeline because we'll want to save the original buffer - * for later. - */ - H5_ASSIGN_OVERFLOW(alloc, ent->chunk_size, uint32_t, size_t); - if(NULL == (buf = H5MM_malloc(alloc))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline") - HDmemcpy(buf, ent->chunk, alloc); - } /* end if */ - else { - /* - * If we are reseting and something goes wrong after this - * point then it's too late to recover because we may have - * destroyed the original data by calling H5Z_pipeline(). - * The only safe option is to continue with the reset - * even if we can't write the data to disk. - */ - point_of_no_return = TRUE; - ent->chunk = NULL; - } /* end else */ - H5_ASSIGN_OVERFLOW(nbytes, udata.nbytes, uint32_t, size_t); - if(H5Z_pipeline(&(io_info->dset->shared->dcpl_cache.pline), 0, &(udata.filter_mask), io_info->dxpl_cache->err_detect, - io_info->dxpl_cache->filter_cb, &nbytes, &alloc, &buf) < 0) - HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "output pipeline failed") - H5_ASSIGN_OVERFLOW(udata.nbytes, nbytes, size_t, uint32_t); - } /* end if */ - - /* - * Create the chunk it if it doesn't exist, or reallocate the chunk if - * its size changed. Then write the data into the file. - */ - if(H5B_insert(io_info->dset->oloc.file, io_info->dxpl_id, H5B_ISTORE, io_info->dset->shared->layout.u.chunk.addr, &udata)<0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to allocate chunk") - H5_CHECK_OVERFLOW(udata.nbytes, uint32_t, size_t); - if(H5F_block_write(io_info->dset->oloc.file, H5FD_MEM_DRAW, udata.addr, (size_t)udata.nbytes, io_info->dxpl_id, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") - - /* Cache the chunk's info, in case it's accessed again shortly */ - H5D_istore_cinfo_cache_update(&io_info->dset->shared->cache.chunk.last, &udata); - - /* Mark cache entry as clean */ - ent->dirty = FALSE; -#ifdef H5D_ISTORE_DEBUG - io_info->dset->shared->cache.chunk.nflushes++; -#endif /* H5D_ISTORE_DEBUG */ - } /* end if */ - - /* Reset, but do not free or removed from list */ - if(reset) { - point_of_no_return = FALSE; - if(buf == ent->chunk) - buf = NULL; - if(ent->chunk != NULL) - ent->chunk = (uint8_t *)H5D_istore_chunk_xfree(ent->chunk, &(io_info->dset->shared->dcpl_cache.pline)); - } /* end if */ - -done: - /* Free the temp buffer only if it's different than the entry chunk */ - if(buf != ent->chunk) - H5MM_xfree(buf); - - /* - * If we reached the point of no return then we have no choice but to - * reset the entry. This can only happen if RESET is true but the - * output pipeline failed. Do not free the entry or remove it from the - * list. - */ - if(ret_value < 0 && point_of_no_return) { - if(ent->chunk) - ent->chunk = (uint8_t *)H5D_istore_chunk_xfree(ent->chunk, &(io_info->dset->shared->dcpl_cache.pline)); - } /* end if */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_flush_entry() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_preempt - * - * Purpose: Preempts the specified entry from the cache, flushing it to - * disk if necessary. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Robb Matzke - * Thursday, May 21, 1998 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D_istore_preempt(const H5D_io_info_t *io_info, H5D_rdcc_ent_t * ent, hbool_t flush) -{ - H5D_rdcc_t *rdcc = &(io_info->dset->shared->cache.chunk); - herr_t ret_value=SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_preempt) - - assert(io_info); - assert(ent); - assert(!ent->locked); - assert(ent->idx < rdcc->nslots); - - if(flush) { - /* Flush */ - if(H5D_istore_flush_entry(io_info, ent, TRUE) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") - } /* end if */ - else { - /* Don't flush, just free chunk */ - if(ent->chunk != NULL) - ent->chunk = (uint8_t *)H5D_istore_chunk_xfree(ent->chunk, &(io_info->dset->shared->dcpl_cache.pline)); - } /* end else */ - - /* Unlink from list */ - if(ent->prev) - ent->prev->next = ent->next; - else - rdcc->head = ent->next; - if(ent->next) - ent->next->prev = ent->prev; - else - rdcc->tail = ent->prev; - ent->prev = ent->next = NULL; - - /* Remove from cache */ - rdcc->slot[ent->idx] = NULL; - ent->idx = UINT_MAX; - rdcc->nbytes -= ent->chunk_size; - --rdcc->nused; - - /* Free */ - H5FL_FREE(H5D_rdcc_ent_t, ent); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_preempt() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_flush - * - * Purpose: Writes all dirty chunks to disk and optionally preempts them - * from the cache. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Robb Matzke - * Thursday, May 21, 1998 - * - *------------------------------------------------------------------------- - */ -herr_t -H5D_istore_flush(H5D_t *dset, hid_t dxpl_id, unsigned flags) -{ - H5D_io_info_t io_info; /* Temporary I/O info object */ - H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ - H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ - H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); - unsigned nerrors = 0; - H5D_rdcc_ent_t *ent, *next; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5D_istore_flush, FAIL) - - /* Fill the DXPL cache values for later use */ - if(H5D_get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") - - /* Construct dataset I/O info */ - H5D_BUILD_IO_INFO_WRT(&io_info, dset, dxpl_cache, dxpl_id, NULL, NULL); - - /* Loop over all entries in the chunk cache */ - for(ent = rdcc->head; ent; ent = next) { - next = ent->next; - if((flags & H5F_FLUSH_INVALIDATE)) { - if(H5D_istore_preempt(&io_info, ent, TRUE) < 0) - nerrors++; - } else { - if(H5D_istore_flush_entry(&io_info, ent, FALSE) < 0) - nerrors++; - } - } /* end for */ - if(nerrors) - HGOTO_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "unable to flush one or more raw data chunks") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_flush() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_dest + * Function: H5D_istore_remove * - * Purpose: Destroy the entire chunk cache by flushing dirty entries, - * preempting all entries, and freeing the cache itself. + * Purpose: Removes chunks that are no longer necessary in the B-tree. * * Return: Non-negative on success/Negative on failure * - * Programmer: Robb Matzke - * Thursday, May 21, 1998 + * Programmer: Robb Matzke + * Pedro Vicente, pvn@ncsa.uiuc.edu + * March 28, 2002 * *------------------------------------------------------------------------- */ -herr_t -H5D_istore_dest(H5D_t *dset, hid_t dxpl_id) +/* ARGSUSED */ +static H5B_ins_t +H5D_istore_remove(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key /*in,out */ , + hbool_t *lt_key_changed /*out */ , + void UNUSED * _udata /*in,out */ , + void UNUSED * _rt_key /*in,out */ , + hbool_t *rt_key_changed /*out */ ) { - H5D_io_info_t io_info; /* Temporary I/O info object */ - H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ - H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ - H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); - int nerrors = 0; - H5D_rdcc_ent_t *ent = NULL, *next = NULL; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5D_istore_dest, FAIL) - - HDassert(dset); - - /* Fill the DXPL cache values for later use */ - if(H5D_get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") - - /* Construct dataset I/O info */ - H5D_BUILD_IO_INFO_WRT(&io_info, dset, dxpl_cache, dxpl_id, NULL, NULL); + H5D_istore_key_t *lt_key = (H5D_istore_key_t *)_lt_key; + H5B_ins_t ret_value=H5B_INS_REMOVE; /* Return value */ - /* Flush all the cached chunks */ - for(ent = rdcc->head; ent; ent = next) { -#ifdef H5D_ISTORE_DEBUG - HDfputc('c', stderr); - HDfflush(stderr); -#endif - next = ent->next; - if(H5D_istore_preempt(&io_info, ent, TRUE) < 0) - nerrors++; - } /* end for */ - if(nerrors) - HGOTO_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "unable to flush one or more raw data chunks") + FUNC_ENTER_NOAPI_NOINIT(H5D_istore_remove) - if(rdcc->slot) - H5FL_SEQ_FREE(H5D_rdcc_ent_ptr_t, rdcc->slot); - HDmemset(rdcc, 0, sizeof(H5D_rdcc_t)); + /* Remove raw data chunk from file */ + H5_CHECK_OVERFLOW(lt_key->nbytes, uint32_t, hsize_t); + if(H5MF_xfree(f, H5FD_MEM_DRAW, dxpl_id, addr, (hsize_t)lt_key->nbytes) < 0) + HGOTO_ERROR(H5E_STORAGE, H5E_CANTFREE, H5B_INS_ERROR, "unable to free chunk") - /* Free the raw B-tree node buffer */ - if(dset->shared->layout.u.chunk.btree_shared == NULL) - HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "ref-counted page nil") - if(H5RC_DEC(dset->shared->layout.u.chunk.btree_shared) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") + /* Mark keys as unchanged */ + *lt_key_changed = FALSE; + *rt_key_changed = FALSE; done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_dest() */ +} /* end H5D_istore_remove() */ /*------------------------------------------------------------------------- @@ -1572,41 +815,28 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D_istore_shared_create (const H5F_t *f, H5O_layout_t *layout) +H5D_istore_shared_create(const H5F_t *f, H5O_layout_t *layout) { H5B_shared_t *shared; /* Shared B-tree node info */ - size_t u; /* Local index variable */ - herr_t ret_value=SUCCEED; /* Return value */ + size_t sizeof_rkey; /* Size of raw (disk) key */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_istore_shared_create) - /* Allocate space for the shared structure */ - if(NULL==(shared=H5FL_MALLOC(H5B_shared_t))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for shared B-tree info") - - /* Set up the "global" information for this file's groups */ - shared->type= H5B_ISTORE; - shared->two_k=2*H5F_KVALUE(f,H5B_ISTORE); - shared->sizeof_rkey = 4 + /*storage size */ - 4 + /*filter mask */ - layout->u.chunk.ndims*8; /*dimension indices */ - assert(shared->sizeof_rkey); - shared->sizeof_rnode = H5B_nodesize(f, shared, &shared->sizeof_keys); - assert(shared->sizeof_rnode); - if(NULL==(shared->page=H5FL_BLK_MALLOC(chunk_page,shared->sizeof_rnode))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree page") -#ifdef H5_CLEAR_MEMORY -HDmemset(shared->page, 0, shared->sizeof_rnode); -#endif /* H5_CLEAR_MEMORY */ - if(NULL==(shared->nkey=H5FL_SEQ_MALLOC(size_t,(size_t)(2*H5F_KVALUE(f,H5B_ISTORE)+1)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for B-tree page") - - /* Initialize the offsets into the native key buffer */ - for(u=0; u<(2*H5F_KVALUE(f,H5B_ISTORE)+1); u++) - shared->nkey[u]=u*H5B_ISTORE[0].sizeof_nkey; + /* Set the raw key size */ + sizeof_rkey = 4 + /*storage size */ + 4 + /*filter mask */ + layout->u.chunk.ndims * 8; /*dimension indices */ + + /* Allocate & initialize global info for the shared structure */ + if(NULL == (shared = H5B_shared_new(f, H5B_ISTORE, sizeof_rkey))) + HGOTO_ERROR(H5E_BTREE, H5E_NOSPACE, FAIL, "memory allocation failed for shared B-tree info") + + /* Set up the "local" information for this dataset's chunks */ + /* <none> */ /* Make shared B-tree info reference counted */ - if(NULL==(layout->u.chunk.btree_shared=H5RC_create(shared,H5D_istore_shared_free))) + if(NULL == (layout->u.chunk.btree_shared = H5RC_create(shared, H5B_shared_free))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't create ref-count wrapper for shared B-tree info") done: @@ -1615,538 +845,49 @@ done: /*------------------------------------------------------------------------- - * Function: H5D_istore_shared_free + * Function: H5D_istore_idx_init * - * Purpose: Free B-tree shared info - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * Thursday, July 8, 2004 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5D_istore_shared_free (void *_shared) -{ - H5B_shared_t *shared = (H5B_shared_t *)_shared; - - FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_shared_free) - - /* Free the raw B-tree node buffer */ - (void)H5FL_BLK_FREE(chunk_page, shared->page); - - /* Free the B-tree native key offsets buffer */ - H5FL_SEQ_FREE(size_t, shared->nkey); - - /* Free the shared B-tree info */ - H5FL_FREE(H5B_shared_t, shared); - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5D_istore_shared_free() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_prune - * - * Purpose: Prune the cache by preempting some things until the cache has - * room for something which is SIZE bytes. Only unlocked - * entries are considered for preemption. + * Purpose: Initialize the indexing information for a dataset. * * Return: Non-negative on success/Negative on failure * * Programmer: Robb Matzke - * Thursday, May 21, 1998 + * Monday, May 18, 1998 * *------------------------------------------------------------------------- */ static herr_t -H5D_istore_prune (const H5D_io_info_t *io_info, size_t size) -{ - int i, j, nerrors=0; - const H5D_rdcc_t *rdcc = &(io_info->dset->shared->cache.chunk); - size_t total = rdcc->nbytes; - const int nmeth=2; /*number of methods */ - int w[1]; /*weighting as an interval */ - H5D_rdcc_ent_t *p[2], *cur; /*list pointers */ - H5D_rdcc_ent_t *n[2]; /*list next pointers */ - herr_t ret_value=SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_prune) - - /* - * Preemption is accomplished by having multiple pointers (currently two) - * slide down the list beginning at the head. Pointer p(N+1) will start - * traversing the list when pointer pN reaches wN percent of the original - * list. In other words, preemption method N gets to consider entries in - * approximate least recently used order w0 percent before method N+1 - * where 100% means tha method N will run to completion before method N+1 - * begins. The pointers participating in the list traversal are each - * given a chance at preemption before any of the pointers are advanced. - */ - w[0] = (int)(rdcc->nused * H5F_RDCC_W0(io_info->dset->oloc.file)); - p[0] = rdcc->head; - p[1] = NULL; - - while ((p[0] || p[1]) && rdcc->nbytes+size>total) { - - /* Introduce new pointers */ - for(i = 0; i < (nmeth - 1); i++) - if(0 == w[i]) - p[i + 1] = rdcc->head; - - /* Compute next value for each pointer */ - for(i = 0; i < nmeth; i++) - n[i] = p[i] ? p[i]->next : NULL; - - /* Give each method a chance */ - for(i = 0; i < nmeth && (rdcc->nbytes + size) > total; i++) { - if(0 == i && p[0] && !p[0]->locked && - ((0 == p[0]->rd_count && 0 == p[0]->wr_count) || - (0 == p[0]->rd_count && p[0]->chunk_size == p[0]->wr_count) || - (p[0]->chunk_size == p[0]->rd_count && 0 == p[0]->wr_count))) { - /* - * Method 0: Preempt entries that have been completely written - * and/or completely read but not entries that are partially - * written or partially read. - */ - cur = p[0]; -#ifdef H5D_ISTORE_DEBUG - HDputc('.', stderr); - HDfflush(stderr); -#endif - - } else if (1==i && p[1] && !p[1]->locked) { - /* - * Method 1: Preempt the entry without regard to - * considerations other than being locked. This is the last - * resort preemption. - */ - cur = p[1]; -#ifdef H5D_ISTORE_DEBUG - HDputc(':', stderr); - HDfflush(stderr); -#endif - - } else { - /* Nothing to preempt at this point */ - cur= NULL; - } - - if (cur) { - for (j=0; j<nmeth; j++) { - if (p[j]==cur) - p[j] = NULL; - if (n[j]==cur) - n[j] = cur->next; - } - if (H5D_istore_preempt(io_info, cur, TRUE)<0) - nerrors++; - } - } - - /* Advance pointers */ - for (i=0; i<nmeth; i++) - p[i] = n[i]; - for (i=0; i<nmeth-1; i++) - w[i] -= 1; - } - - if (nerrors) - HGOTO_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "unable to preempt one or more raw data cache entry") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_prune() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_lock - * - * Purpose: Return a pointer to a dataset chunk. The pointer points - * directly into the chunk cache and should not be freed - * by the caller but will be valid until it is unlocked. The - * input value IDX_HINT is used to speed up cache lookups and - * it's output value should be given to H5F_istore_unlock(). - * IDX_HINT is ignored if it is out of range, and if it points - * to the wrong entry then we fall back to the normal search - * method. - * - * If RELAX is non-zero and the chunk isn't in the cache then - * don't try to read it from the file, but just allocate an - * uninitialized buffer to hold the result. This is intended - * for output functions that are about to overwrite the entire - * chunk. - * - * Return: Success: Ptr to a file chunk. - * - * Failure: NULL - * - * Programmer: Robb Matzke - * Thursday, May 21, 1998 - * - *------------------------------------------------------------------------- - */ -void * -H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, - hbool_t relax, unsigned *idx_hint/*in,out*/) -{ - H5D_t *dset = io_info->dset; /* Local pointer to the dataset info */ - const H5O_pline_t *pline = &(dset->shared->dcpl_cache.pline); /* I/O pipeline info */ - const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ - const H5O_fill_t *fill = &(dset->shared->dcpl_cache.fill); /* Fill value info */ - H5D_fill_buf_info_t fb_info; /* Dataset's fill buffer info */ - hbool_t fb_info_init = FALSE; /* Whether the fill value buffer has been initialized */ - H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache*/ - H5D_rdcc_ent_t *ent = NULL; /*cache entry */ - unsigned idx = 0; /*hash index number */ - hbool_t found = FALSE; /*already in cache? */ - size_t chunk_size; /*size of a chunk */ - void *chunk = NULL; /*the file chunk */ - unsigned u; /*counters */ - void *ret_value; /*return value */ - - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_lock) - - HDassert(io_info); - HDassert(dset); - HDassert(io_info->dxpl_cache); - HDassert(io_info->store); - HDassert(TRUE == H5P_isa_class(io_info->dxpl_id, H5P_DATASET_XFER)); - - /* Get the chunk's size */ - HDassert(layout->u.chunk.size > 0); - H5_ASSIGN_OVERFLOW(chunk_size, layout->u.chunk.size, uint32_t, size_t); - - /* Search for the chunk in the cache */ - if(rdcc->nslots > 0) { - idx = H5D_CHUNK_HASH(dset->shared, io_info->store->chunk.index); - ent = rdcc->slot[idx]; - - if(ent) - for(u = 0, found = TRUE; u < layout->u.chunk.ndims; u++) - if(io_info->store->chunk.offset[u] != ent->offset[u]) { - found = FALSE; - break; - } /* end if */ - } /* end if */ - - if(found) { - /* - * Already in the cache. Count a hit. - */ -#ifdef H5D_ISTORE_DEBUG - rdcc->nhits++; -#endif /* H5D_ISTORE_DEBUG */ - } /* end if */ - else if(relax) { - /* - * Not in the cache, but we're about to overwrite the whole thing - * anyway, so just allocate a buffer for it but don't initialize that - * buffer with the file contents. Count this as a hit instead of a - * miss because we saved ourselves lots of work. - */ -#ifdef H5D_ISTORE_DEBUG - HDputc('w', stderr); - HDfflush(stderr); - rdcc->nhits++; -#endif - if(NULL == (chunk = H5D_istore_chunk_alloc(chunk_size, pline))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk") - - /* In the case that some dataset functions look through this data, - * clear it to all 0s. */ - HDmemset(chunk, 0, chunk_size); - } /* end if */ - else { - H5D_istore_ud1_t tmp_udata; /*B-tree pass-through */ - haddr_t chunk_addr; /* Address of chunk on disk */ - - if(udata!=NULL) - chunk_addr = udata->addr; - else { - /* Point at temporary storage for B-tree pass through */ - udata = &tmp_udata; - - /* - * Not in the cache. Read it from the file and count this as a miss - * if it's in the file or an init if it isn't. - */ - chunk_addr = H5D_istore_get_addr(io_info, udata); - } /* end else */ - - /* Check if the chunk exists on disk */ - if(H5F_addr_defined(chunk_addr)) { - size_t chunk_alloc; /* Allocated chunk size */ - - /* Chunk size on disk isn't [likely] the same size as the final chunk - * size in memory, so allocate memory big enough. */ - H5_ASSIGN_OVERFLOW(chunk_alloc, udata->nbytes, uint32_t, size_t); - if(NULL == (chunk = H5D_istore_chunk_alloc(chunk_alloc, pline))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk") - if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, chunk_addr, chunk_alloc, io_info->dxpl_id, chunk) < 0) - HGOTO_ERROR(H5E_IO, H5E_READERROR, NULL, "unable to read raw data chunk") - - if(pline->nused) { - if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &(udata->filter_mask), io_info->dxpl_cache->err_detect, - io_info->dxpl_cache->filter_cb, &chunk_alloc, &chunk_alloc, &chunk) < 0) - HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, NULL, "data pipeline read failed") - H5_ASSIGN_OVERFLOW(udata->nbytes, chunk_alloc, size_t, uint32_t); - } /* end if */ -#ifdef H5D_ISTORE_DEBUG - rdcc->nmisses++; -#endif /* H5D_ISTORE_DEBUG */ - } else { - H5D_fill_value_t fill_status; - -#ifdef OLD_WAY - /* Clear the error stack from not finding the chunk on disk */ - H5E_clear_stack(NULL); -#endif /* OLD_WAY */ - - /* Chunk size on disk isn't [likely] the same size as the final chunk - * size in memory, so allocate memory big enough. */ - if(NULL == (chunk = H5D_istore_chunk_alloc(chunk_size, pline))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk") - - if(H5P_is_fill_value_defined(fill, &fill_status) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't tell if fill value defined") - - if(fill->fill_time == H5D_FILL_TIME_ALLOC || - (fill->fill_time == H5D_FILL_TIME_IFSET && fill_status == H5D_FILL_VALUE_USER_DEFINED)) { - /* - * The chunk doesn't exist in the file. Replicate the fill - * value throughout the chunk, if the fill value is defined. - */ - - /* Initialize the fill value buffer */ - /* (use the compact dataset storage buffer as the fill value buffer) */ - if(H5D_fill_init(&fb_info, chunk, FALSE, - NULL, NULL, NULL, NULL, - &dset->shared->dcpl_cache.fill, dset->shared->type, - dset->shared->type_id, (size_t)0, chunk_size, io_info->dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "can't initialize fill buffer info") - fb_info_init = TRUE; - - /* Check for VL datatype & non-default fill value */ - if(fb_info.has_vlen_fill_type) - /* Fill the buffer with VL datatype fill values */ - if(H5D_fill_refill_vl(&fb_info, fb_info.elmts_per_buf, io_info->dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, NULL, "can't refill fill value buffer") - } /* end if */ - else - HDmemset(chunk, 0, chunk_size); -#ifdef H5D_ISTORE_DEBUG - rdcc->ninits++; -#endif /* H5D_ISTORE_DEBUG */ - } /* end else */ - } /* end else */ - HDassert(found || chunk_size > 0); - - if(!found && rdcc->nslots > 0 && chunk_size <= rdcc->nbytes && - (!ent || !ent->locked)) { - /* - * Add the chunk to the cache only if the slot is not already locked. - * Preempt enough things from the cache to make room. - */ - if(ent) { -#ifdef H5D_ISTORE_DEBUG - HDputc('#', stderr); - HDfflush(stderr); -#endif - if(H5D_istore_preempt(io_info, ent, TRUE) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk from cache") - } /* end if */ - if(H5D_istore_prune(io_info, chunk_size) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk(s) from cache") - - /* Create a new entry */ - ent = H5FL_MALLOC(H5D_rdcc_ent_t); - ent->locked = 0; - ent->dirty = FALSE; - H5_ASSIGN_OVERFLOW(ent->chunk_size, chunk_size, size_t, uint32_t); - ent->alloc_size = chunk_size; - for(u = 0; u < layout->u.chunk.ndims; u++) - ent->offset[u] = io_info->store->chunk.offset[u]; - H5_ASSIGN_OVERFLOW(ent->rd_count, chunk_size, size_t, uint32_t); - H5_ASSIGN_OVERFLOW(ent->wr_count, chunk_size, size_t, uint32_t); - ent->chunk = (uint8_t *)chunk; - - /* Add it to the cache */ - HDassert(NULL == rdcc->slot[idx]); - rdcc->slot[idx] = ent; - ent->idx = idx; - rdcc->nbytes += chunk_size; - rdcc->nused++; - - /* Add it to the linked list */ - ent->next = NULL; - if(rdcc->tail) { - rdcc->tail->next = ent; - ent->prev = rdcc->tail; - rdcc->tail = ent; - } /* end if */ - else { - rdcc->head = rdcc->tail = ent; - ent->prev = NULL; - } /* end else */ - found = TRUE; - } else if(!found) { - /* - * The chunk is larger than the entire cache so we don't cache it. - * This is the reason all those arguments have to be repeated for the - * unlock function. - */ - ent = NULL; - idx = UINT_MAX; - } else { - /* - * The chunk is not at the beginning of the cache; move it backward - * by one slot. This is how we implement the LRU preemption - * algorithm. - */ - HDassert(ent); - if(ent->next) { - if(ent->next->next) - ent->next->next->prev = ent; - else - rdcc->tail = ent; - ent->next->prev = ent->prev; - if(ent->prev) - ent->prev->next = ent->next; - else - rdcc->head = ent->next; - ent->prev = ent->next; - ent->next = ent->next->next; - ent->prev->next = ent; - } /* end if */ - } /* end else */ - - /* Lock the chunk into the cache */ - if(ent) { - HDassert(!ent->locked); - ent->locked = TRUE; - chunk = ent->chunk; - } /* end if */ - - if(idx_hint) - *idx_hint = idx; - - /* Set return value */ - ret_value = chunk; - -done: - /* Release the fill buffer info, if it's been initialized */ - if(fb_info_init && H5D_fill_term(&fb_info) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, NULL, "Can't release fill buffer info") - - /* Release the chunk allocated, on error */ - if(!ret_value) - if(chunk) - chunk = H5D_istore_chunk_xfree(chunk, pline); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_lock() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_unlock - * - * Purpose: Unlocks a previously locked chunk. The LAYOUT, COMP, and - * OFFSET arguments should be the same as for H5F_rdcc_lock(). - * The DIRTY argument should be set to non-zero if the chunk has - * been modified since it was locked. The IDX_HINT argument is - * the returned index hint from the lock operation and BUF is - * the return value from the lock. - * - * The NACCESSED argument should be the number of bytes accessed - * for reading or writing (depending on the value of DIRTY). - * It's only purpose is to provide additional information to the - * preemption policy. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Robb Matzke - * Thursday, May 21, 1998 - * - *------------------------------------------------------------------------- - */ -herr_t -H5D_istore_unlock(const H5D_io_info_t *io_info, - hbool_t dirty, unsigned idx_hint, void *chunk, uint32_t naccessed) +H5D_istore_idx_init(const H5D_chk_idx_info_t *idx_info) { - const H5O_layout_t *layout=&(io_info->dset->shared->layout); /* Dataset layout */ - const H5D_rdcc_t *rdcc = &(io_info->dset->shared->cache.chunk); - H5D_rdcc_ent_t *ent = NULL; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_unlock) + herr_t ret_value = SUCCEED; /* Return value */ - assert(io_info); + FUNC_ENTER_NOAPI_NOINIT(H5D_istore_idx_init) - if(UINT_MAX == idx_hint) { - /* - * It's not in the cache, probably because it's too big. If it's - * dirty then flush it to disk. In any case, free the chunk. - * Note: we have to copy the layout and filter messages so we - * don't discard the `const' qualifier. - */ - if (dirty) { - H5D_rdcc_ent_t x; - - HDmemset(&x, 0, sizeof(x)); - x.dirty = TRUE; - HDassert(sizeof(x.offset[0]) == sizeof(io_info->store->chunk.offset[0])); - HDmemcpy(x.offset, io_info->store->chunk.offset, layout->u.chunk.ndims * sizeof(x.offset[0])); - HDassert(layout->u.chunk.size > 0); - x.chunk_size = layout->u.chunk.size; - H5_ASSIGN_OVERFLOW(x.alloc_size, x.chunk_size, uint32_t, size_t); - x.chunk = (uint8_t *)chunk; - - if(H5D_istore_flush_entry(io_info, &x, TRUE) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") - } /* end if */ - else { - if(chunk) - chunk = H5D_istore_chunk_xfree(chunk, &(io_info->dset->shared->dcpl_cache.pline)); - } /* end else */ - } /* end if */ - else { - /* Sanity check */ - HDassert(idx_hint < rdcc->nslots); - HDassert(rdcc->slot[idx_hint]); - HDassert(rdcc->slot[idx_hint]->chunk == chunk); + /* Check args */ + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); - /* - * It's in the cache so unlock it. - */ - ent = rdcc->slot[idx_hint]; - HDassert(ent->locked); - if(dirty) { - ent->dirty = TRUE; - ent->wr_count -= MIN(ent->wr_count, naccessed); - } /* end if */ - else - ent->rd_count -= MIN(ent->rd_count, naccessed); - ent->locked = FALSE; - } /* end else */ + /* Allocate the shared structure */ + if(H5D_istore_shared_create(idx_info->f, idx_info->layout) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_unlock() */ +} /* end H5D_istore_idx_init() */ /*------------------------------------------------------------------------- - * Function: H5D_istore_create + * Function: H5D_istore_idx_create * * Purpose: Creates a new indexed-storage B-tree and initializes the - * istore struct with information about the storage. The + * layout struct with information about the storage. The * struct should be immediately written to the object header. * - * This function must be called before passing ISTORE to any of + * This function must be called before passing LAYOUT to any of * the other indexed storage functions! * - * Return: Non-negative on success (with the ISTORE argument initialized + * Return: Non-negative on success (with the LAYOUT argument initialized * and ready to write to an object header). Negative on failure. * * Programmer: Robb Matzke @@ -2154,138 +895,70 @@ done: * *------------------------------------------------------------------------- */ -herr_t -H5D_istore_create(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout /*out */) +static herr_t +H5D_istore_idx_create(const H5D_chk_idx_info_t *idx_info) { - H5D_istore_ud0_t udata; -#ifndef NDEBUG - unsigned u; -#endif - herr_t ret_value = SUCCEED; /* Return value */ + H5D_istore_ud0_t udata; /* User data for B-tree callback */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(H5D_istore_create, FAIL) + FUNC_ENTER_NOAPI_NOINIT(H5D_istore_idx_create) /* Check args */ - HDassert(f); - HDassert(layout && H5D_CHUNKED == layout->type); - HDassert(layout->u.chunk.ndims > 0 && layout->u.chunk.ndims <= H5O_LAYOUT_NDIMS); -#ifndef NDEBUG - for(u = 0; u < layout->u.chunk.ndims; u++) - HDassert(layout->u.chunk.dim[u] > 0); -#endif + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); /* Initialize "user" data for B-tree callbacks, etc. */ - udata.mesg = layout; + udata.mesg = idx_info->layout; - if(H5B_create(f, dxpl_id, H5B_ISTORE, &udata, &(layout->u.chunk.addr)/*out*/) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "can't create B-tree") + /* Create the v1 B-tree for the chunk index */ + if(H5B_create(idx_info->f, idx_info->dxpl_id, H5B_ISTORE, &udata, &(idx_info->layout->u.chunk.addr)/*out*/) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't create B-tree") done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_create() */ +} /* end H5D_istore_idx_create() */ /*------------------------------------------------------------------------- - * Function: H5D_istore_allocated - * - * Purpose: Return the number of bytes allocated in the file for storage - * of raw data under the specified B-tree (ADDR is the address - * of the B-tree). + * Function: H5D_istore_idx_insert * - * Return: Success: Number of bytes stored in all chunks. + * Purpose: Create the chunk it if it doesn't exist, or reallocate the + * chunk if its size changed. * - * Failure: 0 + * Return: Non-negative on success/Negative on failure * * Programmer: Robb Matzke - * Wednesday, April 21, 1999 - * - *------------------------------------------------------------------------- - */ -hsize_t -H5D_istore_allocated(H5D_t *dset, hid_t dxpl_id) -{ - H5D_io_info_t io_info; /* Temporary I/O info object */ - const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache */ - H5D_rdcc_ent_t *ent; /*cache entry */ - H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ - H5D_dxpl_cache_t *dxpl_cache=&_dxpl_cache; /* Data transfer property cache */ - H5D_istore_it_ud1_t udata; - hsize_t ret_value; /* Return value */ - - FUNC_ENTER_NOAPI(H5D_istore_allocated, 0) - - HDassert(dset); - - /* Fill the DXPL cache values for later use */ - if(H5D_get_dxpl_cache(dxpl_id,&dxpl_cache) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "can't fill dxpl cache") - - /* Construct dataset I/O info */ - H5D_BUILD_IO_INFO_WRT(&io_info, dset, dxpl_cache, dxpl_id, NULL, NULL); - - /* Search for cached chunks that haven't been written out */ - for(ent = rdcc->head; ent; ent = ent->next) { - /* Flush the chunk out to disk, to make certain the size is correct later */ - if (H5D_istore_flush_entry(&io_info, ent, FALSE) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, 0, "cannot flush indexed storage buffer") - } /* end for */ - - HDmemset(&udata, 0, sizeof udata); - udata.common.mesg = &dset->shared->layout; - if(H5B_iterate(dset->oloc.file, dxpl_id, H5B_ISTORE, H5D_istore_iter_allocated, dset->shared->layout.u.chunk.addr, &udata) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, 0, "unable to iterate over chunk B-tree") - - /* Set return value */ - ret_value = udata.total_storage; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_allocated() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_chunkmap - * - * Purpose: obtain the chunk address and corresponding chunk index - * - * Return: Success: Non-negative on succeed. - * - * Failure: negative value - * - * Programmer: Kent Yang - * November 15, 2005 + * Thursday, May 21, 1998 * *------------------------------------------------------------------------- */ -herr_t -H5D_istore_chunkmap(const H5D_io_info_t *io_info, haddr_t chunk_addr[], - const hsize_t down_chunks[]) +static herr_t +H5D_istore_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) { - H5D_t *dset = io_info->dset; /* Local pointer to dataset info */ - H5D_istore_it_ud5_t udata; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(H5D_istore_chunkmap, FAIL) + FUNC_ENTER_NOAPI_NOINIT(H5D_istore_idx_insert) - HDassert(dset); - - /* Set up user data for B-tree callback */ - HDmemset(&udata, 0, sizeof(udata)); - udata.common.mesg = &dset->shared->layout; - udata.down_chunks = down_chunks; - udata.chunk_addr = chunk_addr; + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(udata); - /* Build mapping of chunk addresses and indices */ - if(H5B_iterate(dset->oloc.file, io_info->dxpl_id, H5B_ISTORE, H5D_istore_iter_chunkmap, dset->shared->layout.u.chunk.addr, &udata) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to iterate over chunk B-tree") + /* + * Create the chunk it if it doesn't exist, or reallocate the chunk if + * its size changed. + */ + if(H5B_insert(idx_info->f, idx_info->dxpl_id, H5B_ISTORE, idx_info->layout->u.chunk.addr, udata) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to allocate chunk") done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_chunkmap() */ +} /* H5D_istore_idx_insert() */ /*------------------------------------------------------------------------- - * Function: H5D_istore_get_addr + * Function: H5D_istore_idx_get_addr * * Purpose: Get the file address of a chunk if file space has been * assigned. Save the retrieved information in the udata @@ -2298,985 +971,169 @@ done: * *------------------------------------------------------------------------- */ -haddr_t -H5D_istore_get_addr(const H5D_io_info_t *io_info, H5D_istore_ud1_t *_udata) +static haddr_t +H5D_istore_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) { - H5D_istore_ud1_t tmp_udata; /* Information about a chunk */ - H5D_istore_ud1_t *udata; /* Pointer to information about a chunk */ haddr_t ret_value; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_get_addr) - - HDassert(io_info); - HDassert(io_info->dset); - HDassert(io_info->dset->shared->layout.u.chunk.ndims > 0); - HDassert(io_info->store->chunk.offset); - - /* Check for udata struct to return */ - udata = (_udata != NULL ? _udata : &tmp_udata); - - /* Initialize the information about the chunk we are looking for */ - udata->common.mesg = &(io_info->dset->shared->layout); - udata->common.offset = io_info->store->chunk.offset; - udata->nbytes = 0; - udata->filter_mask = 0; - udata->addr = HADDR_UNDEF; - - /* Check for cached information */ - if(!H5D_istore_cinfo_cache_found(&io_info->dset->shared->cache.chunk.last, udata)) { - /* Go get the chunk information */ - if(H5B_find(io_info->dset->oloc.file, io_info->dxpl_id, H5B_ISTORE, io_info->dset->shared->layout.u.chunk.addr, udata) < 0) { - /* Note: don't push error on stack, leave that to next higher level, - * since many times the B-tree is searched in order to determine - * if a chunk exists in the B-tree or not. -QAK - */ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_idx_get_addr) + + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(idx_info->layout->u.chunk.ndims > 0); + HDassert(udata); + + /* Go get the chunk information */ + if(H5B_find(idx_info->f, idx_info->dxpl_id, H5B_ISTORE, idx_info->layout->u.chunk.addr, udata) < 0) { + /* Note: don't push error on stack, leave that to next higher level, + * since many times the B-tree is searched in order to determine + * if a chunk exists in the B-tree or not. -QAK + */ #ifdef OLD_WAY - H5E_clear_stack(NULL); + H5E_clear_stack(NULL); - HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, HADDR_UNDEF, "Can't locate chunk info") + HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, HADDR_UNDEF, "Can't locate chunk info") #else /* OLD_WAY */ - /* Cache the fact that the chunk is not in the B-tree */ - H5D_istore_cinfo_cache_update(&io_info->dset->shared->cache.chunk.last, udata); - - HGOTO_DONE(HADDR_UNDEF) + HGOTO_DONE(HADDR_UNDEF) #endif /* OLD_WAY */ - } /* end if */ - - /* Cache the information retrieved */ - HDassert(H5F_addr_defined(udata->addr)); - H5D_istore_cinfo_cache_update(&io_info->dset->shared->cache.chunk.last, udata); - } /* end else */ + } /* end if */ /* Success! Set the return value */ ret_value = udata->addr; done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5D_istore_get_addr() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_chunk_alloc - * - * Purpose: Allocate space for a chunk in memory. This routine allocates - * memory space for non-filtered chunks from a block free list - * and uses malloc()/free() for filtered chunks. - * - * Return: Pointer to memory for chunk on success/NULL on failure - * - * Programmer: Quincey Koziol - * April 22, 2004 - * - *------------------------------------------------------------------------- - */ -static void * -H5D_istore_chunk_alloc(size_t size, const H5O_pline_t *pline) -{ - void *ret_value = NULL; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_chunk_alloc) - - HDassert(size); - HDassert(pline); - - if(pline->nused > 0) - ret_value = H5MM_malloc(size); - else - ret_value = H5FL_BLK_MALLOC(chunk, size); - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5D_istore_chunk_alloc() */ +} /* H5D_istore_idx_get_addr() */ /*------------------------------------------------------------------------- - * Function: H5D_istore_chunk_xfree + * Function: H5D_istore_idx_iterate_cb * - * Purpose: Free space for a chunk in memory. This routine allocates - * memory space for non-filtered chunks from a block free list - * and uses malloc()/free() for filtered chunks. + * Purpose: Translate the B-tree specific chunk record into a generic + * form and make the callback to the generic chunk callback + * routine. * - * Return: NULL (never fails) + * Return: Success: Non-negative + * Failure: Negative * * Programmer: Quincey Koziol - * April 22, 2004 - * - *------------------------------------------------------------------------- - */ -static void * -H5D_istore_chunk_xfree(void *chk, const H5O_pline_t *pline) -{ - FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_chunk_xfree) - - HDassert(pline); - - if(chk) { - if(pline->nused > 0) - H5MM_xfree(chk); - else - (void)H5FL_BLK_FREE(chunk, chk); - } /* end if */ - - FUNC_LEAVE_NOAPI(NULL) -} /* H5D_istore_chunk_xfree() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_allocate - * - * Purpose: Allocate file space for all chunks that are not allocated yet. - * Return SUCCEED if all needed allocation succeed, otherwise - * FAIL. - * - * Return: Non-negative on success/Negative on failure - * - * Note: Current implementation relies on cache_size being 0, - * thus no chunk is cached and written to disk immediately - * when a chunk is unlocked (via H5F_istore_unlock) - * This should be changed to do a direct flush independent - * of the cache value. - * - * Programmer: Albert Cheng - * June 26, 1998 - * - *------------------------------------------------------------------------- - */ -herr_t -H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) -{ - H5D_io_info_t io_info; /* Dataset I/O info */ - H5D_storage_t store; /* Dataset storage information */ - hsize_t chunk_offset[H5O_LAYOUT_NDIMS]; /* Offset of current chunk */ - size_t orig_chunk_size; /* Original size of chunk in bytes */ - unsigned filter_mask = 0; /* Filter mask for chunks that have them */ - const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ - const H5O_pline_t *pline = &(dset->shared->dcpl_cache.pline); /* I/O pipeline info */ - const H5O_fill_t *fill = &(dset->shared->dcpl_cache.fill); /* Fill value info */ - H5D_fill_value_t fill_status; /* The fill value status */ - hbool_t should_fill = FALSE; /* Whether fill values should be written */ - H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ - H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ -#ifdef H5_HAVE_PARALLEL - MPI_Comm mpi_comm = MPI_COMM_NULL; /* MPI communicator for file */ - int mpi_rank = (-1); /* This process's rank */ - int mpi_code; /* MPI return code */ - hbool_t blocks_written = FALSE; /* Flag to indicate that chunk was actually written */ - hbool_t using_mpi = FALSE; /* Flag to indicate that the file is being accessed with an MPI-capable file driver */ -#endif /* H5_HAVE_PARALLEL */ - hbool_t carry; /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */ - int space_ndims; /* Dataset's space rank */ - hsize_t space_dim[H5O_LAYOUT_NDIMS]; /* Dataset's dataspace dimensions */ - H5D_fill_buf_info_t fb_info; /* Dataset's fill buffer info */ - hbool_t fb_info_init = FALSE; /* Whether the fill value buffer has been initialized */ - hid_t data_dxpl_id; /* DXPL ID to use for raw data I/O operations */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5D_istore_allocate, FAIL) - - /* Check args */ - HDassert(dset && H5D_CHUNKED == layout->type); - HDassert(layout->u.chunk.ndims > 0 && layout->u.chunk.ndims <= H5O_LAYOUT_NDIMS); - HDassert(H5F_addr_defined(layout->u.chunk.addr)); - HDassert(TRUE == H5P_isa_class(dxpl_id, H5P_DATASET_XFER)); - - /* Retrieve the dataset dimensions */ - if((space_ndims = H5S_get_simple_extent_dims(dset->shared->space, space_dim, NULL)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get simple dataspace info") - space_dim[space_ndims] = layout->u.chunk.dim[space_ndims]; - -#ifdef H5_HAVE_PARALLEL - /* Retrieve MPI parameters */ - if(IS_H5FD_MPI(dset->oloc.file)) { - /* Get the MPI communicator */ - if(MPI_COMM_NULL == (mpi_comm = H5F_mpi_get_comm(dset->oloc.file))) - HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI communicator") - - /* Get the MPI rank */ - if((mpi_rank = H5F_mpi_get_rank(dset->oloc.file)) < 0) - HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI rank") - - /* Set the MPI-capable file driver flag */ - using_mpi = TRUE; - - /* Use the internal "independent" DXPL */ - data_dxpl_id = H5AC_ind_dxpl_id; - } /* end if */ - else { -#endif /* H5_HAVE_PARALLEL */ - /* Use the DXPL we were given */ - data_dxpl_id = dxpl_id; -#ifdef H5_HAVE_PARALLEL - } /* end else */ -#endif /* H5_HAVE_PARALLEL */ - - /* Fill the DXPL cache values for later use */ - if(H5D_get_dxpl_cache(data_dxpl_id, &dxpl_cache) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") - - /* Get original chunk size */ - H5_ASSIGN_OVERFLOW(orig_chunk_size, layout->u.chunk.size, uint32_t, size_t); - - /* Check the dataset's fill-value status */ - if(H5P_is_fill_value_defined(fill, &fill_status) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't tell if fill value defined") - - /* If we are filling the dataset on allocation or "if set" and - * the fill value _is_ set, _and_ we are not overwriting the new blocks, - * or if there are any pipeline filters defined, - * set the "should fill" flag - */ - if((!full_overwrite && (fill->fill_time == H5D_FILL_TIME_ALLOC || - (fill->fill_time == H5D_FILL_TIME_IFSET && fill_status == H5D_FILL_VALUE_USER_DEFINED))) - || pline->nused > 0) - should_fill = TRUE; - - /* Check if fill values should be written to chunks */ - if(should_fill) { - /* Initialize the fill value buffer */ - /* (delay allocating fill buffer for VL datatypes until refilling) */ - /* (casting away const OK - QAK) */ - if(H5D_fill_init(&fb_info, NULL, (hbool_t)(pline->nused > 0), - (H5MM_allocate_t)H5D_istore_chunk_alloc, (void *)pline, - (H5MM_free_t)H5D_istore_chunk_xfree, (void *)pline, - &dset->shared->dcpl_cache.fill, dset->shared->type, - dset->shared->type_id, (size_t)0, orig_chunk_size, data_dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill buffer info") - fb_info_init = TRUE; - - /* Check if there are filters which need to be applied to the chunk */ - /* (only do this in advance when the chunk info can be re-used (i.e. - * it doesn't contain any non-default VL datatype fill values) - */ - if(!fb_info.has_vlen_fill_type && pline->nused > 0) { - size_t buf_size = orig_chunk_size; - - /* Push the chunk through the filters */ - if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &orig_chunk_size, &buf_size, &fb_info.fill_buf) < 0) - HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed") - } /* end if */ - } /* end if */ - - /* Set up dataset I/O info */ - store.chunk.offset = chunk_offset; - H5D_BUILD_IO_INFO_WRT(&io_info, dset, dxpl_cache, data_dxpl_id, &store, NULL); - - /* Reset the chunk offset indices */ - HDmemset(chunk_offset, 0, (layout->u.chunk.ndims * sizeof(chunk_offset[0]))); - - /* Loop over all chunks */ - carry = FALSE; - while(!carry) { - int i; /* Local index variable */ - - /* Check if the chunk exists yet on disk */ - if(!H5F_addr_defined(H5D_istore_get_addr(&io_info, NULL))) { - const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /* Raw data chunk cache */ - H5D_rdcc_ent_t *ent; /* Cache entry */ - hbool_t chunk_exists; /* Flag to indicate whether a chunk exists already */ - unsigned u; /* Local index variable */ - - /* Didn't find the chunk on disk */ - chunk_exists = FALSE; - - /* Look for chunk in cache */ - for(ent = rdcc->head; ent && !chunk_exists; ent = ent->next) { - /* Assume a match */ - chunk_exists = TRUE; - for(u = 0; u < layout->u.chunk.ndims; u++) - if(ent->offset[u] != chunk_offset[u]) { - chunk_exists = FALSE; /* Reset if no match */ - break; - } /* end if */ - } /* end for */ - - /* Chunk wasn't in cache either, create it now */ - if(!chunk_exists) { - H5D_istore_ud1_t udata; /* B-tree pass-through for creating chunk */ - size_t chunk_size; /* Size of chunk in bytes, possibly filtered */ - - /* Check for VL datatype & non-default fill value */ - if(fb_info_init && fb_info.has_vlen_fill_type) { - /* Sanity check */ - HDassert(should_fill); - - /* Fill the buffer with VL datatype fill values */ - if(H5D_fill_refill_vl(&fb_info, fb_info.elmts_per_buf, data_dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't refill fill value buffer") - - /* Check if there are filters which need to be applied to the chunk */ - if(pline->nused > 0) { - size_t buf_size = orig_chunk_size; - size_t nbytes = fb_info.fill_buf_size; - - /* Push the chunk through the filters */ - if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &nbytes, &buf_size, &fb_info.fill_buf) < 0) - HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed") - - /* Keep the number of bytes the chunk turned in to */ - chunk_size = nbytes; - } /* end if */ - else - H5_ASSIGN_OVERFLOW(chunk_size, layout->u.chunk.size, uint32_t, size_t); - } /* end if */ - else - chunk_size = orig_chunk_size; - - /* Initialize the chunk information */ - udata.common.mesg = layout; - udata.common.offset = chunk_offset; - H5_ASSIGN_OVERFLOW(udata.nbytes, chunk_size, size_t, uint32_t); - udata.filter_mask = filter_mask; - udata.addr = HADDR_UNDEF; - - /* Allocate the chunk with all processes */ - if(H5B_insert(dset->oloc.file, dxpl_id, H5B_ISTORE, layout->u.chunk.addr, &udata) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to allocate chunk") - - /* Check if fill values should be written to chunks */ - if(should_fill) { - /* Sanity check */ - HDassert(fb_info_init); - -#ifdef H5_HAVE_PARALLEL - /* Check if this file is accessed with an MPI-capable file driver */ - if(using_mpi) { - /* Write the chunks out from only one process */ - /* !! Use the internal "independent" DXPL!! -QAK */ - if(H5_PAR_META_WRITE == mpi_rank) { - HDassert(udata.nbytes == chunk_size); - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, chunk_size, data_dxpl_id, fb_info.fill_buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") - } /* end if */ - - /* Indicate that blocks are being written */ - blocks_written = TRUE; - } /* end if */ - else { -#endif /* H5_HAVE_PARALLEL */ - HDassert(udata.nbytes == chunk_size); - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, chunk_size, data_dxpl_id, fb_info.fill_buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") -#ifdef H5_HAVE_PARALLEL - } /* end else */ -#endif /* H5_HAVE_PARALLEL */ - } /* end if */ - - /* Release the fill buffer if we need to re-allocate it each time */ - if(fb_info_init && fb_info.has_vlen_fill_type && pline->nused > 0) - H5D_fill_release(&fb_info); - } /* end if */ - } /* end if */ - - /* Increment indices */ - carry = TRUE; - for(i = (int)(space_ndims - 1); i >= 0; --i) { - chunk_offset[i] += layout->u.chunk.dim[i]; - if(chunk_offset[i] >= space_dim[i]) - chunk_offset[i] = 0; - else { - carry = FALSE; - break; - } /* end else */ - } /* end for */ - } /* end while */ - -#ifdef H5_HAVE_PARALLEL - /* Only need to block at the barrier if we actually initialized a chunk */ - /* using an MPI-capable file driver */ - if(using_mpi && blocks_written) { - /* Wait at barrier to avoid race conditions where some processes are - * still writing out chunks and other processes race ahead to read - * them in, getting bogus data. - */ - if(MPI_SUCCESS != (mpi_code = MPI_Barrier(mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code) - } /* end if */ -#endif /* H5_HAVE_PARALLEL */ - - /* Reset any cached chunk info for this dataset */ - H5D_istore_cinfo_cache_reset(&dset->shared->cache.chunk.last); - -done: - /* Release the fill buffer info, if it's been initialized */ - if(fb_info_init && H5D_fill_term(&fb_info) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release fill buffer info") - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_allocate() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_prune_check - * - * Purpose: Search for chunks that are no longer necessary in the B-tree. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu - * March 26, 2002 + * Tuesday, May 20, 2008 * *------------------------------------------------------------------------- */ /* ARGSUSED */ static int -H5D_istore_prune_check(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, - const void *_lt_key, haddr_t UNUSED addr, const void UNUSED *_rt_key, +H5D_istore_idx_iterate_cb(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, + const void *_lt_key, haddr_t addr, const void UNUSED *_rt_key, void *_udata) { - H5D_istore_it_ud3_t *udata = (H5D_istore_it_ud3_t *)_udata; - const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; - unsigned rank; /*current # of dimensions */ - unsigned u; - int ret_value = H5_ITER_CONT; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_prune_check) - - /* Figure out what chunks are no longer in use for the specified extent and release them */ - rank = udata->common.mesg->u.chunk.ndims - 1; - for(u = 0; u < rank; u++) - /* The LT_KEY is the left key (the one that describes the chunk). It points to a chunk of - * storage that contains the beginning of the logical address space represented by UDATA. - */ - if((hsize_t)lt_key->offset[u] > udata->dims[u]) { - H5D_istore_sl_ck_t *sl_node; /* Skip list node for chunk to remove */ - - /* Allocate space for the shared structure */ - if(NULL == (sl_node = H5FL_MALLOC(H5D_istore_sl_ck_t))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5_ITER_ERROR, "memory allocation failed for shared B-tree info") + H5D_istore_it_ud_t *udata = (H5D_istore_it_ud_t *)_udata; /* User data */ + const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; /* B-tree key for chunk */ + H5D_chunk_rec_t chunk_rec; /* Generic chunk record for callback */ + int ret_value; /* Return value */ - /* Calculate the index of this chunk */ - if(H5V_chunk_index(rank, lt_key->offset, udata->common.mesg->u.chunk.dim, udata->down_chunks, &sl_node->index) < 0) { - H5FL_FREE(H5D_istore_sl_ck_t, sl_node); - HGOTO_ERROR(H5E_IO, H5E_BADRANGE, H5_ITER_ERROR, "can't get chunk index") - } /* end if */ + FUNC_ENTER_NOAPI_NOINIT(H5D_istore_idx_iterate_cb) - /* Store the key for the chunk */ - sl_node->key = *lt_key; + /* Sanity check for memcpy() */ + HDassert(offsetof(H5D_chunk_rec_t, nbytes) == offsetof(H5D_istore_key_t, nbytes)); + HDassert(offsetof(H5D_chunk_rec_t, offset) == offsetof(H5D_istore_key_t, offset)); + HDassert(offsetof(H5D_chunk_rec_t, filter_mask) == offsetof(H5D_istore_key_t, filter_mask)); - /* Insert the chunk description in the skip list */ - if(H5SL_insert(udata->outside, sl_node, &sl_node->index) < 0) { - H5FL_FREE(H5D_istore_sl_ck_t, sl_node); - HGOTO_ERROR(H5E_IO, H5E_CANTINSERT, H5_ITER_ERROR, "can't insert chunk into skip list") - } /* end if */ + /* Compose generic chunk record for callback */ + HDmemcpy(&chunk_rec, lt_key, sizeof(*lt_key)); + chunk_rec.chunk_addr = addr; - /* Break out of loop, we know the chunk is outside the current dimensions */ - break; - } /* end if */ + /* Make "generic chunk" callback */ + if((ret_value = (udata->cb)(&chunk_rec, udata->udata)) < 0) + HERROR(H5E_DATASET, H5E_CALLBACK, "failure in generic chunk iterator callback"); -done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_prune_check() */ +} /* H5D_istore_idx_iterate_cb() */ /*------------------------------------------------------------------------- - * Function: H5D_istore_prune_remove + * Function: H5D_istore_idx_iterate * - * Purpose: Destroy a skip list node for "pruning" chunks, also removes - * the chunk from the B-tree. + * Purpose: Iterate over the chunks in the B-tree index, making a callback + * for each one. * * Return: Non-negative on success/Negative on failure * - * Programmer: Quincey Koziol, koziol@hdfgroup.org - * May 3, 2007 + * Programmer: Quincey Koziol + * Tuesday, May 20, 2008 * *------------------------------------------------------------------------- */ -static herr_t -H5D_istore_prune_remove(void *item, void UNUSED *key, void *op_data) +static int +H5D_istore_idx_iterate(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_cb_func_t chunk_cb, void *chunk_udata) { - H5D_istore_sl_ck_t *sl_node = (H5D_istore_sl_ck_t *)item; /* Temporary pointer to chunk to remove */ - H5D_istore_sl_rm_t *rm_info = (H5D_istore_sl_rm_t *)op_data; /* Information needed for removing chunk from B-tree */ - H5D_istore_ud0_t bt_udata; /* User data for B-tree removal routine */ - herr_t ret_value = H5_ITER_CONT; /* Return value */ + H5D_istore_it_ud_t udata; /* User data for B-tree iterator callback */ + int ret_value; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_prune_remove) + FUNC_ENTER_NOAPI_NOINIT(H5D_istore_idx_iterate) - /* Sanity checks */ - HDassert(sl_node); - HDassert(rm_info); - - /* Initialize the user data for the B-tree callback */ - bt_udata.mesg = rm_info->mesg; - bt_udata.offset = sl_node->key.offset; - - /* Remove */ - if(H5B_remove(rm_info->f, rm_info->dxpl_id, H5B_ISTORE, rm_info->mesg->u.chunk.addr, &bt_udata) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, H5_ITER_ERROR, "unable to remove entry") - - /* Free the chunk checking node */ - H5FL_FREE(H5D_istore_sl_ck_t, sl_node); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5D_istore_prune_remove() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_prune_by_extent - * - * Purpose: This function searches for chunks that are no longer necessary both in the - * raw data cache and in the B-tree. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu - * Algorithm: Robb Matzke - * March 27, 2002 - * - * The algorithm is: - * - * For chunks that are no longer necessary: - * - * 1. Search in the raw data cache for each chunk - * 2. If found then preempt it from the cache - * 3. Search in the B-tree for each chunk - * 4. If found then remove it from the B-tree and deallocate file storage for the chunk - * - * This example shows a 2d dataset of 90x90 with a chunk size of 20x20. - * - * - * 0 20 40 60 80 90 100 - * 0 +---------+---------+---------+---------+-----+...+ - * |:::::X:::::::::::::: : : | : - * |:::::::X:::::::::::: : : | : Key - * |::::::::::X::::::::: : : | : -------- - * |::::::::::::X::::::: : : | : +-+ Dataset - * 20+::::::::::::::::::::.........:.........:.....+...: | | Extent - * | :::::X::::: : : | : +-+ - * | ::::::::::: : : | : - * | ::::::::::: : : | : ... Chunk - * | :::::::X::: : : | : : : Boundary - * 40+.........:::::::::::.........:.........:.....+...: :.: - * | : : : : | : - * | : : : : | : ... Allocated - * | : : : : | : ::: & Filled - * | : : : : | : ::: Chunk - * 60+.........:.........:.........:.........:.....+...: - * | : :::::::X::: : | : X Element - * | : ::::::::::: : | : Written - * | : ::::::::::: : | : - * | : ::::::::::: : | : - * 80+.........:.........:::::::::::.........:.....+...: O Fill Val - * | : : ::::::::::: | : Explicitly - * | : : ::::::X:::: | : Written - * 90+---------+---------+---------+---------+-----+ : - * : : : ::::::::::: : - * 100:.........:.........:.........:::::::::::.........: - * - * - * We have 25 total chunks for this dataset, 5 of which have space - * allocated in the file because they were written to one or more - * elements. These five chunks (and only these five) also have entries in - * the storage B-tree for this dataset. - * - * Now lets say we want to shrink the dataset down to 70x70: - * - * - * 0 20 40 60 70 80 90 100 - * 0 +---------+---------+---------+----+----+-----+...+ - * |:::::X:::::::::::::: : | : | : - * |:::::::X:::::::::::: : | : | : Key - * |::::::::::X::::::::: : | : | : -------- - * |::::::::::::X::::::: : | : | : +-+ Dataset - * 20+::::::::::::::::::::.........:....+....:.....|...: | | Extent - * | :::::X::::: : | : | : +-+ - * | ::::::::::: : | : | : - * | ::::::::::: : | : | : ... Chunk - * | :::::::X::: : | : | : : : Boundary - * 40+.........:::::::::::.........:....+....:.....|...: :.: - * | : : : | : | : - * | : : : | : | : ... Allocated - * | : : : | : | : ::: & Filled - * | : : : | : | : ::: Chunk - * 60+.........:.........:.........:....+....:.....|...: - * | : :::::::X::: | : | : X Element - * | : ::::::::::: | : | : Written - * +---------+---------+---------+----+ : | : - * | : ::::::::::: : | : - * 80+.........:.........:::::::::X:.........:.....|...: O Fill Val - * | : : ::::::::::: | : Explicitly - * | : : ::::::X:::: | : Written - * 90+---------+---------+---------+---------+-----+ : - * : : : ::::::::::: : - * 100:.........:.........:.........:::::::::::.........: - * - * - * That means that the nine chunks along the bottom and right side should - * no longer exist. Of those nine chunks, (0,80), (20,80), (40,80), - * (60,80), (80,80), (80,60), (80,40), (80,20), and (80,0), one is actually allocated - * that needs to be released. - * To release the chunks, we traverse the B-tree to obtain a list of unused - * allocated chunks, and then call H5B_remove() for each chunk. - * - *------------------------------------------------------------------------- - */ -herr_t -H5D_istore_prune_by_extent(const H5D_io_info_t *io_info, const hsize_t *old_dims) -{ - H5D_t *dset = io_info->dset; /* Local pointer to the dataset info */ - const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache */ - H5D_rdcc_ent_t *ent = NULL, *next = NULL; /*cache entry */ - H5D_istore_it_ud3_t udata; /*B-tree pass-through */ - H5D_istore_sl_rm_t rm_info; /* User data for skip list destroy callback */ - hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /*current dataspace dimensions */ - hsize_t chunks[H5O_LAYOUT_NDIMS]; /*current number of chunks in each dimension */ - hsize_t down_chunks[H5O_LAYOUT_NDIMS]; /* "down" size of number of elements in each dimension */ - unsigned rank; /* Current # of dimensions */ - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5D_istore_prune_by_extent, FAIL) + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(chunk_cb); + HDassert(chunk_udata); - /* Check args */ - HDassert(io_info); - HDassert(dset && H5D_CHUNKED == dset->shared->layout.type); - HDassert(dset->shared->layout.u.chunk.ndims > 0 && dset->shared->layout.u.chunk.ndims <= H5O_LAYOUT_NDIMS); - HDassert(H5F_addr_defined(dset->shared->layout.u.chunk.addr)); - - /* Go get the rank & dimensions */ - rank = dset->shared->layout.u.chunk.ndims - 1; - if(H5S_get_simple_extent_dims(dset->shared->space, curr_dims, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions") - - /*------------------------------------------------------------------------- - * Figure out what chunks are no longer in use for the specified extent - * and release them from the linked list raw data cache - *------------------------------------------------------------------------- - */ - for(ent = rdcc->head; ent; ent = next) { - /* Get pointer to next extry in cache, in case this one is evicted */ - next = ent->next; - - /* Check for chunk offset outside of new dimensions */ - for(u = 0; u < rank; u++) - if((hsize_t)ent->offset[u] > curr_dims[u]) { -#ifdef H5D_ISTORE_DEBUG - HDfputs("cache:remove:[", stderr); - for(u = 0; u < rank; u++) - HDfprintf(stderr, "%s%Hd", (u ? ", " : ""), ent->offset[u]); - HDfputs("]\n", stderr); -#endif - - /* Preempt the entry from the cache, but do not flush it to disk */ - if(H5D_istore_preempt(io_info, ent, FALSE) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to preempt chunk") - - /* Break out of loop, chunk is evicted */ - break; - } /* end if */ - } /* end for */ - - /* Round up to the next integer # of chunks, to accomodate partial chunks */ - for(u = 0; u < rank; u++) - chunks[u] = ((old_dims[u] + dset->shared->layout.u.chunk.dim[u]) - 1) / dset->shared->layout.u.chunk.dim[u]; - - /* Get the "down" sizes for each dimension */ - if(H5V_array_down(rank, chunks, down_chunks) < 0) - HGOTO_ERROR(H5E_IO, H5E_BADVALUE, FAIL, "can't compute 'down' sizes") - - /* Initialize the user data for the iteration */ + /* Initialize userdata */ HDmemset(&udata, 0, sizeof udata); - udata.common.mesg = &dset->shared->layout; - udata.dims = curr_dims; - udata.down_chunks = down_chunks; - - /* Initialize the skip list that will hold the chunks outside the dimensions */ - if(NULL == (udata.outside = H5SL_create(H5SL_TYPE_HSIZE, 0.5, (size_t)H5D_ISTORE_DEFAULT_SKIPLIST_HEIGHT))) - HGOTO_ERROR(H5E_IO, H5E_CANTCREATE, FAIL, "can't create skip list for chunks outside new dimensions") - - /* Iterate over chunks in dataset, creating a list of chunks which are - * now completely outside the dataset's dimensions. - * - * Note: It would be more efficient to create a new B-tree routine that - * performed a "remove if" operation on the B-tree and remove all - * the chunks that were outside the dataset's dimensions through - * that routine. However, that's a fair amount of work and it's - * unlikely that shrinking a dataset is a performance critical - * operation. - QAK - */ - if(H5B_iterate(dset->oloc.file, io_info->dxpl_id, H5B_ISTORE, H5D_istore_prune_check, dset->shared->layout.u.chunk.addr, &udata) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to iterate over B-tree") - - /* Set up user data for skip list callback */ - rm_info.f = dset->oloc.file; - rm_info.dxpl_id = io_info->dxpl_id; - rm_info.mesg = &dset->shared->layout; + udata.common.mesg = idx_info->layout; + udata.cb = chunk_cb; + udata.udata = chunk_udata; - /* Destroy the skip list, deleting the chunks in the callback */ - H5SL_destroy(udata.outside, H5D_istore_prune_remove, &rm_info); + /* Iterate over existing chunks */ + if((ret_value = H5B_iterate(idx_info->f, idx_info->dxpl_id, H5B_ISTORE, idx_info->layout->u.chunk.addr, H5D_istore_idx_iterate_cb, &udata)) < 0) + HERROR(H5E_DATASET, H5E_BADITER, "unable to iterate over chunk B-tree"); - /* Reset any cached chunk info for this dataset */ - H5D_istore_cinfo_cache_reset(&dset->shared->cache.chunk.last); - -done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_prune_by_extent() */ +} /* end H5D_istore_idx_iterate() */ /*------------------------------------------------------------------------- - * Function: H5D_istore_remove + * Function: H5D_istore_idx_remove * - * Purpose: Removes chunks that are no longer necessary in the B-tree. + * Purpose: Remove chunk from v1 B-tree index. * * Return: Non-negative on success/Negative on failure * - * Programmer: Robb Matzke - * Pedro Vicente, pvn@ncsa.uiuc.edu - * March 28, 2002 + * Programmer: Quincey Koziol + * Thursday, May 22, 2008 * *------------------------------------------------------------------------- */ -/* ARGSUSED */ -static H5B_ins_t -H5D_istore_remove(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key /*in,out */ , - hbool_t *lt_key_changed /*out */ , - void UNUSED * _udata /*in,out */ , - void UNUSED * _rt_key /*in,out */ , - hbool_t *rt_key_changed /*out */ ) +static herr_t +H5D_istore_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata) { - H5D_istore_key_t *lt_key = (H5D_istore_key_t *)_lt_key; - H5B_ins_t ret_value = H5B_INS_REMOVE; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_remove) - - /* Remove raw data chunk from file */ - H5_CHECK_OVERFLOW(lt_key->nbytes, uint32_t, hsize_t); - if(H5MF_xfree(f, H5FD_MEM_DRAW, dxpl_id, addr, (hsize_t)lt_key->nbytes) < 0) - HGOTO_ERROR(H5E_STORAGE, H5E_CANTFREE, H5B_INS_ERROR, "unable to free chunk") + FUNC_ENTER_NOAPI_NOINIT(H5D_istore_idx_remove) - /* Mark keys as unchanged */ - *lt_key_changed = FALSE; - *rt_key_changed = FALSE; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_remove() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_initialize_by_extent - * - * Purpose: This function searches for chunks that have to be initialized with the fill - * value both in the raw data cache and in the B-tree. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu - * April 4, 2002 - * - * Comments: - * - * (See the example of H5D_istore_prune_by_extent) - * Next, there are seven chunks where the database extent boundary is - * within the chunk. We find those seven just like we did with the previous nine. - * Fot the ones that are allocated we initialize the part that lies outside the boundary - * with the fill value. - * - *------------------------------------------------------------------------- - */ -herr_t -H5D_istore_initialize_by_extent(H5D_io_info_t *io_info) -{ - H5S_t *space_chunk = NULL; /* Dataspace for a chunk */ - H5D_t *dset = io_info->dset; /* Local pointer to the dataset info */ - const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset's layout */ - H5D_storage_t store; /* Dataset storage information */ - H5D_fill_buf_info_t fb_info; /* Dataset's fill buffer info */ - hbool_t fb_info_init = FALSE; /* Whether the fill value buffer has been initialized */ - hsize_t dset_dims[H5O_LAYOUT_NDIMS]; /* Current dataspace dimensions */ - hsize_t chunk_dims[H5O_LAYOUT_NDIMS]; /* Current chunk dimensions */ - hsize_t chunk_offset[H5O_LAYOUT_NDIMS]; /* Logical location of the chunks */ - hsize_t hyper_start[H5O_LAYOUT_NDIMS]; /* Starting location of hyperslab */ - hsize_t nchunks[H5O_LAYOUT_NDIMS]; /* Current number of chunks in each dimension */ - hsize_t down_chunks[H5O_LAYOUT_NDIMS]; /* "down" size of number of elements in each dimension */ - uint32_t elmts_per_chunk; /* Elements in a chunk */ - int srank; /* # of chunk dimensions (signed) */ - unsigned rank; /* # of chunk dimensions */ - hbool_t carry; /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */ - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5D_istore_initialize_by_extent, FAIL) + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(udata); - /* Check args */ - HDassert(io_info); - HDassert(io_info->dset && H5D_CHUNKED == layout->type); - HDassert(layout->u.chunk.ndims > 0 && layout->u.chunk.ndims <= H5O_LAYOUT_NDIMS); - HDassert(H5F_addr_defined(layout->u.chunk.addr)); - - /* Go get the rank & dimensions */ - if((srank = H5S_get_simple_extent_dims(dset->shared->space, dset_dims, NULL)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions") - H5_ASSIGN_OVERFLOW(rank, srank, int, unsigned); - - /* Set size of lowest chunk dimension (the dataset element size) */ - dset_dims[rank] = layout->u.chunk.dim[rank]; - - /* Compute the number of chunks in dataset & the # of elements in a chunk */ - /* (round up to the next integer # of chunks, to accomodate partial chunks) */ - for(u = 0, elmts_per_chunk = 1; u < rank; u++) { - nchunks[u] = ((dset_dims[u] - 1) / layout->u.chunk.dim[u]) + 1; - elmts_per_chunk *= layout->u.chunk.dim[u]; - } /* end for */ - - /* Get the "down" sizes for each dimension */ - if(H5V_array_down(rank, nchunks, down_chunks) < 0) - HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "can't compute 'down' sizes") - - /* Create a data space for a chunk & set the extent */ - for(u = 0; u < rank; u++) - chunk_dims[u] = layout->u.chunk.dim[u]; - if(NULL == (space_chunk = H5S_create_simple(rank, chunk_dims, NULL))) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, "can't create simple dataspace") - - /* Point to local dataset storage info */ - HDassert(io_info->store == NULL); /* Make certain we aren't blowing anything away */ - io_info->store = &store; - - /* Reset hyperslab start array */ - HDmemset(hyper_start, 0, sizeof(hyper_start)); - - /* Initialize current chunk offset to the origin (0, 0, 0, ...) */ - HDmemset(chunk_offset, 0, sizeof(chunk_offset)); - - /* Loop over all chunks */ - carry = FALSE; - while(!carry) { - hbool_t found; /* Initialize this entry */ - int i; /* Local index variable */ - - /* - * Figure out what chunks have to be initialized. These are the chunks where the dataspace - * extent boundary is within the chunk - */ - found = FALSE; - for(u = 0; u < rank; u++) - if((chunk_offset[u] + layout->u.chunk.dim[u]) > dset_dims[u]) { - found = TRUE; - break; - } /* end if */ - - if(found) { - H5S_sel_iter_t chunk_iter; /* Memory selection iteration info */ - hssize_t nelmts; /* Number of data elements */ - hsize_t count[H5O_LAYOUT_NDIMS]; /* Element count of hyperslab */ - uint8_t *chunk; /* The file chunk */ - unsigned idx_hint; /* Which chunk we're dealing with */ - hsize_t bytes_accessed; /* Bytes accessed in chunk */ - - /* Initialize the fill value buffer, if necessary */ - if(!fb_info_init) { - H5_CHECK_OVERFLOW(elmts_per_chunk, uint32_t, size_t); - if(H5D_fill_init(&fb_info, NULL, FALSE, NULL, NULL, NULL, NULL, - &dset->shared->dcpl_cache.fill, - dset->shared->type, dset->shared->type_id, (size_t)elmts_per_chunk, - io_info->dxpl_cache->max_temp_buf, io_info->dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill buffer info") - fb_info_init = TRUE; - } /* end if */ - - /* Compute the # of elements to leave with existing value, in each dimension */ - for(u = 0; u < rank; u++) - count[u] = MIN(layout->u.chunk.dim[u], (dset_dims[u] - chunk_offset[u])); - -#ifdef H5D_ISTORE_DEBUG - HDfputs("cache:initialize:offset:[", stdout); - for(u = 0; u < rank; u++) - HDfprintf(stdout, "%s%Hd", u ? ", " : "", chunk_offset[u]); - HDfputs("]", stdout); - HDfputs(":count:[", stdout); - for(u = 0; u < rank; u++) - HDfprintf(stdout, "%s%Hd", u ? ", " : "", count[u]); - HDfputs("]\n", stdout); -#endif - - /* Select all elements in chunk, to begin with */ - if(H5S_select_all(space_chunk, TRUE) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to select space") - - /* "Subtract out" the elements to keep */ - if(H5S_select_hyperslab(space_chunk, H5S_SELECT_NOTB, hyper_start, NULL, count, NULL) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to select hyperslab") - - /* Calculate the index of this chunk */ - if(H5V_chunk_index(rank, chunk_offset, layout->u.chunk.dim, down_chunks, &store.chunk.index) < 0) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index") - - /* Lock the chunk into the cache, to get a pointer to the chunk buffer */ - store.chunk.offset = chunk_offset; - if(NULL == (chunk = (uint8_t *)H5D_istore_lock(io_info, NULL, FALSE, &idx_hint))) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to read raw data chunk") - - - /* Fill the selection in the memory buffer */ - /* Use the size of the elements in the chunk directly instead of */ - /* relying on the fill.size, which might be set to 0 if there is */ - /* no fill-value defined for the dataset -QAK */ - - /* Get the number of elements in the selection */ - nelmts = H5S_GET_SELECT_NPOINTS(space_chunk); - HDassert(nelmts >= 0); - H5_CHECK_OVERFLOW(nelmts, hssize_t, size_t); - - /* Check for VL datatype & non-default fill value */ - if(fb_info.has_vlen_fill_type) - /* Re-fill the buffer to use for this I/O operation */ - if(H5D_fill_refill_vl(&fb_info, (size_t)nelmts, io_info->dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't refill fill value buffer") - - /* Create a selection iterator for scattering the elements to memory buffer */ - H5_CHECK_OVERFLOW(layout->u.chunk.dim[rank], uint32_t, size_t); - if(H5S_select_iter_init(&chunk_iter, space_chunk, (size_t)layout->u.chunk.dim[rank]) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize chunk selection information") - - /* Scatter the data into memory */ - if(H5D_scatter_mem(fb_info.fill_buf, space_chunk, &chunk_iter, (size_t)nelmts, io_info->dxpl_cache, chunk/*out*/) < 0) { - H5S_SELECT_ITER_RELEASE(&chunk_iter); - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "scatter failed") - } /* end if */ - - /* Release the selection iterator */ - if(H5S_SELECT_ITER_RELEASE(&chunk_iter) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") - - - /* The number of bytes accessed in the chunk */ - /* (i.e. the bytes replaced with fill values) */ - bytes_accessed = nelmts * layout->u.chunk.dim[rank]; - - /* Release lock on chunk */ - H5_CHECK_OVERFLOW(bytes_accessed, hsize_t, uint32_t); - if(H5D_istore_unlock(io_info, TRUE, idx_hint, chunk, (uint32_t)bytes_accessed) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to unlock raw data chunk") - } /* end if */ - - /* Increment indices */ - carry = TRUE; - for(i = (int)(rank - 1); i >= 0; --i) { - chunk_offset[i] += layout->u.chunk.dim[i]; - if(chunk_offset[i] >= dset_dims[i]) - chunk_offset[i] = 0; - else { - carry = FALSE; - break; - } /* end else */ - } /* end for */ - } /* end while */ + /* Remove the chunk from the v1 B-tree index and release the space for the + * chunk (in the B-tree callback). + */ + if(H5B_remove(idx_info->f, idx_info->dxpl_id, H5B_ISTORE, idx_info->layout->u.chunk.addr, udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDELETE, FAIL, "unable to remove chunk entry") done: - /* Release resources */ - if(space_chunk && H5S_close(space_chunk) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release dataspace") - if(fb_info_init && H5D_fill_term(&fb_info) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release fill buffer info") - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_initialize_by_extent() */ +} /* H5D_istore_idx_remove() */ /*------------------------------------------------------------------------- - * Function: H5D_istore_delete + * Function: H5D_istore_idx_delete * - * Purpose: Delete raw data storage for entire dataset (i.e. all chunks) + * Purpose: Delete v1 B-tree index and raw data storage for entire dataset + * (i.e. all chunks) * * Return: Success: Non-negative * Failure: negative @@ -3286,365 +1143,123 @@ done: * *------------------------------------------------------------------------- */ -herr_t -H5D_istore_delete(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout) +static herr_t +H5D_istore_idx_delete(const H5D_chk_idx_info_t *idx_info) { - herr_t ret_value=SUCCEED; /* Return value */ + H5O_layout_t tmp_layout; /* Local copy of layout info */ + H5D_istore_ud0_t udata; /* User data for B-tree iterator call */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(H5D_istore_delete, FAIL) + FUNC_ENTER_NOAPI_NOINIT(H5D_istore_idx_delete) - /* Check if the B-tree has been created in the file */ - if(H5F_addr_defined(layout->u.chunk.addr)) { - H5O_layout_t tmp_layout=*layout;/* Local copy of layout info */ - H5D_istore_ud0_t udata; /* User data for B-tree iterator call */ + /* Sanity checks */ + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(H5F_addr_defined(idx_info->layout->u.chunk.addr)); - /* Set up user data for B-tree deletion */ - HDmemset(&udata, 0, sizeof udata); - udata.mesg = &tmp_layout; + /* Set up user data for B-tree deletion */ + HDmemset(&udata, 0, sizeof udata); + tmp_layout = *idx_info->layout; + udata.mesg = &tmp_layout; - /* Allocate the shared structure */ - if(H5D_istore_shared_create(f, &tmp_layout)<0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") + /* Set up the shared structure */ + if(H5D_istore_shared_create(idx_info->f, &tmp_layout) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") - /* Delete entire B-tree */ - if(H5B_delete(f, dxpl_id, H5B_ISTORE, tmp_layout.u.chunk.addr, &udata)<0) - HGOTO_ERROR(H5E_IO, H5E_CANTDELETE, 0, "unable to delete chunk B-tree") + /* Delete entire B-tree */ + if(H5B_delete(idx_info->f, idx_info->dxpl_id, H5B_ISTORE, tmp_layout.u.chunk.addr, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDELETE, FAIL, "unable to delete chunk B-tree") - /* Free the raw B-tree node buffer */ - if(tmp_layout.u.chunk.btree_shared==NULL) - HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "ref-counted page nil") - if(H5RC_DEC(tmp_layout.u.chunk.btree_shared)<0) - HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") - } /* end if */ + /* Free the raw B-tree node buffer */ + if(NULL == tmp_layout.u.chunk.btree_shared) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "ref-counted page nil") + if(H5RC_DEC(tmp_layout.u.chunk.btree_shared) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_delete() */ +} /* end H5D_istore_idx_delete() */ /*------------------------------------------------------------------------- - * Function: H5D_istore_update_cache + * Function: H5D_istore_idx_copy_setup * - * Purpose: Update any cached chunks index values after the dataspace - * size has changed + * Purpose: Set up any necessary information for copying chunks * - * Return: Success: Non-negative - * Failure: negative + * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol - * Saturday, May 29, 2004 + * Thursday, May 29, 2008 * *------------------------------------------------------------------------- */ -herr_t -H5D_istore_update_cache(H5D_t *dset, hid_t dxpl_id) +static herr_t +H5D_istore_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, + const H5D_chk_idx_info_t *idx_info_dst) { - H5D_io_info_t io_info; /* Temporary I/O info object */ - H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache */ - H5D_rdcc_ent_t *ent, *next; /*cache entry */ - H5D_rdcc_ent_t *old_ent; /* Old cache entry */ - H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ - H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ - unsigned rank; /*current # of dimensions */ - hsize_t curr_dims[H5O_LAYOUT_NDIMS]; /*current dataspace dimensions */ - hsize_t chunks[H5O_LAYOUT_NDIMS]; /*current number of chunks in each dimension */ - hsize_t down_chunks[H5O_LAYOUT_NDIMS]; /* "down" size of number of elements in each dimension */ - unsigned u; /*counters */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5D_istore_update_cache, FAIL) - - /* Check args */ - HDassert(dset && H5D_CHUNKED == dset->shared->layout.type); - HDassert(dset->shared->layout.u.chunk.ndims > 0 && dset->shared->layout.u.chunk.ndims <= H5O_LAYOUT_NDIMS); - - /* Get the rank */ - rank = dset->shared->layout.u.chunk.ndims-1; - HDassert(rank > 0); + herr_t ret_value = SUCCEED; /* Return value */ - /* 1-D dataset's chunks can't have their index change */ - if(rank == 1) - HGOTO_DONE(SUCCEED) + FUNC_ENTER_NOAPI_NOINIT(H5D_istore_idx_copy_setup) - /* Go get the dimensions */ - if(H5S_get_simple_extent_dims(dset->shared->space, curr_dims, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions") + HDassert(idx_info_src); + HDassert(idx_info_src->f); + HDassert(idx_info_src->layout); + HDassert(idx_info_dst); + HDassert(idx_info_dst->f); + HDassert(idx_info_dst->layout); + HDassert(!H5F_addr_defined(idx_info_dst->layout->u.chunk.addr)); - /* Round up to the next integer # of chunks, to accomodate partial chunks */ - for(u = 0; u < rank; u++) - chunks[u] = ((curr_dims[u] + dset->shared->layout.u.chunk.dim[u]) - 1) / dset->shared->layout.u.chunk.dim[u]; - - /* Get the "down" sizes for each dimension */ - if(H5V_array_down(rank, chunks, down_chunks) < 0) - HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "can't compute 'down' sizes") - - /* Fill the DXPL cache values for later use */ - if(H5D_get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") - - /* Construct dataset I/O info */ - H5D_BUILD_IO_INFO_WRT(&io_info, dset, dxpl_cache, dxpl_id, NULL, NULL); - - /* Recompute the index for each cached chunk that is in a dataset */ - for(ent = rdcc->head; ent; ent = next) { - hsize_t idx; /* Chunk index */ - unsigned old_idx; /* Previous index number */ - - /* Get the pointer to the next cache entry */ - next = ent->next; - - /* Calculate the index of this chunk */ - if(H5V_chunk_index(rank, ent->offset, dset->shared->layout.u.chunk.dim, down_chunks, &idx) < 0) - HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index") - - /* Compute the index for the chunk entry */ - old_idx = ent->idx; /* Save for later */ - ent->idx = H5D_CHUNK_HASH(dset->shared, idx); - - if(old_idx != ent->idx) { - /* Check if there is already a chunk at this chunk's new location */ - old_ent = rdcc->slot[ent->idx]; - if(old_ent != NULL) { - HDassert(old_ent->locked == 0); - - /* Check if we are removing the entry we would walk to next */ - if(old_ent == next) - next = old_ent->next; - - /* Remove the old entry from the cache */ - if(H5D_istore_preempt(&io_info, old_ent, TRUE) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTFLUSH, FAIL, "unable to flush one or more raw data chunks") - } /* end if */ + /* Create shared B-tree info for each file */ + if(H5D_istore_shared_create(idx_info_src->f, idx_info_src->layout) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for source shared B-tree info") + if(H5D_istore_shared_create(idx_info_dst->f, idx_info_dst->layout) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for destination shared B-tree info") - /* Insert this chunk into correct location in hash table */ - rdcc->slot[ent->idx] = ent; - - /* Null out previous location */ - rdcc->slot[old_idx] = NULL; - } /* end if */ - } /* end for */ + /* Create the root of the B-tree that describes chunked storage in the dest. file */ + if(H5D_istore_idx_create(idx_info_dst) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to initialize chunked storage") done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5F_istore_update_cache() */ +} /* end H5D_istore_idx_copy_setup() */ /*------------------------------------------------------------------------- - * Function: H5D_istore_copy + * Function: H5D_istore_idx_copy_shutdown * - * Purpose: copy an indexed storage B-tree from SRC file to DST file. + * Purpose: Shutdown any information from copying chunks * - * Return: Non-negative on success (with the ISTORE argument initialized - * and ready to write to an object header). Negative on failure. + * Return: Non-negative on success/Negative on failure * - * Programmer: Peter Cao - * August 20, 2005 + * Programmer: Quincey Koziol + * Thursday, May 29, 2008 * *------------------------------------------------------------------------- */ -herr_t -H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, - H5O_layout_t *layout_dst, H5T_t *dt_src, H5O_copy_t *cpy_info, H5O_pline_t *pline, hid_t dxpl_id) +static herr_t +H5D_istore_idx_copy_shutdown(H5O_layout_t *layout_src, H5O_layout_t *layout_dst) { - H5D_istore_it_ud4_t udata; - H5T_path_t *tpath_src_mem = NULL, *tpath_mem_dst = NULL; /* Datatype conversion paths */ - hid_t tid_src = -1; /* Datatype ID for source datatype */ - hid_t tid_dst = -1; /* Datatype ID for destination datatype */ - hid_t tid_mem = -1; /* Datatype ID for memory datatype */ - size_t buf_size; /* Size of copy buffer */ - size_t reclaim_buf_size; /* Size of reclaim buffer */ - void *buf = NULL; /* Buffer for copying data */ - void *bkg = NULL; /* Buffer for background during type conversion */ - void *reclaim_buf = NULL; /* Buffer for reclaiming data */ - H5S_t *buf_space = NULL; /* Dataspace describing buffer */ - hid_t sid_buf = -1; /* ID for buffer dataspace */ - uint32_t nelmts = 0; /* Number of elements in buffer */ - hbool_t do_convert = FALSE; /* Indicate that type conversions should be performed */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5D_istore_copy, FAIL) - - /* Check args */ - HDassert(f_src); - HDassert(f_dst); - HDassert(layout_src && H5D_CHUNKED == layout_src->type); - HDassert(layout_dst && H5D_CHUNKED == layout_dst->type); - HDassert(dt_src); - - /* Create datatype ID for src datatype */ - if((tid_src = H5I_register(H5I_DATATYPE, dt_src)) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register source file datatype") - - /* Create shared B-tree info for each file */ - if(H5D_istore_shared_create(f_src, layout_src) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") - if(H5D_istore_shared_create(f_dst, layout_dst) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") - - /* Check if we need to create the B-tree in the dest. file */ - if(layout_dst->u.chunk.addr == HADDR_UNDEF) { - /* Create the root of the B-tree that describes chunked storage */ - if(H5D_istore_create(f_dst, dxpl_id, layout_dst) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to initialize chunked storage") - } /* end if */ - - /* If there's a VLEN source datatype, set up type conversion information */ - if(H5T_detect_class(dt_src, H5T_VLEN) > 0) { - H5T_t *dt_dst; /* Destination datatype */ - H5T_t *dt_mem; /* Memory datatype */ - size_t mem_dt_size; /* Memory datatype size */ - size_t tmp_dt_size; /* Temp. datatype size */ - size_t max_dt_size; /* Max atatype size */ - hsize_t buf_dim; /* Dimension for buffer */ - unsigned u; - - /* create a memory copy of the variable-length datatype */ - if(NULL == (dt_mem = H5T_copy(dt_src, H5T_COPY_TRANSIENT))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy") - if((tid_mem = H5I_register(H5I_DATATYPE, dt_mem)) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register memory datatype") - - /* create variable-length datatype at the destinaton file */ - if(NULL == (dt_dst = H5T_copy(dt_src, H5T_COPY_TRANSIENT))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy") - if(H5T_set_loc(dt_dst, f_dst, H5T_LOC_DISK) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "cannot mark datatype on disk") - if((tid_dst = H5I_register(H5I_DATATYPE, dt_dst)) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register destination file datatype") - - /* Set up the conversion functions */ - if(NULL == (tpath_src_mem = H5T_path_find(dt_src, dt_mem, NULL, NULL, dxpl_id, FALSE))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and mem datatypes") - if(NULL == (tpath_mem_dst = H5T_path_find(dt_mem, dt_dst, NULL, NULL, dxpl_id, FALSE))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between mem and dst datatypes") - - /* Determine largest datatype size */ - if(0 == (max_dt_size = H5T_get_size(dt_src))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") - if(0 == (mem_dt_size = H5T_get_size(dt_mem))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") - max_dt_size = MAX(max_dt_size, mem_dt_size); - if(0 == (tmp_dt_size = H5T_get_size(dt_dst))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") - max_dt_size = MAX(max_dt_size, tmp_dt_size); - - /* Compute the number of elements per chunk */ - nelmts = 1; - for(u = 0; u < (layout_src->u.chunk.ndims - 1); u++) - nelmts *= layout_src->u.chunk.dim[u]; - - /* Create the space and set the initial extent */ - buf_dim = nelmts; - if(NULL == (buf_space = H5S_create_simple((unsigned)1, &buf_dim, NULL))) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, "can't create simple dataspace") - - /* Atomize */ - if((sid_buf = H5I_register(H5I_DATASPACE, buf_space)) < 0) { - H5S_close(buf_space); - HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register dataspace ID") - } /* end if */ - - /* Set initial buffer sizes */ - buf_size = nelmts * max_dt_size; - reclaim_buf_size = nelmts * mem_dt_size; - - /* Allocate memory for reclaim buf */ - if(NULL == (reclaim_buf = H5MM_malloc(reclaim_buf_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") - - /* Indicate that type conversion should be performed */ - do_convert = TRUE; - } /* end if */ - else { - if(H5T_get_class(dt_src, FALSE) == H5T_REFERENCE) { - /* Indicate that type conversion should be performed */ - do_convert = TRUE; - } /* end if */ - - H5_ASSIGN_OVERFLOW(buf_size, layout_src->u.chunk.size, uint32_t, size_t); - reclaim_buf_size = 0; - } /* end else */ - - /* Set up conversion buffer, if appropriate */ - if(do_convert) { - /* Allocate background memory for converting the chunk */ - if(NULL == (bkg = H5MM_malloc(buf_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") - - /* Check for reference datatype and no expanding references & clear background buffer */ - if(!cpy_info->expand_ref && - ((H5T_get_class(dt_src, FALSE) == H5T_REFERENCE) && (f_src != f_dst))) - /* Reset value to zero */ - HDmemset(bkg, 0, buf_size); - } /* end if */ + herr_t ret_value = SUCCEED; /* Return value */ - /* Allocate memory for copying the chunk */ - if(NULL == (buf = H5MM_malloc(buf_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + FUNC_ENTER_NOAPI_NOINIT(H5D_istore_idx_copy_shutdown) - /* Initialize the callback structure for the source */ - HDmemset(&udata, 0, sizeof udata); - udata.common.mesg = layout_src; - udata.file_src = f_src; - udata.addr_dst = layout_dst->u.chunk.addr; - udata.buf = buf; - udata.bkg = bkg; - udata.buf_size = buf_size; - udata.tid_src = tid_src; - udata.tid_mem = tid_mem; - udata.tid_dst = tid_dst; - udata.dt_src = dt_src; - udata.do_convert = do_convert; - udata.tpath_src_mem = tpath_src_mem; - udata.tpath_mem_dst = tpath_mem_dst; - udata.reclaim_buf = reclaim_buf; - udata.reclaim_buf_size = reclaim_buf_size; - udata.buf_space = buf_space; - udata.nelmts = nelmts; - udata.pline = pline; - udata.file_dst = f_dst; - udata.cpy_info = cpy_info; - - /* copy the chunked data by iteration */ - if(H5B_iterate(f_src, dxpl_id, H5B_ISTORE, H5D_istore_iter_copy, layout_src->u.chunk.addr, &udata) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to iterate over chunk B-tree") - - /* I/O buffers may have been re-allocated */ - buf = udata.buf; - bkg = udata.bkg; - -done: - if(sid_buf > 0) - if(H5I_dec_ref(sid_buf) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary dataspace ID") - if(tid_src > 0) - if(H5I_dec_ref(tid_src) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") - if(tid_dst > 0) - if(H5I_dec_ref(tid_dst) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") - if(tid_mem > 0) - if(H5I_dec_ref(tid_mem) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") - if(buf) - H5MM_xfree(buf); - if(bkg) - H5MM_xfree(bkg); - if(reclaim_buf) - H5MM_xfree(reclaim_buf); + HDassert(layout_src); + HDassert(layout_dst); + /* Decrement refcount on shared B-tree info */ if(H5RC_DEC(layout_src->u.chunk.btree_shared) < 0) - HDONE_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") - + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "unable to decrement ref-counted page") if(H5RC_DEC(layout_dst->u.chunk.btree_shared) < 0) - HDONE_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "unable to decrement ref-counted page") +done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_copy() */ +} /* end H5D_istore_idx_copy_shutdown() */ /*------------------------------------------------------------------------- - * Function: H5D_istore_bh_size + * Function: H5D_istore_idx_size * * Purpose: Retrieve the amount of B-tree storage for chunked dataset * @@ -3657,88 +1272,53 @@ done: *------------------------------------------------------------------------- */ herr_t -H5D_istore_bh_info(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout, hsize_t *btree_size) +H5D_istore_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size) { - H5D_istore_it_ud1_t udata; /* User-data for loading istore nodes */ - H5B_info_ud_t bh_udata; /* User-data for B-tree size iteration */ - herr_t ret_value = SUCCEED; + H5D_istore_ud0_t udata; /* User-data for loading istore nodes */ + H5B_info_t bt_info; /* B-tree info */ + hbool_t shared_init = FALSE; /* Whether shared B-tree info is initialized */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(H5D_istore_bh_info, FAIL) + FUNC_ENTER_NOAPI(H5D_istore_idx_size, FAIL) /* Check args */ - HDassert(f); - HDassert(layout); - HDassert(btree_size); + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(index_size); /* Initialize the shared info for the B-tree traversal */ - if(H5D_istore_shared_create(f, layout) < 0) + if(H5D_istore_shared_create(idx_info->f, idx_info->layout) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") + shared_init = TRUE; /* Initialize istore node user-data */ HDmemset(&udata, 0, sizeof udata); - udata.common.mesg = layout; + udata.mesg = idx_info->layout; - /* Iterate over B-tree, accumulating metadata size */ - bh_udata.udata = &udata; - bh_udata.btree_size = btree_size; - if(H5B_iterate_size(f, dxpl_id, H5B_ISTORE, NULL, layout->u.chunk.addr, &bh_udata) < 0) + /* Get metadata information for B-tree */ + if(H5B_get_info(idx_info->f, idx_info->dxpl_id, H5B_ISTORE, idx_info->layout->u.chunk.addr, &bt_info, NULL, &udata) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "unable to iterate over chunk B-tree") -done: - if(layout->u.chunk.btree_shared == NULL) - HDONE_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "ref-counted page nil") - if(H5RC_DEC(layout->u.chunk.btree_shared) < 0) - HDONE_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_bh_info() */ - - -/*------------------------------------------------------------------------- - * Function: H5D_istore_dump_btree - * - * Purpose: Prints information about the storage B-tree to the specified - * stream. - * - * Return: Success: Non-negative - * - * Failure: negative - * - * Programmer: Robb Matzke - * Wednesday, April 28, 1999 - * - *------------------------------------------------------------------------- - */ -herr_t -H5D_istore_dump_btree(H5F_t *f, hid_t dxpl_id, FILE *stream, unsigned ndims, haddr_t addr) -{ - H5O_layout_t layout; - H5D_istore_it_ud2_t udata; - herr_t ret_value=SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5D_istore_dump_btree, FAIL) - - HDmemset(&udata, 0, sizeof udata); - layout.u.chunk.ndims = ndims; - udata.common.mesg = &layout; - udata.stream = stream; - if(stream) - HDfprintf(stream, " Address: %a\n", addr); - if(H5B_iterate(f, dxpl_id, H5B_ISTORE, H5D_istore_iter_dump, addr, &udata)<0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, 0, "unable to iterate over chunk B-tree") + /* Set the size of the B-tree */ + *index_size = bt_info.size; done: + if(shared_init) { + if(idx_info->layout->u.chunk.btree_shared == NULL) + HDONE_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "ref-counted page nil") + if(H5RC_DEC(idx_info->layout->u.chunk.btree_shared) < 0) + HDONE_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_dump_btree() */ +} /* end H5D_istore_idx_size() */ -#ifdef H5D_ISTORE_DEBUG /*------------------------------------------------------------------------- - * Function: H5D_istore_stats + * Function: H5D_istore_idx_dest * - * Purpose: Print raw data cache statistics to the debug stream. If - * HEADERS is non-zero then print table column headers, - * otherwise assume that the H5AC layer has already printed them. + * Purpose: Release indexing information in memory. * * Return: Non-negative on success/Negative on failure * @@ -3747,53 +1327,26 @@ done: * *------------------------------------------------------------------------- */ -herr_t -H5D_istore_stats (H5D_t *dset, hbool_t headers) +static herr_t +H5D_istore_idx_dest(const H5D_chk_idx_info_t *idx_info) { - H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); - double miss_rate; - char ascii[32]; - herr_t ret_value=SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(H5D_istore_stats, FAIL) - - if (!H5DEBUG(AC)) - HGOTO_DONE(SUCCEED) - - if (headers) { - fprintf(H5DEBUG(AC), "H5D: raw data cache statistics\n"); - fprintf(H5DEBUG(AC), " %-18s %8s %8s %8s %8s+%-8s\n", - "Layer", "Hits", "Misses", "MissRate", "Inits", "Flushes"); - fprintf(H5DEBUG(AC), " %-18s %8s %8s %8s %8s-%-8s\n", - "-----", "----", "------", "--------", "-----", "-------"); - } + herr_t ret_value = SUCCEED; /* Return value */ -#ifdef H5AC_DEBUG - if (H5DEBUG(AC)) headers = TRUE; -#endif + FUNC_ENTER_NOAPI_NOINIT(H5D_istore_idx_dest) - if (headers) { - if (rdcc->nhits>0 || rdcc->nmisses>0) { - miss_rate = 100.0 * rdcc->nmisses / - (rdcc->nhits + rdcc->nmisses); - } else { - miss_rate = 0.0; - } - if (miss_rate > 100) { - sprintf(ascii, "%7d%%", (int) (miss_rate + 0.5)); - } else { - sprintf(ascii, "%7.2f%%", miss_rate); - } + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); - fprintf(H5DEBUG(AC), " %-18s %8u %8u %7s %8d+%-9ld\n", - "raw data chunks", rdcc->nhits, rdcc->nmisses, ascii, - rdcc->ninits, (long)(rdcc->nflushes)-(long)(rdcc->ninits)); - } + /* Free the raw B-tree node buffer */ + if(idx_info->layout->u.chunk.btree_shared == NULL) + HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "ref-counted page nil") + if(H5RC_DEC(idx_info->layout->u.chunk.btree_shared) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_stats() */ -#endif /* H5D_ISTORE_DEBUG */ +} /* end H5D_istore_idx_dest() */ /*------------------------------------------------------------------------- @@ -3812,17 +1365,20 @@ herr_t H5D_istore_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int indent, int fwidth, unsigned ndims) { - H5O_layout_t layout; H5D_istore_ud0_t udata; /* B-tree user data */ + H5O_layout_t layout; /* Layout information for B-tree callback */ + hbool_t shared_init = FALSE; /* Whether B-tree shared info is initialized */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(H5D_istore_debug,FAIL) + FUNC_ENTER_NOAPI(H5D_istore_debug, FAIL) + /* Set up "fake" layout info */ layout.u.chunk.ndims = ndims; /* Allocate the shared structure */ - if(H5D_istore_shared_create(f, &layout)<0) + if(H5D_istore_shared_create(f, &layout) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") + shared_init = TRUE; /* Set up B-tree user data */ HDmemset(&udata, 0, sizeof udata); @@ -3830,13 +1386,15 @@ H5D_istore_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int inden (void)H5B_debug(f, dxpl_id, addr, stream, indent, fwidth, H5B_ISTORE, &udata); - /* Free the raw B-tree node buffer */ - if(layout.u.chunk.btree_shared==NULL) - HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "ref-counted page nil") - if(H5RC_DEC(layout.u.chunk.btree_shared)<0) - HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") - done: + if(shared_init) { + /* Free the raw B-tree node buffer */ + if(layout.u.chunk.btree_shared == NULL) + HDONE_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "ref-counted page nil") + if(H5RC_DEC(layout.u.chunk.btree_shared) < 0) + HDONE_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") + } /* end if */ + FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_debug() */ |