From 29d45ed0d13493d07d7d744950e1fe1b6d5bbff1 Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Wed, 27 May 2009 12:00:29 -0500 Subject: [svn-r16985] Description: Interim checkin of work toward closing race condition window which can cause errors when reading a file that is used for SWMR-write access. This change introduces a chunk proxy in the metadata cache, which participates in the metadata cache's flush dependencies while representing a raw data chunk in a dataset's chunk cache. Also, the extensible array's SWMR behavior is only invoked when the file is opened for SWMR-write access, allowing more flexibility in flushing extensible array data structure pieces when SWMR-write is not enabled. Tested on: FreeBSD/32 6.3 (duty) in debug mode FreeBSD/64 6.3 (liberty) w/C++ & FORTRAN, in debug mode Linux/32 2.6 (jam) w/PGI compilers, w/C++ & FORTRAN, w/threadsafe, in debug mode Solaris/32 2.10 (linew) w/deprecated symbols disabled, w/C++ & FORTRAN, w/szip filter, in production mode Linux/64-ia64 2.6 (cobalt) w/Intel compilers, w/C++ & FORTRAN, in production mode Linux/64-ia64 2.4 (tg-login3) w/parallel, w/FORTRAN, in debug mode Linux/64-amd64 2.6 (abe) w/parallel, w/FORTRAN, in production mode Mac OS X/32 10.5.6 (amazon) in debug mode Mac OS X/32 10.5.6 (amazon) w/C++ & FORTRAN, w/threadsafe, in production mode --- MANIFEST | 1 + src/H5AC.c | 3 +- src/H5ACprivate.h | 1 + src/H5Cpkg.h | 2 +- src/H5Dbtree.c | 3 + src/H5Dchunk.c | 45 ++++- src/H5Dearray.c | 109 ++++++++++++ src/H5Dpkg.h | 37 +++- src/H5Dproxy.c | 520 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/H5EAcache.c | 156 ++++++++-------- src/H5EAhdr.c | 1 + src/H5EAint.c | 3 +- src/H5EApkg.h | 1 + src/H5F.c | 1 + src/H5Fpkg.h | 1 + src/H5Fprivate.h | 3 + src/H5Fquery.c | 29 +++ src/Makefile.am | 2 +- src/Makefile.in | 5 +- test/dsets.c | 281 +++++++++++++++-------------- 20 files changed, 986 insertions(+), 218 deletions(-) create mode 100644 src/H5Dproxy.c diff --git a/MANIFEST b/MANIFEST index 9445116..a3b9125 100644 --- a/MANIFEST +++ b/MANIFEST @@ -487,6 +487,7 @@ ./src/H5Doh.c ./src/H5Dpkg.h ./src/H5Dprivate.h +./src/H5Dproxy.c ./src/H5Dpublic.h ./src/H5Dscatgath.c ./src/H5Dselect.c diff --git a/src/H5AC.c b/src/H5AC.c index 145daa5..a8c0f07 100644 --- a/src/H5AC.c +++ b/src/H5AC.c @@ -496,6 +496,7 @@ static const char * H5AC_entry_type_names[H5AC_NTYPES] = "extensible array super blocks", "extensible array data blocks", "extensible array data block pages", + "chunk proxy", "test entry" /* for testing only -- not used for actual files */ }; @@ -1623,7 +1624,7 @@ done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC_mark_pinned_entry_dirty() */ +} /* H5AC_mark_pinned_or_protected_entry_dirty() */ /*------------------------------------------------------------------------- diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 6bf1afe..8122567 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -67,6 +67,7 @@ typedef enum { H5AC_EARRAY_SBLOCK_ID, /*extensible array super block */ H5AC_EARRAY_DBLOCK_ID, /*extensible array data block */ H5AC_EARRAY_DBLK_PAGE_ID, /*extensible array data block page */ + H5AC_CHUNK_PROXY_ID, /*chunk proxy */ H5AC_TEST_ID, /*test entry -- not used for actual files */ H5AC_NTYPES /* Number of types, must be last */ } H5AC_type_t; diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index 5e7b1a7..de1e87a 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -857,7 +857,7 @@ ****************************************************************************/ #define H5C__H5C_T_MAGIC 0x005CAC0E -#define H5C__MAX_NUM_TYPE_IDS 21 +#define H5C__MAX_NUM_TYPE_IDS 22 #define H5C__PREFIX_LEN 32 struct H5C_t diff --git a/src/H5Dbtree.c b/src/H5Dbtree.c index fe0349c..3674ae3 100644 --- a/src/H5Dbtree.c +++ b/src/H5Dbtree.c @@ -165,6 +165,7 @@ static herr_t H5D_btree_idx_dest(const H5D_chk_idx_info_t *idx_info); /* v1 B-tree indexed chunk I/O ops */ const H5D_chunk_ops_t H5D_COPS_BTREE[1] = {{ + FALSE, /* v1 B-tree indices don't support SWMR access */ H5D_btree_idx_init, H5D_btree_idx_create, H5D_btree_idx_is_space_alloc, @@ -177,6 +178,8 @@ const H5D_chunk_ops_t H5D_COPS_BTREE[1] = {{ H5D_btree_idx_copy_shutdown, H5D_btree_idx_size, H5D_btree_idx_reset, + NULL, + NULL, H5D_btree_idx_dump, H5D_btree_idx_dest }}; diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 237a6e5..a5a76b2 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -254,7 +254,7 @@ const H5D_layout_ops_t H5D_LOPS_NONEXISTENT[1] = {{ /* Declare a free list to manage the H5F_rdcc_ent_ptr_t sequence information */ H5FL_SEQ_DEFINE_STATIC(H5D_rdcc_ent_ptr_t); -/* Declare a free list to manage H5F_rdcc_ent_t objects */ +/* Declare a free list to manage H5D_rdcc_ent_t objects */ H5FL_DEFINE_STATIC(H5D_rdcc_ent_t); /* Declare a free list to manage the H5D_chunk_info_t struct */ @@ -1692,7 +1692,7 @@ H5D_chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_chunk_info_t *chunk_info; /* Chunk information */ H5D_io_info_t *chk_io_info; /* Pointer to I/O info object for this chunk */ void *chunk; /* Pointer to locked chunk buffer */ - H5D_chunk_ud_t udata; /* B-tree pass-through */ + H5D_chunk_ud_t udata; /* Index pass-through */ htri_t cacheable; /* Whether the chunk is cacheable */ /* Get the actual chunk information from the skip list node */ @@ -2198,7 +2198,7 @@ done: * *------------------------------------------------------------------------- */ -static herr_t +herr_t H5D_chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t *dxpl_cache, H5D_rdcc_ent_t *ent, hbool_t reset) { @@ -2300,7 +2300,7 @@ H5D_chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t * /* Write the data to the file */ HDassert(H5F_addr_defined(udata.addr)); if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.nbytes, dxpl_id, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write raw data to file") /* Cache the chunk's info, in case it's accessed again shortly */ H5D_chunk_cinfo_cache_update(&dset->shared->cache.chunk.last, &udata); @@ -2308,6 +2308,13 @@ H5D_chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t * /* Mark cache entry as clean */ ent->dirty = FALSE; + /* Check for SWMR writes to the file */ + if(dset->shared->layout.u.chunk.ops->can_swim && H5F_INTENT(dset->oloc.file) & H5F_ACC_SWMR_WRITE) { + /* Mark the proxy entry in the cache as clean */ + if(H5D_chunk_proxy_mark(dset, dxpl_id, ent, FALSE) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTMARKDIRTY, FAIL, "can't mark proxy for chunk from metadata cache as clean") + } /* end if */ + /* Increment # of flushed entries */ dset->shared->cache.chunk.stats.nflushes++; } /* end if */ @@ -2380,6 +2387,13 @@ H5D_chunk_cache_evict(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t * ent->chunk = (uint8_t *)H5D_chunk_xfree(ent->chunk, &(dset->shared->dcpl_cache.pline)); } /* end else */ + /* Check for SWMR writes to the file */ + if(dset->shared->layout.u.chunk.ops->can_swim && H5F_INTENT(dset->oloc.file) & H5F_ACC_SWMR_WRITE) { + /* Remove the proxy entry in the cache */ + if(H5D_chunk_proxy_remove(dset, dxpl_id, ent) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTREMOVE, FAIL, "can't remove proxy for chunk from metadata cache") + } /* end if */ + /* Unlink from list */ if(ent->prev) ent->prev->next = ent->next; @@ -2699,6 +2713,8 @@ H5D_chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, ent->locked = 0; ent->dirty = FALSE; ent->chunk_addr = chunk_addr; + ent->proxy_addr = HADDR_UNDEF; + ent->proxy = NULL; for(u = 0; u < layout->u.chunk.ndims; u++) ent->offset[u] = io_info->store->chunk.offset[u]; H5_ASSIGN_OVERFLOW(ent->rd_count, chunk_size, size_t, uint32_t); @@ -2723,6 +2739,19 @@ H5D_chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, rdcc->head = rdcc->tail = ent; ent->prev = NULL; } /* end else */ + + /* Check for SWMR writes to the file */ + if(io_info->dset->shared->layout.u.chunk.ops->can_swim + && H5F_INTENT(io_info->dset->oloc.file) & H5F_ACC_SWMR_WRITE) { + /* Insert a proxy entry in the cache, to make certain that the + * flush dependencies are maintained in the proper way for SWMR + * access to work. + */ + if(H5D_chunk_proxy_create(io_info->dset, io_info->dxpl_id, udata, ent) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, NULL, "can't insert proxy for chunk in metadata cache") + } /* end if */ + + /* Indicate that the chunk is in the cache now */ found = TRUE; } else if(!found) { /* @@ -2857,6 +2886,14 @@ H5D_chunk_unlock(const H5D_io_info_t *io_info, const H5D_chunk_ud_t *udata, if(dirty) { ent->dirty = TRUE; ent->wr_count -= MIN(ent->wr_count, naccessed); + + /* Check for SWMR writes to the file */ + if(io_info->dset->shared->layout.u.chunk.ops->can_swim + && H5F_INTENT(io_info->dset->oloc.file) & H5F_ACC_SWMR_WRITE) { + /* Mark the proxy entry in the cache as dirty */ + if(H5D_chunk_proxy_mark(io_info->dset, io_info->dxpl_id, ent, TRUE) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTMARKDIRTY, FAIL, "can't mark proxy for chunk from metadata cache as dirty") + } /* end if */ } /* end if */ else ent->rd_count -= MIN(ent->rd_count, naccessed); diff --git a/src/H5Dearray.c b/src/H5Dearray.c index 7ae62e4..9862a15 100644 --- a/src/H5Dearray.c +++ b/src/H5Dearray.c @@ -132,6 +132,10 @@ static herr_t H5D_earray_idx_copy_shutdown(H5O_layout_t *layout_src, static herr_t H5D_earray_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *size); static herr_t H5D_earray_idx_reset(H5O_layout_t *layout, hbool_t reset_addr); +static herr_t H5D_earray_idx_depend(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_common_ud_t *udata, H5AC_info_t *child_entry); +static herr_t H5D_earray_idx_undepend(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_common_ud_t *udata, H5AC_info_t *child_entry); static herr_t H5D_earray_idx_dump(const H5D_chk_idx_info_t *idx_info, FILE *stream); static herr_t H5D_earray_idx_dest(const H5D_chk_idx_info_t *idx_info); @@ -143,6 +147,7 @@ static herr_t H5D_earray_idx_dest(const H5D_chk_idx_info_t *idx_info); /* Extensible array indexed chunk I/O ops */ const H5D_chunk_ops_t H5D_COPS_EARRAY[1] = {{ + TRUE, /* Extensible array indices support SWMR access */ NULL, H5D_earray_idx_create, H5D_earray_idx_is_space_alloc, @@ -155,6 +160,8 @@ const H5D_chunk_ops_t H5D_COPS_EARRAY[1] = {{ H5D_earray_idx_copy_shutdown, H5D_earray_idx_size, H5D_earray_idx_reset, + H5D_earray_idx_depend, + H5D_earray_idx_undepend, H5D_earray_idx_dump, H5D_earray_idx_dest }}; @@ -1432,6 +1439,108 @@ H5D_earray_idx_reset(H5O_layout_t *layout, hbool_t reset_addr) /*------------------------------------------------------------------------- + * Function: H5D_earray_idx_depend + * + * Purpose: Create a dependency between a chunk [proxy] and the index + * metadata that contains the record for the chunk. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, May 21, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_depend(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_common_ud_t *udata, H5AC_info_t *child_entry) +{ + H5EA_t *ea; /* Pointer to extensible array structure */ + hsize_t idx; /* Array index of chunk */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_depend) + + HDassert(idx_info); + HDassert(udata); + HDassert(child_entry); + + /* Check if the extensible array is open yet */ + if(NULL == idx_info->layout->u.chunk.u.earray.ea) { + /* Open the extensible array in file */ + if(H5D_earray_idx_open(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array") + } /* end if */ + + /* Set convenience pointer to extensible array structure */ + ea = idx_info->layout->u.chunk.u.earray.ea; + + /* Compute array index for chunk offset */ + idx = udata->offset[0] / idx_info->layout->u.chunk.dim[0]; + + /* Create flush dependency between the child_entry and the piece of metadata + * in the extensible array that contains the entry for this chunk. + */ + if(H5EA_depend(ea, idx_info->dxpl_id, idx, child_entry) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, "unable to create flush dependency on extensible array metadata") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_idx_depend() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_undepend + * + * Purpose: Remove a dependency between a chunk [proxy] and the index + * metadata that contains the record for the chunk. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, May 21, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_undepend(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_common_ud_t *udata, H5AC_info_t *child_entry) +{ + H5EA_t *ea; /* Pointer to extensible array structure */ + hsize_t idx; /* Array index of chunk */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_undepend) + + HDassert(idx_info); + HDassert(udata); + HDassert(child_entry); + + /* Check if the extensible array is open yet */ + if(NULL == idx_info->layout->u.chunk.u.earray.ea) { + /* Open the extensible array in file */ + if(H5D_earray_idx_open(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array") + } /* end if */ + + /* Set convenience pointer to extensible array structure */ + ea = idx_info->layout->u.chunk.u.earray.ea; + + /* Compute array index for chunk offset */ + idx = udata->offset[0] / idx_info->layout->u.chunk.dim[0]; + + /* Remove flush dependency between the child_entry and the piece of metadata + * in the extensible array that contains the entry for this chunk. + */ + if(H5EA_undepend(ea, idx_info->dxpl_id, idx, child_entry) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTUNDEPEND, FAIL, "unable to remove flush dependency on extensible array metadata") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_idx_undepend() */ + + +/*------------------------------------------------------------------------- * Function: H5D_earray_idx_dump * * Purpose: Dump indexing information to a stream. diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 155c20d..f67c7e1 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -33,8 +33,6 @@ /* Other private headers needed by this file */ #include "H5Gprivate.h" /* Groups */ -#include "H5Oprivate.h" /* Object headers */ -#include "H5Sprivate.h" /* Dataspaces */ #include "H5SLprivate.h" /* Skip lists */ #include "H5Tprivate.h" /* Datatypes */ @@ -284,12 +282,17 @@ typedef herr_t (*H5D_chunk_copy_shutdown_func_t)(H5O_layout_t *layout_src, typedef herr_t (*H5D_chunk_size_func_t)(const H5D_chk_idx_info_t *idx_info, hsize_t *idx_size); typedef herr_t (*H5D_chunk_reset_func_t)(H5O_layout_t *layout, hbool_t reset_addr); +typedef herr_t (*H5D_chunk_depend_func_t)(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_common_ud_t *udata, H5AC_info_t *child_entry); +typedef herr_t (*H5D_chunk_undepend_func_t)(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_common_ud_t *udata, H5AC_info_t *child_entry); typedef herr_t (*H5D_chunk_dump_func_t)(const H5D_chk_idx_info_t *idx_info, FILE *stream); typedef herr_t (*H5D_chunk_dest_func_t)(const H5D_chk_idx_info_t *idx_info); /* Typedef for grouping chunk I/O routines */ typedef struct H5D_chunk_ops_t { + hbool_t can_swim; /* Flag to indicate that the index supports SWMR access */ H5D_chunk_init_func_t init; /* Routine to initialize indexing information in memory */ H5D_chunk_create_func_t create; /* Routine to create chunk index */ H5D_chunk_is_space_alloc_func_t is_space_alloc; /* Query routine to determine if storage/index is allocated */ @@ -302,6 +305,8 @@ typedef struct H5D_chunk_ops_t { H5D_chunk_copy_shutdown_func_t copy_shutdown; /* Routine to perform any necessary shutdown for copying chunks */ H5D_chunk_size_func_t size; /* Routine to get size of indexing information */ H5D_chunk_reset_func_t reset; /* Routine to reset indexing information */ + H5D_chunk_depend_func_t depend; /* Routine to create dependency between chunk [proxy] and index metadata */ + H5D_chunk_undepend_func_t undepend; /* Routine to remove dependency between chunk [proxy] and index metadata */ H5D_chunk_dump_func_t dump; /* Routine to dump indexing information */ H5D_chunk_dest_func_t dest; /* Routine to destroy indexing information in memory */ } H5D_chunk_ops_t; @@ -370,15 +375,15 @@ typedef struct H5D_rdcc_t { } stats; size_t nbytes_max; /* Maximum cached raw data in bytes */ size_t nslots; /* Number of chunk slots allocated */ - double w0; /* Chunk preemption policy */ + double w0; /* Chunk preemption policy */ struct H5D_rdcc_ent_t *head; /* Head of doubly linked list */ struct H5D_rdcc_ent_t *tail; /* Tail of doubly linked list */ size_t nbytes_used; /* Current cached raw data in bytes */ - int nused; /* Number of chunk slots in use */ + int nused; /* Number of chunk slots in use */ H5D_chunk_cached_t last; /* Cached copy of last chunk information */ struct H5D_rdcc_ent_t **slot; /* Chunk slots, each points to a chunk*/ - H5SL_t *sel_chunks; /* Skip list containing information for each chunk selected */ - H5S_t *single_space; /* Dataspace for single element I/O on chunks */ + H5SL_t *sel_chunks; /* Skip list containing information for each chunk selected */ + H5S_t *single_space; /* Dataspace for single element I/O on chunks */ H5D_chunk_info_t *single_chunk_info; /* Pointer to single chunk's info */ } H5D_rdcc_t; @@ -488,12 +493,22 @@ typedef struct H5D_rdcc_ent_t { uint32_t wr_count; /*bytes remaining to be written */ haddr_t chunk_addr; /*address of chunk in file */ uint8_t *chunk; /*the unfiltered chunk data */ + haddr_t proxy_addr; /*address of chunk proxy in file */ + struct H5D_chunk_proxy_t *proxy; /*pointer to chunk proxy in memory */ unsigned idx; /*index in hash table */ struct H5D_rdcc_ent_t *next;/*next item in doubly-linked list */ struct H5D_rdcc_ent_t *prev;/*previous item in doubly-linked list */ } H5D_rdcc_ent_t; typedef H5D_rdcc_ent_t *H5D_rdcc_ent_ptr_t; /* For free lists */ +/* Metadata cache chunk proxy type */ +typedef struct H5D_chunk_proxy_t { + H5AC_info_t cache_info; /* Information for H5AC cache functions, _must_ be */ + /* first field in structure */ + H5D_t *dset; /* Pointer to dataset that chunk proxies are related to */ + H5D_rdcc_ent_t *ent; /* Pointer to chunk cache entry this proxy is standing in for */ +} H5D_chunk_proxy_t; + /*****************************/ /* Package Private Variables */ @@ -588,6 +603,8 @@ H5_DLL herr_t H5D_chunk_unlock(const H5D_io_info_t *io_info, const H5D_chunk_ud_t *udata, hbool_t dirty, unsigned idx_hint, void *chunk, uint32_t naccessed); H5_DLL herr_t H5D_chunk_flush(H5D_t *dset, hid_t dxpl_id, unsigned flags); +H5_DLL herr_t H5D_chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, + const H5D_dxpl_cache_t *dxpl_cache, H5D_rdcc_ent_t *ent, hbool_t reset); H5_DLL herr_t H5D_chunk_allocated(H5D_t *dset, hid_t dxpl_id, hsize_t *nbytes); H5_DLL herr_t H5D_chunk_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite); H5_DLL herr_t H5D_chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, @@ -627,6 +644,14 @@ H5_DLL herr_t H5D_fill_refill_vl(H5D_fill_buf_info_t *fb_info, size_t nelmts, H5_DLL herr_t H5D_fill_release(H5D_fill_buf_info_t *fb_info); H5_DLL herr_t H5D_fill_term(H5D_fill_buf_info_t *fb_info); +/* Functions that operate on chunk proxy objects */ +H5_DLL herr_t H5D_chunk_proxy_create(H5D_t *dset, hid_t dxpl_id, + H5D_chunk_common_ud_t *udata, H5D_rdcc_ent_t *ent); +H5_DLL herr_t H5D_chunk_proxy_remove(const H5D_t *dset, hid_t dxpl_it, + H5D_rdcc_ent_t *ent); +H5_DLL herr_t H5D_chunk_proxy_mark(const H5D_t *dset, hid_t dxpl_id, + const H5D_rdcc_ent_t *ent, hbool_t dirty); + #ifdef H5_HAVE_PARALLEL #ifdef H5S_DEBUG diff --git a/src/H5Dproxy.c b/src/H5Dproxy.c new file mode 100644 index 0000000..e2c6450 --- /dev/null +++ b/src/H5Dproxy.c @@ -0,0 +1,520 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5Dproxy.c + * May 19 2009 + * Quincey Koziol + * + * Purpose: Implement dataset's metadata cache proxy cache methods. + * + * Note: Chunk proxies exist only to make integrating the chunk + * cache with the metadata cache's flush dependencies + * easier and less coupled than directly tying them + * together. + * + * Chunk proxies never exist on disk (hence their lack of + * a 'load' callback) and their 'flush' callback just + * triggers a flush of the chunk it's a a proxy for. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#define H5D_PACKAGE /*suppress error about including H5Dpkg */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Dpkg.h" /* Dataset functions */ +#include "H5Eprivate.h" /* Error handling */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + +/* Local routines */ +static herr_t H5D_chunk_proxy_destroy(H5D_chunk_proxy_t *proxy); + +/* Metadata cache (H5AC) callbacks */ +static H5D_chunk_proxy_t *H5D_cache_proxy_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *udata, void *udata2); +static herr_t H5D_cache_proxy_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5D_chunk_proxy_t *proxy, unsigned UNUSED * flags_ptr); +static herr_t H5D_cache_proxy_dest(H5F_t *f, H5D_chunk_proxy_t *proxy); +static herr_t H5D_cache_proxy_clear(H5F_t *f, H5D_chunk_proxy_t *proxy, hbool_t destroy); +static herr_t H5D_cache_proxy_size(const H5F_t *f, const H5D_chunk_proxy_t *proxy, size_t *size_ptr); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* H5D chunk proxy inherits cache-like properties from H5AC */ +const H5AC_class_t H5AC_CHUNK_PROXY[1] = {{ + H5AC_CHUNK_PROXY_ID, + (H5AC_load_func_t)H5D_cache_proxy_load, + (H5AC_flush_func_t)H5D_cache_proxy_flush, + (H5AC_dest_func_t)H5D_cache_proxy_dest, + (H5AC_clear_func_t)H5D_cache_proxy_clear, + (H5AC_notify_func_t)NULL, + (H5AC_size_func_t)H5D_cache_proxy_size, +}}; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Declare a free list to manage H5D_chunk_proxy_t objects */ +H5FL_DEFINE_STATIC(H5D_chunk_proxy_t); + + + +/*------------------------------------------------------------------------- + * Function: H5D_cache_proxy_load + * + * Purpose: Loads a chunk proxy from the disk. + * + * Note: This routine should never be invoked + * + * Return: Success: Pointer to a new chunk proxy + * Failure: NULL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * May 21 2009 + * + *------------------------------------------------------------------------- + */ +static H5D_chunk_proxy_t * +H5D_cache_proxy_load(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, haddr_t UNUSED addr, + const void UNUSED *udata1, void UNUSED *udata2) +{ + H5D_chunk_proxy_t *ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_cache_proxy_load) + + /* This routine should never be invoked! */ + HDassert(0 && "H5D_cache_proxy_load called!?!"); + HGOTO_ERROR(H5E_DATASET, H5E_CANTLOAD, NULL, "unable to load chunk proxy") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D_cache_proxy_load() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_cache_proxy_flush + * + * Purpose: Proxy for flushing a chunk in chunk cache under control + * of the metadata cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * May 19 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_cache_proxy_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, + H5D_chunk_proxy_t *proxy, unsigned UNUSED * flags_ptr) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_cache_proxy_flush) +#ifdef QAK +HDfprintf(stderr, "%s: Flushing chunk proxy, addr = %a, destroy = %u\n", FUNC, addr, (unsigned)destroy); +#endif /* QAK */ + + /* check arguments */ + HDassert(f); + HDassert(H5F_addr_defined(addr)); + HDassert(proxy); + + if(proxy->cache_info.is_dirty) { + H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ + H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ + + /* Fill the DXPL cache values for later use */ + if(H5D_get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") + + /* Flush the chunk for the proxy */ + /* (This must be safe from actually performing I/O when the chunk is + * clean - QAK, 5/21/2009) + */ + if(H5D_chunk_flush_entry(proxy->dset, dxpl_id, dxpl_cache, proxy->ent, FALSE) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFLUSH, FAIL, "can't flush chunk via proxy") + + /* Mark the chunk proxy as clean now */ + proxy->cache_info.is_dirty = FALSE; + } /* end if */ + + if(destroy) + if(H5D_cache_proxy_dest(f, proxy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to destroy chunk proxy") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D_cache_proxy_flush() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_cache_proxy_dest + * + * Purpose: Destroys a chunk proxy in memory. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * May 19 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_cache_proxy_dest(H5F_t UNUSED *f, H5D_chunk_proxy_t *proxy) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_cache_proxy_dest) + + /* + * Check arguments. + */ + HDassert(proxy); + + /* Free the chunk proxy itself */ + if(H5D_chunk_proxy_destroy(proxy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to destroy chunk proxy") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_cache_proxy_dest() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_cache_proxy_clear + * + * Purpose: Mark a chunk proxy in memory as non-dirty. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * May 19 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_cache_proxy_clear(H5F_t *f, H5D_chunk_proxy_t *proxy, hbool_t destroy) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_cache_proxy_clear) + + /* + * Check arguments. + */ + HDassert(proxy); + + /* Reset the dirty flag. */ + proxy->cache_info.is_dirty = FALSE; + + if(destroy) + if(H5D_cache_proxy_dest(f, proxy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to destroy chunk proxy") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_cache_proxy_clear() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_cache_proxy_size + * + * Purpose: Compute the size in bytes of a chunk proxy + * on disk, and return it in *size_ptr. On failure, + * the value of *size_ptr is undefined. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * May 19 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_cache_proxy_size(const H5F_t UNUSED *f, const H5D_chunk_proxy_t UNUSED *proxy, + size_t *size_ptr) +{ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_cache_proxy_size) + + /* check arguments */ + HDassert(f); + HDassert(proxy); + HDassert(size_ptr); + + /* Chunk proxies are represented as 1 byte in cache */ + /* (would be 0 bytes, but cache won't allow it currently -QAK, 5/19/09) */ + *size_ptr = 1; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5D_cache_proxy_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_chunk_proxy_create + * + * Purpose: Create a proxy for the chunk and insert it into the metadata cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Tuesday, May 19, 2009 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_chunk_proxy_create(H5D_t *dset, hid_t dxpl_id, H5D_chunk_common_ud_t *udata, + H5D_rdcc_ent_t *ent) +{ + H5D_chunk_proxy_t *proxy = NULL; /* Chunk proxy */ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_chunk_proxy_create) + + HDassert(dset); + HDassert(ent); + + /* Get the chunk proxy address & adjust for next address */ + ent->proxy_addr = H5F_GET_NEXT_PROXY_ADDR(dset->oloc.file); +#ifdef QAK +HDfprintf(stderr, "%s: ent->proxy_addr = %a\n", FUNC, ent->proxy_addr); +#endif /* QAK */ + + /* Create chunk proxy object & initialize fields to zero */ + if(NULL == (proxy = H5FL_CALLOC(H5D_chunk_proxy_t))) + HGOTO_ERROR(H5E_DATASET, H5E_NOSPACE, FAIL, "can't allocate chunk proxy") + ent->proxy = proxy; + + /* Point chunk proxy to chunk cache entry it's representing and dataset + * it's related to + */ + proxy->dset = dset; + proxy->ent = ent; + + /* Insert chunk proxy into metadata cache, pinned */ + if(H5AC_set(dset->oloc.file, dxpl_id, H5AC_CHUNK_PROXY, ent->proxy_addr, proxy, H5AC__PIN_ENTRY_FLAG) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't add chunk proxy to cache") + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.dxpl_id = dxpl_id; + idx_info.pline = &(dset->shared->dcpl_cache.pline); + idx_info.layout = &(dset->shared->layout); + + /* Create a flush dependency between the proxy (as the child) and the + * metadata object in the index (as the parent). + */ + if((dset->shared->layout.u.chunk.ops->depend)(&idx_info, udata, (H5AC_info_t *)proxy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, "unable to create flush dependency for chunk proxy") + +done: + if(ret_value < 0) { + if(proxy) + proxy = H5FL_FREE(H5D_chunk_proxy_t, proxy); + ent->proxy_addr = HADDR_UNDEF; + ent->proxy = NULL; + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_chunk_proxy_create() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_chunk_proxy_remove + * + * Purpose: Remove a proxy for the chunk from the metadata cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Tuesday, May 19, 2009 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_chunk_proxy_remove(const H5D_t *dset, hid_t dxpl_id, H5D_rdcc_ent_t *ent) +{ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + H5D_chunk_common_ud_t udata; /* User-data for chunk */ + H5D_chunk_proxy_t *proxy = NULL; /* Chunk proxy */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_chunk_proxy_remove) + + HDassert(dset); + HDassert(ent); +#ifdef QAK +HDfprintf(stderr, "%s: ent->proxy_addr = %a\n", FUNC, ent->proxy_addr); +#endif /* QAK */ + + /* Protect the chunk proxy */ + if(NULL == (proxy = (H5D_chunk_proxy_t *)H5AC_protect(dset->oloc.file, dxpl_id, H5AC_CHUNK_PROXY, ent->proxy_addr, NULL, NULL, H5AC_WRITE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTPROTECT, FAIL, "unable to protect chunk proxy"); + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.dxpl_id = dxpl_id; + idx_info.pline = &(dset->shared->dcpl_cache.pline); + idx_info.layout = &(dset->shared->layout); + + /* Compose user-data for chunk */ + udata.mesg = &(dset->shared->layout); + udata.offset = ent->offset; + + /* Remove flush dependency between the proxy (as the child) and the + * metadata object in the index (as the parent). + */ + if((dset->shared->layout.u.chunk.ops->undepend)(&idx_info, &udata, (H5AC_info_t *)proxy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEPEND, FAIL, "unable to create flush dependency for chunk proxy") + + /* Unpin & delete chunk proxy from metadata cache, taking ownership of it */ + if(H5AC_unprotect(dset->oloc.file, dxpl_id, H5AC_CHUNK_PROXY, ent->proxy_addr, proxy, (H5AC__UNPIN_ENTRY_FLAG | H5AC__DELETED_FLAG | H5AC__TAKE_OWNERSHIP_FLAG)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTUNPROTECT, FAIL, "unable to release chunk proxy"); + + /* Reset information in chunk cache entry */ + ent->proxy_addr = HADDR_UNDEF; + ent->proxy = NULL; + + /* Release the chunk proxy object */ + if(H5D_chunk_proxy_destroy(proxy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to destroy chunk proxy") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_chunk_proxy_remove() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_chunk_proxy_mark + * + * Purpose: Mark a proxy for the chunk in the metadata cache as clean or + * dirty. + * + * Note: Currently, this code does not mark a chunk proxy as clean in + * the metadata cache. Doing so would require that this routine + * be invoked collectively when operating in parallel I/O mode + * and it's possible that this routine can be invoked during + * indepedent raw data I/O. + * + * So, the chunk proxy's dirty state in the metadata cache may + * be out of sync with the chunk itself, but only in the direction + * of being dirty when the chunk itself is clean. We'll call + * call the 'flush' callback for the chunk proxies more often + * than necessary, but as long as the chunk entry flushing + * routine doesn't actually flush the chunk when it's clean, + * there won't be too much overhead. - QAK, 5/21/2009 + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Tuesday, May 19, 2009 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_chunk_proxy_mark(const H5D_t *dset, hid_t dxpl_id, const H5D_rdcc_ent_t *ent, + hbool_t dirty) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_chunk_proxy_mark) + + HDassert(dset); + HDassert(ent); +#ifdef QAK +HDfprintf(stderr, "%s: ent->proxy_addr = %a, dirty = %t\n", FUNC, ent->proxy_addr, dirty); +#endif /* QAK */ + + /* Check whether to mark the proxy as dirty */ + if(dirty) { + if(H5AC_mark_pinned_or_protected_entry_dirty(dset->oloc.file, ent->proxy) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTMARKDIRTY, FAIL, "can't mark chunk proxy entry in metadata cache as dirty") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_chunk_proxy_mark() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_chunk_proxy_destroy + * + * Purpose: Destroy a chunk proxy object + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, May 21, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_chunk_proxy_destroy(H5D_chunk_proxy_t *proxy) +{ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_chunk_proxy_destroy) + + HDassert(proxy); + + /* Free the chunk proxy object */ + proxy->dset = NULL; + proxy->ent = NULL; + proxy = H5FL_FREE(H5D_chunk_proxy_t, proxy); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_chunk_proxy_destroy() */ + diff --git a/src/H5EAcache.c b/src/H5EAcache.c index 00d3089..a254e84 100644 --- a/src/H5EAcache.c +++ b/src/H5EAcache.c @@ -857,27 +857,30 @@ H5EA__cache_iblock_notify(H5AC_notify_action_t action, H5EA_iblock_t *iblock)) /* Sanity check */ HDassert(iblock); - /* Determine which action to take */ - switch(action) { - case H5AC_NOTIFY_ACTION_AFTER_INSERT: - /* Create flush dependency on extensible array header */ - if(H5EA__create_flush_depend(iblock->hdr, (H5AC_info_t *)iblock->hdr, (H5AC_info_t *)iblock) < 0) - H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between index block and header, address = %llu", (unsigned long long)iblock->addr) - break; - - case H5AC_NOTIFY_ACTION_BEFORE_EVICT: - /* Destroy flush dependency on extensible array header */ - if(H5EA__destroy_flush_depend(iblock->hdr, (H5AC_info_t *)iblock->hdr, (H5AC_info_t *)iblock) < 0) - H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between index block and header, address = %llu", (unsigned long long)iblock->addr) - break; - - default: + /* Check if the file was opened with SWMR-write access */ + if(iblock->hdr->swmr_write) { + /* Determine which action to take */ + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + /* Create flush dependency on extensible array header */ + if(H5EA__create_flush_depend(iblock->hdr, (H5AC_info_t *)iblock->hdr, (H5AC_info_t *)iblock) < 0) + H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between index block and header, address = %llu", (unsigned long long)iblock->addr) + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* Destroy flush dependency on extensible array header */ + if(H5EA__destroy_flush_depend(iblock->hdr, (H5AC_info_t *)iblock->hdr, (H5AC_info_t *)iblock) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between index block and header, address = %llu", (unsigned long long)iblock->addr) + break; + + default: #ifdef NDEBUG - H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") + H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") #else /* NDEBUG */ - HDassert(0 && "Unknown action?!?"); + HDassert(0 && "Unknown action?!?"); #endif /* NDEBUG */ - } /* end switch */ + } /* end switch */ + } /* end if */ CATCH @@ -1288,27 +1291,30 @@ H5EA__cache_sblock_notify(H5AC_notify_action_t action, H5EA_sblock_t *sblock)) /* Sanity check */ HDassert(sblock); - /* Determine which action to take */ - switch(action) { - case H5AC_NOTIFY_ACTION_AFTER_INSERT: - /* Create flush dependency on index block */ - if(H5EA__create_flush_depend(sblock->hdr, (H5AC_info_t *)sblock->parent, (H5AC_info_t *)sblock) < 0) - H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between super block and index block, address = %llu", (unsigned long long)sblock->addr) - break; - - case H5AC_NOTIFY_ACTION_BEFORE_EVICT: - /* Destroy flush dependency on index block */ - if(H5EA__destroy_flush_depend(sblock->hdr, (H5AC_info_t *)sblock->parent, (H5AC_info_t *)sblock) < 0) - H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between super block and index block, address = %llu", (unsigned long long)sblock->addr) - break; - - default: + /* Check if the file was opened with SWMR-write access */ + if(sblock->hdr->swmr_write) { + /* Determine which action to take */ + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + /* Create flush dependency on index block */ + if(H5EA__create_flush_depend(sblock->hdr, (H5AC_info_t *)sblock->parent, (H5AC_info_t *)sblock) < 0) + H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between super block and index block, address = %llu", (unsigned long long)sblock->addr) + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* Destroy flush dependency on index block */ + if(H5EA__destroy_flush_depend(sblock->hdr, (H5AC_info_t *)sblock->parent, (H5AC_info_t *)sblock) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between super block and index block, address = %llu", (unsigned long long)sblock->addr) + break; + + default: #ifdef NDEBUG - H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") + H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") #else /* NDEBUG */ - HDassert(0 && "Unknown action?!?"); + HDassert(0 && "Unknown action?!?"); #endif /* NDEBUG */ - } /* end switch */ + } /* end switch */ + } /* end if */ CATCH @@ -1653,27 +1659,30 @@ H5EA__cache_dblock_notify(H5AC_notify_action_t action, H5EA_dblock_t *dblock)) /* Sanity check */ HDassert(dblock); - /* Determine which action to take */ - switch(action) { - case H5AC_NOTIFY_ACTION_AFTER_INSERT: - /* Create flush dependency on parent */ - if(H5EA__create_flush_depend(dblock->hdr, (H5AC_info_t *)dblock->parent, (H5AC_info_t *)dblock) < 0) - H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between data block and parent, address = %llu", (unsigned long long)dblock->addr) - break; - - case H5AC_NOTIFY_ACTION_BEFORE_EVICT: - /* Destroy flush dependency on parent */ - if(H5EA__destroy_flush_depend(dblock->hdr, (H5AC_info_t *)dblock->parent, (H5AC_info_t *)dblock) < 0) - H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between data block and parent, address = %llu", (unsigned long long)dblock->addr) - break; - - default: + /* Check if the file was opened with SWMR-write access */ + if(dblock->hdr->swmr_write) { + /* Determine which action to take */ + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + /* Create flush dependency on parent */ + if(H5EA__create_flush_depend(dblock->hdr, (H5AC_info_t *)dblock->parent, (H5AC_info_t *)dblock) < 0) + H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between data block and parent, address = %llu", (unsigned long long)dblock->addr) + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* Destroy flush dependency on parent */ + if(H5EA__destroy_flush_depend(dblock->hdr, (H5AC_info_t *)dblock->parent, (H5AC_info_t *)dblock) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between data block and parent, address = %llu", (unsigned long long)dblock->addr) + break; + + default: #ifdef NDEBUG - H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") + H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") #else /* NDEBUG */ - HDassert(0 && "Unknown action?!?"); + HDassert(0 && "Unknown action?!?"); #endif /* NDEBUG */ - } /* end switch */ + } /* end switch */ + } /* end if */ CATCH @@ -2007,27 +2016,30 @@ H5EA__cache_dblk_page_notify(H5AC_notify_action_t action, H5EA_dblk_page_t *dblk /* Sanity check */ HDassert(dblk_page); - /* Determine which action to take */ - switch(action) { - case H5AC_NOTIFY_ACTION_AFTER_INSERT: - /* Create flush dependency on parent */ - if(H5EA__create_flush_depend(dblk_page->hdr, (H5AC_info_t *)dblk_page->parent, (H5AC_info_t *)dblk_page) < 0) - H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between data block page and parent, address = %llu", (unsigned long long)dblk_page->addr) - break; - - case H5AC_NOTIFY_ACTION_BEFORE_EVICT: - /* Destroy flush dependency on parent */ - if(H5EA__destroy_flush_depend(dblk_page->hdr, (H5AC_info_t *)dblk_page->parent, (H5AC_info_t *)dblk_page) < 0) - H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between data block page and parent, address = %llu", (unsigned long long)dblk_page->addr) - break; - - default: + /* Check if the file was opened with SWMR-write access */ + if(dblk_page->hdr->swmr_write) { + /* Determine which action to take */ + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + /* Create flush dependency on parent */ + if(H5EA__create_flush_depend(dblk_page->hdr, (H5AC_info_t *)dblk_page->parent, (H5AC_info_t *)dblk_page) < 0) + H5E_THROW(H5E_CANTDEPEND, "unable to create flush dependency between data block page and parent, address = %llu", (unsigned long long)dblk_page->addr) + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* Destroy flush dependency on parent */ + if(H5EA__destroy_flush_depend(dblk_page->hdr, (H5AC_info_t *)dblk_page->parent, (H5AC_info_t *)dblk_page) < 0) + H5E_THROW(H5E_CANTUNDEPEND, "unable to destroy flush dependency between data block page and parent, address = %llu", (unsigned long long)dblk_page->addr) + break; + + default: #ifdef NDEBUG - H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") + H5E_THROW(H5E_BADVALUE, "unknown action from metadata cache") #else /* NDEBUG */ - HDassert(0 && "Unknown action?!?"); + HDassert(0 && "Unknown action?!?"); #endif /* NDEBUG */ - } /* end switch */ + } /* end switch */ + } /* end if */ CATCH diff --git a/src/H5EAhdr.c b/src/H5EAhdr.c index 1e01601..e6c33b1 100644 --- a/src/H5EAhdr.c +++ b/src/H5EAhdr.c @@ -134,6 +134,7 @@ H5EA__hdr_alloc(H5F_t *f, const H5EA_class_t *cls, void *udata)) /* Set the internal parameters for the array */ hdr->f = f; + hdr->swmr_write = (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) > 0; hdr->sizeof_addr = H5F_SIZEOF_ADDR(f); hdr->sizeof_size = H5F_SIZEOF_SIZE(f); diff --git a/src/H5EAint.c b/src/H5EAint.c index 5ce88fc..28681e8 100644 --- a/src/H5EAint.c +++ b/src/H5EAint.c @@ -127,7 +127,8 @@ END_FUNC(PKG) /* end H5EA__create_flush_depend() */ */ BEGIN_FUNC(PKG, ERR, herr_t, SUCCEED, FAIL, -H5EA__destroy_flush_depend(H5EA_hdr_t *hdr, H5AC_info_t *parent_entry, H5AC_info_t *child_entry)) +H5EA__destroy_flush_depend(H5EA_hdr_t *hdr, H5AC_info_t *parent_entry, + H5AC_info_t *child_entry)) /* Sanity check */ HDassert(hdr); diff --git a/src/H5EApkg.h b/src/H5EApkg.h index bed47e2..e74e3b4 100644 --- a/src/H5EApkg.h +++ b/src/H5EApkg.h @@ -495,6 +495,7 @@ typedef struct H5EA_hdr_t { haddr_t addr; /* Address of header in file */ size_t size; /* Size of header in file */ H5F_t *f; /* Pointer to file for extensible array */ + hbool_t swmr_write; /* Flag indicating the file is opened with SWMR-write access */ size_t file_rc; /* Reference count of files using array header */ hbool_t pending_delete; /* Array is pending deletion */ size_t sizeof_addr; /* Size of file addresses */ diff --git a/src/H5F.c b/src/H5F.c index a367428..334a27e 100644 --- a/src/H5F.c +++ b/src/H5F.c @@ -894,6 +894,7 @@ H5F_new(H5F_file_t *shared, hid_t fcpl_id, hid_t fapl_id, H5FD_t *lf) f->shared->accum.loc = HADDR_UNDEF; f->shared->lf = lf; f->shared->root_addr = HADDR_UNDEF; + f->shared->next_proxy_addr = HADDR_MAX; /* * Copy the file creation and file access property lists into the diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 0d5a593..f6809e8 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -162,6 +162,7 @@ typedef struct H5F_file_t { haddr_t root_addr; /* Root group address */ H5FO_t *open_objs; /* Open objects in file */ H5RC_t *grp_btree_shared; /* Ref-counted group B-tree node info */ + haddr_t next_proxy_addr; /* Next address to use for metadata cache proxy entries */ /* File space allocation information */ unsigned fs_aggr_merge[H5FD_MEM_NTYPES]; /* Flags for whether free space can merge with aggregator(s) */ diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index cd6bcd0..59bdd10 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -265,6 +265,7 @@ typedef struct H5F_blk_aggr_t H5F_blk_aggr_t; #define H5F_HAS_FEATURE(F,FL) ((F)->shared->lf->feature_flags & (FL)) #define H5F_DRIVER_ID(F) ((F)->shared->lf->driver_id) #define H5F_GET_FILENO(F,FILENUM) ((FILENUM) = (F)->shared->lf->fileno) +#define H5F_GET_NEXT_PROXY_ADDR(F) ((F)->shared->next_proxy_addr--) #else /* H5F_PACKAGE */ #define H5F_INTENT(F) (H5F_get_intent(F)) #define H5F_FCPL(F) (H5F_get_fcpl(F)) @@ -287,6 +288,7 @@ typedef struct H5F_blk_aggr_t H5F_blk_aggr_t; #define H5F_HAS_FEATURE(F,FL) (H5F_has_feature(F,FL)) #define H5F_DRIVER_ID(F) (H5F_get_driver_id(F)) #define H5F_GET_FILENO(F,FILENUM) (H5F_get_fileno((F), &(FILENUM))) +#define H5F_GET_NEXT_PROXY_ADDR(F) (H5F_get_next_proxy_addr(F)) #endif /* H5F_PACKAGE */ @@ -472,6 +474,7 @@ H5_DLL char *H5F_get_name(const H5F_t *f); H5_DLL hid_t H5F_get_id(H5F_t *file, hbool_t app_ref); H5_DLL size_t H5F_get_obj_count(const H5F_t *f, unsigned types, hbool_t app_ref); H5_DLL size_t H5F_get_obj_ids(const H5F_t *f, unsigned types, size_t max_objs, hid_t *obj_id_list, hbool_t app_ref); +H5_DLL haddr_t H5F_get_next_proxy_addr(const H5F_t *f); /* Functions than retrieve values set/cached from the superblock/FCPL */ H5_DLL hid_t H5F_get_fcpl(const H5F_t *f); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 275061d..d070b94 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -687,3 +687,32 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_get_fileno() */ + +/*------------------------------------------------------------------------- + * Function: H5F_get_next_proxy_addr + * + * Purpose: Quick and dirty routine to retrieve the next metadata proxy + * address for a file. + * (Mainly added to stop non-file routines from poking about in the + * H5F_t data structure) + * + * Return: Success: Address to use for metadata cache proxy + * Failure: abort (should not happen) + * + * Programmer: Quincey Koziol + * May 19, 2009 + * + *------------------------------------------------------------------------- + */ +haddr_t +H5F_get_next_proxy_addr(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOFUNC here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5F_get_next_proxy_addr) + + HDassert(f); + HDassert(f->shared); + + FUNC_LEAVE_NOAPI(f->shared->next_proxy_addr--) +} /* end H5F_get_next_proxy_addr() */ + diff --git a/src/Makefile.am b/src/Makefile.am index b893950..5614e9e 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -48,7 +48,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5C.c H5CS.c \ H5D.c H5Dbtree.c H5Dchunk.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \ H5Ddeprec.c H5Dearray.c H5Defl.c H5Dfill.c H5Dint.c H5Dio.c \ - H5Dmpio.c H5Doh.c H5Dscatgath.c H5Dselect.c H5Dtest.c \ + H5Dmpio.c H5Doh.c H5Dproxy.c H5Dscatgath.c H5Dselect.c H5Dtest.c \ H5E.c H5Edeprec.c H5Eint.c \ H5EA.c H5EAcache.c H5EAdbg.c H5EAdblkpage.c H5EAdblock.c H5EAhdr.c \ H5EAiblock.c H5EAint.c H5EAsblock.c H5EAstat.c H5EAtest.c \ diff --git a/src/Makefile.in b/src/Makefile.in index 97e9c67..2709f3b 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -85,7 +85,7 @@ am_libhdf5_la_OBJECTS = H5.lo H5checksum.lo H5dbg.lo H5system.lo \ H5B2stat.lo H5B2test.lo H5C.lo H5CS.lo H5D.lo H5Dbtree.lo \ H5Dchunk.lo H5Dcompact.lo H5Dcontig.lo H5Ddbg.lo H5Ddeprec.lo \ H5Dearray.lo H5Defl.lo H5Dfill.lo H5Dint.lo H5Dio.lo \ - H5Dmpio.lo H5Doh.lo H5Dscatgath.lo H5Dselect.lo H5Dtest.lo \ + H5Dmpio.lo H5Doh.lo H5Dproxy.lo H5Dscatgath.lo H5Dselect.lo H5Dtest.lo \ H5E.lo H5Edeprec.lo H5Eint.lo H5EA.lo H5EAcache.lo H5EAdbg.lo \ H5EAdblkpage.lo H5EAdblock.lo H5EAhdr.lo H5EAiblock.lo \ H5EAint.lo H5EAsblock.lo H5EAstat.lo H5EAtest.lo H5F.lo \ @@ -432,7 +432,7 @@ libhdf5_la_SOURCES = H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5C.c H5CS.c \ H5D.c H5Dbtree.c H5Dchunk.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \ H5Ddeprec.c H5Dearray.c H5Defl.c H5Dfill.c H5Dint.c H5Dio.c \ - H5Dmpio.c H5Doh.c H5Dscatgath.c H5Dselect.c H5Dtest.c \ + H5Dmpio.c H5Doh.c H5Dproxy.c H5Dscatgath.c H5Dselect.c H5Dtest.c \ H5E.c H5Edeprec.c H5Eint.c \ H5EA.c H5EAcache.c H5EAdbg.c H5EAdblkpage.c H5EAdblock.c H5EAhdr.c \ H5EAiblock.c H5EAint.c H5EAsblock.c H5EAstat.c H5EAtest.c \ @@ -648,6 +648,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dio.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dmpio.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Doh.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dproxy.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dscatgath.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dselect.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dtest.Plo@am__quote@ diff --git a/test/dsets.c b/test/dsets.c index 3a8ab0e..b509d9a 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -6912,6 +6912,7 @@ test_chunk_fast(hid_t fapl) { char filename[FILENAME_BUF_SIZE]; hid_t fid = -1; /* File ID */ + hid_t my_fapl = -1; /* File access property list ID */ hid_t dcpl = -1; /* Dataset creation property list ID */ hid_t sid = -1; /* Dataspace ID */ hid_t scalar_sid = -1;/* Scalar dataspace ID */ @@ -6922,115 +6923,92 @@ test_chunk_fast(hid_t fapl) hsize_t hs_offset; /* Hyperslab offset */ hsize_t hs_size; /* Hyperslab size */ H5D_alloc_time_t alloc_time; /* Storage allocation time */ + hbool_t swmr; /* Whether file should be written with SWMR access enabled */ #ifdef H5_HAVE_FILTER_DEFLATE hbool_t compress; /* Whether chunks should be compressed */ #endif /* H5_HAVE_FILTER_DEFLATE */ unsigned write_elem, read_elem; /* Element written/read */ + int mdc_nelmts; /* # of elements in metadata cache */ + size_t rdcc_nelmts; /* # of chunks in chunk cache */ + size_t rdcc_nbytes; /* # of bytes in chunk cache */ + double rdcc_w0; /* write-ratio for chunk cache */ unsigned u; /* Local index variable */ TESTING("datasets w/extensible array as chunk index"); h5_fixname(FILENAME[9], fapl, filename, sizeof filename); + /* Copy the file access property list */ + if((my_fapl = H5Pcopy(fapl)) < 0) FAIL_STACK_ERROR + + /* Turn on the chunk cache again */ + if(H5Pget_cache(my_fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0) FAIL_STACK_ERROR + rdcc_nbytes = 1048576; + if(H5Pset_cache(my_fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0) FAIL_STACK_ERROR + /* Check if we are using the latest version of the format */ - if(H5Pget_libver_bounds(fapl, &low, &high) < 0) FAIL_STACK_ERROR + if(H5Pget_libver_bounds(my_fapl, &low, &high) < 0) FAIL_STACK_ERROR + /* Loop over using SWMR access to write */ + for(swmr = FALSE; swmr <= TRUE; swmr++) { #ifdef H5_HAVE_FILTER_DEFLATE - /* Loop over compressing chunks */ - for(compress = FALSE; compress <= TRUE; compress++) { + /* Loop over compressing chunks */ + for(compress = FALSE; compress <= TRUE; compress++) { #endif /* H5_HAVE_FILTER_DEFLATE */ - /* Loop over storage allocation time */ - for(alloc_time = H5D_ALLOC_TIME_EARLY; alloc_time <= H5D_ALLOC_TIME_INCR; alloc_time++) { - /* Create file */ - if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR + /* Loop over storage allocation time */ + for(alloc_time = H5D_ALLOC_TIME_EARLY; alloc_time <= H5D_ALLOC_TIME_INCR; alloc_time++) { + /* Create file */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC | (swmr ? H5F_ACC_SWMR_WRITE : 0), H5P_DEFAULT, my_fapl)) < 0) FAIL_STACK_ERROR - /* Create dataset creation property list */ - if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) FAIL_STACK_ERROR + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) FAIL_STACK_ERROR - /* Set chunking */ - chunk_dim = 10; - if(H5Pset_chunk(dcpl, 1, &chunk_dim) < 0) FAIL_STACK_ERROR + /* Set chunking */ + chunk_dim = 10; + if(H5Pset_chunk(dcpl, 1, &chunk_dim) < 0) FAIL_STACK_ERROR #ifdef H5_HAVE_FILTER_DEFLATE - /* Check if we should compress the chunks */ - if(compress) - if(H5Pset_deflate(dcpl, 9) < 0) FAIL_STACK_ERROR + /* Check if we should compress the chunks */ + if(compress) + if(H5Pset_deflate(dcpl, 9) < 0) FAIL_STACK_ERROR #endif /* H5_HAVE_FILTER_DEFLATE */ - /* Set fill time */ - if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) FAIL_STACK_ERROR - - /* Set allocation time */ - if(H5Pset_alloc_time(dcpl, alloc_time) < 0) FAIL_STACK_ERROR + /* Set fill time */ + if(H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) FAIL_STACK_ERROR - /* Create scalar dataspace */ - if((scalar_sid = H5Screate(H5S_SCALAR)) < 0) FAIL_STACK_ERROR + /* Set allocation time */ + if(H5Pset_alloc_time(dcpl, alloc_time) < 0) FAIL_STACK_ERROR - /* Create 1-D dataspace */ - dim = 100; - max_dim = H5S_UNLIMITED; - if((sid = H5Screate_simple(1, &dim, &max_dim)) < 0) FAIL_STACK_ERROR - - /* Create chunked dataset */ - if((dsid = H5Dcreate2(fid, "dset", H5T_NATIVE_UINT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) - FAIL_STACK_ERROR - - /* Get the chunk index type */ - if(H5D_layout_idx_type_test(dsid, &idx_type) < 0) FAIL_STACK_ERROR - - /* Chunk index tyepe expected depends on whether we are using the latest version of the format */ - if(low == H5F_LIBVER_LATEST) { - /* Verify index type */ - if(idx_type != H5D_CHUNK_EARRAY) FAIL_PUTS_ERROR("should be using extensible array as index"); - } /* end if */ - else { - /* Verify index type */ - if(idx_type != H5D_CHUNK_BTREE) FAIL_PUTS_ERROR("should be using v1 B-tree as index"); - } /* end else */ - - /* Fill existing elements */ - hs_size = 1; - for(u = 0; u < 100; u++) { - /* Select a single element in the dataset */ - hs_offset = u; - if(H5Sselect_hyperslab(sid, H5S_SELECT_SET, &hs_offset, NULL, &hs_size, NULL) < 0) FAIL_STACK_ERROR - - /* Read (unwritten) element from dataset */ - read_elem = 1; - if(H5Dread(dsid, H5T_NATIVE_UINT, scalar_sid, sid, H5P_DEFAULT, &read_elem) < 0) FAIL_STACK_ERROR - - /* Verify unwritten element is fill value (0) */ - if(read_elem != 0) FAIL_PUTS_ERROR("invalid unwritten element read"); - - /* Write element to dataset */ - write_elem = u; - if(H5Dwrite(dsid, H5T_NATIVE_UINT, scalar_sid, sid, H5P_DEFAULT, &write_elem) < 0) FAIL_STACK_ERROR - - /* Read element from dataset */ - read_elem = write_elem + 1; - if(H5Dread(dsid, H5T_NATIVE_UINT, scalar_sid, sid, H5P_DEFAULT, &read_elem) < 0) FAIL_STACK_ERROR + /* Create scalar dataspace */ + if((scalar_sid = H5Screate(H5S_SCALAR)) < 0) FAIL_STACK_ERROR - /* Verify written element is read in */ - if(read_elem != write_elem) FAIL_PUTS_ERROR("invalid written element read"); - } /* end for */ + /* Create 1-D dataspace */ + dim = 100; + max_dim = H5S_UNLIMITED; + if((sid = H5Screate_simple(1, &dim, &max_dim)) < 0) FAIL_STACK_ERROR - /* Incrementally extend dataset and verify write/reads */ - while(dim < 1000) { - /* Extend dataset */ - dim += 100; - if(H5Dset_extent(dsid, &dim) < 0) FAIL_STACK_ERROR + /* Create chunked dataset */ + if((dsid = H5Dcreate2(fid, "dset", H5T_NATIVE_UINT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR - /* Close old dataspace */ - if(H5Sclose(sid) < 0) FAIL_STACK_ERROR + /* Get the chunk index type */ + if(H5D_layout_idx_type_test(dsid, &idx_type) < 0) FAIL_STACK_ERROR - /* Get dataspace for dataset now */ - if((sid = H5Dget_space(dsid)) < 0) FAIL_STACK_ERROR + /* Chunk index tyepe expected depends on whether we are using the latest version of the format */ + if(low == H5F_LIBVER_LATEST) { + /* Verify index type */ + if(idx_type != H5D_CHUNK_EARRAY) FAIL_PUTS_ERROR("should be using extensible array as index"); + } /* end if */ + else { + /* Verify index type */ + if(idx_type != H5D_CHUNK_BTREE) FAIL_PUTS_ERROR("should be using v1 B-tree as index"); + } /* end else */ - /* Fill new elements */ + /* Fill existing elements */ hs_size = 1; for(u = 0; u < 100; u++) { /* Select a single element in the dataset */ - hs_offset = (dim + u) - 100; + hs_offset = u; if(H5Sselect_hyperslab(sid, H5S_SELECT_SET, &hs_offset, NULL, &hs_size, NULL) < 0) FAIL_STACK_ERROR /* Read (unwritten) element from dataset */ @@ -7051,73 +7029,115 @@ test_chunk_fast(hid_t fapl) /* Verify written element is read in */ if(read_elem != write_elem) FAIL_PUTS_ERROR("invalid written element read"); } /* end for */ - } /* end while */ - /* Close everything */ - if(H5Dclose(dsid) < 0) FAIL_STACK_ERROR - if(H5Sclose(sid) < 0) FAIL_STACK_ERROR - if(H5Sclose(scalar_sid) < 0) FAIL_STACK_ERROR - if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR - if(H5Fclose(fid) < 0) FAIL_STACK_ERROR + /* Incrementally extend dataset and verify write/reads */ + while(dim < 1000) { + /* Extend dataset */ + dim += 100; + if(H5Dset_extent(dsid, &dim) < 0) FAIL_STACK_ERROR - /* Re-open file & dataset */ - if((fid = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) FAIL_STACK_ERROR + /* Close old dataspace */ + if(H5Sclose(sid) < 0) FAIL_STACK_ERROR - /* Open dataset */ - if((dsid = H5Dopen2(fid, "dset", H5P_DEFAULT)) < 0) FAIL_STACK_ERROR + /* Get dataspace for dataset now */ + if((sid = H5Dget_space(dsid)) < 0) FAIL_STACK_ERROR - /* Get the chunk index type */ - if(H5D_layout_idx_type_test(dsid, &idx_type) < 0) FAIL_STACK_ERROR + /* Fill new elements */ + hs_size = 1; + for(u = 0; u < 100; u++) { + /* Select a single element in the dataset */ + hs_offset = (dim + u) - 100; + if(H5Sselect_hyperslab(sid, H5S_SELECT_SET, &hs_offset, NULL, &hs_size, NULL) < 0) FAIL_STACK_ERROR - /* Chunk index tyepe expected depends on whether we are using the latest version of the format */ - if(low == H5F_LIBVER_LATEST) { - /* Verify index type */ - if(idx_type != H5D_CHUNK_EARRAY) FAIL_PUTS_ERROR("should be using extensible array as index"); - } /* end if */ - else { - /* Verify index type */ - if(idx_type != H5D_CHUNK_BTREE) FAIL_PUTS_ERROR("should be using v1 B-tree as index"); - } /* end else */ + /* Read (unwritten) element from dataset */ + read_elem = 1; + if(H5Dread(dsid, H5T_NATIVE_UINT, scalar_sid, sid, H5P_DEFAULT, &read_elem) < 0) FAIL_STACK_ERROR - /* Create scalar dataspace */ - if((scalar_sid = H5Screate(H5S_SCALAR)) < 0) FAIL_STACK_ERROR + /* Verify unwritten element is fill value (0) */ + if(read_elem != 0) FAIL_PUTS_ERROR("invalid unwritten element read"); - /* Get dataspace for dataset now */ - if((sid = H5Dget_space(dsid)) < 0) FAIL_STACK_ERROR + /* Write element to dataset */ + write_elem = u; + if(H5Dwrite(dsid, H5T_NATIVE_UINT, scalar_sid, sid, H5P_DEFAULT, &write_elem) < 0) FAIL_STACK_ERROR - /* Read elements */ - hs_size = 1; - for(u = 0; u < 1000; u++) { - /* Select a single element in the dataset */ - hs_offset = u; - if(H5Sselect_hyperslab(sid, H5S_SELECT_SET, &hs_offset, NULL, &hs_size, NULL) < 0) FAIL_STACK_ERROR + /* Read element from dataset */ + read_elem = write_elem + 1; + if(H5Dread(dsid, H5T_NATIVE_UINT, scalar_sid, sid, H5P_DEFAULT, &read_elem) < 0) FAIL_STACK_ERROR - /* Read (unwritten) element from dataset */ - read_elem = u + 1; - if(H5Dread(dsid, H5T_NATIVE_UINT, scalar_sid, sid, H5P_DEFAULT, &read_elem) < 0) FAIL_STACK_ERROR + /* Verify written element is read in */ + if(read_elem != write_elem) FAIL_PUTS_ERROR("invalid written element read"); + } /* end for */ + } /* end while */ - /* Verify unwritten element is fill value (0) */ - if(read_elem != (u % 100)) FAIL_PUTS_ERROR("invalid element read"); - } /* end for */ + /* Close everything */ + if(H5Dclose(dsid) < 0) FAIL_STACK_ERROR + if(H5Sclose(sid) < 0) FAIL_STACK_ERROR + if(H5Sclose(scalar_sid) < 0) FAIL_STACK_ERROR + if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR + if(H5Fclose(fid) < 0) FAIL_STACK_ERROR - /* Close everything */ - if(H5Dclose(dsid) < 0) FAIL_STACK_ERROR - if(H5Sclose(sid) < 0) FAIL_STACK_ERROR - if(H5Sclose(scalar_sid) < 0) FAIL_STACK_ERROR - if(H5Fclose(fid) < 0) FAIL_STACK_ERROR + /* Re-open file & dataset */ + if((fid = H5Fopen(filename, H5F_ACC_RDONLY, my_fapl)) < 0) FAIL_STACK_ERROR - /* Re-open file */ - if((fid = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) FAIL_STACK_ERROR + /* Open dataset */ + if((dsid = H5Dopen2(fid, "dset", H5P_DEFAULT)) < 0) FAIL_STACK_ERROR - /* Delete dataset */ - if(H5Ldelete(fid, "dset", H5P_DEFAULT) < 0) FAIL_STACK_ERROR + /* Get the chunk index type */ + if(H5D_layout_idx_type_test(dsid, &idx_type) < 0) FAIL_STACK_ERROR - /* Close everything */ - if(H5Fclose(fid) < 0) FAIL_STACK_ERROR - } /* end for */ + /* Chunk index tyepe expected depends on whether we are using the latest version of the format */ + if(low == H5F_LIBVER_LATEST) { + /* Verify index type */ + if(idx_type != H5D_CHUNK_EARRAY) FAIL_PUTS_ERROR("should be using extensible array as index"); + } /* end if */ + else { + /* Verify index type */ + if(idx_type != H5D_CHUNK_BTREE) FAIL_PUTS_ERROR("should be using v1 B-tree as index"); + } /* end else */ + + /* Create scalar dataspace */ + if((scalar_sid = H5Screate(H5S_SCALAR)) < 0) FAIL_STACK_ERROR + + /* Get dataspace for dataset now */ + if((sid = H5Dget_space(dsid)) < 0) FAIL_STACK_ERROR + + /* Read elements */ + hs_size = 1; + for(u = 0; u < 1000; u++) { + /* Select a single element in the dataset */ + hs_offset = u; + if(H5Sselect_hyperslab(sid, H5S_SELECT_SET, &hs_offset, NULL, &hs_size, NULL) < 0) FAIL_STACK_ERROR + + /* Read (unwritten) element from dataset */ + read_elem = u + 1; + if(H5Dread(dsid, H5T_NATIVE_UINT, scalar_sid, sid, H5P_DEFAULT, &read_elem) < 0) FAIL_STACK_ERROR + + /* Verify unwritten element is fill value (0) */ + if(read_elem != (u % 100)) FAIL_PUTS_ERROR("invalid element read"); + } /* end for */ + + /* Close everything */ + if(H5Dclose(dsid) < 0) FAIL_STACK_ERROR + if(H5Sclose(sid) < 0) FAIL_STACK_ERROR + if(H5Sclose(scalar_sid) < 0) FAIL_STACK_ERROR + if(H5Fclose(fid) < 0) FAIL_STACK_ERROR + + /* Re-open file */ + if((fid = H5Fopen(filename, H5F_ACC_RDWR, my_fapl)) < 0) FAIL_STACK_ERROR + + /* Delete dataset */ + if(H5Ldelete(fid, "dset", H5P_DEFAULT) < 0) FAIL_STACK_ERROR + + /* Close everything */ + if(H5Fclose(fid) < 0) FAIL_STACK_ERROR + } /* end for */ #ifdef H5_HAVE_FILTER_DEFLATE - } /* end for */ + } /* end for */ #endif /* H5_HAVE_FILTER_DEFLATE */ + } /* end for */ + + /* Close FAPL copy */ + if(H5Pclose(my_fapl) < 0) FAIL_STACK_ERROR PASSED(); return 0; @@ -7129,6 +7149,7 @@ error: H5Sclose(sid); H5Sclose(scalar_sid); H5Fclose(fid); + H5Pclose(my_fapl); } H5E_END_TRY; return -1; } /* end test_chunk_fast() */ -- cgit v0.12