diff options
author | Vailin Choi <vchoi@hdfgroup.org> | 2015-05-01 19:51:47 (GMT) |
---|---|---|
committer | Vailin Choi <vchoi@hdfgroup.org> | 2015-05-01 19:51:47 (GMT) |
commit | a2aa6d4e6454375e804c63318dde42074da155e3 (patch) | |
tree | 3cbf15330b98dc0caaa6ec6f0f846b432e48df6b | |
parent | e0e28a7aae2fabf3ae9d03a1bd5809176935466a (diff) | |
download | hdf5-a2aa6d4e6454375e804c63318dde42074da155e3.zip hdf5-a2aa6d4e6454375e804c63318dde42074da155e3.tar.gz hdf5-a2aa6d4e6454375e804c63318dde42074da155e3.tar.bz2 |
[svn-r26992] Bring revisions #26785 - #26837 from trunk to revise_chunks.
h5committested.
-rw-r--r-- | CMakeLists.txt | 2 | ||||
-rw-r--r-- | config/cmake/ConfigureChecks.cmake | 39 | ||||
-rw-r--r-- | config/cmake/H5pubconf.h.in | 35 | ||||
-rw-r--r-- | fortran/src/H5Zf.c | 2 | ||||
-rw-r--r-- | release_docs/INSTALL_parallel | 9 | ||||
-rw-r--r-- | src/H5Dbtree.c | 30 | ||||
-rw-r--r-- | src/H5Dbtree2.c | 22 | ||||
-rw-r--r-- | src/H5Dchunk.c | 742 | ||||
-rw-r--r-- | src/H5Dearray.c | 20 | ||||
-rw-r--r-- | src/H5Dfarray.c | 20 | ||||
-rw-r--r-- | src/H5Dmpio.c | 4 | ||||
-rw-r--r-- | src/H5Dnone.c | 4 | ||||
-rw-r--r-- | src/H5Dpkg.h | 10 | ||||
-rw-r--r-- | src/H5Fio.c | 3 | ||||
-rw-r--r-- | src/H5Fprivate.h | 6 | ||||
-rw-r--r-- | src/H5Rpublic.h | 2 | ||||
-rw-r--r-- | src/libhdf5.settings.in | 2 |
17 files changed, 457 insertions, 495 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index a6994ef..a4135f4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -763,7 +763,7 @@ endif (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN" OR HDF5_ALLOW_EXTERNAL_SUPPORT #----------------------------------------------------------------------------- # Dashboard and Testing Settings #----------------------------------------------------------------------------- -option (BUILD_TESTING "Build HDF5 Unit Testing" OFF) +option (BUILD_TESTING "Build HDF5 Unit Testing" ON) if (BUILD_TESTING) set (DART_TESTING_TIMEOUT 1200 CACHE INTEGER diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index b193ff4..e223553 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -92,45 +92,6 @@ endif (WINDOWS) # ---------------------------------------------------------------------- CHECK_FUNCTION_EXISTS (difftime H5_HAVE_DIFFTIME) -#CHECK_FUNCTION_EXISTS (gettimeofday H5_HAVE_GETTIMEOFDAY) -# Since gettimeofday is not defined any where standard, lets look in all the -# usual places. On MSVC we are just going to use ::clock() -if (NOT MSVC) - if ("H5_HAVE_TIME_GETTIMEOFDAY" MATCHES "^H5_HAVE_TIME_GETTIMEOFDAY$") - TRY_COMPILE (HAVE_TIME_GETTIMEOFDAY - ${CMAKE_BINARY_DIR} - ${HDF_RESOURCES_EXT_DIR}/GetTimeOfDayTest.cpp - COMPILE_DEFINITIONS -DTRY_TIME_H - OUTPUT_VARIABLE OUTPUT - ) - if (HAVE_TIME_GETTIMEOFDAY STREQUAL "TRUE") - set (H5_HAVE_TIME_GETTIMEOFDAY "1" CACHE INTERNAL "H5_HAVE_TIME_GETTIMEOFDAY") - set (H5_HAVE_GETTIMEOFDAY "1" CACHE INTERNAL "H5_HAVE_GETTIMEOFDAY") - endif (HAVE_TIME_GETTIMEOFDAY STREQUAL "TRUE") - endif ("H5_HAVE_TIME_GETTIMEOFDAY" MATCHES "^H5_HAVE_TIME_GETTIMEOFDAY$") - - if ("H5_HAVE_SYS_TIME_GETTIMEOFDAY" MATCHES "^H5_HAVE_SYS_TIME_GETTIMEOFDAY$") - TRY_COMPILE (HAVE_SYS_TIME_GETTIMEOFDAY - ${CMAKE_BINARY_DIR} - ${HDF_RESOURCES_EXT_DIR}/GetTimeOfDayTest.cpp - COMPILE_DEFINITIONS -DTRY_SYS_TIME_H - OUTPUT_VARIABLE OUTPUT - ) - if (HAVE_SYS_TIME_GETTIMEOFDAY STREQUAL "TRUE") - set (H5_HAVE_SYS_TIME_GETTIMEOFDAY "1" CACHE INTERNAL "H5_HAVE_SYS_TIME_GETTIMEOFDAY") - set (H5_HAVE_GETTIMEOFDAY "1" CACHE INTERNAL "H5_HAVE_GETTIMEOFDAY") - endif (HAVE_SYS_TIME_GETTIMEOFDAY STREQUAL "TRUE") - endif ("H5_HAVE_SYS_TIME_GETTIMEOFDAY" MATCHES "^H5_HAVE_SYS_TIME_GETTIMEOFDAY$") - - if (NOT HAVE_SYS_TIME_GETTIMEOFDAY AND NOT H5_HAVE_GETTIMEOFDAY) - message (STATUS "---------------------------------------------------------------") - message (STATUS "Function 'gettimeofday()' was not found. HDF5 will use its") - message (STATUS " own implementation.. This can happen on older versions of") - message (STATUS " MinGW on Windows. Consider upgrading your MinGW installation") - message (STATUS " to a newer version such as MinGW 3.12") - message (STATUS "---------------------------------------------------------------") - endif (NOT HAVE_SYS_TIME_GETTIMEOFDAY AND NOT H5_HAVE_GETTIMEOFDAY) -endif (NOT MSVC) # Find the library containing clock_gettime() if (NOT WINDOWS) diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index c47b7c4..fcbdbd5 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -88,10 +88,6 @@ /* Define if Darwin or Mac OS X */ #cmakedefine H5_HAVE_DARWIN @H5_HAVE_DARWIN@ -/* Define to 1 if you have the declaration of `tzname', and to 0 if you don't. - */ -#cmakedefine H5_HAVE_DECL_TZNAME @H5_HAVE_DECL_TZNAME@ - /* Define to 1 if you have the `difftime' function. */ #cmakedefine H5_HAVE_DIFFTIME @H5_HAVE_DIFFTIME@ @@ -167,12 +163,6 @@ /* Define to 1 if you have the `gettimeofday' function. */ #cmakedefine H5_HAVE_GETTIMEOFDAY @H5_HAVE_GETTIMEOFDAY@ -/* Define to 1 if you have the `gettimeofday' function declared in time.h . */ -#cmakedefine H5_HAVE_TIME_GETTIMEOFDAY @H5_HAVE_TIME_GETTIMEOFDAY@ - -/* Define to 1 if you have the `gettimeofday' function declared in sys/time.h . */ -#cmakedefine H5_HAVE_SYS_TIME_GETTIMEOFDAY @H5_HAVE_SYS_TIME_GETTIMEOFDAY@ - /* Define to 1 if you have the `get_fpc_csr' function. */ #cmakedefine H5_HAVE_GET_FPC_CSR @H5_HAVE_GET_FPC_CSR@ @@ -201,15 +191,6 @@ /* Define to 1 if you have the `mpe' library (-lmpe). */ #cmakedefine H5_HAVE_LIBMPE @H5_HAVE_LIBMPE@ -/* Define to 1 if you have the `mpi' library (-lmpi). */ -#cmakedefine H5_HAVE_LIBMPI @H5_HAVE_LIBMPI@ - -/* Define to 1 if you have the `mpich' library (-lmpich). */ -#cmakedefine H5_HAVE_LIBMPICH @H5_HAVE_LIBMPICH@ - -/* Define to 1 if you have the `mpio' library (-lmpio). */ -#cmakedefine H5_HAVE_LIBMPIO @H5_HAVE_LIBMPIO@ - /* Define to 1 if you have the `nsl' library (-lnsl). */ #cmakedefine H5_HAVE_LIBNSL @H5_HAVE_LIBNSL@ @@ -279,9 +260,6 @@ /* Define to 1 if you have the `setsysinfo' function. */ #cmakedefine H5_HAVE_SETSYSINFO @H5_HAVE_SETSYSINFO@ -/* Define to 1 if you have the `sigaction' function. */ -#cmakedefine H5_HAVE_SIGACTION @H5_HAVE_SIGACTION@ - /* Define to 1 if you have the `siglongjmp' function. */ #cmakedefine H5_HAVE_SIGLONGJMP @H5_HAVE_SIGLONGJMP@ @@ -363,9 +341,6 @@ /* Define to 1 if you have the <sys/timeb.h> header file. */ #cmakedefine H5_HAVE_SYS_TIMEB_H @H5_HAVE_SYS_TIMEB_H@ -/* Define to 1 if you have the <time.h> header file. */ -#cmakedefine H5_HAVE_TIME_H @H5_HAVE_TIME_H@ - /* Define to 1 if you have the <sys/time.h> header file. */ #cmakedefine H5_HAVE_SYS_TIME_H @H5_HAVE_SYS_TIME_H@ @@ -396,9 +371,6 @@ /* Define to 1 if you have the <unistd.h> header file. */ #cmakedefine H5_HAVE_UNISTD_H @H5_HAVE_UNISTD_H@ -/* Define to 1 if you have the `asprintf' function. */ -#cmakedefine H5_HAVE_ASPRINTF @H5_HAVE_ASPRINTF@ - /* Define to 1 if you have the `vasprintf' function. */ #cmakedefine H5_HAVE_VASPRINTF @H5_HAVE_VASPRINTF@ @@ -620,9 +592,6 @@ /* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */ #cmakedefine H5_TIME_WITH_SYS_TIME @H5_TIME_WITH_SYS_TIME@ -/* Define to 1 if your <sys/time.h> declares `struct tm'. */ -#cmakedefine H5_TM_IN_SYS_TIME @H5_TM_IN_SYS_TIME@ - /* Define using v1.6 public API symbols by default */ #cmakedefine H5_USE_16_API_DEFAULT @H5_USE_16_API_DEFAULT@ @@ -681,8 +650,4 @@ /* Define to `long' if <sys/types.h> does not define. */ #cmakedefine H5_ssize_t -#if defined(__cplusplus) && defined(inline) -#undef inline -#endif - #endif diff --git a/fortran/src/H5Zf.c b/fortran/src/H5Zf.c index 0427c23..169e018 100644 --- a/fortran/src/H5Zf.c +++ b/fortran/src/H5Zf.c @@ -51,9 +51,7 @@ nh5zunregister_c (int_f *filter) * Call H5Zunregister function. */ c_filter = (H5Z_filter_t)*filter; - printf(" filter # %d \n", (int)c_filter); status = H5Zunregister(c_filter); - printf("From C zunregister %d \n", status); if (status < 0) return ret_value; ret_value = 0; return ret_value; diff --git a/release_docs/INSTALL_parallel b/release_docs/INSTALL_parallel index c38c3fc..e4570b7 100644 --- a/release_docs/INSTALL_parallel +++ b/release_docs/INSTALL_parallel @@ -49,9 +49,8 @@ more detailed explanations. ---------------------------- HDF5 knows several parallel compilers: mpicc, hcc, mpcc, mpcc_r. To build parallel HDF5 with one of the above, just set CC as it and configure. -The "--enable-parallel" is optional in this case. - $ CC=/usr/local/mpi/bin/mpicc ./configure --prefix=<install-directory> + $ CC=/usr/local/mpi/bin/mpicc ./configure --enable-parallel --prefix=<install-directory> $ make # build the library $ make check # verify the correctness # Read the Details section about parallel tests. @@ -79,7 +78,7 @@ is supported. Then do the following steps: - $ ./configure --disable-shared --prefix=<install-directory> + $ ./configure --enable-parallel --disable-shared --prefix=<install-directory> $ make # build the library $ make check # verify the correctness # Read the Details section about parallel tests. @@ -176,8 +175,8 @@ a properly installed parallel compiler (e.g., MPICH's mpicc or IBM's mpcc_r) and supply the compiler name as the value of the CC environment variable. For examples, - $ CC=mpcc_r ./configure - $ CC=/usr/local/mpi/bin/mpicc ./configure + $ CC=mpcc_r ./configure --enable-parallel + $ CC=/usr/local/mpi/bin/mpicc ./configure --enable-parallel If no such a compiler command is available then you must use your normal C compiler along with the location(s) of MPI/MPI-IO files to be used. diff --git a/src/H5Dbtree.c b/src/H5Dbtree.c index a3019c3..328795e 100644 --- a/src/H5Dbtree.c +++ b/src/H5Dbtree.c @@ -74,8 +74,8 @@ * The chunk's file address is part of the B-tree and not part of the key. */ typedef struct H5D_btree_key_t { - uint32_t nbytes; /*size of stored data */ hsize_t offset[H5O_LAYOUT_NDIMS]; /*logical offset to start*/ + uint32_t nbytes; /*size of stored data */ unsigned filter_mask; /*excluded filters */ } H5D_btree_key_t; @@ -277,15 +277,15 @@ H5D__btree_new_node(H5F_t *f, hid_t UNUSED dxpl_id, H5B_ins_t op, HDassert(addr_p); /* Set address */ - HDassert(udata->nbytes > 0); - HDassert(H5F_addr_defined(udata->addr)); - *addr_p = udata->addr; + HDassert(H5F_addr_defined(udata->chunk_block.offset)); + HDassert(udata->chunk_block.length > 0); + *addr_p = udata->chunk_block.offset; /* * The left key describes the storage of the UDATA chunk being * inserted into the tree. */ - lt_key->nbytes = udata->nbytes; + H5_ASSIGN_OVERFLOW(lt_key->nbytes, udata->chunk_block.length, hsize_t, uint32_t); lt_key->filter_mask = udata->filter_mask; for(u = 0; u < udata->common.layout->ndims; u++) lt_key->offset[u] = udata->common.offset[u]; @@ -466,8 +466,8 @@ H5D__btree_found(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, haddr_t addr, const void /* Initialize return values */ HDassert(lt_key->nbytes > 0); - udata->addr = addr; - udata->nbytes = lt_key->nbytes; + udata->chunk_block.offset = addr; + udata->chunk_block.length = lt_key->nbytes; udata->filter_mask = lt_key->filter_mask; done: @@ -545,17 +545,17 @@ H5D__btree_insert(H5F_t *f, hid_t UNUSED dxpl_id, haddr_t addr, void *_lt_key, * Already exists. If the new size is not the same as the old size * then we should reallocate storage. */ - if(lt_key->nbytes != udata->nbytes) { + if(lt_key->nbytes != udata->chunk_block.length) { /* Set node's address (already re-allocated by main chunk routines) */ - HDassert(H5F_addr_defined(udata->addr)); - *new_node_p = udata->addr; - lt_key->nbytes = udata->nbytes; + HDassert(H5F_addr_defined(udata->chunk_block.offset)); + *new_node_p = udata->chunk_block.offset; + H5_ASSIGN_OVERFLOW(lt_key->nbytes, udata->chunk_block.length, hsize_t, uint32_t); lt_key->filter_mask = udata->filter_mask; *lt_key_changed = TRUE; ret_value = H5B_INS_CHANGE; } else { /* Already have address in udata, from main chunk routines */ - HDassert(H5F_addr_defined(udata->addr)); + HDassert(H5F_addr_defined(udata->chunk_block.offset)); ret_value = H5B_INS_NOOP; } @@ -569,15 +569,15 @@ H5D__btree_insert(H5F_t *f, hid_t UNUSED dxpl_id, haddr_t addr, void *_lt_key, * Split this node, inserting the new new node to the right of the * current node. The MD_KEY is where the split occurs. */ - md_key->nbytes = udata->nbytes; + H5_ASSIGN_OVERFLOW(md_key->nbytes, udata->chunk_block.length, hsize_t, uint32_t); md_key->filter_mask = udata->filter_mask; for(u = 0; u < udata->common.layout->ndims; u++) { HDassert(0 == udata->common.offset[u] % udata->common.layout->dim[u]); md_key->offset[u] = udata->common.offset[u]; } /* end for */ - HDassert(H5F_addr_defined(udata->addr)); - *new_node_p = udata->addr; + HDassert(H5F_addr_defined(udata->chunk_block.offset)); + *new_node_p = udata->chunk_block.offset; ret_value = H5B_INS_RIGHT; } else { diff --git a/src/H5Dbtree2.c b/src/H5Dbtree2.c index e90aebd..fb79046 100644 --- a/src/H5Dbtree2.c +++ b/src/H5Dbtree2.c @@ -1035,8 +1035,7 @@ H5D_bt2_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat HDassert(idx_info->storage); HDassert(H5F_addr_defined(idx_info->storage->idx_addr)); HDassert(udata); - HDassert(udata->need_insert || udata->need_modify); - HDassert(H5F_addr_defined(udata->addr)); + HDassert(H5F_addr_defined(udata->chunk_block.offset)); /* Check if the v2 B-tree is open yet */ if(NULL == idx_info->storage->u.btree2.bt2) { @@ -1054,16 +1053,15 @@ H5D_bt2_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat if(idx_info->pline->nused > 0) { /* filtered chunk */ bt2_udata.rec = &filt_rec; - - filt_rec.addr = udata->addr; - filt_rec.nbytes = udata->nbytes; + filt_rec.addr = udata->chunk_block.offset; + filt_rec.nbytes = udata->chunk_block.length; filt_rec.filter_mask = udata->filter_mask; for(u = 0; u < (idx_info->layout->ndims - 1); u++) filt_rec.offset[u] = udata->common.offset[u]; } else { /* non-filtered chunk */ bt2_udata.rec = &rec; - rec.addr = udata->addr; + rec.addr = udata->chunk_block.offset; for(u = 0; u < (idx_info->layout->ndims - 1); u++) rec.offset[u] = udata->common.offset[u]; } @@ -1075,7 +1073,6 @@ H5D_bt2_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat if(H5B2_modify(bt2, idx_info->dxpl_id, &bt2_udata, H5D_bt2_mod_filt_cb, &filt_rec) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to modify record in v2 B-tree") } else { - HDassert(udata->need_insert); /* Insert record for object in v2 B-tree */ if(H5B2_insert(bt2, idx_info->dxpl_id, &bt2_udata) < 0) @@ -1205,8 +1202,8 @@ H5D_bt2_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) HGOTO_ERROR(H5E_HEAP, H5E_NOTFOUND, FAIL, "can't find object in v2 B-tree") /* Set info for the chunk */ - udata->addr = found_rec.addr; - udata->nbytes = found_rec.nbytes; + udata->chunk_block.offset = found_rec.addr; + udata->chunk_block.length = found_rec.nbytes; udata->filter_mask = found_rec.filter_mask; } /* end if */ else { /* non-filtered chunk */ @@ -1230,13 +1227,16 @@ H5D_bt2_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) if(H5B2_find(bt2, idx_info->dxpl_id, &bt2_udata, H5D_bt2_found_cb, &found_rec) < 0) HGOTO_ERROR(H5E_HEAP, H5E_NOTFOUND, FAIL, "can't find object in v2 B-tree") - udata->addr = found_rec.addr; + udata->chunk_block.offset = found_rec.addr; /* Update the other (constant) information for the chunk */ - udata->nbytes = idx_info->layout->size; + udata->chunk_block.length = idx_info->layout->size; udata->filter_mask = 0; } /* end else */ + if(!H5F_addr_defined(udata->chunk_block.offset)) + udata->chunk_block.length = 0; + done: FUNC_LEAVE_NOAPI(ret_value) } /* H5D_bt2_idx_get_addr() */ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index bdd9c9c..39e3e48 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -214,9 +214,9 @@ H5D__nonexistent_readvv(const H5D_io_info_t *io_info, /* Helper routines */ static herr_t H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims, const hsize_t *curr_dims, const hsize_t *max_dims); -static void *H5D__chunk_alloc(size_t size, const H5O_pline_t *pline); -static void *H5D__chunk_xfree(void *chk, const H5O_pline_t *pline); -static void *H5D__chunk_realloc(void *chk, size_t size, +static void *H5D__chunk_mem_alloc(size_t size, const H5O_pline_t *pline); +static void *H5D__chunk_mem_xfree(void *chk, const H5O_pline_t *pline); +static void *H5D__chunk_mem_realloc(void *chk, size_t size, const H5O_pline_t *pline); static herr_t H5D__chunk_cinfo_cache_reset(H5D_chunk_cached_t *last); static herr_t H5D__chunk_cinfo_cache_update(H5D_chunk_cached_t *last, @@ -241,7 +241,9 @@ static hbool_t H5D__chunk_is_partial_edge_chunk(const hsize_t *chunk_offset, static herr_t H5D__chunk_cache_prune(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t *dxpl_cache, size_t size); static herr_t H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, hbool_t new_unfilt_chunk); -static herr_t H5D__idx_chunk_alloc(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata); +static herr_t H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, + const H5F_block_t *old_chunk, H5F_block_t *new_chunk, hbool_t *need_insert, + hbool_t *need_bt2_modify, hsize_t scaled[]); #ifdef H5_HAVE_PARALLEL static herr_t H5D__chunk_collective_fill(const H5D_t *dset, hid_t dxpl_id, H5D_chunk_coll_info_t *chunk_info, size_t chunk_size, const void *fill_buf); @@ -329,12 +331,14 @@ H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, uint32_t filters, hsiz const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache */ H5D_chk_idx_info_t idx_info; /* Chunked index info */ hsize_t scaled[H5S_MAX_RANK]; /* Scaled coordinates for this chunk */ + H5F_block_t old_chunk; /* Offset/length of old chunk */ + hbool_t need_insert = FALSE; /* Whether the chunk needs to be inserted into the index */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC_TAG(dxpl_id, dset->oloc.addr, FAIL) /* Allocate dataspace and initialize it if it hasn't been. */ - if(!(*dset->shared->layout.ops->is_space_alloc)(&dset->shared->layout.storage)) + if(!(*layout->ops->is_space_alloc)(&layout->storage)) /* Allocate storage */ if(H5D__alloc_storage(dset, dxpl_id, H5D_ALLOC_WRITE, FALSE, NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize storage") @@ -346,52 +350,65 @@ H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, uint32_t filters, hsiz if(H5D__chunk_lookup(dset, dxpl_id, offset, scaled, &udata) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address") - udata.filter_mask = filters; - - /* Check if the chunk needs to be 'inserted' (could exist already and - * the 'insert' operation could resize it) - */ - { + /* Sanity check */ + HDassert((H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length > 0) || + (!H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length == 0)); - /* Compose chunked index info struct */ - idx_info.f = dset->oloc.file; - idx_info.dxpl_id = dxpl_id; - idx_info.pline = &(dset->shared->dcpl_cache.pline); - idx_info.layout = &(dset->shared->layout.u.chunk); - idx_info.storage = &(dset->shared->layout.storage.u.chunk); + /* Set the file block information for the old chunk */ + /* (Which is only defined when overwriting an existing chunk) */ + old_chunk.offset = udata.chunk_block.offset; + old_chunk.length = udata.chunk_block.length; - /* Set up the size of chunk for user data */ - udata.nbytes = data_size; + /* Check if the chunk needs to be inserted (it also could exist already + * and the chunk allocate operation could resize it) + */ + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.dxpl_id = dxpl_id; + idx_info.pline = &(dset->shared->dcpl_cache.pline); + idx_info.layout = &(dset->shared->layout.u.chunk); + idx_info.storage = &(dset->shared->layout.storage.u.chunk); - /* Create the chunk it if it doesn't exist, or reallocate the chunk - * if its size changed. - */ - if(H5D__idx_chunk_alloc(&idx_info, &udata) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk on chunk level") + /* Set up the size of chunk for user data */ + udata.chunk_block.length = data_size; - /* Make sure the address of the chunk is returned. */ - if(!H5F_addr_defined(udata.addr)) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "chunk address isn't defined") - } /* end if */ + /* Create the chunk it if it doesn't exist, or reallocate the chunk + * if its size changed. + */ + if(H5D__chunk_file_alloc(&idx_info, &old_chunk, &udata.chunk_block, &need_insert, &udata.need_modify, scaled) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk") - /* Fill the DXPL cache values for later use */ - if(H5D__get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") + /* Make sure the address of the chunk is returned. */ + if(!H5F_addr_defined(udata.chunk_block.offset)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "chunk address isn't defined") /* Evict the (old) entry from the cache if present, but do not flush * it to disk */ - if(UINT_MAX != udata.idx_hint) + if(UINT_MAX != udata.idx_hint) { + H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ + H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ + const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache */ + + /* Fill the DXPL cache values for later use */ + if(H5D__get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") + if(H5D__chunk_cache_evict(dset, dxpl_id, dxpl_cache, rdcc->slot[udata.idx_hint], FALSE) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTREMOVE, FAIL, "unable to evict chunk") + } /* end if */ /* Write the data to the file */ - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, data_size, dxpl_id, buf) < 0) + if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.chunk_block.offset, data_size, dxpl_id, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write raw data to file") - if((udata.need_insert || udata.need_modify) && dset->shared->layout.storage.u.chunk.ops->insert_addr) { - if((dset->shared->layout.storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0) + /* Insert the chunk record into the index */ + if((need_insert || udata.need_modify) && layout->storage.u.chunk.ops->insert_addr) { + /* Set the chunk's filter mask to the new settings */ + udata.filter_mask = filters; + + if((layout->storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index") - } + } /* end if */ done: FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) @@ -984,7 +1001,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5D__chunk_alloc + * Function: H5D__chunk_mem_alloc * * Purpose: Allocate space for a chunk in memory. This routine allocates * memory space for non-filtered chunks from a block free list @@ -998,7 +1015,7 @@ done: *------------------------------------------------------------------------- */ static void * -H5D__chunk_alloc(size_t size, const H5O_pline_t *pline) +H5D__chunk_mem_alloc(size_t size, const H5O_pline_t *pline) { void *ret_value = NULL; /* Return value */ @@ -1012,11 +1029,11 @@ H5D__chunk_alloc(size_t size, const H5O_pline_t *pline) ret_value = H5FL_BLK_MALLOC(chunk, size); FUNC_LEAVE_NOAPI(ret_value) -} /* H5D__chunk_alloc() */ +} /* H5D__chunk_mem_alloc() */ /*------------------------------------------------------------------------- - * Function: H5D__chunk_xfree + * Function: H5D__chunk_mem_xfree * * Purpose: Free space for a chunk in memory. This routine allocates * memory space for non-filtered chunks from a block free list @@ -1030,7 +1047,7 @@ H5D__chunk_alloc(size_t size, const H5O_pline_t *pline) *------------------------------------------------------------------------- */ static void * -H5D__chunk_xfree(void *chk, const H5O_pline_t *pline) +H5D__chunk_mem_xfree(void *chk, const H5O_pline_t *pline) { FUNC_ENTER_STATIC_NOERR @@ -1042,11 +1059,11 @@ H5D__chunk_xfree(void *chk, const H5O_pline_t *pline) } /* end if */ FUNC_LEAVE_NOAPI(NULL) -} /* H5D__chunk_xfree() */ +} /* H5D__chunk_mem_xfree() */ /*------------------------------------------------------------------------- - * Function: H5D__chunk_realloc + * Function: H5D__chunk_mem_realloc * * Purpose: Reallocate space for a chunk in memory. This routine allocates * memory space for non-filtered chunks from a block free list @@ -1060,7 +1077,7 @@ H5D__chunk_xfree(void *chk, const H5O_pline_t *pline) *------------------------------------------------------------------------- */ static void * -H5D__chunk_realloc(void *chk, size_t size, const H5O_pline_t *pline) +H5D__chunk_mem_realloc(void *chk, size_t size, const H5O_pline_t *pline) { void *ret_value = NULL; /* Return value */ @@ -1075,7 +1092,7 @@ H5D__chunk_realloc(void *chk, size_t size, const H5O_pline_t *pline) ret_value = H5FL_BLK_REALLOC(chunk, chk, size); FUNC_LEAVE_NOAPI(ret_value) -} /* H5D__chunk_realloc() */ +} /* H5D__chunk_mem_realloc() */ /*-------------------------------------------------------------------------- @@ -1860,10 +1877,7 @@ H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, chunk_node = H5D_CHUNK_GET_FIRST_NODE(fm); while(chunk_node) { H5D_chunk_info_t *chunk_info; /* Chunk information */ - H5D_io_info_t *chk_io_info; /* Pointer to I/O info object for this chunk */ - void *chunk; /* Pointer to locked chunk buffer */ - H5D_chunk_ud_t udata; /* B-tree pass-through */ - htri_t cacheable; /* Whether the chunk is cacheable */ + H5D_chunk_ud_t udata; /* Chunk index pass-through */ /* Get the actual chunk information from the skip list node */ chunk_info = H5D_CHUNK_GET_NODE_INFO(fm, chunk_node); @@ -1872,18 +1886,26 @@ H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, if(H5D__chunk_lookup(io_info->dset, io_info->dxpl_id, chunk_info->coords, chunk_info->scaled, &udata) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address") + /* Sanity check */ + HDassert((H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length > 0) || + (!H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length == 0)); + /* Check for non-existant chunk & skip it if appropriate */ - if(H5F_addr_defined(udata.addr) || UINT_MAX != udata.idx_hint + if(H5F_addr_defined(udata.chunk_block.offset) || UINT_MAX != udata.idx_hint || !skip_missing_chunks) { - /* Pass in chunk's coordinates. */ - io_info->store->chunk.offset = chunk_info->coords; + H5D_io_info_t *chk_io_info; /* Pointer to I/O info object for this chunk */ + void *chunk = NULL; /* Pointer to locked chunk buffer */ + htri_t cacheable; /* Whether the chunk is cacheable */ + + /* Pass in chunk's coordinates in a union. */ + io_info->store->chunk.offset = chunk_info->coords; + io_info->store->chunk.index = chunk_info->index; - /* Determine if we should use the chunk cache */ - if((cacheable = H5D__chunk_cacheable(io_info, udata.addr, FALSE)) < 0) + /* Load the chunk into cache and lock it. */ + if((cacheable = H5D__chunk_cacheable(io_info, udata.chunk_block.offset, FALSE)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't tell if chunk is cacheable") if(cacheable) { - /* Load the chunk into cache and lock it. */ /* Compute # of bytes accessed in chunk */ H5_CHECK_OVERFLOW(type_info->src_type_size, /*From:*/ size_t, /*To:*/ uint32_t); @@ -1899,20 +1921,14 @@ H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, /* Point I/O info at contiguous I/O info for this chunk */ chk_io_info = &cpt_io_info; } /* end if */ - else if(H5F_addr_defined(udata.addr)) { + else if(H5F_addr_defined(udata.chunk_block.offset)) { /* Set up the storage address information for this chunk */ - ctg_store.contig.dset_addr = udata.addr; - - /* No chunk cached */ - chunk = NULL; + ctg_store.contig.dset_addr = udata.chunk_block.offset; /* Point I/O info at temporary I/O info for this chunk */ chk_io_info = &ctg_io_info; } /* end else if */ else { - /* No chunk cached */ - chunk = NULL; - /* Point I/O info at "nonexistent" I/O info for this chunk */ chk_io_info = &nonexistent_io_info; } /* end else */ @@ -1990,11 +2006,12 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, chunk_node = H5D_CHUNK_GET_FIRST_NODE(fm); while(chunk_node) { H5D_chunk_info_t *chunk_info; /* Chunk information */ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ H5D_io_info_t *chk_io_info; /* Pointer to I/O info object for this chunk */ void *chunk; /* Pointer to locked chunk buffer */ H5D_chunk_ud_t udata; /* Index pass-through */ htri_t cacheable; /* Whether the chunk is cacheable */ - H5D_chk_idx_info_t idx_info; /* Chunked index info */ + hbool_t need_insert = FALSE; /* Whether the chunk needs to be inserted into the index */ /* Get the actual chunk information from the skip list node */ chunk_info = H5D_CHUNK_GET_NODE_INFO(fm, chunk_node); @@ -2003,13 +2020,16 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, if(H5D__chunk_lookup(io_info->dset, io_info->dxpl_id, chunk_info->coords, chunk_info->scaled, &udata) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address") - /* Pass in chunk's coordinates. */ - io_info->store->chunk.offset = chunk_info->coords; + /* Sanity check */ + HDassert((H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length > 0) || + (!H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length == 0)); - /* Determine if we should use the chunk cache */ - if((cacheable = H5D__chunk_cacheable(io_info, udata.addr, TRUE)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't tell if chunk is cacheable") + /* Pass in chunk's coordinates in a union. */ + io_info->store->chunk.offset = &chunk_info->coords[0]; + io_info->store->chunk.index = chunk_info->index; + if((cacheable = H5D__chunk_cacheable(io_info, udata.chunk_block.offset, TRUE)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't tell if chunk is cacheable") if(cacheable) { /* Load the chunk into cache. But if the whole chunk is written, * simply allocate space instead of load the chunk. */ @@ -2036,8 +2056,7 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, } /* end if */ else { /* If the chunk hasn't been allocated on disk, do so now. */ - if(!H5F_addr_defined(udata.addr)) { - + if(!H5F_addr_defined(udata.chunk_block.offset)) { /* Compose chunked index info struct */ idx_info.f = io_info->dset->oloc.file; idx_info.dxpl_id = io_info->dxpl_id; @@ -2046,15 +2065,14 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, idx_info.storage = &(io_info->dset->shared->layout.storage.u.chunk); /* Set up the size of chunk for user data */ - udata.nbytes = io_info->dset->shared->layout.u.chunk.size; + udata.chunk_block.length = io_info->dset->shared->layout.u.chunk.size; - /* Create the chunk */ - if(H5D__idx_chunk_alloc(&idx_info, &udata) < 0) + /* Allocate the chunk */ + if(H5D__chunk_file_alloc(&idx_info, NULL, &udata.chunk_block, &need_insert, &udata.need_modify, chunk_info->scaled) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk on chunk level") - /* Make sure the address of the chunk is returned. */ - if(!H5F_addr_defined(udata.addr)) + if(!H5F_addr_defined(udata.chunk_block.offset)) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "chunk address isn't defined") /* Cache the new chunk information */ @@ -2062,7 +2080,7 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, } /* end if */ /* Set up the storage address information for this chunk */ - ctg_store.contig.dset_addr = udata.addr; + ctg_store.contig.dset_addr = udata.chunk_block.offset; /* No chunk cached */ chunk = NULL; @@ -2078,11 +2096,11 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, /* Release the cache lock on the chunk, or insert chunk into index. */ if(chunk) { - if(chunk && H5D__chunk_unlock(io_info, &udata, TRUE, chunk, dst_accessed_bytes) < 0) + if(H5D__chunk_unlock(io_info, &udata, TRUE, chunk, dst_accessed_bytes) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to unlock raw data chunk") } /* end if */ else { - if((udata.need_insert || udata.need_modify) && io_info->dset->shared->layout.storage.u.chunk.ops->insert_addr) + if((need_insert || udata.need_modify) && io_info->dset->shared->layout.storage.u.chunk.ops->insert_addr) if((io_info->dset->shared->layout.storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index") } /* end else */ @@ -2292,8 +2310,8 @@ H5D__chunk_cinfo_cache_update(H5D_chunk_cached_t *last, const H5D_chunk_ud_t *ud /* Stored the information to cache */ HDmemcpy(last->offset, udata->common.offset, sizeof(hsize_t) * udata->common.layout->ndims); - last->addr = udata->addr; - last->nbytes = udata->nbytes; + last->addr = udata->chunk_block.offset; + H5_ASSIGN_OVERFLOW(last->nbytes, udata->chunk_block.length, hsize_t, uint32_t); last->chunk_idx = udata->chunk_idx; last->filter_mask = udata->filter_mask; @@ -2340,8 +2358,8 @@ H5D__chunk_cinfo_cache_found(const H5D_chunk_cached_t *last, H5D_chunk_ud_t *uda HGOTO_DONE(FALSE) /* Retrieve the information from the cache */ - udata->addr = last->addr; - udata->nbytes = last->nbytes; + udata->chunk_block.offset = last->addr; + udata->chunk_block.length = last->nbytes; udata->chunk_idx = last->chunk_idx; udata->filter_mask = last->filter_mask; @@ -2511,11 +2529,10 @@ H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id, const hsize_t *chunk_offset, udata->common.scaled = scaled; /* Reset information about the chunk we are looking for */ - udata->addr = HADDR_UNDEF; - udata->nbytes = 0; + udata->chunk_block.offset = HADDR_UNDEF; + udata->chunk_block.length = 0; udata->filter_mask = 0; udata->new_unfilt_chunk = FALSE; - udata->need_insert = FALSE; udata->need_modify = FALSE; /* Check for chunk in cache */ @@ -2533,9 +2550,11 @@ H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id, const hsize_t *chunk_offset, /* Find chunk addr */ if(found) { - udata->addr = ent->chunk_addr; + udata->chunk_block.offset = ent->chunk_block.offset; + udata->chunk_block.length = ent->chunk_block.length;; udata->chunk_idx = ent->chunk_idx; - } else { + } /* end if */ + else { /* Invalidate idx_hint, to signal that the chunk is not in cache */ udata->idx_hint = UINT_MAX; @@ -2607,9 +2626,10 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t buf = ent->chunk; if(ent->dirty && !ent->deleted) { + H5D_chk_idx_info_t idx_info; /* Chunked index info */ H5D_chunk_ud_t udata; /* pass through B-tree */ - hbool_t must_insert = FALSE; /* Whether the chunk must go through the "insert" method */ - H5D_chk_idx_info_t idx_info; /* Chunked index info */ + hbool_t must_alloc = FALSE; /* Whether the chunk must be allocated */ + hbool_t need_insert = FALSE; /* Whether the chunk needs to be inserted into the index */ /* Set up user data for index callbacks */ udata.common.layout = &dset->shared->layout.u.chunk; @@ -2617,18 +2637,16 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t udata.common.offset = ent->offset; udata.common.rdcc = &(dset->shared->cache.chunk); udata.common.scaled = ent->scaled; - + udata.chunk_block.offset = ent->chunk_block.offset; + udata.chunk_block.length = dset->shared->layout.u.chunk.size; udata.filter_mask = 0; - udata.nbytes = dset->shared->layout.u.chunk.size; - udata.addr = ent->chunk_addr; udata.chunk_idx = ent->chunk_idx; - udata.need_insert = FALSE; udata.need_modify = FALSE; /* Should the chunk be filtered before writing it to disk? */ if(dset->shared->dcpl_cache.pline.nused && !(ent->edge_chunk_state & H5D_RDCC_DISABLE_FILTERS)) { - size_t alloc = udata.nbytes; /* Bytes allocated for BUF */ + size_t alloc = udata.chunk_block.length; /* Bytes allocated for BUF */ size_t nbytes; /* Chunk size (in bytes) */ if(!reset) { @@ -2637,10 +2655,9 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t * the pipeline because we'll want to save the original buffer * for later. */ - H5_ASSIGN_OVERFLOW(alloc, udata.nbytes, uint32_t, size_t); if(NULL == (buf = H5MM_malloc(alloc))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline") - HDmemcpy(buf, ent->chunk, udata.nbytes); + HDmemcpy(buf, ent->chunk, alloc); } /* end if */ else { /* @@ -2653,7 +2670,7 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t point_of_no_return = TRUE; ent->chunk = NULL; } /* end else */ - H5_ASSIGN_OVERFLOW(nbytes, udata.nbytes, uint32_t, size_t); + H5_ASSIGN_OVERFLOW(nbytes, udata.chunk_block.length, uint32_t, size_t); if(H5Z_pipeline(&(dset->shared->dcpl_cache.pline), 0, &(udata.filter_mask), dxpl_cache->err_detect, dxpl_cache->filter_cb, &nbytes, &alloc, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "output pipeline failed") @@ -2662,14 +2679,14 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t if(nbytes > ((size_t)0xffffffff)) HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk too large for 32-bit length") #endif /* H5_SIZEOF_SIZE_T > 4 */ - H5_ASSIGN_OVERFLOW(udata.nbytes, nbytes, size_t, uint32_t); + H5_ASSIGN_OVERFLOW(udata.chunk_block.length, nbytes, size_t, uint32_t); - /* Indicate that the chunk must go through 'insert' method */ - must_insert = TRUE; + /* Indicate that the chunk must be allocated */ + must_alloc = TRUE; } /* end if */ - else if(!H5F_addr_defined(udata.addr)) { - /* Indicate that the chunk must go through 'insert' method */ - must_insert = TRUE; + else if(!H5F_addr_defined(udata.chunk_block.offset)) { + /* Indicate that the chunk must be allocated */ + must_alloc = TRUE; /* This flag could be set for this chunk, just remove and ignore it */ @@ -2678,7 +2695,7 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t else if(ent->edge_chunk_state & H5D_RDCC_NEWLY_DISABLED_FILTERS) { /* Chunk on disk is still filtered, must insert to allocate correct * size */ - must_insert = TRUE; + must_alloc = TRUE; /* Set the disable filters field back to the standard disable * filters setting, as it no longer needs to be inserted with every @@ -2688,11 +2705,10 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t HDassert(!(ent->edge_chunk_state & H5D_RDCC_NEWLY_DISABLED_FILTERS)); - /* Check if the chunk needs to be 'inserted' (could exist already and - * the 'insert' operation could resize it) + /* Check if the chunk needs to be allocated (it also could exist already + * and the chunk alloc operation could resize it) */ - if(must_insert) { - + if(must_alloc) { /* Compose chunked index info struct */ idx_info.f = dset->oloc.file; idx_info.dxpl_id = dxpl_id; @@ -2703,24 +2719,23 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t /* Create the chunk it if it doesn't exist, or reallocate the chunk * if its size changed. */ - if(H5D__idx_chunk_alloc(&idx_info, &udata) < 0) + if(H5D__chunk_file_alloc(&idx_info, &(ent->chunk_block), &udata.chunk_block, &need_insert, &udata.need_modify, ent->scaled) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk on chunk level") - /* Update the chunk entry's address, in case it was allocated or relocated */ - ent->chunk_addr = udata.addr; + /* Update the chunk entry's info, in case it was allocated or relocated */ + ent->chunk_block.offset = udata.chunk_block.offset; + ent->chunk_block.length = udata.chunk_block.length; } /* end if */ /* Write the data to the file */ - HDassert(H5F_addr_defined(udata.addr)); - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.nbytes, dxpl_id, buf) < 0) + HDassert(H5F_addr_defined(udata.chunk_block.offset)); + if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.chunk_block.offset, udata.chunk_block.length, dxpl_id, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write raw data to file") - if(udata.need_insert || udata.need_modify) { - if(dset->shared->layout.storage.u.chunk.ops->insert_addr) { - if((dset->shared->layout.storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index") - } - } + /* Insert the chunk record into the index */ + if((need_insert || udata.need_modify) && dset->shared->layout.storage.u.chunk.ops->insert_addr) + if((dset->shared->layout.storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index") /* Cache the chunk's info, in case it's accessed again shortly */ H5D__chunk_cinfo_cache_update(&dset->shared->cache.chunk.last, &udata); @@ -2738,7 +2753,7 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t if(buf == ent->chunk) buf = NULL; if(ent->chunk != NULL) - ent->chunk = (uint8_t *)H5D__chunk_xfree(ent->chunk, + ent->chunk = (uint8_t *)H5D__chunk_mem_xfree(ent->chunk, ((ent->edge_chunk_state & H5D_RDCC_DISABLE_FILTERS) ? NULL : &(dset->shared->dcpl_cache.pline))); } /* end if */ @@ -2756,7 +2771,7 @@ done: */ if(ret_value < 0 && point_of_no_return) if(ent->chunk) - ent->chunk = (uint8_t *)H5D__chunk_xfree(ent->chunk, + ent->chunk = (uint8_t *)H5D__chunk_mem_xfree(ent->chunk, ((ent->edge_chunk_state & H5D_RDCC_DISABLE_FILTERS) ? NULL : &(dset->shared->dcpl_cache.pline))); @@ -2800,7 +2815,7 @@ H5D__chunk_cache_evict(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t else { /* Don't flush, just free chunk */ if(ent->chunk != NULL) - ent->chunk = (uint8_t *)H5D__chunk_xfree(ent->chunk, + ent->chunk = (uint8_t *)H5D__chunk_mem_xfree(ent->chunk, ((ent->edge_chunk_state & H5D_RDCC_DISABLE_FILTERS) ? NULL : &(dset->shared->dcpl_cache.pline))); } /* end else */ @@ -2984,15 +2999,14 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, hbool_t relax, hbool_t prev_unfilt_chunk) { const H5D_t *dset = io_info->dset; /* Local pointer to the dataset info */ - const H5O_pline_t *pline = &(dset->shared->dcpl_cache.pline); /* I/O pipeline info - always equal to the pline passed to H5D__chunk_alloc */ + const H5O_pline_t *pline = &(dset->shared->dcpl_cache.pline); /* I/O pipeline info - always equal to the pline passed to H5D__chunk_mem_alloc */ const H5O_pline_t *old_pline = pline; /* Old pipeline, i.e. pipeline used to read the chunk */ const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ const H5O_fill_t *fill = &(dset->shared->dcpl_cache.fill); /* Fill value info */ H5D_fill_buf_info_t fb_info; /* Dataset's fill buffer info */ hbool_t fb_info_init = FALSE; /* Whether the fill value buffer has been initialized */ H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache*/ - H5D_rdcc_ent_t *ent = NULL; /*cache entry */ - haddr_t chunk_addr = HADDR_UNDEF; /* Address of chunk on disk */ + H5D_rdcc_ent_t *ent; /*cache entry */ size_t chunk_size; /*size of a chunk */ hbool_t disable_filters = FALSE; /* Whether to disable filters (when adding to cache) */ void *chunk = NULL; /*the file chunk */ @@ -3055,12 +3069,12 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, * code is changed in an inappropriate/incomplete way. */ pline = NULL; - /* Reallocate the chunk so H5D__chunk_xfree doesn't get confused + /* Reallocate the chunk so H5D__chunk_mem_xfree doesn't get confused */ - if(NULL == (chunk = H5D__chunk_alloc(chunk_size, pline))) + if(NULL == (chunk = H5D__chunk_mem_alloc(chunk_size, pline))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk") HDmemcpy(chunk, ent->chunk, chunk_size); - ent->chunk = (uint8_t *)H5D__chunk_xfree(ent->chunk, old_pline); + ent->chunk = (uint8_t *)H5D__chunk_mem_xfree(ent->chunk, old_pline); ent->chunk = (uint8_t *)chunk; chunk = NULL; @@ -3081,13 +3095,13 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, /* Mark the old pipeline as having been disabled */ old_pline = NULL; - /* Reallocate the chunk so H5D__chunk_xfree doesn't get confused + /* Reallocate the chunk so H5D__chunk_mem_xfree doesn't get confused */ - if(NULL == (chunk = H5D__chunk_alloc(chunk_size, pline))) + if(NULL == (chunk = H5D__chunk_mem_alloc(chunk_size, pline))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk") HDmemcpy(chunk, ent->chunk, chunk_size); - ent->chunk = (uint8_t *)H5D__chunk_xfree(ent->chunk, old_pline); + ent->chunk = (uint8_t *)H5D__chunk_mem_xfree(ent->chunk, old_pline); ent->chunk = (uint8_t *)chunk; chunk = NULL; @@ -3096,8 +3110,35 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, | H5D_RDCC_NEWLY_DISABLED_FILTERS); } /* end else */ } /* end if */ + + /* + * If the chunk is not at the beginning of the cache; move it backward + * by one slot. This is how we implement the LRU preemption + * algorithm. + */ + if(ent->next) { + if(ent->next->next) + ent->next->next->prev = ent; + else + rdcc->tail = ent; + ent->next->prev = ent->prev; + if(ent->prev) + ent->prev->next = ent->next; + else + rdcc->head = ent->next; + ent->prev = ent->next; + ent->next = ent->next->next; + ent->prev->next = ent; + } /* end if */ } /* end if */ else { + haddr_t chunk_addr; /* Address of chunk on disk */ + hsize_t chunk_alloc; /* Length of chunk on disk */ + + /* Save the chunk info so the cache stays consistent */ + chunk_addr = udata->chunk_block.offset; + chunk_alloc = udata->chunk_block.length; + /* Check if we should disable filters on this chunk */ if(pline->nused) { if(udata->new_unfilt_chunk) { @@ -3135,21 +3176,18 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, if(relax) { /* - * Not in the cache, but we're about to overwrite the whole thing - * anyway, so just allocate a buffer for it but don't initialize that - * buffer with the file contents. Count this as a hit instead of a - * miss because we saved ourselves lots of work. - */ + * Not in the cache, but we're about to overwrite the whole thing + * anyway, so just allocate a buffer for it but don't initialize that + * buffer with the file contents. Count this as a hit instead of a + * miss because we saved ourselves lots of work. + */ rdcc->stats.nhits++; - /* Still save the chunk address so the cache stays consistent */ - chunk_addr = udata->addr; - - if(NULL == (chunk = H5D__chunk_alloc(chunk_size, pline))) + if(NULL == (chunk = H5D__chunk_mem_alloc(chunk_size, pline))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk") /* In the case that some dataset functions look through this data, - * clear it to all 0s. */ + * clear it to all 0s. */ HDmemset(chunk, 0, chunk_size); } /* end if */ else { @@ -3158,41 +3196,36 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, * or an init if it isn't. */ - /* Save the chunk address */ - chunk_addr = udata->addr; - /* Check if the chunk exists on disk */ if(H5F_addr_defined(chunk_addr)) { - size_t chunk_alloc = 0; /*allocated chunk size */ + size_t my_chunk_alloc = chunk_alloc; /* Allocated buffer size */ + size_t buf_alloc = chunk_alloc; /* [Re-]allocated buffer size */ /* Chunk size on disk isn't [likely] the same size as the final chunk * size in memory, so allocate memory big enough. */ - H5_ASSIGN_OVERFLOW(chunk_alloc, udata->nbytes, uint32_t, size_t); - if(NULL == (chunk = H5D__chunk_alloc(chunk_alloc, - (udata->new_unfilt_chunk ? old_pline : pline)))) + if(NULL == (chunk = H5D__chunk_mem_alloc(my_chunk_alloc, (udata->new_unfilt_chunk ? old_pline : pline)))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk") - if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, chunk_addr, chunk_alloc, io_info->dxpl_id, chunk) < 0) + if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, chunk_addr, my_chunk_alloc, io_info->dxpl_id, chunk) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, NULL, "unable to read raw data chunk") - if(old_pline && old_pline->nused) { + if(old_pline && old_pline->nused) { if(H5Z_pipeline(old_pline, H5Z_FLAG_REVERSE, &(udata->filter_mask), io_info->dxpl_cache->err_detect, io_info->dxpl_cache->filter_cb, - &chunk_alloc, &chunk_alloc, &chunk) < 0) + &my_chunk_alloc, &buf_alloc, &chunk) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, NULL, "data pipeline read failed") - H5_ASSIGN_OVERFLOW(udata->nbytes, chunk_alloc, size_t, uint32_t); /* Reallocate chunk if necessary */ if(udata->new_unfilt_chunk) { void *tmp_chunk = chunk; - if(NULL == (chunk = H5D__chunk_alloc(chunk_alloc, pline))) { - (void)H5D__chunk_xfree(tmp_chunk, old_pline); + if(NULL == (chunk = H5D__chunk_mem_alloc(my_chunk_alloc, pline))) { + (void)H5D__chunk_mem_xfree(tmp_chunk, old_pline); HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk") } /* end if */ HDmemcpy(chunk, tmp_chunk, chunk_size); - (void)H5D__chunk_xfree(tmp_chunk, old_pline); + (void)H5D__chunk_mem_xfree(tmp_chunk, old_pline); } /* end if */ } /* end if */ @@ -3202,21 +3235,21 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, else { H5D_fill_value_t fill_status; - /* Sanity check */ - HDassert(fill->alloc_time != H5D_ALLOC_TIME_EARLY); + /* Sanity check */ + HDassert(fill->alloc_time != H5D_ALLOC_TIME_EARLY); /* Chunk size on disk isn't [likely] the same size as the final chunk * size in memory, so allocate memory big enough. */ - if(NULL == (chunk = H5D__chunk_alloc(chunk_size, pline))) + if(NULL == (chunk = H5D__chunk_mem_alloc(chunk_size, pline))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk") if(H5P_is_fill_value_defined(fill, &fill_status) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't tell if fill value defined") if(fill->fill_time == H5D_FILL_TIME_ALLOC || - (fill->fill_time == H5D_FILL_TIME_IFSET && - (fill_status == H5D_FILL_VALUE_USER_DEFINED || - fill_status == H5D_FILL_VALUE_DEFAULT))) { + (fill->fill_time == H5D_FILL_TIME_IFSET && + (fill_status == H5D_FILL_VALUE_USER_DEFINED || + fill_status == H5D_FILL_VALUE_DEFAULT))) { /* * The chunk doesn't exist in the file. Replicate the fill * value throughout the chunk, if the fill value is defined. @@ -3243,105 +3276,85 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata, rdcc->stats.ninits++; } /* end else */ } /* end else */ - } /* end else */ - HDassert(chunk_size > 0); - - if(ent) { - /* - * The chunk is not at the beginning of the cache; move it backward - * by one slot. This is how we implement the LRU preemption - * algorithm. - */ - HDassert(ent); - if(ent->next) { - if(ent->next->next) - ent->next->next->prev = ent; - else - rdcc->tail = ent; - ent->next->prev = ent->prev; - if(ent->prev) - ent->prev->next = ent->next; - else - rdcc->head = ent->next; - ent->prev = ent->next; - ent->next = ent->next->next; - ent->prev->next = ent; - } /* end if */ - } /* end if */ - else if(rdcc->nslots > 0 && chunk_size <= rdcc->nbytes_max) { - /* Calculate the index */ - udata->idx_hint = H5D__chunk_hash_val(io_info->dset->shared, udata->common.scaled); - /* Add the chunk to the cache only if the slot is not already locked */ - ent = rdcc->slot[udata->idx_hint]; - if(!ent || !ent->locked) { - /* Preempt enough things from the cache to make room */ - if(ent) { - if(H5D__chunk_cache_evict(io_info->dset, io_info->dxpl_id, io_info->dxpl_cache, ent, TRUE) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk from cache") - } /* end if */ - if(H5D__chunk_cache_prune(io_info->dset, io_info->dxpl_id, io_info->dxpl_cache, chunk_size) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk(s) from cache") + /* See if the chunk can be cached */ + if(rdcc->nslots > 0 && chunk_size <= rdcc->nbytes_max) { + /* Calculate the index */ + udata->idx_hint = H5D__chunk_hash_val(io_info->dset->shared, udata->common.scaled); + + /* Add the chunk to the cache only if the slot is not already locked */ + ent = rdcc->slot[udata->idx_hint]; + if(!ent || !ent->locked) { + /* Preempt enough things from the cache to make room */ + if(ent) { + if(H5D__chunk_cache_evict(io_info->dset, io_info->dxpl_id, io_info->dxpl_cache, ent, TRUE) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk from cache") + } /* end if */ + if(H5D__chunk_cache_prune(io_info->dset, io_info->dxpl_id, io_info->dxpl_cache, chunk_size) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk(s) from cache") + + /* Create a new entry */ + if(NULL == (ent = H5FL_CALLOC(H5D_rdcc_ent_t))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "can't allocate raw data chunk entry") + + ent->locked = 0; + ent->edge_chunk_state = disable_filters ? H5D_RDCC_DISABLE_FILTERS : 0; + if(udata->new_unfilt_chunk) + ent->edge_chunk_state |= H5D_RDCC_NEWLY_DISABLED_FILTERS; + ent->dirty = FALSE; + ent->deleted = FALSE; + + /* Initialize the new entry */ + ent->chunk_block.offset = chunk_addr; + ent->chunk_block.length = chunk_alloc; + ent->chunk_idx = udata->chunk_idx; + HDmemcpy(ent->offset, io_info->store->chunk.offset, sizeof(hsize_t) * layout->u.chunk.ndims); + HDmemcpy(ent->scaled, udata->common.scaled, sizeof(hsize_t) * layout->u.chunk.ndims); + + H5_ASSIGN_OVERFLOW(ent->rd_count, chunk_size, size_t, uint32_t); + H5_ASSIGN_OVERFLOW(ent->wr_count, chunk_size, size_t, uint32_t); + ent->chunk = (uint8_t *)chunk; - /* Create a new entry */ - if(NULL == (ent = H5FL_CALLOC(H5D_rdcc_ent_t))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "can't allocate raw data chunk entry") + /* Add it to the cache */ + HDassert(NULL == rdcc->slot[udata->idx_hint]); + rdcc->slot[udata->idx_hint] = ent; + ent->idx = udata->idx_hint; + rdcc->nbytes_used += chunk_size; + rdcc->nused++; + + /* Add it to the linked list */ + if(rdcc->tail) { + rdcc->tail->next = ent; + ent->prev = rdcc->tail; + rdcc->tail = ent; + } /* end if */ + else + rdcc->head = rdcc->tail = ent; + ent->tmp_next = NULL; + ent->tmp_prev = NULL; - ent->locked = 0; - ent->edge_chunk_state = disable_filters ? H5D_RDCC_DISABLE_FILTERS : 0; - if(udata->new_unfilt_chunk) - ent->edge_chunk_state |= H5D_RDCC_NEWLY_DISABLED_FILTERS; - ent->dirty = FALSE; - ent->deleted = FALSE; - - /* Initialize the new entry */ - ent->chunk_addr = chunk_addr; - ent->chunk_idx = udata->chunk_idx; - HDmemcpy(ent->offset, io_info->store->chunk.offset, sizeof(hsize_t) * layout->u.chunk.ndims); - HDmemcpy(ent->scaled, udata->common.scaled, sizeof(hsize_t) * layout->u.chunk.ndims); - - H5_ASSIGN_OVERFLOW(ent->rd_count, chunk_size, size_t, uint32_t); - H5_ASSIGN_OVERFLOW(ent->wr_count, chunk_size, size_t, uint32_t); - ent->chunk = (uint8_t *)chunk; - - /* Add it to the cache */ - HDassert(NULL == rdcc->slot[udata->idx_hint]); - rdcc->slot[udata->idx_hint] = ent; - ent->idx = udata->idx_hint; - rdcc->nbytes_used += chunk_size; - rdcc->nused++; - - /* Add it to the linked list */ - if(rdcc->tail) { - rdcc->tail->next = ent; - ent->prev = rdcc->tail; - rdcc->tail = ent; } /* end if */ else - rdcc->head = rdcc->tail = ent; - ent->tmp_next = NULL; - ent->tmp_prev = NULL; - - } /* end if */ - else - /* We did not add the chunk to cache */ + /* We did not add the chunk to cache */ + ent = NULL; + } /* end else */ + else /* No cache set up, or chunk is too large: chunk is uncacheable */ ent = NULL; } /* end else */ - if(!ent) - /* - * The chunk cannot be placed in cache so we don't cache it. This is the - * reason all those arguments have to be repeated for the unlock - * function. - */ - udata->idx_hint = UINT_MAX; - /* Lock the chunk into the cache */ if(ent) { HDassert(!ent->locked); ent->locked = TRUE; chunk = ent->chunk; } /* end if */ + else + /* + * The chunk cannot be placed in cache so we don't cache it. This is the + * reason all those arguments have to be repeated for the unlock + * function. + */ + udata->idx_hint = UINT_MAX; /* Set return value */ ret_value = chunk; @@ -3354,7 +3367,7 @@ done: /* Release the chunk allocated, on error */ if(!ret_value) if(chunk) - chunk = H5D__chunk_xfree(chunk, pline); + chunk = H5D__chunk_mem_xfree(chunk, pline); FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_lock() */ @@ -3432,7 +3445,8 @@ H5D__chunk_unlock(const H5D_io_info_t *io_info, const H5D_chunk_ud_t *udata, HDmemcpy(fake_ent.scaled, udata->common.scaled, sizeof(hsize_t) * layout->u.chunk.ndims); HDassert(layout->u.chunk.size > 0); fake_ent.chunk_idx = udata->chunk_idx; - fake_ent.chunk_addr = udata->addr; + fake_ent.chunk_block.offset = udata->chunk_block.offset; + fake_ent.chunk_block.length = udata->chunk_block.length; fake_ent.chunk = (uint8_t *)chunk; if(H5D__chunk_flush_entry(io_info->dset, io_info->dxpl_id, io_info->dxpl_cache, &fake_ent, TRUE) < 0) @@ -3440,8 +3454,8 @@ H5D__chunk_unlock(const H5D_io_info_t *io_info, const H5D_chunk_ud_t *udata, } /* end if */ else { if(chunk) - chunk = H5D__chunk_xfree(chunk, (is_unfiltered_edge_chunk ? NULL - : &(io_info->dset->shared->dcpl_cache.pline))); + chunk = H5D__chunk_mem_xfree(chunk, (is_unfiltered_edge_chunk ? NULL + : &(io_info->dset->shared->dcpl_cache.pline))); } /* end else */ } /* end if */ else { @@ -3720,8 +3734,8 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, /* Initialize the fill value buffer */ /* (delay allocating fill buffer for VL datatypes until refilling) */ /* (casting away const OK - QAK) */ - if(H5D__fill_init(&fb_info, NULL, (H5MM_allocate_t)H5D__chunk_alloc, - (void *)pline, (H5MM_free_t)H5D__chunk_xfree, (void *)pline, + if(H5D__fill_init(&fb_info, NULL, (H5MM_allocate_t)H5D__chunk_mem_alloc, + (void *)pline, (H5MM_free_t)H5D__chunk_mem_xfree, (void *)pline, &dset->shared->dcpl_cache.fill, dset->shared->type, dset->shared->type_id, (size_t)0, orig_chunk_size, dxpl_id) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill buffer info") @@ -3742,7 +3756,7 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, /* If the dataset has disabled partial chunk filters, create a copy * of the unfiltered fill_buf to use for partial chunks */ if(has_unfilt_edge_chunks) { - if(NULL == (unfilt_fill_buf = H5D__chunk_alloc(orig_chunk_size, &def_pline))) + if(NULL == (unfilt_fill_buf = H5D__chunk_mem_alloc(orig_chunk_size, &def_pline))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") HDmemcpy(unfilt_fill_buf, fb_info.fill_buf, orig_chunk_size); } /* end if */ @@ -3826,6 +3840,10 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, while(!carry) { /* None of the chunks should be allocated */ hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Scaled coordinates for this chunk */ + hbool_t need_insert = FALSE; /* Whether the chunk needs to be inserted into the index */ + + /* Reset size of chunk in bytes, in case filtered size changes */ + /* chunk_size = orig_chunk_size; */ /* Look up this chunk */ H5VM_chunk_scaled(space_ndims, chunk_offset, layout->u.chunk.dim, scaled); @@ -3834,7 +3852,7 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, #ifndef NDEBUG if(H5D_CHUNK_IDX_NONE != layout->storage.u.chunk.idx_type) - HDassert(!H5F_addr_defined(udata.addr)); + HDassert(!H5F_addr_defined(udata.chunk_block.offset)); /* Make sure the chunk is really in the dataset and outside the * original dimensions */ @@ -3863,7 +3881,7 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, * possible (though ill-advised) for the filter to shrink the * buffer. */ if(fb_info.fill_buf_size < orig_chunk_size) { - if(NULL == (fb_info.fill_buf = H5D__chunk_realloc(fb_info.fill_buf, orig_chunk_size, pline))) + if(NULL == (fb_info.fill_buf = H5D__chunk_mem_realloc(fb_info.fill_buf, orig_chunk_size, pline))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory reallocation failed for raw data chunk") fb_info.fill_buf_size = orig_chunk_size; } /* end if */ @@ -3900,23 +3918,21 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, udata.common.storage = &layout->storage.u.chunk; udata.common.offset = chunk_offset; udata.common.rdcc = &dset->shared->cache.chunk; - udata.addr = HADDR_UNDEF; - H5_ASSIGN_OVERFLOW(udata.nbytes, chunk_size, size_t, uint32_t); + udata.chunk_block.offset = HADDR_UNDEF; + H5_ASSIGN_OVERFLOW(udata.chunk_block.length, chunk_size, size_t, uint32_t); udata.filter_mask = filter_mask; - udata.need_insert = FALSE; udata.need_modify = FALSE; /* Allocate the chunk (with all processes) */ - if(H5D__idx_chunk_alloc(&idx_info, &udata) < 0) + if(H5D__chunk_file_alloc(&idx_info, NULL, &udata.chunk_block, &need_insert, &udata.need_modify, scaled) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk on chunk level") - - HDassert(H5F_addr_defined(udata.addr)); + HDassert(H5F_addr_defined(udata.chunk_block.offset)); /* Check if fill values should be written to chunks */ if(should_fill) { /* Sanity check */ HDassert(fb_info_init); - HDassert(udata.nbytes == chunk_size); + HDassert(udata.chunk_block.length == chunk_size); #ifdef H5_HAVE_PARALLEL /* Check if this file is accessed with an MPI-capable file driver */ @@ -3929,7 +3945,7 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "memory allocation failed for chunk addresses") /* Store the chunk's address for later */ - chunk_info.addr[chunk_info.num_io] = udata.addr; + chunk_info.addr[chunk_info.num_io] = udata.chunk_block.offset; chunk_info.num_io++; /* Indicate that blocks will be written */ @@ -3937,17 +3953,20 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, } /* end if */ else { #endif /* H5_HAVE_PARALLEL */ - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, chunk_size, dxpl_id, *fill_buf) < 0) + if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.chunk_block.offset, chunk_size, dxpl_id, *fill_buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") #ifdef H5_HAVE_PARALLEL } /* end else */ #endif /* H5_HAVE_PARALLEL */ } /* end if */ - if((udata.need_insert || udata.need_modify) && ops->insert_addr) { + /* Insert the chunk record into the index */ + /* (Note that this isn't safe, from a SWMR perspective, unlike + * serial operation. -QAK + */ + if((need_insert || udata.need_modify) && ops->insert_addr) if((ops->insert_addr)(&idx_info, &udata) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index") - } /* Increment indices and adjust the edge chunk state */ carry = TRUE; @@ -4021,7 +4040,7 @@ done: HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release fill buffer info") /* Free the unfiltered fill value buffer */ - unfilt_fill_buf = H5D__chunk_xfree(unfilt_fill_buf, &def_pline); + unfilt_fill_buf = H5D__chunk_mem_xfree(unfilt_fill_buf, &def_pline); #ifdef H5_HAVE_PARALLEL if(using_mpi && chunk_info.addr) @@ -4181,7 +4200,7 @@ H5D__chunk_update_old_edge_chunks(H5D_t *dset, hid_t dxpl_id, hsize_t old_dim[]) /* If this chunk does not exist in cache or on disk, no need to do * anything */ - if(H5F_addr_defined(chk_udata.addr) + if(H5F_addr_defined(chk_udata.chunk_block.offset) || (UINT_MAX != chk_udata.idx_hint)) { /* Lock the chunk into cache. H5D__chunk_lock will take care of * updating the chunk to no longer be an edge chunk. */ @@ -4424,7 +4443,7 @@ H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, hbool_t new_unfilt_chunk) chk_udata.new_unfilt_chunk = new_unfilt_chunk; /* If this chunk does not exist in cache or on disk, no need to do anything */ - if(!H5F_addr_defined(chk_udata.addr) && UINT_MAX == chk_udata.idx_hint) + if(!H5F_addr_defined(chk_udata.chunk_block.offset) && UINT_MAX == chk_udata.idx_hint) HGOTO_DONE(SUCCEED) /* Initialize the fill value buffer, if necessary */ @@ -4826,7 +4845,7 @@ H5D__chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, const hsize_t *old_dim) dims_outside_fill[u] = FALSE; carry = FALSE; - } /* end if */ + } /* end if */ while(!carry) { hsize_t scaled[H5S_MAX_RANK]; /* Scaled coordinates for the current chunk */ @@ -4893,7 +4912,7 @@ H5D__chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, const hsize_t *old_dim) HGOTO_ERROR(H5E_DATASET, H5E_CANTREMOVE, FAIL, "unable to evict chunk") /* Remove the chunk from disk, if present */ - if(H5F_addr_defined(chk_udata.addr)) { + if(H5F_addr_defined(chk_udata.chunk_block.offset)) { /* Update the offset in idx_udata */ idx_udata.offset = chunk_offset; idx_udata.scaled = udata.common.scaled; @@ -5287,6 +5306,7 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) hsize_t scaled[H5S_MAX_RANK]; /* Scaled coordinates of chunk */ hbool_t is_vlen = FALSE; /* Whether datatype is variable-length */ hbool_t fix_ref = FALSE; /* Whether to fix up references in the dest. file */ + hbool_t need_insert = FALSE; /* Whether the chunk needs to be inserted into the index */ /* General information about chunk copy */ void *bkg = udata->bkg; /* Background buffer for datatype conversion */ @@ -5418,8 +5438,8 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) udata_dst.common.storage = udata->idx_info_dst->storage; udata_dst.common.offset = chunk_rec->offset; udata_dst.common.rdcc = NULL; - udata_dst.addr = HADDR_UNDEF; - udata_dst.nbytes = chunk_rec->nbytes; + udata_dst.chunk_block.offset = HADDR_UNDEF; + udata_dst.chunk_block.length = chunk_rec->nbytes; udata_dst.filter_mask = chunk_rec->filter_mask; /* Need to compress variable-length & reference data elements before writing to file */ @@ -5431,7 +5451,7 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) if(nbytes > ((size_t)0xffffffff)) HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, H5_ITER_ERROR, "chunk too large for 32-bit length") #endif /* H5_SIZEOF_SIZE_T > 4 */ - H5_ASSIGN_OVERFLOW(udata_dst.nbytes, nbytes, size_t, uint32_t); + H5_ASSIGN_OVERFLOW(udata_dst.chunk_block.length, nbytes, size_t, uint32_t); udata->buf = buf; udata->buf_size = buf_size; } /* end if */ @@ -5440,22 +5460,23 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) udata_dst.chunk_idx = H5VM_chunk_index_scaled(udata_dst.common.layout->ndims - 1, chunk_rec->offset, udata_dst.common.layout->dim, udata_dst.common.layout->down_chunks, scaled); udata_dst.common.scaled = scaled; - /* Insert chunk into the destination index */ - if(H5D__idx_chunk_alloc(udata->idx_info_dst, &udata_dst) < 0) + /* Allocate chunk in the file */ + if(H5D__chunk_file_alloc(udata->idx_info_dst, NULL, &udata_dst.chunk_block, &need_insert, &udata_dst.need_modify, scaled) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk on chunk level") /* Write chunk data to destination file */ - HDassert(H5F_addr_defined(udata_dst.addr)); - if(H5F_block_write(udata->idx_info_dst->f, H5FD_MEM_DRAW, udata_dst.addr, nbytes, udata->idx_info_dst->dxpl_id, buf) < 0) + HDassert(H5F_addr_defined(udata_dst.chunk_block.offset)); + if(H5F_block_write(udata->idx_info_dst->f, H5FD_MEM_DRAW, udata_dst.chunk_block.offset, nbytes, udata->idx_info_dst->dxpl_id, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, H5_ITER_ERROR, "unable to write raw data to file") /* Set metadata tag in dxpl_id */ H5_BEGIN_TAG(udata->idx_info_dst->dxpl_id, H5AC__COPIED_TAG, H5_ITER_ERROR); - if((udata_dst.need_insert || udata_dst.need_modify) && udata->idx_info_dst->storage->ops->insert_addr) { + /* Insert chunk record into index */ + if((need_insert || udata_dst.need_modify) && udata->idx_info_dst->storage->ops->insert_addr) if((udata->idx_info_dst->storage->ops->insert_addr)(udata->idx_info_dst, &udata_dst) < 0) HGOTO_ERROR_TAG(H5E_DATASET, H5E_CANTINSERT, H5_ITER_ERROR, "unable to insert chunk addr into index") - } + /* Reset metadata tag in dxpl_id */ H5_END_TAG(H5_ITER_ERROR); @@ -6197,7 +6218,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5D_idx_chunk_alloc() + * Function: H5D__chunk_file_alloc() * * Purpose: Chunk allocation: * Create the chunk if it doesn't exist, or reallocate the @@ -6211,10 +6232,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__idx_chunk_alloc(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) +H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old_chunk, + H5F_block_t *new_chunk, hbool_t *need_insert, hbool_t *need_modify, hsize_t scaled[]) { hbool_t alloc_chunk = FALSE; /* Whether to allocate chunk */ - hbool_t need_modify = FALSE; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -6226,111 +6247,116 @@ H5D__idx_chunk_alloc(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) HDassert(idx_info->layout); HDassert(idx_info->storage); HDassert(H5F_addr_defined(idx_info->storage->idx_addr)); - HDassert(udata); + HDassert(new_chunk); + HDassert(need_insert); + HDassert(need_modify); - udata->need_modify = FALSE; /* this is mainly for V2-btree */ - udata->need_insert = FALSE; + *need_modify = FALSE; /* this is mainly for V2-btree */ + *need_insert = FALSE; + /* Check for filters on chunks */ if(idx_info->pline->nused > 0) { - unsigned allow_chunk_size_len; /* Allowed size of encoded chunk size */ - unsigned new_chunk_size_len; /* Size of encoded chunk size */ - H5D_chunk_ud_t found_udata; HDassert(idx_info->storage->idx_type != H5D_CHUNK_IDX_NONE); + /* Sanity/error checking block */ + { + unsigned allow_chunk_size_len; /* Allowed size of encoded chunk size */ + unsigned new_chunk_size_len; /* Size of encoded chunk size */ - /* Compute the size required for encoding the size of a chunk, allowing - * for an extra byte, in case the filter makes the chunk larger. - */ - allow_chunk_size_len = 1 + ((H5VM_log2_gen((uint64_t)(idx_info->layout->size)) + 8) / 8); - if(allow_chunk_size_len > 8) - allow_chunk_size_len = 8; - - /* Compute encoded size of chunk */ - new_chunk_size_len = (H5VM_log2_gen((uint64_t)(udata->nbytes)) + 8) / 8; - if(new_chunk_size_len > 8) - HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "encoded chunk size is more than 8 bytes?!?") - - /* Check if the chunk became too large to be encoded */ - if(new_chunk_size_len > allow_chunk_size_len) - HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk size can't be encoded") - - /* Initialize found_udata for ops->get_addr() */ - HDmemset(&found_udata, 0, sizeof found_udata); - found_udata.common.offset = udata->common.offset; - found_udata.common.scaled = udata->common.scaled; - found_udata.common.storage = udata->common.storage; - found_udata.common.layout = udata->common.layout; - found_udata.addr = HADDR_UNDEF; - - /* Get chunk information */ - if((idx_info->storage->ops->get_addr)(idx_info, &found_udata) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query chunk address") - - if(H5F_addr_defined(found_udata.addr)) { + /* Compute the size required for encoding the size of a chunk, allowing + * for an extra byte, in case the filter makes the chunk larger. + */ + allow_chunk_size_len = 1 + ((H5VM_log2_gen((uint64_t)(idx_info->layout->size)) + 8) / 8); + if(allow_chunk_size_len > 8) + allow_chunk_size_len = 8; + + /* Compute encoded size of chunk */ + new_chunk_size_len = (H5VM_log2_gen((uint64_t)(new_chunk->length)) + 8) / 8; + if(new_chunk_size_len > 8) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "encoded chunk size is more than 8 bytes?!?") + + /* Check if the chunk became too large to be encoded */ + if(new_chunk_size_len > allow_chunk_size_len) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk size can't be encoded") + } /* end block */ + + if(old_chunk && H5F_addr_defined(old_chunk->offset)) { /* Sanity check */ - HDassert(!H5F_addr_defined(udata->addr) || H5F_addr_eq(udata->addr, found_udata.addr)); + HDassert(!H5F_addr_defined(new_chunk->offset) || H5F_addr_eq(new_chunk->offset, old_chunk->offset)); /* Check for chunk being same size */ - if(udata->nbytes != found_udata.nbytes) { + if(new_chunk->length != old_chunk->length) { /* Release previous chunk */ /* Only free the old location if not doing SWMR writes - otherwise - * we must keep the old chunk around in case a reader has an - * outdated version of the b-tree node - */ - if(!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) { - H5_CHECK_OVERFLOW(found_udata.nbytes, uint32_t, hsize_t); - if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, found_udata.addr, (hsize_t)found_udata.nbytes) < 0) + * we must keep the old chunk around in case a reader has an + * outdated version of the b-tree node + */ + if(!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) { + if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, old_chunk->offset, old_chunk->length) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") - } /* end if */ + } alloc_chunk = TRUE; - need_modify = TRUE; - - } else { + *need_modify = TRUE; + } /* end if */ + else { /* Don't need to reallocate chunk, but send its address back up */ - if(!H5F_addr_defined(udata->addr)) - udata->addr = found_udata.addr; - } - } else + if(!H5F_addr_defined(new_chunk->offset)) + new_chunk->offset = old_chunk->offset; + } /* end else */ + } /* end if */ + else { + HDassert(!H5F_addr_defined(new_chunk->offset)); alloc_chunk = TRUE; - - } else { - HDassert(!H5F_addr_defined(udata->addr)); - HDassert(udata->nbytes == idx_info->layout->size); + } /* end else */ + } /* end if */ + else { + HDassert(!H5F_addr_defined(new_chunk->offset)); + HDassert(new_chunk->length == idx_info->layout->size); alloc_chunk = TRUE; - } + } /* end else */ + /* Actually allocate space for the chunk in the file */ if(alloc_chunk) { switch(idx_info->storage->idx_type) { case H5D_CHUNK_IDX_NONE: - if((idx_info->storage->ops->get_addr)(idx_info, udata) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query chunk address") - break; + { + H5D_chunk_ud_t udata; + + udata.common.scaled = scaled; + if((idx_info->storage->ops->get_addr)(idx_info, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query chunk address") + new_chunk->offset = udata.chunk_block.offset; + HDassert(new_chunk->length == udata.chunk_block.length); + break; + } case H5D_CHUNK_IDX_EARRAY: - case H5D_CHUNK_IDX_FARRAY: + case H5D_CHUNK_IDX_FARRAY: + case H5D_CHUNK_IDX_BT2: case H5D_CHUNK_IDX_BTREE: - case H5D_CHUNK_IDX_BT2: - H5_CHECK_OVERFLOW(udata->nbytes, /*From: */uint32_t, /*To: */hsize_t); - udata->addr = H5MF_alloc(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, (hsize_t)udata->nbytes); - if(!H5F_addr_defined(udata->addr)) - HGOTO_ERROR (H5E_DATASET, H5E_CANTALLOC, FAIL, "file allocation failed") + HDassert(new_chunk->length > 0); + H5_CHECK_OVERFLOW(new_chunk->length, /*From: */uint32_t, /*To: */hsize_t); + new_chunk->offset = H5MF_alloc(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, (hsize_t)new_chunk->length); + if(!H5F_addr_defined(new_chunk->offset)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "file allocation failed") if(idx_info->storage->idx_type == H5D_CHUNK_IDX_BT2) { - /* This can be done together with other index types when Quincy checks into H5B2_modify() */ - if(!(udata->need_modify = need_modify)) - udata->need_insert = TRUE; - } else - udata->need_insert = TRUE; + /* This can be done together with other index types when Quincy checks into H5B2_modify() */ + if(!(*need_modify)) + *need_insert = TRUE; + } else + *need_insert = TRUE; + break; case H5D_CHUNK_IDX_NTYPES: default: - HDassert(0 && "This Should never be executed!"); + HDassert(0 && "This should never be executed!"); break; } /* end switch */ } /* end if */ - HDassert(H5F_addr_defined(udata->addr)); + + HDassert(H5F_addr_defined(new_chunk->offset)); done: FUNC_LEAVE_NOAPI(ret_value) - -} /* H5D__idx_chunk_alloc() */ +} /* H5D__chunk_file_alloc() */ diff --git a/src/H5Dearray.c b/src/H5Dearray.c index fcb60aa..a6c6535 100644 --- a/src/H5Dearray.c +++ b/src/H5Dearray.c @@ -1077,7 +1077,6 @@ H5D_earray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u HDassert(idx_info->storage); HDassert(H5F_addr_defined(idx_info->storage->idx_addr)); HDassert(udata); - HDassert(udata->need_insert); /* Check if the extensible array is open yet */ if(NULL == idx_info->storage->u.earray.ea) { @@ -1089,7 +1088,7 @@ H5D_earray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u /* Set convenience pointer to extensible array structure */ ea = idx_info->storage->u.earray.ea; - if(!H5F_addr_defined(udata->addr)) + if(!H5F_addr_defined(udata->chunk_block.offset)) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "The chunk should have allocated already") if(udata->chunk_idx != (udata->chunk_idx & 0xffffffff)) /* negative value */ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "chunk index must be less than 2^32") @@ -1097,8 +1096,8 @@ H5D_earray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u if(idx_info->pline->nused > 0) { H5D_earray_filt_elmt_t elmt; /* Extensible array element */ - elmt.addr = udata->addr; - elmt.nbytes = udata->nbytes; + elmt.addr = udata->chunk_block.offset; + elmt.nbytes = udata->chunk_block.length; elmt.filter_mask = udata->filter_mask; /* Set the info for the chunk */ @@ -1106,7 +1105,7 @@ H5D_earray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set chunk info") } else { /* Set the address for the chunk */ - if(H5EA_set(ea, idx_info->dxpl_id, udata->chunk_idx, &udata->addr) < 0) + if(H5EA_set(ea, idx_info->dxpl_id, udata->chunk_idx, &udata->chunk_block.offset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set chunk address") } @@ -1184,20 +1183,23 @@ H5D_earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info") /* Set the info for the chunk */ - udata->addr = elmt.addr; - udata->nbytes = elmt.nbytes; + udata->chunk_block.offset = elmt.addr; + udata->chunk_block.length = elmt.nbytes; udata->filter_mask = elmt.filter_mask; } /* end if */ else { /* Get the address for the chunk */ - if(H5EA_get(ea, idx_info->dxpl_id, idx, &udata->addr) < 0) + if(H5EA_get(ea, idx_info->dxpl_id, idx, &udata->chunk_block.offset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") /* Update the other (constant) information for the chunk */ - udata->nbytes = idx_info->layout->size; + udata->chunk_block.length = idx_info->layout->size; udata->filter_mask = 0; } /* end else */ + if(!H5F_addr_defined(udata->chunk_block.offset)) + udata->chunk_block.length = 0; + done: FUNC_LEAVE_NOAPI(ret_value) } /* H5D_earray_idx_get_addr() */ diff --git a/src/H5Dfarray.c b/src/H5Dfarray.c index 09476ab..de184ea 100644 --- a/src/H5Dfarray.c +++ b/src/H5Dfarray.c @@ -1047,7 +1047,6 @@ H5D_farray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u HDassert(idx_info->storage); HDassert(H5F_addr_defined(idx_info->storage->idx_addr)); HDassert(udata); - HDassert(udata->need_insert); /* Check if the fixed array is open yet */ if(NULL == idx_info->storage->u.farray.fa) { @@ -1059,7 +1058,7 @@ H5D_farray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u /* Set convenience pointer to fixed array structure */ fa = idx_info->storage->u.farray.fa; - if(!H5F_addr_defined(udata->addr)) + if(!H5F_addr_defined(udata->chunk_block.offset)) HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "The chunk should have allocated already") if(udata->chunk_idx != (udata->chunk_idx & 0xffffffff)) /* negative value */ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "chunk index must be less than 2^32") @@ -1068,8 +1067,8 @@ H5D_farray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u if(idx_info->pline->nused > 0) { H5D_farray_filt_elmt_t elmt; /* Fixed array element */ - elmt.addr = udata->addr; - elmt.nbytes = udata->nbytes; + elmt.addr = udata->chunk_block.offset; + elmt.nbytes = udata->chunk_block.length; elmt.filter_mask = udata->filter_mask; /* Set the info for the chunk */ @@ -1078,7 +1077,7 @@ H5D_farray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u } /* end if */ else { /* Set the address for the chunk */ - if(H5FA_set(fa, idx_info->dxpl_id, udata->chunk_idx, &udata->addr) < 0) + if(H5FA_set(fa, idx_info->dxpl_id, udata->chunk_idx, &udata->chunk_block.offset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set chunk address") } /* end else */ @@ -1147,20 +1146,23 @@ H5D_farray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info") /* Set the info for the chunk */ - udata->addr = elmt.addr; - udata->nbytes = elmt.nbytes; + udata->chunk_block.offset = elmt.addr; + udata->chunk_block.length = elmt.nbytes; udata->filter_mask = elmt.filter_mask; } /* end if */ else { /* Get the address for the chunk */ - if(H5FA_get(fa, idx_info->dxpl_id, idx, &udata->addr) < 0) + if(H5FA_get(fa, idx_info->dxpl_id, idx, &udata->chunk_block.offset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") /* Update the other (constant) information for the chunk */ - udata->nbytes = idx_info->layout->size; + udata->chunk_block.length = idx_info->layout->size; udata->filter_mask = 0; } /* end else */ + if(!H5F_addr_defined(udata->chunk_block.offset)) + udata->chunk_block.length = 0; + done: FUNC_LEAVE_NOAPI(ret_value) } /* H5D_farray_idx_get_addr() */ diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 89b7728..e94d2fa 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -863,7 +863,7 @@ H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *typ if(H5D__chunk_lookup(io_info->dset, io_info->dxpl_id, chunk_info->coords, chunk_info->scaled, &udata) < 0) HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL, "couldn't get chunk address") - ctg_store.contig.dset_addr = udata.addr; + ctg_store.contig.dset_addr = udata.chunk_block.offset; } /* end else */ /* Set up the base storage address for this chunk */ @@ -1591,7 +1591,7 @@ if(H5DEBUG(D)) if(H5D__chunk_lookup(io_info->dset, io_info->dxpl_id, chunk_info->coords, chunk_info->scaled, &udata) < 0) HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL, "couldn't get chunk info from skipped list") - chunk_addr = udata.addr; + chunk_addr = udata.chunk_block.offset; } /* end if */ else chunk_addr = total_chunk_addr_array[chunk_info->index]; diff --git a/src/H5Dnone.c b/src/H5Dnone.c index c059066..1bea20a 100644 --- a/src/H5Dnone.c +++ b/src/H5Dnone.c @@ -207,10 +207,10 @@ H5D_none_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) udata->chunk_idx = H5VM_array_offset_pre((idx_info->layout->ndims - 1), idx_info->layout->max_down_chunks, udata->common.scaled); /* Calculate the address of the chunk */ - udata->addr = idx_info->storage->idx_addr + udata->chunk_idx * idx_info->layout->size; + udata->chunk_block.offset = idx_info->storage->idx_addr + udata->chunk_idx * idx_info->layout->size; /* Update the other (constant) information for the chunk */ - udata->nbytes = idx_info->layout->size; + udata->chunk_block.length = idx_info->layout->size; udata->filter_mask = 0; FUNC_LEAVE_NOAPI(SUCCEED) diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 764030b..6681d18 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -34,6 +34,7 @@ /* Other private headers needed by this file */ #include "H5ACprivate.h" /* Metadata cache */ #include "H5B2private.h" /* v2 B-trees */ +#include "H5Fprivate.h" /* File access */ #include "H5Gprivate.h" /* Groups */ #include "H5SLprivate.h" /* Skip lists */ #include "H5Tprivate.h" /* Datatypes */ @@ -184,6 +185,7 @@ typedef struct { } H5D_contig_storage_t; typedef struct { + hsize_t index; /* "Index" of chunk in dataset (must be first for TBBT routines) */ hsize_t *offset; /* Chunk's coordinates in elements */ } H5D_chunk_storage_t; @@ -257,8 +259,8 @@ typedef struct H5D_chk_idx_info_t { * The chunk's file address, filter mask and size on disk are not key values. */ typedef struct H5D_chunk_rec_t { - uint32_t nbytes; /* Size of stored data */ hsize_t offset[H5O_LAYOUT_NDIMS]; /* Logical offset to start */ + uint32_t nbytes; /* Size of stored data */ unsigned filter_mask; /* Excluded filters */ haddr_t chunk_addr; /* Address of chunk in file */ } H5D_chunk_rec_t; @@ -286,12 +288,10 @@ typedef struct H5D_chunk_ud_t { /* Upward */ unsigned idx_hint; /*index of chunk in cache, if present */ - haddr_t addr; /*file address of chunk */ - uint32_t nbytes; /*size of stored data */ + H5F_block_t chunk_block; /*offset/length of chunk in file */ unsigned filter_mask; /*excluded filters */ hbool_t new_unfilt_chunk; /*whether the chunk just became unfiltered */ hsize_t chunk_idx; /*chunk index for EA, FA indexing */ - hbool_t need_insert; hbool_t need_modify; } H5D_chunk_ud_t; @@ -540,7 +540,7 @@ typedef struct H5D_rdcc_ent_t { hsize_t scaled[H5O_LAYOUT_NDIMS]; /*scaled chunk 'name' (coordinates) */ uint32_t rd_count; /*bytes remaining to be read */ uint32_t wr_count; /*bytes remaining to be written */ - haddr_t chunk_addr; /*address of chunk in file */ + H5F_block_t chunk_block; /*offset/length of chunk in file */ hsize_t chunk_idx; /*index of chunk in dataset */ uint8_t *chunk; /*the unfiltered chunk data */ unsigned idx; /*index in hash table */ diff --git a/src/H5Fio.c b/src/H5Fio.c index cca0077..3b6c079 100644 --- a/src/H5Fio.c +++ b/src/H5Fio.c @@ -100,6 +100,9 @@ H5F_block_read(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) +#ifdef QAK +HDfprintf(stderr, "%s: read from addr = %a, size = %Zu\n", FUNC, addr, size); +#endif /* QAK */ HDassert(f); HDassert(f->shared); diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 4009ca8..725812f 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -573,6 +573,12 @@ typedef struct H5F_io_info_t { const struct H5P_genplist_t *dxpl; /* DXPL object */ } H5F_io_info_t; +/* Concise info about a block of bytes in a file */ +typedef struct H5F_block_t { + haddr_t offset; /* Offset of the block in the file */ + hsize_t length; /* Length of the block in the file */ +} H5F_block_t; + /*****************************/ /* Library-private Variables */ diff --git a/src/H5Rpublic.h b/src/H5Rpublic.h index 997309f..e990661 100644 --- a/src/H5Rpublic.h +++ b/src/H5Rpublic.h @@ -14,7 +14,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * This file contains public declarations for the H5S module. + * This file contains public declarations for the H5R module. */ #ifndef _H5Rpublic_H #define _H5Rpublic_H diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in index 1b1ae96..1eed645 100644 --- a/src/libhdf5.settings.in +++ b/src/libhdf5.settings.in @@ -51,7 +51,7 @@ Languages: @BUILD_CXX_CONDITIONAL_TRUE@ C++ Flags: @CXXFLAGS@ @BUILD_CXX_CONDITIONAL_TRUE@ H5 C++ Flags: @H5_CXXFLAGS@ @BUILD_CXX_CONDITIONAL_TRUE@ AM C++ Flags: @AM_CXXFLAGS@ -@BUILD_CXX_CONDITIONAL_TRUE@ Shared C++ Library: @H5_CXX_SHARED@ +@BUILD_CXX_CONDITIONAL_TRUE@ Shared C++ Library: @enable_shared@ @BUILD_CXX_CONDITIONAL_TRUE@ Static C++ Library: @enable_static@ Features: |