summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorVailin Choi <vchoi@hdfgroup.org>2015-05-01 19:51:47 (GMT)
committerVailin Choi <vchoi@hdfgroup.org>2015-05-01 19:51:47 (GMT)
commita2aa6d4e6454375e804c63318dde42074da155e3 (patch)
tree3cbf15330b98dc0caaa6ec6f0f846b432e48df6b
parente0e28a7aae2fabf3ae9d03a1bd5809176935466a (diff)
downloadhdf5-a2aa6d4e6454375e804c63318dde42074da155e3.zip
hdf5-a2aa6d4e6454375e804c63318dde42074da155e3.tar.gz
hdf5-a2aa6d4e6454375e804c63318dde42074da155e3.tar.bz2
[svn-r26992] Bring revisions #26785 - #26837 from trunk to revise_chunks.
h5committested.
-rw-r--r--CMakeLists.txt2
-rw-r--r--config/cmake/ConfigureChecks.cmake39
-rw-r--r--config/cmake/H5pubconf.h.in35
-rw-r--r--fortran/src/H5Zf.c2
-rw-r--r--release_docs/INSTALL_parallel9
-rw-r--r--src/H5Dbtree.c30
-rw-r--r--src/H5Dbtree2.c22
-rw-r--r--src/H5Dchunk.c742
-rw-r--r--src/H5Dearray.c20
-rw-r--r--src/H5Dfarray.c20
-rw-r--r--src/H5Dmpio.c4
-rw-r--r--src/H5Dnone.c4
-rw-r--r--src/H5Dpkg.h10
-rw-r--r--src/H5Fio.c3
-rw-r--r--src/H5Fprivate.h6
-rw-r--r--src/H5Rpublic.h2
-rw-r--r--src/libhdf5.settings.in2
17 files changed, 457 insertions, 495 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a6994ef..a4135f4 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -763,7 +763,7 @@ endif (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN" OR HDF5_ALLOW_EXTERNAL_SUPPORT
#-----------------------------------------------------------------------------
# Dashboard and Testing Settings
#-----------------------------------------------------------------------------
-option (BUILD_TESTING "Build HDF5 Unit Testing" OFF)
+option (BUILD_TESTING "Build HDF5 Unit Testing" ON)
if (BUILD_TESTING)
set (DART_TESTING_TIMEOUT 1200
CACHE INTEGER
diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake
index b193ff4..e223553 100644
--- a/config/cmake/ConfigureChecks.cmake
+++ b/config/cmake/ConfigureChecks.cmake
@@ -92,45 +92,6 @@ endif (WINDOWS)
# ----------------------------------------------------------------------
CHECK_FUNCTION_EXISTS (difftime H5_HAVE_DIFFTIME)
-#CHECK_FUNCTION_EXISTS (gettimeofday H5_HAVE_GETTIMEOFDAY)
-# Since gettimeofday is not defined any where standard, lets look in all the
-# usual places. On MSVC we are just going to use ::clock()
-if (NOT MSVC)
- if ("H5_HAVE_TIME_GETTIMEOFDAY" MATCHES "^H5_HAVE_TIME_GETTIMEOFDAY$")
- TRY_COMPILE (HAVE_TIME_GETTIMEOFDAY
- ${CMAKE_BINARY_DIR}
- ${HDF_RESOURCES_EXT_DIR}/GetTimeOfDayTest.cpp
- COMPILE_DEFINITIONS -DTRY_TIME_H
- OUTPUT_VARIABLE OUTPUT
- )
- if (HAVE_TIME_GETTIMEOFDAY STREQUAL "TRUE")
- set (H5_HAVE_TIME_GETTIMEOFDAY "1" CACHE INTERNAL "H5_HAVE_TIME_GETTIMEOFDAY")
- set (H5_HAVE_GETTIMEOFDAY "1" CACHE INTERNAL "H5_HAVE_GETTIMEOFDAY")
- endif (HAVE_TIME_GETTIMEOFDAY STREQUAL "TRUE")
- endif ("H5_HAVE_TIME_GETTIMEOFDAY" MATCHES "^H5_HAVE_TIME_GETTIMEOFDAY$")
-
- if ("H5_HAVE_SYS_TIME_GETTIMEOFDAY" MATCHES "^H5_HAVE_SYS_TIME_GETTIMEOFDAY$")
- TRY_COMPILE (HAVE_SYS_TIME_GETTIMEOFDAY
- ${CMAKE_BINARY_DIR}
- ${HDF_RESOURCES_EXT_DIR}/GetTimeOfDayTest.cpp
- COMPILE_DEFINITIONS -DTRY_SYS_TIME_H
- OUTPUT_VARIABLE OUTPUT
- )
- if (HAVE_SYS_TIME_GETTIMEOFDAY STREQUAL "TRUE")
- set (H5_HAVE_SYS_TIME_GETTIMEOFDAY "1" CACHE INTERNAL "H5_HAVE_SYS_TIME_GETTIMEOFDAY")
- set (H5_HAVE_GETTIMEOFDAY "1" CACHE INTERNAL "H5_HAVE_GETTIMEOFDAY")
- endif (HAVE_SYS_TIME_GETTIMEOFDAY STREQUAL "TRUE")
- endif ("H5_HAVE_SYS_TIME_GETTIMEOFDAY" MATCHES "^H5_HAVE_SYS_TIME_GETTIMEOFDAY$")
-
- if (NOT HAVE_SYS_TIME_GETTIMEOFDAY AND NOT H5_HAVE_GETTIMEOFDAY)
- message (STATUS "---------------------------------------------------------------")
- message (STATUS "Function 'gettimeofday()' was not found. HDF5 will use its")
- message (STATUS " own implementation.. This can happen on older versions of")
- message (STATUS " MinGW on Windows. Consider upgrading your MinGW installation")
- message (STATUS " to a newer version such as MinGW 3.12")
- message (STATUS "---------------------------------------------------------------")
- endif (NOT HAVE_SYS_TIME_GETTIMEOFDAY AND NOT H5_HAVE_GETTIMEOFDAY)
-endif (NOT MSVC)
# Find the library containing clock_gettime()
if (NOT WINDOWS)
diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in
index c47b7c4..fcbdbd5 100644
--- a/config/cmake/H5pubconf.h.in
+++ b/config/cmake/H5pubconf.h.in
@@ -88,10 +88,6 @@
/* Define if Darwin or Mac OS X */
#cmakedefine H5_HAVE_DARWIN @H5_HAVE_DARWIN@
-/* Define to 1 if you have the declaration of `tzname', and to 0 if you don't.
- */
-#cmakedefine H5_HAVE_DECL_TZNAME @H5_HAVE_DECL_TZNAME@
-
/* Define to 1 if you have the `difftime' function. */
#cmakedefine H5_HAVE_DIFFTIME @H5_HAVE_DIFFTIME@
@@ -167,12 +163,6 @@
/* Define to 1 if you have the `gettimeofday' function. */
#cmakedefine H5_HAVE_GETTIMEOFDAY @H5_HAVE_GETTIMEOFDAY@
-/* Define to 1 if you have the `gettimeofday' function declared in time.h . */
-#cmakedefine H5_HAVE_TIME_GETTIMEOFDAY @H5_HAVE_TIME_GETTIMEOFDAY@
-
-/* Define to 1 if you have the `gettimeofday' function declared in sys/time.h . */
-#cmakedefine H5_HAVE_SYS_TIME_GETTIMEOFDAY @H5_HAVE_SYS_TIME_GETTIMEOFDAY@
-
/* Define to 1 if you have the `get_fpc_csr' function. */
#cmakedefine H5_HAVE_GET_FPC_CSR @H5_HAVE_GET_FPC_CSR@
@@ -201,15 +191,6 @@
/* Define to 1 if you have the `mpe' library (-lmpe). */
#cmakedefine H5_HAVE_LIBMPE @H5_HAVE_LIBMPE@
-/* Define to 1 if you have the `mpi' library (-lmpi). */
-#cmakedefine H5_HAVE_LIBMPI @H5_HAVE_LIBMPI@
-
-/* Define to 1 if you have the `mpich' library (-lmpich). */
-#cmakedefine H5_HAVE_LIBMPICH @H5_HAVE_LIBMPICH@
-
-/* Define to 1 if you have the `mpio' library (-lmpio). */
-#cmakedefine H5_HAVE_LIBMPIO @H5_HAVE_LIBMPIO@
-
/* Define to 1 if you have the `nsl' library (-lnsl). */
#cmakedefine H5_HAVE_LIBNSL @H5_HAVE_LIBNSL@
@@ -279,9 +260,6 @@
/* Define to 1 if you have the `setsysinfo' function. */
#cmakedefine H5_HAVE_SETSYSINFO @H5_HAVE_SETSYSINFO@
-/* Define to 1 if you have the `sigaction' function. */
-#cmakedefine H5_HAVE_SIGACTION @H5_HAVE_SIGACTION@
-
/* Define to 1 if you have the `siglongjmp' function. */
#cmakedefine H5_HAVE_SIGLONGJMP @H5_HAVE_SIGLONGJMP@
@@ -363,9 +341,6 @@
/* Define to 1 if you have the <sys/timeb.h> header file. */
#cmakedefine H5_HAVE_SYS_TIMEB_H @H5_HAVE_SYS_TIMEB_H@
-/* Define to 1 if you have the <time.h> header file. */
-#cmakedefine H5_HAVE_TIME_H @H5_HAVE_TIME_H@
-
/* Define to 1 if you have the <sys/time.h> header file. */
#cmakedefine H5_HAVE_SYS_TIME_H @H5_HAVE_SYS_TIME_H@
@@ -396,9 +371,6 @@
/* Define to 1 if you have the <unistd.h> header file. */
#cmakedefine H5_HAVE_UNISTD_H @H5_HAVE_UNISTD_H@
-/* Define to 1 if you have the `asprintf' function. */
-#cmakedefine H5_HAVE_ASPRINTF @H5_HAVE_ASPRINTF@
-
/* Define to 1 if you have the `vasprintf' function. */
#cmakedefine H5_HAVE_VASPRINTF @H5_HAVE_VASPRINTF@
@@ -620,9 +592,6 @@
/* Define to 1 if you can safely include both <sys/time.h> and <time.h>. */
#cmakedefine H5_TIME_WITH_SYS_TIME @H5_TIME_WITH_SYS_TIME@
-/* Define to 1 if your <sys/time.h> declares `struct tm'. */
-#cmakedefine H5_TM_IN_SYS_TIME @H5_TM_IN_SYS_TIME@
-
/* Define using v1.6 public API symbols by default */
#cmakedefine H5_USE_16_API_DEFAULT @H5_USE_16_API_DEFAULT@
@@ -681,8 +650,4 @@
/* Define to `long' if <sys/types.h> does not define. */
#cmakedefine H5_ssize_t
-#if defined(__cplusplus) && defined(inline)
-#undef inline
-#endif
-
#endif
diff --git a/fortran/src/H5Zf.c b/fortran/src/H5Zf.c
index 0427c23..169e018 100644
--- a/fortran/src/H5Zf.c
+++ b/fortran/src/H5Zf.c
@@ -51,9 +51,7 @@ nh5zunregister_c (int_f *filter)
* Call H5Zunregister function.
*/
c_filter = (H5Z_filter_t)*filter;
- printf(" filter # %d \n", (int)c_filter);
status = H5Zunregister(c_filter);
- printf("From C zunregister %d \n", status);
if (status < 0) return ret_value;
ret_value = 0;
return ret_value;
diff --git a/release_docs/INSTALL_parallel b/release_docs/INSTALL_parallel
index c38c3fc..e4570b7 100644
--- a/release_docs/INSTALL_parallel
+++ b/release_docs/INSTALL_parallel
@@ -49,9 +49,8 @@ more detailed explanations.
----------------------------
HDF5 knows several parallel compilers: mpicc, hcc, mpcc, mpcc_r. To build
parallel HDF5 with one of the above, just set CC as it and configure.
-The "--enable-parallel" is optional in this case.
- $ CC=/usr/local/mpi/bin/mpicc ./configure --prefix=<install-directory>
+ $ CC=/usr/local/mpi/bin/mpicc ./configure --enable-parallel --prefix=<install-directory>
$ make # build the library
$ make check # verify the correctness
# Read the Details section about parallel tests.
@@ -79,7 +78,7 @@ is supported.
Then do the following steps:
- $ ./configure --disable-shared --prefix=<install-directory>
+ $ ./configure --enable-parallel --disable-shared --prefix=<install-directory>
$ make # build the library
$ make check # verify the correctness
# Read the Details section about parallel tests.
@@ -176,8 +175,8 @@ a properly installed parallel compiler (e.g., MPICH's mpicc or IBM's mpcc_r)
and supply the compiler name as the value of the CC environment variable.
For examples,
- $ CC=mpcc_r ./configure
- $ CC=/usr/local/mpi/bin/mpicc ./configure
+ $ CC=mpcc_r ./configure --enable-parallel
+ $ CC=/usr/local/mpi/bin/mpicc ./configure --enable-parallel
If no such a compiler command is available then you must use your normal
C compiler along with the location(s) of MPI/MPI-IO files to be used.
diff --git a/src/H5Dbtree.c b/src/H5Dbtree.c
index a3019c3..328795e 100644
--- a/src/H5Dbtree.c
+++ b/src/H5Dbtree.c
@@ -74,8 +74,8 @@
* The chunk's file address is part of the B-tree and not part of the key.
*/
typedef struct H5D_btree_key_t {
- uint32_t nbytes; /*size of stored data */
hsize_t offset[H5O_LAYOUT_NDIMS]; /*logical offset to start*/
+ uint32_t nbytes; /*size of stored data */
unsigned filter_mask; /*excluded filters */
} H5D_btree_key_t;
@@ -277,15 +277,15 @@ H5D__btree_new_node(H5F_t *f, hid_t UNUSED dxpl_id, H5B_ins_t op,
HDassert(addr_p);
/* Set address */
- HDassert(udata->nbytes > 0);
- HDassert(H5F_addr_defined(udata->addr));
- *addr_p = udata->addr;
+ HDassert(H5F_addr_defined(udata->chunk_block.offset));
+ HDassert(udata->chunk_block.length > 0);
+ *addr_p = udata->chunk_block.offset;
/*
* The left key describes the storage of the UDATA chunk being
* inserted into the tree.
*/
- lt_key->nbytes = udata->nbytes;
+ H5_ASSIGN_OVERFLOW(lt_key->nbytes, udata->chunk_block.length, hsize_t, uint32_t);
lt_key->filter_mask = udata->filter_mask;
for(u = 0; u < udata->common.layout->ndims; u++)
lt_key->offset[u] = udata->common.offset[u];
@@ -466,8 +466,8 @@ H5D__btree_found(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, haddr_t addr, const void
/* Initialize return values */
HDassert(lt_key->nbytes > 0);
- udata->addr = addr;
- udata->nbytes = lt_key->nbytes;
+ udata->chunk_block.offset = addr;
+ udata->chunk_block.length = lt_key->nbytes;
udata->filter_mask = lt_key->filter_mask;
done:
@@ -545,17 +545,17 @@ H5D__btree_insert(H5F_t *f, hid_t UNUSED dxpl_id, haddr_t addr, void *_lt_key,
* Already exists. If the new size is not the same as the old size
* then we should reallocate storage.
*/
- if(lt_key->nbytes != udata->nbytes) {
+ if(lt_key->nbytes != udata->chunk_block.length) {
/* Set node's address (already re-allocated by main chunk routines) */
- HDassert(H5F_addr_defined(udata->addr));
- *new_node_p = udata->addr;
- lt_key->nbytes = udata->nbytes;
+ HDassert(H5F_addr_defined(udata->chunk_block.offset));
+ *new_node_p = udata->chunk_block.offset;
+ H5_ASSIGN_OVERFLOW(lt_key->nbytes, udata->chunk_block.length, hsize_t, uint32_t);
lt_key->filter_mask = udata->filter_mask;
*lt_key_changed = TRUE;
ret_value = H5B_INS_CHANGE;
} else {
/* Already have address in udata, from main chunk routines */
- HDassert(H5F_addr_defined(udata->addr));
+ HDassert(H5F_addr_defined(udata->chunk_block.offset));
ret_value = H5B_INS_NOOP;
}
@@ -569,15 +569,15 @@ H5D__btree_insert(H5F_t *f, hid_t UNUSED dxpl_id, haddr_t addr, void *_lt_key,
* Split this node, inserting the new new node to the right of the
* current node. The MD_KEY is where the split occurs.
*/
- md_key->nbytes = udata->nbytes;
+ H5_ASSIGN_OVERFLOW(md_key->nbytes, udata->chunk_block.length, hsize_t, uint32_t);
md_key->filter_mask = udata->filter_mask;
for(u = 0; u < udata->common.layout->ndims; u++) {
HDassert(0 == udata->common.offset[u] % udata->common.layout->dim[u]);
md_key->offset[u] = udata->common.offset[u];
} /* end for */
- HDassert(H5F_addr_defined(udata->addr));
- *new_node_p = udata->addr;
+ HDassert(H5F_addr_defined(udata->chunk_block.offset));
+ *new_node_p = udata->chunk_block.offset;
ret_value = H5B_INS_RIGHT;
} else {
diff --git a/src/H5Dbtree2.c b/src/H5Dbtree2.c
index e90aebd..fb79046 100644
--- a/src/H5Dbtree2.c
+++ b/src/H5Dbtree2.c
@@ -1035,8 +1035,7 @@ H5D_bt2_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat
HDassert(idx_info->storage);
HDassert(H5F_addr_defined(idx_info->storage->idx_addr));
HDassert(udata);
- HDassert(udata->need_insert || udata->need_modify);
- HDassert(H5F_addr_defined(udata->addr));
+ HDassert(H5F_addr_defined(udata->chunk_block.offset));
/* Check if the v2 B-tree is open yet */
if(NULL == idx_info->storage->u.btree2.bt2) {
@@ -1054,16 +1053,15 @@ H5D_bt2_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat
if(idx_info->pline->nused > 0) { /* filtered chunk */
bt2_udata.rec = &filt_rec;
-
- filt_rec.addr = udata->addr;
- filt_rec.nbytes = udata->nbytes;
+ filt_rec.addr = udata->chunk_block.offset;
+ filt_rec.nbytes = udata->chunk_block.length;
filt_rec.filter_mask = udata->filter_mask;
for(u = 0; u < (idx_info->layout->ndims - 1); u++)
filt_rec.offset[u] = udata->common.offset[u];
} else { /* non-filtered chunk */
bt2_udata.rec = &rec;
- rec.addr = udata->addr;
+ rec.addr = udata->chunk_block.offset;
for(u = 0; u < (idx_info->layout->ndims - 1); u++)
rec.offset[u] = udata->common.offset[u];
}
@@ -1075,7 +1073,6 @@ H5D_bt2_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat
if(H5B2_modify(bt2, idx_info->dxpl_id, &bt2_udata, H5D_bt2_mod_filt_cb, &filt_rec) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to modify record in v2 B-tree")
} else {
- HDassert(udata->need_insert);
/* Insert record for object in v2 B-tree */
if(H5B2_insert(bt2, idx_info->dxpl_id, &bt2_udata) < 0)
@@ -1205,8 +1202,8 @@ H5D_bt2_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata)
HGOTO_ERROR(H5E_HEAP, H5E_NOTFOUND, FAIL, "can't find object in v2 B-tree")
/* Set info for the chunk */
- udata->addr = found_rec.addr;
- udata->nbytes = found_rec.nbytes;
+ udata->chunk_block.offset = found_rec.addr;
+ udata->chunk_block.length = found_rec.nbytes;
udata->filter_mask = found_rec.filter_mask;
} /* end if */
else { /* non-filtered chunk */
@@ -1230,13 +1227,16 @@ H5D_bt2_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata)
if(H5B2_find(bt2, idx_info->dxpl_id, &bt2_udata, H5D_bt2_found_cb, &found_rec) < 0)
HGOTO_ERROR(H5E_HEAP, H5E_NOTFOUND, FAIL, "can't find object in v2 B-tree")
- udata->addr = found_rec.addr;
+ udata->chunk_block.offset = found_rec.addr;
/* Update the other (constant) information for the chunk */
- udata->nbytes = idx_info->layout->size;
+ udata->chunk_block.length = idx_info->layout->size;
udata->filter_mask = 0;
} /* end else */
+ if(!H5F_addr_defined(udata->chunk_block.offset))
+ udata->chunk_block.length = 0;
+
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5D_bt2_idx_get_addr() */
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index bdd9c9c..39e3e48 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -214,9 +214,9 @@ H5D__nonexistent_readvv(const H5D_io_info_t *io_info,
/* Helper routines */
static herr_t H5D__chunk_set_info_real(H5O_layout_chunk_t *layout, unsigned ndims,
const hsize_t *curr_dims, const hsize_t *max_dims);
-static void *H5D__chunk_alloc(size_t size, const H5O_pline_t *pline);
-static void *H5D__chunk_xfree(void *chk, const H5O_pline_t *pline);
-static void *H5D__chunk_realloc(void *chk, size_t size,
+static void *H5D__chunk_mem_alloc(size_t size, const H5O_pline_t *pline);
+static void *H5D__chunk_mem_xfree(void *chk, const H5O_pline_t *pline);
+static void *H5D__chunk_mem_realloc(void *chk, size_t size,
const H5O_pline_t *pline);
static herr_t H5D__chunk_cinfo_cache_reset(H5D_chunk_cached_t *last);
static herr_t H5D__chunk_cinfo_cache_update(H5D_chunk_cached_t *last,
@@ -241,7 +241,9 @@ static hbool_t H5D__chunk_is_partial_edge_chunk(const hsize_t *chunk_offset,
static herr_t H5D__chunk_cache_prune(const H5D_t *dset, hid_t dxpl_id,
const H5D_dxpl_cache_t *dxpl_cache, size_t size);
static herr_t H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, hbool_t new_unfilt_chunk);
-static herr_t H5D__idx_chunk_alloc(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata);
+static herr_t H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info,
+ const H5F_block_t *old_chunk, H5F_block_t *new_chunk, hbool_t *need_insert,
+ hbool_t *need_bt2_modify, hsize_t scaled[]);
#ifdef H5_HAVE_PARALLEL
static herr_t H5D__chunk_collective_fill(const H5D_t *dset, hid_t dxpl_id,
H5D_chunk_coll_info_t *chunk_info, size_t chunk_size, const void *fill_buf);
@@ -329,12 +331,14 @@ H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, uint32_t filters, hsiz
const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache */
H5D_chk_idx_info_t idx_info; /* Chunked index info */
hsize_t scaled[H5S_MAX_RANK]; /* Scaled coordinates for this chunk */
+ H5F_block_t old_chunk; /* Offset/length of old chunk */
+ hbool_t need_insert = FALSE; /* Whether the chunk needs to be inserted into the index */
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC_TAG(dxpl_id, dset->oloc.addr, FAIL)
/* Allocate dataspace and initialize it if it hasn't been. */
- if(!(*dset->shared->layout.ops->is_space_alloc)(&dset->shared->layout.storage))
+ if(!(*layout->ops->is_space_alloc)(&layout->storage))
/* Allocate storage */
if(H5D__alloc_storage(dset, dxpl_id, H5D_ALLOC_WRITE, FALSE, NULL) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize storage")
@@ -346,52 +350,65 @@ H5D__chunk_direct_write(const H5D_t *dset, hid_t dxpl_id, uint32_t filters, hsiz
if(H5D__chunk_lookup(dset, dxpl_id, offset, scaled, &udata) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address")
- udata.filter_mask = filters;
-
- /* Check if the chunk needs to be 'inserted' (could exist already and
- * the 'insert' operation could resize it)
- */
- {
+ /* Sanity check */
+ HDassert((H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length > 0) ||
+ (!H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length == 0));
- /* Compose chunked index info struct */
- idx_info.f = dset->oloc.file;
- idx_info.dxpl_id = dxpl_id;
- idx_info.pline = &(dset->shared->dcpl_cache.pline);
- idx_info.layout = &(dset->shared->layout.u.chunk);
- idx_info.storage = &(dset->shared->layout.storage.u.chunk);
+ /* Set the file block information for the old chunk */
+ /* (Which is only defined when overwriting an existing chunk) */
+ old_chunk.offset = udata.chunk_block.offset;
+ old_chunk.length = udata.chunk_block.length;
- /* Set up the size of chunk for user data */
- udata.nbytes = data_size;
+ /* Check if the chunk needs to be inserted (it also could exist already
+ * and the chunk allocate operation could resize it)
+ */
+ /* Compose chunked index info struct */
+ idx_info.f = dset->oloc.file;
+ idx_info.dxpl_id = dxpl_id;
+ idx_info.pline = &(dset->shared->dcpl_cache.pline);
+ idx_info.layout = &(dset->shared->layout.u.chunk);
+ idx_info.storage = &(dset->shared->layout.storage.u.chunk);
- /* Create the chunk it if it doesn't exist, or reallocate the chunk
- * if its size changed.
- */
- if(H5D__idx_chunk_alloc(&idx_info, &udata) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk on chunk level")
+ /* Set up the size of chunk for user data */
+ udata.chunk_block.length = data_size;
- /* Make sure the address of the chunk is returned. */
- if(!H5F_addr_defined(udata.addr))
- HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "chunk address isn't defined")
- } /* end if */
+ /* Create the chunk it if it doesn't exist, or reallocate the chunk
+ * if its size changed.
+ */
+ if(H5D__chunk_file_alloc(&idx_info, &old_chunk, &udata.chunk_block, &need_insert, &udata.need_modify, scaled) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate chunk")
- /* Fill the DXPL cache values for later use */
- if(H5D__get_dxpl_cache(dxpl_id, &dxpl_cache) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache")
+ /* Make sure the address of the chunk is returned. */
+ if(!H5F_addr_defined(udata.chunk_block.offset))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "chunk address isn't defined")
/* Evict the (old) entry from the cache if present, but do not flush
* it to disk */
- if(UINT_MAX != udata.idx_hint)
+ if(UINT_MAX != udata.idx_hint) {
+ H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */
+ H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */
+ const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache */
+
+ /* Fill the DXPL cache values for later use */
+ if(H5D__get_dxpl_cache(dxpl_id, &dxpl_cache) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache")
+
if(H5D__chunk_cache_evict(dset, dxpl_id, dxpl_cache, rdcc->slot[udata.idx_hint], FALSE) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTREMOVE, FAIL, "unable to evict chunk")
+ } /* end if */
/* Write the data to the file */
- if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, data_size, dxpl_id, buf) < 0)
+ if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.chunk_block.offset, data_size, dxpl_id, buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write raw data to file")
- if((udata.need_insert || udata.need_modify) && dset->shared->layout.storage.u.chunk.ops->insert_addr) {
- if((dset->shared->layout.storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0)
+ /* Insert the chunk record into the index */
+ if((need_insert || udata.need_modify) && layout->storage.u.chunk.ops->insert_addr) {
+ /* Set the chunk's filter mask to the new settings */
+ udata.filter_mask = filters;
+
+ if((layout->storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index")
- }
+ } /* end if */
done:
FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
@@ -984,7 +1001,7 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5D__chunk_alloc
+ * Function: H5D__chunk_mem_alloc
*
* Purpose: Allocate space for a chunk in memory. This routine allocates
* memory space for non-filtered chunks from a block free list
@@ -998,7 +1015,7 @@ done:
*-------------------------------------------------------------------------
*/
static void *
-H5D__chunk_alloc(size_t size, const H5O_pline_t *pline)
+H5D__chunk_mem_alloc(size_t size, const H5O_pline_t *pline)
{
void *ret_value = NULL; /* Return value */
@@ -1012,11 +1029,11 @@ H5D__chunk_alloc(size_t size, const H5O_pline_t *pline)
ret_value = H5FL_BLK_MALLOC(chunk, size);
FUNC_LEAVE_NOAPI(ret_value)
-} /* H5D__chunk_alloc() */
+} /* H5D__chunk_mem_alloc() */
/*-------------------------------------------------------------------------
- * Function: H5D__chunk_xfree
+ * Function: H5D__chunk_mem_xfree
*
* Purpose: Free space for a chunk in memory. This routine allocates
* memory space for non-filtered chunks from a block free list
@@ -1030,7 +1047,7 @@ H5D__chunk_alloc(size_t size, const H5O_pline_t *pline)
*-------------------------------------------------------------------------
*/
static void *
-H5D__chunk_xfree(void *chk, const H5O_pline_t *pline)
+H5D__chunk_mem_xfree(void *chk, const H5O_pline_t *pline)
{
FUNC_ENTER_STATIC_NOERR
@@ -1042,11 +1059,11 @@ H5D__chunk_xfree(void *chk, const H5O_pline_t *pline)
} /* end if */
FUNC_LEAVE_NOAPI(NULL)
-} /* H5D__chunk_xfree() */
+} /* H5D__chunk_mem_xfree() */
/*-------------------------------------------------------------------------
- * Function: H5D__chunk_realloc
+ * Function: H5D__chunk_mem_realloc
*
* Purpose: Reallocate space for a chunk in memory. This routine allocates
* memory space for non-filtered chunks from a block free list
@@ -1060,7 +1077,7 @@ H5D__chunk_xfree(void *chk, const H5O_pline_t *pline)
*-------------------------------------------------------------------------
*/
static void *
-H5D__chunk_realloc(void *chk, size_t size, const H5O_pline_t *pline)
+H5D__chunk_mem_realloc(void *chk, size_t size, const H5O_pline_t *pline)
{
void *ret_value = NULL; /* Return value */
@@ -1075,7 +1092,7 @@ H5D__chunk_realloc(void *chk, size_t size, const H5O_pline_t *pline)
ret_value = H5FL_BLK_REALLOC(chunk, chk, size);
FUNC_LEAVE_NOAPI(ret_value)
-} /* H5D__chunk_realloc() */
+} /* H5D__chunk_mem_realloc() */
/*--------------------------------------------------------------------------
@@ -1860,10 +1877,7 @@ H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
chunk_node = H5D_CHUNK_GET_FIRST_NODE(fm);
while(chunk_node) {
H5D_chunk_info_t *chunk_info; /* Chunk information */
- H5D_io_info_t *chk_io_info; /* Pointer to I/O info object for this chunk */
- void *chunk; /* Pointer to locked chunk buffer */
- H5D_chunk_ud_t udata; /* B-tree pass-through */
- htri_t cacheable; /* Whether the chunk is cacheable */
+ H5D_chunk_ud_t udata; /* Chunk index pass-through */
/* Get the actual chunk information from the skip list node */
chunk_info = H5D_CHUNK_GET_NODE_INFO(fm, chunk_node);
@@ -1872,18 +1886,26 @@ H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
if(H5D__chunk_lookup(io_info->dset, io_info->dxpl_id, chunk_info->coords, chunk_info->scaled, &udata) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address")
+ /* Sanity check */
+ HDassert((H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length > 0) ||
+ (!H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length == 0));
+
/* Check for non-existant chunk & skip it if appropriate */
- if(H5F_addr_defined(udata.addr) || UINT_MAX != udata.idx_hint
+ if(H5F_addr_defined(udata.chunk_block.offset) || UINT_MAX != udata.idx_hint
|| !skip_missing_chunks) {
- /* Pass in chunk's coordinates. */
- io_info->store->chunk.offset = chunk_info->coords;
+ H5D_io_info_t *chk_io_info; /* Pointer to I/O info object for this chunk */
+ void *chunk = NULL; /* Pointer to locked chunk buffer */
+ htri_t cacheable; /* Whether the chunk is cacheable */
+
+ /* Pass in chunk's coordinates in a union. */
+ io_info->store->chunk.offset = chunk_info->coords;
+ io_info->store->chunk.index = chunk_info->index;
- /* Determine if we should use the chunk cache */
- if((cacheable = H5D__chunk_cacheable(io_info, udata.addr, FALSE)) < 0)
+ /* Load the chunk into cache and lock it. */
+ if((cacheable = H5D__chunk_cacheable(io_info, udata.chunk_block.offset, FALSE)) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't tell if chunk is cacheable")
if(cacheable) {
- /* Load the chunk into cache and lock it. */
/* Compute # of bytes accessed in chunk */
H5_CHECK_OVERFLOW(type_info->src_type_size, /*From:*/ size_t, /*To:*/ uint32_t);
@@ -1899,20 +1921,14 @@ H5D__chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
/* Point I/O info at contiguous I/O info for this chunk */
chk_io_info = &cpt_io_info;
} /* end if */
- else if(H5F_addr_defined(udata.addr)) {
+ else if(H5F_addr_defined(udata.chunk_block.offset)) {
/* Set up the storage address information for this chunk */
- ctg_store.contig.dset_addr = udata.addr;
-
- /* No chunk cached */
- chunk = NULL;
+ ctg_store.contig.dset_addr = udata.chunk_block.offset;
/* Point I/O info at temporary I/O info for this chunk */
chk_io_info = &ctg_io_info;
} /* end else if */
else {
- /* No chunk cached */
- chunk = NULL;
-
/* Point I/O info at "nonexistent" I/O info for this chunk */
chk_io_info = &nonexistent_io_info;
} /* end else */
@@ -1990,11 +2006,12 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
chunk_node = H5D_CHUNK_GET_FIRST_NODE(fm);
while(chunk_node) {
H5D_chunk_info_t *chunk_info; /* Chunk information */
+ H5D_chk_idx_info_t idx_info; /* Chunked index info */
H5D_io_info_t *chk_io_info; /* Pointer to I/O info object for this chunk */
void *chunk; /* Pointer to locked chunk buffer */
H5D_chunk_ud_t udata; /* Index pass-through */
htri_t cacheable; /* Whether the chunk is cacheable */
- H5D_chk_idx_info_t idx_info; /* Chunked index info */
+ hbool_t need_insert = FALSE; /* Whether the chunk needs to be inserted into the index */
/* Get the actual chunk information from the skip list node */
chunk_info = H5D_CHUNK_GET_NODE_INFO(fm, chunk_node);
@@ -2003,13 +2020,16 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
if(H5D__chunk_lookup(io_info->dset, io_info->dxpl_id, chunk_info->coords, chunk_info->scaled, &udata) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error looking up chunk address")
- /* Pass in chunk's coordinates. */
- io_info->store->chunk.offset = chunk_info->coords;
+ /* Sanity check */
+ HDassert((H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length > 0) ||
+ (!H5F_addr_defined(udata.chunk_block.offset) && udata.chunk_block.length == 0));
- /* Determine if we should use the chunk cache */
- if((cacheable = H5D__chunk_cacheable(io_info, udata.addr, TRUE)) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't tell if chunk is cacheable")
+ /* Pass in chunk's coordinates in a union. */
+ io_info->store->chunk.offset = &chunk_info->coords[0];
+ io_info->store->chunk.index = chunk_info->index;
+ if((cacheable = H5D__chunk_cacheable(io_info, udata.chunk_block.offset, TRUE)) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't tell if chunk is cacheable")
if(cacheable) {
/* Load the chunk into cache. But if the whole chunk is written,
* simply allocate space instead of load the chunk. */
@@ -2036,8 +2056,7 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
} /* end if */
else {
/* If the chunk hasn't been allocated on disk, do so now. */
- if(!H5F_addr_defined(udata.addr)) {
-
+ if(!H5F_addr_defined(udata.chunk_block.offset)) {
/* Compose chunked index info struct */
idx_info.f = io_info->dset->oloc.file;
idx_info.dxpl_id = io_info->dxpl_id;
@@ -2046,15 +2065,14 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
idx_info.storage = &(io_info->dset->shared->layout.storage.u.chunk);
/* Set up the size of chunk for user data */
- udata.nbytes = io_info->dset->shared->layout.u.chunk.size;
+ udata.chunk_block.length = io_info->dset->shared->layout.u.chunk.size;
- /* Create the chunk */
- if(H5D__idx_chunk_alloc(&idx_info, &udata) < 0)
+ /* Allocate the chunk */
+ if(H5D__chunk_file_alloc(&idx_info, NULL, &udata.chunk_block, &need_insert, &udata.need_modify, chunk_info->scaled) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk on chunk level")
-
/* Make sure the address of the chunk is returned. */
- if(!H5F_addr_defined(udata.addr))
+ if(!H5F_addr_defined(udata.chunk_block.offset))
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "chunk address isn't defined")
/* Cache the new chunk information */
@@ -2062,7 +2080,7 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
} /* end if */
/* Set up the storage address information for this chunk */
- ctg_store.contig.dset_addr = udata.addr;
+ ctg_store.contig.dset_addr = udata.chunk_block.offset;
/* No chunk cached */
chunk = NULL;
@@ -2078,11 +2096,11 @@ H5D__chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
/* Release the cache lock on the chunk, or insert chunk into index. */
if(chunk) {
- if(chunk && H5D__chunk_unlock(io_info, &udata, TRUE, chunk, dst_accessed_bytes) < 0)
+ if(H5D__chunk_unlock(io_info, &udata, TRUE, chunk, dst_accessed_bytes) < 0)
HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to unlock raw data chunk")
} /* end if */
else {
- if((udata.need_insert || udata.need_modify) && io_info->dset->shared->layout.storage.u.chunk.ops->insert_addr)
+ if((need_insert || udata.need_modify) && io_info->dset->shared->layout.storage.u.chunk.ops->insert_addr)
if((io_info->dset->shared->layout.storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index")
} /* end else */
@@ -2292,8 +2310,8 @@ H5D__chunk_cinfo_cache_update(H5D_chunk_cached_t *last, const H5D_chunk_ud_t *ud
/* Stored the information to cache */
HDmemcpy(last->offset, udata->common.offset, sizeof(hsize_t) * udata->common.layout->ndims);
- last->addr = udata->addr;
- last->nbytes = udata->nbytes;
+ last->addr = udata->chunk_block.offset;
+ H5_ASSIGN_OVERFLOW(last->nbytes, udata->chunk_block.length, hsize_t, uint32_t);
last->chunk_idx = udata->chunk_idx;
last->filter_mask = udata->filter_mask;
@@ -2340,8 +2358,8 @@ H5D__chunk_cinfo_cache_found(const H5D_chunk_cached_t *last, H5D_chunk_ud_t *uda
HGOTO_DONE(FALSE)
/* Retrieve the information from the cache */
- udata->addr = last->addr;
- udata->nbytes = last->nbytes;
+ udata->chunk_block.offset = last->addr;
+ udata->chunk_block.length = last->nbytes;
udata->chunk_idx = last->chunk_idx;
udata->filter_mask = last->filter_mask;
@@ -2511,11 +2529,10 @@ H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id, const hsize_t *chunk_offset,
udata->common.scaled = scaled;
/* Reset information about the chunk we are looking for */
- udata->addr = HADDR_UNDEF;
- udata->nbytes = 0;
+ udata->chunk_block.offset = HADDR_UNDEF;
+ udata->chunk_block.length = 0;
udata->filter_mask = 0;
udata->new_unfilt_chunk = FALSE;
- udata->need_insert = FALSE;
udata->need_modify = FALSE;
/* Check for chunk in cache */
@@ -2533,9 +2550,11 @@ H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id, const hsize_t *chunk_offset,
/* Find chunk addr */
if(found) {
- udata->addr = ent->chunk_addr;
+ udata->chunk_block.offset = ent->chunk_block.offset;
+ udata->chunk_block.length = ent->chunk_block.length;;
udata->chunk_idx = ent->chunk_idx;
- } else {
+ } /* end if */
+ else {
/* Invalidate idx_hint, to signal that the chunk is not in cache */
udata->idx_hint = UINT_MAX;
@@ -2607,9 +2626,10 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
buf = ent->chunk;
if(ent->dirty && !ent->deleted) {
+ H5D_chk_idx_info_t idx_info; /* Chunked index info */
H5D_chunk_ud_t udata; /* pass through B-tree */
- hbool_t must_insert = FALSE; /* Whether the chunk must go through the "insert" method */
- H5D_chk_idx_info_t idx_info; /* Chunked index info */
+ hbool_t must_alloc = FALSE; /* Whether the chunk must be allocated */
+ hbool_t need_insert = FALSE; /* Whether the chunk needs to be inserted into the index */
/* Set up user data for index callbacks */
udata.common.layout = &dset->shared->layout.u.chunk;
@@ -2617,18 +2637,16 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
udata.common.offset = ent->offset;
udata.common.rdcc = &(dset->shared->cache.chunk);
udata.common.scaled = ent->scaled;
-
+ udata.chunk_block.offset = ent->chunk_block.offset;
+ udata.chunk_block.length = dset->shared->layout.u.chunk.size;
udata.filter_mask = 0;
- udata.nbytes = dset->shared->layout.u.chunk.size;
- udata.addr = ent->chunk_addr;
udata.chunk_idx = ent->chunk_idx;
- udata.need_insert = FALSE;
udata.need_modify = FALSE;
/* Should the chunk be filtered before writing it to disk? */
if(dset->shared->dcpl_cache.pline.nused
&& !(ent->edge_chunk_state & H5D_RDCC_DISABLE_FILTERS)) {
- size_t alloc = udata.nbytes; /* Bytes allocated for BUF */
+ size_t alloc = udata.chunk_block.length; /* Bytes allocated for BUF */
size_t nbytes; /* Chunk size (in bytes) */
if(!reset) {
@@ -2637,10 +2655,9 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
* the pipeline because we'll want to save the original buffer
* for later.
*/
- H5_ASSIGN_OVERFLOW(alloc, udata.nbytes, uint32_t, size_t);
if(NULL == (buf = H5MM_malloc(alloc)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline")
- HDmemcpy(buf, ent->chunk, udata.nbytes);
+ HDmemcpy(buf, ent->chunk, alloc);
} /* end if */
else {
/*
@@ -2653,7 +2670,7 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
point_of_no_return = TRUE;
ent->chunk = NULL;
} /* end else */
- H5_ASSIGN_OVERFLOW(nbytes, udata.nbytes, uint32_t, size_t);
+ H5_ASSIGN_OVERFLOW(nbytes, udata.chunk_block.length, uint32_t, size_t);
if(H5Z_pipeline(&(dset->shared->dcpl_cache.pline), 0, &(udata.filter_mask), dxpl_cache->err_detect,
dxpl_cache->filter_cb, &nbytes, &alloc, &buf) < 0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "output pipeline failed")
@@ -2662,14 +2679,14 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
if(nbytes > ((size_t)0xffffffff))
HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk too large for 32-bit length")
#endif /* H5_SIZEOF_SIZE_T > 4 */
- H5_ASSIGN_OVERFLOW(udata.nbytes, nbytes, size_t, uint32_t);
+ H5_ASSIGN_OVERFLOW(udata.chunk_block.length, nbytes, size_t, uint32_t);
- /* Indicate that the chunk must go through 'insert' method */
- must_insert = TRUE;
+ /* Indicate that the chunk must be allocated */
+ must_alloc = TRUE;
} /* end if */
- else if(!H5F_addr_defined(udata.addr)) {
- /* Indicate that the chunk must go through 'insert' method */
- must_insert = TRUE;
+ else if(!H5F_addr_defined(udata.chunk_block.offset)) {
+ /* Indicate that the chunk must be allocated */
+ must_alloc = TRUE;
/* This flag could be set for this chunk, just remove and ignore it
*/
@@ -2678,7 +2695,7 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
else if(ent->edge_chunk_state & H5D_RDCC_NEWLY_DISABLED_FILTERS) {
/* Chunk on disk is still filtered, must insert to allocate correct
* size */
- must_insert = TRUE;
+ must_alloc = TRUE;
/* Set the disable filters field back to the standard disable
* filters setting, as it no longer needs to be inserted with every
@@ -2688,11 +2705,10 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
HDassert(!(ent->edge_chunk_state & H5D_RDCC_NEWLY_DISABLED_FILTERS));
- /* Check if the chunk needs to be 'inserted' (could exist already and
- * the 'insert' operation could resize it)
+ /* Check if the chunk needs to be allocated (it also could exist already
+ * and the chunk alloc operation could resize it)
*/
- if(must_insert) {
-
+ if(must_alloc) {
/* Compose chunked index info struct */
idx_info.f = dset->oloc.file;
idx_info.dxpl_id = dxpl_id;
@@ -2703,24 +2719,23 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
/* Create the chunk it if it doesn't exist, or reallocate the chunk
* if its size changed.
*/
- if(H5D__idx_chunk_alloc(&idx_info, &udata) < 0)
+ if(H5D__chunk_file_alloc(&idx_info, &(ent->chunk_block), &udata.chunk_block, &need_insert, &udata.need_modify, ent->scaled) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk on chunk level")
- /* Update the chunk entry's address, in case it was allocated or relocated */
- ent->chunk_addr = udata.addr;
+ /* Update the chunk entry's info, in case it was allocated or relocated */
+ ent->chunk_block.offset = udata.chunk_block.offset;
+ ent->chunk_block.length = udata.chunk_block.length;
} /* end if */
/* Write the data to the file */
- HDassert(H5F_addr_defined(udata.addr));
- if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.nbytes, dxpl_id, buf) < 0)
+ HDassert(H5F_addr_defined(udata.chunk_block.offset));
+ if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.chunk_block.offset, udata.chunk_block.length, dxpl_id, buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write raw data to file")
- if(udata.need_insert || udata.need_modify) {
- if(dset->shared->layout.storage.u.chunk.ops->insert_addr) {
- if((dset->shared->layout.storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index")
- }
- }
+ /* Insert the chunk record into the index */
+ if((need_insert || udata.need_modify) && dset->shared->layout.storage.u.chunk.ops->insert_addr)
+ if((dset->shared->layout.storage.u.chunk.ops->insert_addr)(&idx_info, &udata) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index")
/* Cache the chunk's info, in case it's accessed again shortly */
H5D__chunk_cinfo_cache_update(&dset->shared->cache.chunk.last, &udata);
@@ -2738,7 +2753,7 @@ H5D__chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
if(buf == ent->chunk)
buf = NULL;
if(ent->chunk != NULL)
- ent->chunk = (uint8_t *)H5D__chunk_xfree(ent->chunk,
+ ent->chunk = (uint8_t *)H5D__chunk_mem_xfree(ent->chunk,
((ent->edge_chunk_state & H5D_RDCC_DISABLE_FILTERS) ? NULL
: &(dset->shared->dcpl_cache.pline)));
} /* end if */
@@ -2756,7 +2771,7 @@ done:
*/
if(ret_value < 0 && point_of_no_return)
if(ent->chunk)
- ent->chunk = (uint8_t *)H5D__chunk_xfree(ent->chunk,
+ ent->chunk = (uint8_t *)H5D__chunk_mem_xfree(ent->chunk,
((ent->edge_chunk_state & H5D_RDCC_DISABLE_FILTERS) ? NULL
: &(dset->shared->dcpl_cache.pline)));
@@ -2800,7 +2815,7 @@ H5D__chunk_cache_evict(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t
else {
/* Don't flush, just free chunk */
if(ent->chunk != NULL)
- ent->chunk = (uint8_t *)H5D__chunk_xfree(ent->chunk,
+ ent->chunk = (uint8_t *)H5D__chunk_mem_xfree(ent->chunk,
((ent->edge_chunk_state & H5D_RDCC_DISABLE_FILTERS) ? NULL
: &(dset->shared->dcpl_cache.pline)));
} /* end else */
@@ -2984,15 +2999,14 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata,
hbool_t relax, hbool_t prev_unfilt_chunk)
{
const H5D_t *dset = io_info->dset; /* Local pointer to the dataset info */
- const H5O_pline_t *pline = &(dset->shared->dcpl_cache.pline); /* I/O pipeline info - always equal to the pline passed to H5D__chunk_alloc */
+ const H5O_pline_t *pline = &(dset->shared->dcpl_cache.pline); /* I/O pipeline info - always equal to the pline passed to H5D__chunk_mem_alloc */
const H5O_pline_t *old_pline = pline; /* Old pipeline, i.e. pipeline used to read the chunk */
const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */
const H5O_fill_t *fill = &(dset->shared->dcpl_cache.fill); /* Fill value info */
H5D_fill_buf_info_t fb_info; /* Dataset's fill buffer info */
hbool_t fb_info_init = FALSE; /* Whether the fill value buffer has been initialized */
H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache*/
- H5D_rdcc_ent_t *ent = NULL; /*cache entry */
- haddr_t chunk_addr = HADDR_UNDEF; /* Address of chunk on disk */
+ H5D_rdcc_ent_t *ent; /*cache entry */
size_t chunk_size; /*size of a chunk */
hbool_t disable_filters = FALSE; /* Whether to disable filters (when adding to cache) */
void *chunk = NULL; /*the file chunk */
@@ -3055,12 +3069,12 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata,
* code is changed in an inappropriate/incomplete way. */
pline = NULL;
- /* Reallocate the chunk so H5D__chunk_xfree doesn't get confused
+ /* Reallocate the chunk so H5D__chunk_mem_xfree doesn't get confused
*/
- if(NULL == (chunk = H5D__chunk_alloc(chunk_size, pline)))
+ if(NULL == (chunk = H5D__chunk_mem_alloc(chunk_size, pline)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk")
HDmemcpy(chunk, ent->chunk, chunk_size);
- ent->chunk = (uint8_t *)H5D__chunk_xfree(ent->chunk, old_pline);
+ ent->chunk = (uint8_t *)H5D__chunk_mem_xfree(ent->chunk, old_pline);
ent->chunk = (uint8_t *)chunk;
chunk = NULL;
@@ -3081,13 +3095,13 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata,
/* Mark the old pipeline as having been disabled */
old_pline = NULL;
- /* Reallocate the chunk so H5D__chunk_xfree doesn't get confused
+ /* Reallocate the chunk so H5D__chunk_mem_xfree doesn't get confused
*/
- if(NULL == (chunk = H5D__chunk_alloc(chunk_size, pline)))
+ if(NULL == (chunk = H5D__chunk_mem_alloc(chunk_size, pline)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk")
HDmemcpy(chunk, ent->chunk, chunk_size);
- ent->chunk = (uint8_t *)H5D__chunk_xfree(ent->chunk, old_pline);
+ ent->chunk = (uint8_t *)H5D__chunk_mem_xfree(ent->chunk, old_pline);
ent->chunk = (uint8_t *)chunk;
chunk = NULL;
@@ -3096,8 +3110,35 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata,
| H5D_RDCC_NEWLY_DISABLED_FILTERS);
} /* end else */
} /* end if */
+
+ /*
+ * If the chunk is not at the beginning of the cache; move it backward
+ * by one slot. This is how we implement the LRU preemption
+ * algorithm.
+ */
+ if(ent->next) {
+ if(ent->next->next)
+ ent->next->next->prev = ent;
+ else
+ rdcc->tail = ent;
+ ent->next->prev = ent->prev;
+ if(ent->prev)
+ ent->prev->next = ent->next;
+ else
+ rdcc->head = ent->next;
+ ent->prev = ent->next;
+ ent->next = ent->next->next;
+ ent->prev->next = ent;
+ } /* end if */
} /* end if */
else {
+ haddr_t chunk_addr; /* Address of chunk on disk */
+ hsize_t chunk_alloc; /* Length of chunk on disk */
+
+ /* Save the chunk info so the cache stays consistent */
+ chunk_addr = udata->chunk_block.offset;
+ chunk_alloc = udata->chunk_block.length;
+
/* Check if we should disable filters on this chunk */
if(pline->nused) {
if(udata->new_unfilt_chunk) {
@@ -3135,21 +3176,18 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata,
if(relax) {
/*
- * Not in the cache, but we're about to overwrite the whole thing
- * anyway, so just allocate a buffer for it but don't initialize that
- * buffer with the file contents. Count this as a hit instead of a
- * miss because we saved ourselves lots of work.
- */
+ * Not in the cache, but we're about to overwrite the whole thing
+ * anyway, so just allocate a buffer for it but don't initialize that
+ * buffer with the file contents. Count this as a hit instead of a
+ * miss because we saved ourselves lots of work.
+ */
rdcc->stats.nhits++;
- /* Still save the chunk address so the cache stays consistent */
- chunk_addr = udata->addr;
-
- if(NULL == (chunk = H5D__chunk_alloc(chunk_size, pline)))
+ if(NULL == (chunk = H5D__chunk_mem_alloc(chunk_size, pline)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk")
/* In the case that some dataset functions look through this data,
- * clear it to all 0s. */
+ * clear it to all 0s. */
HDmemset(chunk, 0, chunk_size);
} /* end if */
else {
@@ -3158,41 +3196,36 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata,
* or an init if it isn't.
*/
- /* Save the chunk address */
- chunk_addr = udata->addr;
-
/* Check if the chunk exists on disk */
if(H5F_addr_defined(chunk_addr)) {
- size_t chunk_alloc = 0; /*allocated chunk size */
+ size_t my_chunk_alloc = chunk_alloc; /* Allocated buffer size */
+ size_t buf_alloc = chunk_alloc; /* [Re-]allocated buffer size */
/* Chunk size on disk isn't [likely] the same size as the final chunk
* size in memory, so allocate memory big enough. */
- H5_ASSIGN_OVERFLOW(chunk_alloc, udata->nbytes, uint32_t, size_t);
- if(NULL == (chunk = H5D__chunk_alloc(chunk_alloc,
- (udata->new_unfilt_chunk ? old_pline : pline))))
+ if(NULL == (chunk = H5D__chunk_mem_alloc(my_chunk_alloc, (udata->new_unfilt_chunk ? old_pline : pline))))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk")
- if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, chunk_addr, chunk_alloc, io_info->dxpl_id, chunk) < 0)
+ if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, chunk_addr, my_chunk_alloc, io_info->dxpl_id, chunk) < 0)
HGOTO_ERROR(H5E_IO, H5E_READERROR, NULL, "unable to read raw data chunk")
- if(old_pline && old_pline->nused) {
+ if(old_pline && old_pline->nused) {
if(H5Z_pipeline(old_pline, H5Z_FLAG_REVERSE,
&(udata->filter_mask),
io_info->dxpl_cache->err_detect,
io_info->dxpl_cache->filter_cb,
- &chunk_alloc, &chunk_alloc, &chunk) < 0)
+ &my_chunk_alloc, &buf_alloc, &chunk) < 0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, NULL, "data pipeline read failed")
- H5_ASSIGN_OVERFLOW(udata->nbytes, chunk_alloc, size_t, uint32_t);
/* Reallocate chunk if necessary */
if(udata->new_unfilt_chunk) {
void *tmp_chunk = chunk;
- if(NULL == (chunk = H5D__chunk_alloc(chunk_alloc, pline))) {
- (void)H5D__chunk_xfree(tmp_chunk, old_pline);
+ if(NULL == (chunk = H5D__chunk_mem_alloc(my_chunk_alloc, pline))) {
+ (void)H5D__chunk_mem_xfree(tmp_chunk, old_pline);
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk")
} /* end if */
HDmemcpy(chunk, tmp_chunk, chunk_size);
- (void)H5D__chunk_xfree(tmp_chunk, old_pline);
+ (void)H5D__chunk_mem_xfree(tmp_chunk, old_pline);
} /* end if */
} /* end if */
@@ -3202,21 +3235,21 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata,
else {
H5D_fill_value_t fill_status;
- /* Sanity check */
- HDassert(fill->alloc_time != H5D_ALLOC_TIME_EARLY);
+ /* Sanity check */
+ HDassert(fill->alloc_time != H5D_ALLOC_TIME_EARLY);
/* Chunk size on disk isn't [likely] the same size as the final chunk
* size in memory, so allocate memory big enough. */
- if(NULL == (chunk = H5D__chunk_alloc(chunk_size, pline)))
+ if(NULL == (chunk = H5D__chunk_mem_alloc(chunk_size, pline)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk")
if(H5P_is_fill_value_defined(fill, &fill_status) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't tell if fill value defined")
if(fill->fill_time == H5D_FILL_TIME_ALLOC ||
- (fill->fill_time == H5D_FILL_TIME_IFSET &&
- (fill_status == H5D_FILL_VALUE_USER_DEFINED ||
- fill_status == H5D_FILL_VALUE_DEFAULT))) {
+ (fill->fill_time == H5D_FILL_TIME_IFSET &&
+ (fill_status == H5D_FILL_VALUE_USER_DEFINED ||
+ fill_status == H5D_FILL_VALUE_DEFAULT))) {
/*
* The chunk doesn't exist in the file. Replicate the fill
* value throughout the chunk, if the fill value is defined.
@@ -3243,105 +3276,85 @@ H5D__chunk_lock(const H5D_io_info_t *io_info, H5D_chunk_ud_t *udata,
rdcc->stats.ninits++;
} /* end else */
} /* end else */
- } /* end else */
- HDassert(chunk_size > 0);
-
- if(ent) {
- /*
- * The chunk is not at the beginning of the cache; move it backward
- * by one slot. This is how we implement the LRU preemption
- * algorithm.
- */
- HDassert(ent);
- if(ent->next) {
- if(ent->next->next)
- ent->next->next->prev = ent;
- else
- rdcc->tail = ent;
- ent->next->prev = ent->prev;
- if(ent->prev)
- ent->prev->next = ent->next;
- else
- rdcc->head = ent->next;
- ent->prev = ent->next;
- ent->next = ent->next->next;
- ent->prev->next = ent;
- } /* end if */
- } /* end if */
- else if(rdcc->nslots > 0 && chunk_size <= rdcc->nbytes_max) {
- /* Calculate the index */
- udata->idx_hint = H5D__chunk_hash_val(io_info->dset->shared, udata->common.scaled);
- /* Add the chunk to the cache only if the slot is not already locked */
- ent = rdcc->slot[udata->idx_hint];
- if(!ent || !ent->locked) {
- /* Preempt enough things from the cache to make room */
- if(ent) {
- if(H5D__chunk_cache_evict(io_info->dset, io_info->dxpl_id, io_info->dxpl_cache, ent, TRUE) < 0)
- HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk from cache")
- } /* end if */
- if(H5D__chunk_cache_prune(io_info->dset, io_info->dxpl_id, io_info->dxpl_cache, chunk_size) < 0)
- HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk(s) from cache")
+ /* See if the chunk can be cached */
+ if(rdcc->nslots > 0 && chunk_size <= rdcc->nbytes_max) {
+ /* Calculate the index */
+ udata->idx_hint = H5D__chunk_hash_val(io_info->dset->shared, udata->common.scaled);
+
+ /* Add the chunk to the cache only if the slot is not already locked */
+ ent = rdcc->slot[udata->idx_hint];
+ if(!ent || !ent->locked) {
+ /* Preempt enough things from the cache to make room */
+ if(ent) {
+ if(H5D__chunk_cache_evict(io_info->dset, io_info->dxpl_id, io_info->dxpl_cache, ent, TRUE) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk from cache")
+ } /* end if */
+ if(H5D__chunk_cache_prune(io_info->dset, io_info->dxpl_id, io_info->dxpl_cache, chunk_size) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk(s) from cache")
+
+ /* Create a new entry */
+ if(NULL == (ent = H5FL_CALLOC(H5D_rdcc_ent_t)))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "can't allocate raw data chunk entry")
+
+ ent->locked = 0;
+ ent->edge_chunk_state = disable_filters ? H5D_RDCC_DISABLE_FILTERS : 0;
+ if(udata->new_unfilt_chunk)
+ ent->edge_chunk_state |= H5D_RDCC_NEWLY_DISABLED_FILTERS;
+ ent->dirty = FALSE;
+ ent->deleted = FALSE;
+
+ /* Initialize the new entry */
+ ent->chunk_block.offset = chunk_addr;
+ ent->chunk_block.length = chunk_alloc;
+ ent->chunk_idx = udata->chunk_idx;
+ HDmemcpy(ent->offset, io_info->store->chunk.offset, sizeof(hsize_t) * layout->u.chunk.ndims);
+ HDmemcpy(ent->scaled, udata->common.scaled, sizeof(hsize_t) * layout->u.chunk.ndims);
+
+ H5_ASSIGN_OVERFLOW(ent->rd_count, chunk_size, size_t, uint32_t);
+ H5_ASSIGN_OVERFLOW(ent->wr_count, chunk_size, size_t, uint32_t);
+ ent->chunk = (uint8_t *)chunk;
- /* Create a new entry */
- if(NULL == (ent = H5FL_CALLOC(H5D_rdcc_ent_t)))
- HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "can't allocate raw data chunk entry")
+ /* Add it to the cache */
+ HDassert(NULL == rdcc->slot[udata->idx_hint]);
+ rdcc->slot[udata->idx_hint] = ent;
+ ent->idx = udata->idx_hint;
+ rdcc->nbytes_used += chunk_size;
+ rdcc->nused++;
+
+ /* Add it to the linked list */
+ if(rdcc->tail) {
+ rdcc->tail->next = ent;
+ ent->prev = rdcc->tail;
+ rdcc->tail = ent;
+ } /* end if */
+ else
+ rdcc->head = rdcc->tail = ent;
+ ent->tmp_next = NULL;
+ ent->tmp_prev = NULL;
- ent->locked = 0;
- ent->edge_chunk_state = disable_filters ? H5D_RDCC_DISABLE_FILTERS : 0;
- if(udata->new_unfilt_chunk)
- ent->edge_chunk_state |= H5D_RDCC_NEWLY_DISABLED_FILTERS;
- ent->dirty = FALSE;
- ent->deleted = FALSE;
-
- /* Initialize the new entry */
- ent->chunk_addr = chunk_addr;
- ent->chunk_idx = udata->chunk_idx;
- HDmemcpy(ent->offset, io_info->store->chunk.offset, sizeof(hsize_t) * layout->u.chunk.ndims);
- HDmemcpy(ent->scaled, udata->common.scaled, sizeof(hsize_t) * layout->u.chunk.ndims);
-
- H5_ASSIGN_OVERFLOW(ent->rd_count, chunk_size, size_t, uint32_t);
- H5_ASSIGN_OVERFLOW(ent->wr_count, chunk_size, size_t, uint32_t);
- ent->chunk = (uint8_t *)chunk;
-
- /* Add it to the cache */
- HDassert(NULL == rdcc->slot[udata->idx_hint]);
- rdcc->slot[udata->idx_hint] = ent;
- ent->idx = udata->idx_hint;
- rdcc->nbytes_used += chunk_size;
- rdcc->nused++;
-
- /* Add it to the linked list */
- if(rdcc->tail) {
- rdcc->tail->next = ent;
- ent->prev = rdcc->tail;
- rdcc->tail = ent;
} /* end if */
else
- rdcc->head = rdcc->tail = ent;
- ent->tmp_next = NULL;
- ent->tmp_prev = NULL;
-
- } /* end if */
- else
- /* We did not add the chunk to cache */
+ /* We did not add the chunk to cache */
+ ent = NULL;
+ } /* end else */
+ else /* No cache set up, or chunk is too large: chunk is uncacheable */
ent = NULL;
} /* end else */
- if(!ent)
- /*
- * The chunk cannot be placed in cache so we don't cache it. This is the
- * reason all those arguments have to be repeated for the unlock
- * function.
- */
- udata->idx_hint = UINT_MAX;
-
/* Lock the chunk into the cache */
if(ent) {
HDassert(!ent->locked);
ent->locked = TRUE;
chunk = ent->chunk;
} /* end if */
+ else
+ /*
+ * The chunk cannot be placed in cache so we don't cache it. This is the
+ * reason all those arguments have to be repeated for the unlock
+ * function.
+ */
+ udata->idx_hint = UINT_MAX;
/* Set return value */
ret_value = chunk;
@@ -3354,7 +3367,7 @@ done:
/* Release the chunk allocated, on error */
if(!ret_value)
if(chunk)
- chunk = H5D__chunk_xfree(chunk, pline);
+ chunk = H5D__chunk_mem_xfree(chunk, pline);
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5D__chunk_lock() */
@@ -3432,7 +3445,8 @@ H5D__chunk_unlock(const H5D_io_info_t *io_info, const H5D_chunk_ud_t *udata,
HDmemcpy(fake_ent.scaled, udata->common.scaled, sizeof(hsize_t) * layout->u.chunk.ndims);
HDassert(layout->u.chunk.size > 0);
fake_ent.chunk_idx = udata->chunk_idx;
- fake_ent.chunk_addr = udata->addr;
+ fake_ent.chunk_block.offset = udata->chunk_block.offset;
+ fake_ent.chunk_block.length = udata->chunk_block.length;
fake_ent.chunk = (uint8_t *)chunk;
if(H5D__chunk_flush_entry(io_info->dset, io_info->dxpl_id, io_info->dxpl_cache, &fake_ent, TRUE) < 0)
@@ -3440,8 +3454,8 @@ H5D__chunk_unlock(const H5D_io_info_t *io_info, const H5D_chunk_ud_t *udata,
} /* end if */
else {
if(chunk)
- chunk = H5D__chunk_xfree(chunk, (is_unfiltered_edge_chunk ? NULL
- : &(io_info->dset->shared->dcpl_cache.pline)));
+ chunk = H5D__chunk_mem_xfree(chunk, (is_unfiltered_edge_chunk ? NULL
+ : &(io_info->dset->shared->dcpl_cache.pline)));
} /* end else */
} /* end if */
else {
@@ -3720,8 +3734,8 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite,
/* Initialize the fill value buffer */
/* (delay allocating fill buffer for VL datatypes until refilling) */
/* (casting away const OK - QAK) */
- if(H5D__fill_init(&fb_info, NULL, (H5MM_allocate_t)H5D__chunk_alloc,
- (void *)pline, (H5MM_free_t)H5D__chunk_xfree, (void *)pline,
+ if(H5D__fill_init(&fb_info, NULL, (H5MM_allocate_t)H5D__chunk_mem_alloc,
+ (void *)pline, (H5MM_free_t)H5D__chunk_mem_xfree, (void *)pline,
&dset->shared->dcpl_cache.fill, dset->shared->type,
dset->shared->type_id, (size_t)0, orig_chunk_size, dxpl_id) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill buffer info")
@@ -3742,7 +3756,7 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite,
/* If the dataset has disabled partial chunk filters, create a copy
* of the unfiltered fill_buf to use for partial chunks */
if(has_unfilt_edge_chunks) {
- if(NULL == (unfilt_fill_buf = H5D__chunk_alloc(orig_chunk_size, &def_pline)))
+ if(NULL == (unfilt_fill_buf = H5D__chunk_mem_alloc(orig_chunk_size, &def_pline)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk")
HDmemcpy(unfilt_fill_buf, fb_info.fill_buf, orig_chunk_size);
} /* end if */
@@ -3826,6 +3840,10 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite,
while(!carry) {
/* None of the chunks should be allocated */
hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Scaled coordinates for this chunk */
+ hbool_t need_insert = FALSE; /* Whether the chunk needs to be inserted into the index */
+
+ /* Reset size of chunk in bytes, in case filtered size changes */
+ /* chunk_size = orig_chunk_size; */
/* Look up this chunk */
H5VM_chunk_scaled(space_ndims, chunk_offset, layout->u.chunk.dim, scaled);
@@ -3834,7 +3852,7 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite,
#ifndef NDEBUG
if(H5D_CHUNK_IDX_NONE != layout->storage.u.chunk.idx_type)
- HDassert(!H5F_addr_defined(udata.addr));
+ HDassert(!H5F_addr_defined(udata.chunk_block.offset));
/* Make sure the chunk is really in the dataset and outside the
* original dimensions */
@@ -3863,7 +3881,7 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite,
* possible (though ill-advised) for the filter to shrink the
* buffer. */
if(fb_info.fill_buf_size < orig_chunk_size) {
- if(NULL == (fb_info.fill_buf = H5D__chunk_realloc(fb_info.fill_buf, orig_chunk_size, pline)))
+ if(NULL == (fb_info.fill_buf = H5D__chunk_mem_realloc(fb_info.fill_buf, orig_chunk_size, pline)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory reallocation failed for raw data chunk")
fb_info.fill_buf_size = orig_chunk_size;
} /* end if */
@@ -3900,23 +3918,21 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite,
udata.common.storage = &layout->storage.u.chunk;
udata.common.offset = chunk_offset;
udata.common.rdcc = &dset->shared->cache.chunk;
- udata.addr = HADDR_UNDEF;
- H5_ASSIGN_OVERFLOW(udata.nbytes, chunk_size, size_t, uint32_t);
+ udata.chunk_block.offset = HADDR_UNDEF;
+ H5_ASSIGN_OVERFLOW(udata.chunk_block.length, chunk_size, size_t, uint32_t);
udata.filter_mask = filter_mask;
- udata.need_insert = FALSE;
udata.need_modify = FALSE;
/* Allocate the chunk (with all processes) */
- if(H5D__idx_chunk_alloc(&idx_info, &udata) < 0)
+ if(H5D__chunk_file_alloc(&idx_info, NULL, &udata.chunk_block, &need_insert, &udata.need_modify, scaled) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk on chunk level")
-
- HDassert(H5F_addr_defined(udata.addr));
+ HDassert(H5F_addr_defined(udata.chunk_block.offset));
/* Check if fill values should be written to chunks */
if(should_fill) {
/* Sanity check */
HDassert(fb_info_init);
- HDassert(udata.nbytes == chunk_size);
+ HDassert(udata.chunk_block.length == chunk_size);
#ifdef H5_HAVE_PARALLEL
/* Check if this file is accessed with an MPI-capable file driver */
@@ -3929,7 +3945,7 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite,
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "memory allocation failed for chunk addresses")
/* Store the chunk's address for later */
- chunk_info.addr[chunk_info.num_io] = udata.addr;
+ chunk_info.addr[chunk_info.num_io] = udata.chunk_block.offset;
chunk_info.num_io++;
/* Indicate that blocks will be written */
@@ -3937,17 +3953,20 @@ H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite,
} /* end if */
else {
#endif /* H5_HAVE_PARALLEL */
- if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, chunk_size, dxpl_id, *fill_buf) < 0)
+ if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.chunk_block.offset, chunk_size, dxpl_id, *fill_buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file")
#ifdef H5_HAVE_PARALLEL
} /* end else */
#endif /* H5_HAVE_PARALLEL */
} /* end if */
- if((udata.need_insert || udata.need_modify) && ops->insert_addr) {
+ /* Insert the chunk record into the index */
+ /* (Note that this isn't safe, from a SWMR perspective, unlike
+ * serial operation. -QAK
+ */
+ if((need_insert || udata.need_modify) && ops->insert_addr)
if((ops->insert_addr)(&idx_info, &udata) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert chunk addr into index")
- }
/* Increment indices and adjust the edge chunk state */
carry = TRUE;
@@ -4021,7 +4040,7 @@ done:
HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release fill buffer info")
/* Free the unfiltered fill value buffer */
- unfilt_fill_buf = H5D__chunk_xfree(unfilt_fill_buf, &def_pline);
+ unfilt_fill_buf = H5D__chunk_mem_xfree(unfilt_fill_buf, &def_pline);
#ifdef H5_HAVE_PARALLEL
if(using_mpi && chunk_info.addr)
@@ -4181,7 +4200,7 @@ H5D__chunk_update_old_edge_chunks(H5D_t *dset, hid_t dxpl_id, hsize_t old_dim[])
/* If this chunk does not exist in cache or on disk, no need to do
* anything */
- if(H5F_addr_defined(chk_udata.addr)
+ if(H5F_addr_defined(chk_udata.chunk_block.offset)
|| (UINT_MAX != chk_udata.idx_hint)) {
/* Lock the chunk into cache. H5D__chunk_lock will take care of
* updating the chunk to no longer be an edge chunk. */
@@ -4424,7 +4443,7 @@ H5D__chunk_prune_fill(H5D_chunk_it_ud1_t *udata, hbool_t new_unfilt_chunk)
chk_udata.new_unfilt_chunk = new_unfilt_chunk;
/* If this chunk does not exist in cache or on disk, no need to do anything */
- if(!H5F_addr_defined(chk_udata.addr) && UINT_MAX == chk_udata.idx_hint)
+ if(!H5F_addr_defined(chk_udata.chunk_block.offset) && UINT_MAX == chk_udata.idx_hint)
HGOTO_DONE(SUCCEED)
/* Initialize the fill value buffer, if necessary */
@@ -4826,7 +4845,7 @@ H5D__chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, const hsize_t *old_dim)
dims_outside_fill[u] = FALSE;
carry = FALSE;
- } /* end if */
+ } /* end if */
while(!carry) {
hsize_t scaled[H5S_MAX_RANK]; /* Scaled coordinates for the current chunk */
@@ -4893,7 +4912,7 @@ H5D__chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, const hsize_t *old_dim)
HGOTO_ERROR(H5E_DATASET, H5E_CANTREMOVE, FAIL, "unable to evict chunk")
/* Remove the chunk from disk, if present */
- if(H5F_addr_defined(chk_udata.addr)) {
+ if(H5F_addr_defined(chk_udata.chunk_block.offset)) {
/* Update the offset in idx_udata */
idx_udata.offset = chunk_offset;
idx_udata.scaled = udata.common.scaled;
@@ -5287,6 +5306,7 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
hsize_t scaled[H5S_MAX_RANK]; /* Scaled coordinates of chunk */
hbool_t is_vlen = FALSE; /* Whether datatype is variable-length */
hbool_t fix_ref = FALSE; /* Whether to fix up references in the dest. file */
+ hbool_t need_insert = FALSE; /* Whether the chunk needs to be inserted into the index */
/* General information about chunk copy */
void *bkg = udata->bkg; /* Background buffer for datatype conversion */
@@ -5418,8 +5438,8 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
udata_dst.common.storage = udata->idx_info_dst->storage;
udata_dst.common.offset = chunk_rec->offset;
udata_dst.common.rdcc = NULL;
- udata_dst.addr = HADDR_UNDEF;
- udata_dst.nbytes = chunk_rec->nbytes;
+ udata_dst.chunk_block.offset = HADDR_UNDEF;
+ udata_dst.chunk_block.length = chunk_rec->nbytes;
udata_dst.filter_mask = chunk_rec->filter_mask;
/* Need to compress variable-length & reference data elements before writing to file */
@@ -5431,7 +5451,7 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
if(nbytes > ((size_t)0xffffffff))
HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, H5_ITER_ERROR, "chunk too large for 32-bit length")
#endif /* H5_SIZEOF_SIZE_T > 4 */
- H5_ASSIGN_OVERFLOW(udata_dst.nbytes, nbytes, size_t, uint32_t);
+ H5_ASSIGN_OVERFLOW(udata_dst.chunk_block.length, nbytes, size_t, uint32_t);
udata->buf = buf;
udata->buf_size = buf_size;
} /* end if */
@@ -5440,22 +5460,23 @@ H5D__chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
udata_dst.chunk_idx = H5VM_chunk_index_scaled(udata_dst.common.layout->ndims - 1, chunk_rec->offset, udata_dst.common.layout->dim, udata_dst.common.layout->down_chunks, scaled);
udata_dst.common.scaled = scaled;
- /* Insert chunk into the destination index */
- if(H5D__idx_chunk_alloc(udata->idx_info_dst, &udata_dst) < 0)
+ /* Allocate chunk in the file */
+ if(H5D__chunk_file_alloc(udata->idx_info_dst, NULL, &udata_dst.chunk_block, &need_insert, &udata_dst.need_modify, scaled) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINSERT, FAIL, "unable to insert/resize chunk on chunk level")
/* Write chunk data to destination file */
- HDassert(H5F_addr_defined(udata_dst.addr));
- if(H5F_block_write(udata->idx_info_dst->f, H5FD_MEM_DRAW, udata_dst.addr, nbytes, udata->idx_info_dst->dxpl_id, buf) < 0)
+ HDassert(H5F_addr_defined(udata_dst.chunk_block.offset));
+ if(H5F_block_write(udata->idx_info_dst->f, H5FD_MEM_DRAW, udata_dst.chunk_block.offset, nbytes, udata->idx_info_dst->dxpl_id, buf) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, H5_ITER_ERROR, "unable to write raw data to file")
/* Set metadata tag in dxpl_id */
H5_BEGIN_TAG(udata->idx_info_dst->dxpl_id, H5AC__COPIED_TAG, H5_ITER_ERROR);
- if((udata_dst.need_insert || udata_dst.need_modify) && udata->idx_info_dst->storage->ops->insert_addr) {
+ /* Insert chunk record into index */
+ if((need_insert || udata_dst.need_modify) && udata->idx_info_dst->storage->ops->insert_addr)
if((udata->idx_info_dst->storage->ops->insert_addr)(udata->idx_info_dst, &udata_dst) < 0)
HGOTO_ERROR_TAG(H5E_DATASET, H5E_CANTINSERT, H5_ITER_ERROR, "unable to insert chunk addr into index")
- }
+
/* Reset metadata tag in dxpl_id */
H5_END_TAG(H5_ITER_ERROR);
@@ -6197,7 +6218,7 @@ done:
/*-------------------------------------------------------------------------
- * Function: H5D_idx_chunk_alloc()
+ * Function: H5D__chunk_file_alloc()
*
* Purpose: Chunk allocation:
* Create the chunk if it doesn't exist, or reallocate the
@@ -6211,10 +6232,10 @@ done:
*-------------------------------------------------------------------------
*/
static herr_t
-H5D__idx_chunk_alloc(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata)
+H5D__chunk_file_alloc(const H5D_chk_idx_info_t *idx_info, const H5F_block_t *old_chunk,
+ H5F_block_t *new_chunk, hbool_t *need_insert, hbool_t *need_modify, hsize_t scaled[])
{
hbool_t alloc_chunk = FALSE; /* Whether to allocate chunk */
- hbool_t need_modify = FALSE;
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_STATIC
@@ -6226,111 +6247,116 @@ H5D__idx_chunk_alloc(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata)
HDassert(idx_info->layout);
HDassert(idx_info->storage);
HDassert(H5F_addr_defined(idx_info->storage->idx_addr));
- HDassert(udata);
+ HDassert(new_chunk);
+ HDassert(need_insert);
+ HDassert(need_modify);
- udata->need_modify = FALSE; /* this is mainly for V2-btree */
- udata->need_insert = FALSE;
+ *need_modify = FALSE; /* this is mainly for V2-btree */
+ *need_insert = FALSE;
+ /* Check for filters on chunks */
if(idx_info->pline->nused > 0) {
- unsigned allow_chunk_size_len; /* Allowed size of encoded chunk size */
- unsigned new_chunk_size_len; /* Size of encoded chunk size */
- H5D_chunk_ud_t found_udata;
HDassert(idx_info->storage->idx_type != H5D_CHUNK_IDX_NONE);
+ /* Sanity/error checking block */
+ {
+ unsigned allow_chunk_size_len; /* Allowed size of encoded chunk size */
+ unsigned new_chunk_size_len; /* Size of encoded chunk size */
- /* Compute the size required for encoding the size of a chunk, allowing
- * for an extra byte, in case the filter makes the chunk larger.
- */
- allow_chunk_size_len = 1 + ((H5VM_log2_gen((uint64_t)(idx_info->layout->size)) + 8) / 8);
- if(allow_chunk_size_len > 8)
- allow_chunk_size_len = 8;
-
- /* Compute encoded size of chunk */
- new_chunk_size_len = (H5VM_log2_gen((uint64_t)(udata->nbytes)) + 8) / 8;
- if(new_chunk_size_len > 8)
- HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "encoded chunk size is more than 8 bytes?!?")
-
- /* Check if the chunk became too large to be encoded */
- if(new_chunk_size_len > allow_chunk_size_len)
- HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk size can't be encoded")
-
- /* Initialize found_udata for ops->get_addr() */
- HDmemset(&found_udata, 0, sizeof found_udata);
- found_udata.common.offset = udata->common.offset;
- found_udata.common.scaled = udata->common.scaled;
- found_udata.common.storage = udata->common.storage;
- found_udata.common.layout = udata->common.layout;
- found_udata.addr = HADDR_UNDEF;
-
- /* Get chunk information */
- if((idx_info->storage->ops->get_addr)(idx_info, &found_udata) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query chunk address")
-
- if(H5F_addr_defined(found_udata.addr)) {
+ /* Compute the size required for encoding the size of a chunk, allowing
+ * for an extra byte, in case the filter makes the chunk larger.
+ */
+ allow_chunk_size_len = 1 + ((H5VM_log2_gen((uint64_t)(idx_info->layout->size)) + 8) / 8);
+ if(allow_chunk_size_len > 8)
+ allow_chunk_size_len = 8;
+
+ /* Compute encoded size of chunk */
+ new_chunk_size_len = (H5VM_log2_gen((uint64_t)(new_chunk->length)) + 8) / 8;
+ if(new_chunk_size_len > 8)
+ HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "encoded chunk size is more than 8 bytes?!?")
+
+ /* Check if the chunk became too large to be encoded */
+ if(new_chunk_size_len > allow_chunk_size_len)
+ HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk size can't be encoded")
+ } /* end block */
+
+ if(old_chunk && H5F_addr_defined(old_chunk->offset)) {
/* Sanity check */
- HDassert(!H5F_addr_defined(udata->addr) || H5F_addr_eq(udata->addr, found_udata.addr));
+ HDassert(!H5F_addr_defined(new_chunk->offset) || H5F_addr_eq(new_chunk->offset, old_chunk->offset));
/* Check for chunk being same size */
- if(udata->nbytes != found_udata.nbytes) {
+ if(new_chunk->length != old_chunk->length) {
/* Release previous chunk */
/* Only free the old location if not doing SWMR writes - otherwise
- * we must keep the old chunk around in case a reader has an
- * outdated version of the b-tree node
- */
- if(!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) {
- H5_CHECK_OVERFLOW(found_udata.nbytes, uint32_t, hsize_t);
- if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, found_udata.addr, (hsize_t)found_udata.nbytes) < 0)
+ * we must keep the old chunk around in case a reader has an
+ * outdated version of the b-tree node
+ */
+ if(!(H5F_INTENT(idx_info->f) & H5F_ACC_SWMR_WRITE)) {
+ if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, old_chunk->offset, old_chunk->length) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk")
- } /* end if */
+ }
alloc_chunk = TRUE;
- need_modify = TRUE;
-
- } else {
+ *need_modify = TRUE;
+ } /* end if */
+ else {
/* Don't need to reallocate chunk, but send its address back up */
- if(!H5F_addr_defined(udata->addr))
- udata->addr = found_udata.addr;
- }
- } else
+ if(!H5F_addr_defined(new_chunk->offset))
+ new_chunk->offset = old_chunk->offset;
+ } /* end else */
+ } /* end if */
+ else {
+ HDassert(!H5F_addr_defined(new_chunk->offset));
alloc_chunk = TRUE;
-
- } else {
- HDassert(!H5F_addr_defined(udata->addr));
- HDassert(udata->nbytes == idx_info->layout->size);
+ } /* end else */
+ } /* end if */
+ else {
+ HDassert(!H5F_addr_defined(new_chunk->offset));
+ HDassert(new_chunk->length == idx_info->layout->size);
alloc_chunk = TRUE;
- }
+ } /* end else */
+ /* Actually allocate space for the chunk in the file */
if(alloc_chunk) {
switch(idx_info->storage->idx_type) {
case H5D_CHUNK_IDX_NONE:
- if((idx_info->storage->ops->get_addr)(idx_info, udata) < 0)
- HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query chunk address")
- break;
+ {
+ H5D_chunk_ud_t udata;
+
+ udata.common.scaled = scaled;
+ if((idx_info->storage->ops->get_addr)(idx_info, &udata) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query chunk address")
+ new_chunk->offset = udata.chunk_block.offset;
+ HDassert(new_chunk->length == udata.chunk_block.length);
+ break;
+ }
case H5D_CHUNK_IDX_EARRAY:
- case H5D_CHUNK_IDX_FARRAY:
+ case H5D_CHUNK_IDX_FARRAY:
+ case H5D_CHUNK_IDX_BT2:
case H5D_CHUNK_IDX_BTREE:
- case H5D_CHUNK_IDX_BT2:
- H5_CHECK_OVERFLOW(udata->nbytes, /*From: */uint32_t, /*To: */hsize_t);
- udata->addr = H5MF_alloc(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, (hsize_t)udata->nbytes);
- if(!H5F_addr_defined(udata->addr))
- HGOTO_ERROR (H5E_DATASET, H5E_CANTALLOC, FAIL, "file allocation failed")
+ HDassert(new_chunk->length > 0);
+ H5_CHECK_OVERFLOW(new_chunk->length, /*From: */uint32_t, /*To: */hsize_t);
+ new_chunk->offset = H5MF_alloc(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, (hsize_t)new_chunk->length);
+ if(!H5F_addr_defined(new_chunk->offset))
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "file allocation failed")
if(idx_info->storage->idx_type == H5D_CHUNK_IDX_BT2) {
- /* This can be done together with other index types when Quincy checks into H5B2_modify() */
- if(!(udata->need_modify = need_modify))
- udata->need_insert = TRUE;
- } else
- udata->need_insert = TRUE;
+ /* This can be done together with other index types when Quincy checks into H5B2_modify() */
+ if(!(*need_modify))
+ *need_insert = TRUE;
+ } else
+ *need_insert = TRUE;
+
break;
case H5D_CHUNK_IDX_NTYPES:
default:
- HDassert(0 && "This Should never be executed!");
+ HDassert(0 && "This should never be executed!");
break;
} /* end switch */
} /* end if */
- HDassert(H5F_addr_defined(udata->addr));
+
+ HDassert(H5F_addr_defined(new_chunk->offset));
done:
FUNC_LEAVE_NOAPI(ret_value)
-
-} /* H5D__idx_chunk_alloc() */
+} /* H5D__chunk_file_alloc() */
diff --git a/src/H5Dearray.c b/src/H5Dearray.c
index fcb60aa..a6c6535 100644
--- a/src/H5Dearray.c
+++ b/src/H5Dearray.c
@@ -1077,7 +1077,6 @@ H5D_earray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u
HDassert(idx_info->storage);
HDassert(H5F_addr_defined(idx_info->storage->idx_addr));
HDassert(udata);
- HDassert(udata->need_insert);
/* Check if the extensible array is open yet */
if(NULL == idx_info->storage->u.earray.ea) {
@@ -1089,7 +1088,7 @@ H5D_earray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u
/* Set convenience pointer to extensible array structure */
ea = idx_info->storage->u.earray.ea;
- if(!H5F_addr_defined(udata->addr))
+ if(!H5F_addr_defined(udata->chunk_block.offset))
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "The chunk should have allocated already")
if(udata->chunk_idx != (udata->chunk_idx & 0xffffffff)) /* negative value */
HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "chunk index must be less than 2^32")
@@ -1097,8 +1096,8 @@ H5D_earray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u
if(idx_info->pline->nused > 0) {
H5D_earray_filt_elmt_t elmt; /* Extensible array element */
- elmt.addr = udata->addr;
- elmt.nbytes = udata->nbytes;
+ elmt.addr = udata->chunk_block.offset;
+ elmt.nbytes = udata->chunk_block.length;
elmt.filter_mask = udata->filter_mask;
/* Set the info for the chunk */
@@ -1106,7 +1105,7 @@ H5D_earray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u
HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set chunk info")
} else {
/* Set the address for the chunk */
- if(H5EA_set(ea, idx_info->dxpl_id, udata->chunk_idx, &udata->addr) < 0)
+ if(H5EA_set(ea, idx_info->dxpl_id, udata->chunk_idx, &udata->chunk_block.offset) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set chunk address")
}
@@ -1184,20 +1183,23 @@ H5D_earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info")
/* Set the info for the chunk */
- udata->addr = elmt.addr;
- udata->nbytes = elmt.nbytes;
+ udata->chunk_block.offset = elmt.addr;
+ udata->chunk_block.length = elmt.nbytes;
udata->filter_mask = elmt.filter_mask;
} /* end if */
else {
/* Get the address for the chunk */
- if(H5EA_get(ea, idx_info->dxpl_id, idx, &udata->addr) < 0)
+ if(H5EA_get(ea, idx_info->dxpl_id, idx, &udata->chunk_block.offset) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address")
/* Update the other (constant) information for the chunk */
- udata->nbytes = idx_info->layout->size;
+ udata->chunk_block.length = idx_info->layout->size;
udata->filter_mask = 0;
} /* end else */
+ if(!H5F_addr_defined(udata->chunk_block.offset))
+ udata->chunk_block.length = 0;
+
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5D_earray_idx_get_addr() */
diff --git a/src/H5Dfarray.c b/src/H5Dfarray.c
index 09476ab..de184ea 100644
--- a/src/H5Dfarray.c
+++ b/src/H5Dfarray.c
@@ -1047,7 +1047,6 @@ H5D_farray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u
HDassert(idx_info->storage);
HDassert(H5F_addr_defined(idx_info->storage->idx_addr));
HDassert(udata);
- HDassert(udata->need_insert);
/* Check if the fixed array is open yet */
if(NULL == idx_info->storage->u.farray.fa) {
@@ -1059,7 +1058,7 @@ H5D_farray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u
/* Set convenience pointer to fixed array structure */
fa = idx_info->storage->u.farray.fa;
- if(!H5F_addr_defined(udata->addr))
+ if(!H5F_addr_defined(udata->chunk_block.offset))
HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "The chunk should have allocated already")
if(udata->chunk_idx != (udata->chunk_idx & 0xffffffff)) /* negative value */
HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "chunk index must be less than 2^32")
@@ -1068,8 +1067,8 @@ H5D_farray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u
if(idx_info->pline->nused > 0) {
H5D_farray_filt_elmt_t elmt; /* Fixed array element */
- elmt.addr = udata->addr;
- elmt.nbytes = udata->nbytes;
+ elmt.addr = udata->chunk_block.offset;
+ elmt.nbytes = udata->chunk_block.length;
elmt.filter_mask = udata->filter_mask;
/* Set the info for the chunk */
@@ -1078,7 +1077,7 @@ H5D_farray_idx_insert_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *u
} /* end if */
else {
/* Set the address for the chunk */
- if(H5FA_set(fa, idx_info->dxpl_id, udata->chunk_idx, &udata->addr) < 0)
+ if(H5FA_set(fa, idx_info->dxpl_id, udata->chunk_idx, &udata->chunk_block.offset) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "can't set chunk address")
} /* end else */
@@ -1147,20 +1146,23 @@ H5D_farray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udat
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info")
/* Set the info for the chunk */
- udata->addr = elmt.addr;
- udata->nbytes = elmt.nbytes;
+ udata->chunk_block.offset = elmt.addr;
+ udata->chunk_block.length = elmt.nbytes;
udata->filter_mask = elmt.filter_mask;
} /* end if */
else {
/* Get the address for the chunk */
- if(H5FA_get(fa, idx_info->dxpl_id, idx, &udata->addr) < 0)
+ if(H5FA_get(fa, idx_info->dxpl_id, idx, &udata->chunk_block.offset) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address")
/* Update the other (constant) information for the chunk */
- udata->nbytes = idx_info->layout->size;
+ udata->chunk_block.length = idx_info->layout->size;
udata->filter_mask = 0;
} /* end else */
+ if(!H5F_addr_defined(udata->chunk_block.offset))
+ udata->chunk_block.length = 0;
+
done:
FUNC_LEAVE_NOAPI(ret_value)
} /* H5D_farray_idx_get_addr() */
diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c
index 89b7728..e94d2fa 100644
--- a/src/H5Dmpio.c
+++ b/src/H5Dmpio.c
@@ -863,7 +863,7 @@ H5D__link_chunk_collective_io(H5D_io_info_t *io_info, const H5D_type_info_t *typ
if(H5D__chunk_lookup(io_info->dset, io_info->dxpl_id, chunk_info->coords,
chunk_info->scaled, &udata) < 0)
HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL, "couldn't get chunk address")
- ctg_store.contig.dset_addr = udata.addr;
+ ctg_store.contig.dset_addr = udata.chunk_block.offset;
} /* end else */
/* Set up the base storage address for this chunk */
@@ -1591,7 +1591,7 @@ if(H5DEBUG(D))
if(H5D__chunk_lookup(io_info->dset, io_info->dxpl_id,
chunk_info->coords, chunk_info->scaled, &udata) < 0)
HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL, "couldn't get chunk info from skipped list")
- chunk_addr = udata.addr;
+ chunk_addr = udata.chunk_block.offset;
} /* end if */
else
chunk_addr = total_chunk_addr_array[chunk_info->index];
diff --git a/src/H5Dnone.c b/src/H5Dnone.c
index c059066..1bea20a 100644
--- a/src/H5Dnone.c
+++ b/src/H5Dnone.c
@@ -207,10 +207,10 @@ H5D_none_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata)
udata->chunk_idx = H5VM_array_offset_pre((idx_info->layout->ndims - 1), idx_info->layout->max_down_chunks, udata->common.scaled);
/* Calculate the address of the chunk */
- udata->addr = idx_info->storage->idx_addr + udata->chunk_idx * idx_info->layout->size;
+ udata->chunk_block.offset = idx_info->storage->idx_addr + udata->chunk_idx * idx_info->layout->size;
/* Update the other (constant) information for the chunk */
- udata->nbytes = idx_info->layout->size;
+ udata->chunk_block.length = idx_info->layout->size;
udata->filter_mask = 0;
FUNC_LEAVE_NOAPI(SUCCEED)
diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h
index 764030b..6681d18 100644
--- a/src/H5Dpkg.h
+++ b/src/H5Dpkg.h
@@ -34,6 +34,7 @@
/* Other private headers needed by this file */
#include "H5ACprivate.h" /* Metadata cache */
#include "H5B2private.h" /* v2 B-trees */
+#include "H5Fprivate.h" /* File access */
#include "H5Gprivate.h" /* Groups */
#include "H5SLprivate.h" /* Skip lists */
#include "H5Tprivate.h" /* Datatypes */
@@ -184,6 +185,7 @@ typedef struct {
} H5D_contig_storage_t;
typedef struct {
+ hsize_t index; /* "Index" of chunk in dataset (must be first for TBBT routines) */
hsize_t *offset; /* Chunk's coordinates in elements */
} H5D_chunk_storage_t;
@@ -257,8 +259,8 @@ typedef struct H5D_chk_idx_info_t {
* The chunk's file address, filter mask and size on disk are not key values.
*/
typedef struct H5D_chunk_rec_t {
- uint32_t nbytes; /* Size of stored data */
hsize_t offset[H5O_LAYOUT_NDIMS]; /* Logical offset to start */
+ uint32_t nbytes; /* Size of stored data */
unsigned filter_mask; /* Excluded filters */
haddr_t chunk_addr; /* Address of chunk in file */
} H5D_chunk_rec_t;
@@ -286,12 +288,10 @@ typedef struct H5D_chunk_ud_t {
/* Upward */
unsigned idx_hint; /*index of chunk in cache, if present */
- haddr_t addr; /*file address of chunk */
- uint32_t nbytes; /*size of stored data */
+ H5F_block_t chunk_block; /*offset/length of chunk in file */
unsigned filter_mask; /*excluded filters */
hbool_t new_unfilt_chunk; /*whether the chunk just became unfiltered */
hsize_t chunk_idx; /*chunk index for EA, FA indexing */
- hbool_t need_insert;
hbool_t need_modify;
} H5D_chunk_ud_t;
@@ -540,7 +540,7 @@ typedef struct H5D_rdcc_ent_t {
hsize_t scaled[H5O_LAYOUT_NDIMS]; /*scaled chunk 'name' (coordinates) */
uint32_t rd_count; /*bytes remaining to be read */
uint32_t wr_count; /*bytes remaining to be written */
- haddr_t chunk_addr; /*address of chunk in file */
+ H5F_block_t chunk_block; /*offset/length of chunk in file */
hsize_t chunk_idx; /*index of chunk in dataset */
uint8_t *chunk; /*the unfiltered chunk data */
unsigned idx; /*index in hash table */
diff --git a/src/H5Fio.c b/src/H5Fio.c
index cca0077..3b6c079 100644
--- a/src/H5Fio.c
+++ b/src/H5Fio.c
@@ -100,6 +100,9 @@ H5F_block_read(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size,
herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI(FAIL)
+#ifdef QAK
+HDfprintf(stderr, "%s: read from addr = %a, size = %Zu\n", FUNC, addr, size);
+#endif /* QAK */
HDassert(f);
HDassert(f->shared);
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 4009ca8..725812f 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -573,6 +573,12 @@ typedef struct H5F_io_info_t {
const struct H5P_genplist_t *dxpl; /* DXPL object */
} H5F_io_info_t;
+/* Concise info about a block of bytes in a file */
+typedef struct H5F_block_t {
+ haddr_t offset; /* Offset of the block in the file */
+ hsize_t length; /* Length of the block in the file */
+} H5F_block_t;
+
/*****************************/
/* Library-private Variables */
diff --git a/src/H5Rpublic.h b/src/H5Rpublic.h
index 997309f..e990661 100644
--- a/src/H5Rpublic.h
+++ b/src/H5Rpublic.h
@@ -14,7 +14,7 @@
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
/*
- * This file contains public declarations for the H5S module.
+ * This file contains public declarations for the H5R module.
*/
#ifndef _H5Rpublic_H
#define _H5Rpublic_H
diff --git a/src/libhdf5.settings.in b/src/libhdf5.settings.in
index 1b1ae96..1eed645 100644
--- a/src/libhdf5.settings.in
+++ b/src/libhdf5.settings.in
@@ -51,7 +51,7 @@ Languages:
@BUILD_CXX_CONDITIONAL_TRUE@ C++ Flags: @CXXFLAGS@
@BUILD_CXX_CONDITIONAL_TRUE@ H5 C++ Flags: @H5_CXXFLAGS@
@BUILD_CXX_CONDITIONAL_TRUE@ AM C++ Flags: @AM_CXXFLAGS@
-@BUILD_CXX_CONDITIONAL_TRUE@ Shared C++ Library: @H5_CXX_SHARED@
+@BUILD_CXX_CONDITIONAL_TRUE@ Shared C++ Library: @enable_shared@
@BUILD_CXX_CONDITIONAL_TRUE@ Static C++ Library: @enable_static@
Features: