summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--src/H5Dchunk.c78
-rw-r--r--src/H5Dint.c14
-rw-r--r--src/H5Distore.c246
-rw-r--r--src/H5Dmpio.c12
-rw-r--r--src/H5Dpkg.h18
-rw-r--r--src/H5Oprivate.h4
-rw-r--r--src/H5Pdcpl.c54
-rw-r--r--src/H5V.c31
-rw-r--r--src/H5Vprivate.h6
-rw-r--r--test/Makefile.am2
-rw-r--r--test/dsets.c150
-rw-r--r--test/istore.c7
-rw-r--r--test/tsohm.c12
13 files changed, 399 insertions, 235 deletions
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index baaa5f1..ba07725 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -189,16 +189,16 @@ H5D_chunk_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info
HGOTO_ERROR(H5E_DATASET, H5E_BADSELECT, FAIL, "unable to normalize dataspace by offset")
/* Decide the number of chunks in each dimension*/
- for(u=0; u<f_ndims; u++) {
+ for(u = 0; u < f_ndims; u++) {
/* Keep the size of the chunk dimensions as hsize_t for various routines */
- fm->chunk_dim[u]=fm->layout->u.chunk.dim[u];
+ fm->chunk_dim[u] = fm->layout->u.chunk.dim[u];
/* Round up to the next integer # of chunks, to accomodate partial chunks */
- fm->chunks[u] = ((fm->f_dims[u]+dataset->shared->layout.u.chunk.dim[u])-1) / dataset->shared->layout.u.chunk.dim[u];
+ fm->chunks[u] = ((fm->f_dims[u] + dataset->shared->layout.u.chunk.dim[u]) - 1) / dataset->shared->layout.u.chunk.dim[u];
} /* end for */
/* Compute the "down" size of 'chunks' information */
- if(H5V_array_down(f_ndims,fm->chunks,fm->down_chunks) < 0)
+ if(H5V_array_down(f_ndims, fm->chunks, fm->down_chunks) < 0)
HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "can't compute 'down' sizes")
#ifdef H5_HAVE_PARALLEL
@@ -591,16 +591,15 @@ H5D_create_chunk_file_map_hyper(H5D_chunk_map_t *fm, const H5D_io_info_t
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get file selection bound info")
/* Set initial chunk location & hyperslab size */
-
- for(u=0; u<fm->f_ndims; u++) {
- start_coords[u]=(sel_start[u]/fm->layout->u.chunk.dim[u])*fm->layout->u.chunk.dim[u];
- coords[u]=start_coords[u];
- end[u]=(coords[u]+fm->chunk_dim[u])-1;
+ for(u = 0; u < fm->f_ndims; u++) {
+ start_coords[u] = (sel_start[u] / fm->layout->u.chunk.dim[u]) * fm->layout->u.chunk.dim[u];
+ coords[u] = start_coords[u];
+ end[u] = (coords[u] + fm->chunk_dim[u]) - 1;
} /* end for */
/* Calculate the index of this chunk */
- if(H5V_chunk_index(fm->f_ndims,coords,fm->layout->u.chunk.dim,fm->down_chunks,&chunk_index) < 0)
+ if(H5V_chunk_index(fm->f_ndims, coords, fm->layout->u.chunk.dim, fm->down_chunks, &chunk_index) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index")
/* Iterate through each chunk in the dataset */
@@ -679,17 +678,16 @@ H5D_create_chunk_file_map_hyper(H5D_chunk_map_t *fm, const H5D_io_info_t
} /* end if */
/* Get number of elements selected in chunk */
- if((schunk_points=H5S_GET_SELECT_NPOINTS(tmp_fchunk)) < 0)
+ if((schunk_points = H5S_GET_SELECT_NPOINTS(tmp_fchunk)) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get file selection # of elements")
- H5_ASSIGN_OVERFLOW(new_chunk_info->chunk_points,schunk_points,hssize_t,size_t);
+ H5_ASSIGN_OVERFLOW(new_chunk_info->chunk_points, schunk_points, hssize_t, uint32_t);
/* Decrement # of points left in file selection */
- sel_points-=(hsize_t)schunk_points;
+ sel_points -= (hsize_t)schunk_points;
/* Leave if we are done */
- if(sel_points==0)
+ if(sel_points == 0)
HGOTO_DONE(SUCCEED)
- assert(sel_points>0);
} /* end if */
/* Increment chunk index */
@@ -719,7 +717,7 @@ H5D_create_chunk_file_map_hyper(H5D_chunk_map_t *fm, const H5D_io_info_t
} while(coords[curr_dim]>sel_end[curr_dim]);
/* Re-Calculate the index of this chunk */
- if(H5V_chunk_index(fm->f_ndims,coords,fm->layout->u.chunk.dim,fm->down_chunks,&chunk_index) < 0)
+ if(H5V_chunk_index(fm->f_ndims, coords, fm->layout->u.chunk.dim, fm->down_chunks, &chunk_index) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index")
} /* end if */
} /* end while */
@@ -867,7 +865,7 @@ H5D_chunk_file_cb(void UNUSED *elem, hid_t UNUSED type_id, unsigned ndims, const
FUNC_ENTER_NOAPI_NOINIT(H5D_chunk_file_cb)
/* Calculate the index of this chunk */
- if(H5V_chunk_index(ndims,coords,fm->layout->u.chunk.dim,fm->down_chunks,&chunk_index) < 0)
+ if(H5V_chunk_index(ndims, coords, fm->layout->u.chunk.dim, fm->down_chunks, &chunk_index) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index")
/* Find correct chunk in file & memory skip list */
@@ -912,18 +910,18 @@ H5D_chunk_file_cb(void UNUSED *elem, hid_t UNUSED type_id, unsigned ndims, const
chunk_info->fspace_shared = FALSE;
/* Set the memory chunk dataspace */
- chunk_info->mspace=NULL;
+ chunk_info->mspace = NULL;
chunk_info->mspace_shared = FALSE;
/* Set the number of selected elements in chunk to zero */
- chunk_info->chunk_points=0;
+ chunk_info->chunk_points = 0;
/* Compute the chunk's coordinates */
- for(u=0; u<fm->f_ndims; u++) {
- H5_CHECK_OVERFLOW(fm->layout->u.chunk.dim[u],hsize_t,hssize_t);
- chunk_info->coords[u]=(coords[u]/(hssize_t)fm->layout->u.chunk.dim[u])*(hssize_t)fm->layout->u.chunk.dim[u];
+ for(u = 0; u < fm->f_ndims; u++) {
+ H5_CHECK_OVERFLOW(fm->layout->u.chunk.dim[u], hsize_t, hssize_t);
+ chunk_info->coords[u] = (coords[u] / (hssize_t)fm->layout->u.chunk.dim[u]) * (hssize_t)fm->layout->u.chunk.dim[u];
} /* end for */
- chunk_info->coords[fm->f_ndims]=0;
+ chunk_info->coords[fm->f_ndims] = 0;
/* Insert the new chunk into the skip list */
if(H5SL_insert(fm->sel_chunks,chunk_info,&chunk_info->index) < 0) {
@@ -938,8 +936,8 @@ H5D_chunk_file_cb(void UNUSED *elem, hid_t UNUSED type_id, unsigned ndims, const
} /* end else */
/* Get the coordinates of the element in the chunk */
- for(u=0; u<fm->f_ndims; u++)
- coords_in_chunk[u]=coords[u]%fm->layout->u.chunk.dim[u];
+ for(u = 0; u < fm->f_ndims; u++)
+ coords_in_chunk[u] = coords[u] % fm->layout->u.chunk.dim[u];
/* Add point to file selection for chunk */
if(H5S_select_elements(chunk_info->fspace, H5S_SELECT_APPEND, (size_t)1, coords_in_chunk) < 0)
@@ -979,7 +977,7 @@ H5D_chunk_mem_cb(void UNUSED *elem, hid_t UNUSED type_id, unsigned ndims, const
FUNC_ENTER_NOAPI_NOINIT(H5D_chunk_mem_cb)
/* Calculate the index of this chunk */
- if(H5V_chunk_index(ndims,coords,fm->layout->u.chunk.dim,fm->down_chunks,&chunk_index) < 0)
+ if(H5V_chunk_index(ndims, coords, fm->layout->u.chunk.dim, fm->down_chunks, &chunk_index) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index")
/* Find correct chunk in file & memory skip list */
@@ -1059,7 +1057,7 @@ H5D_chunk_cacheable(const H5D_io_info_t *io_info, haddr_t caddr)
/* Must bring the whole chunk in if there are any filters */
if(dataset->shared->dcpl_cache.pline.nused > 0)
ret_value = TRUE;
- else
+ else {
#ifdef H5_HAVE_PARALLEL
/* If MPI based VFD is used and the file is opened for write access, must
* bypass the chunk-cache scheme because other MPI processes could
@@ -1068,17 +1066,22 @@ H5D_chunk_cacheable(const H5D_io_info_t *io_info, haddr_t caddr)
*/
if(io_info->using_mpi_vfd && (H5F_ACC_RDWR & H5F_INTENT(dataset->oloc.file)))
ret_value = FALSE;
- else
+ else {
#endif /* H5_HAVE_PARALLEL */
/* If the chunk is too large to keep in the cache and if the address
* for the chunk has been defined, then don't load the chunk into the
* cache, just write the data to it directly.
*/
- if(dataset->shared->layout.u.chunk.size > dataset->shared->cache.chunk.nbytes
+ H5_CHECK_OVERFLOW(dataset->shared->layout.u.chunk.size, uint32_t, size_t);
+ if((size_t)dataset->shared->layout.u.chunk.size > dataset->shared->cache.chunk.nbytes
&& H5F_addr_defined(caddr))
ret_value = FALSE;
else
ret_value = TRUE;
+#ifdef H5_HAVE_PARALLEL
+ } /* end else */
+#endif /* H5_HAVE_PARALLEL */
+ } /* end else */
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5D_chunk_cacheable() */
@@ -1152,7 +1155,7 @@ H5D_chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
H5D_io_info_t cpt_io_info; /* Compact I/O info object */
H5D_storage_t cpt_store; /* Chunk storage information as compact dataset */
hbool_t cpt_dirty; /* Temporary placeholder for compact storage "dirty" flag */
- size_t src_accessed_bytes = 0; /* Total accessed size in a chunk */
+ uint32_t src_accessed_bytes = 0; /* Total accessed size in a chunk */
hbool_t skip_missing_chunks = FALSE; /* Whether to skip missing chunks */
unsigned idx_hint = 0; /* Cache index hint */
herr_t ret_value = SUCCEED; /*return value */
@@ -1175,7 +1178,7 @@ H5D_chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
ctg_io_info.layout_ops = *H5D_LOPS_CONTIG;
/* Initialize temporary contiguous storage info */
- ctg_store.contig.dset_size = (hsize_t)io_info->dset->shared->layout.u.chunk.size;
+ H5_ASSIGN_OVERFLOW(ctg_store.contig.dset_size, io_info->dset->shared->layout.u.chunk.size, uint32_t, hsize_t);
/* Set up compact I/O info object */
HDmemcpy(&cpt_io_info, io_info, sizeof(cpt_io_info));
@@ -1232,8 +1235,11 @@ H5D_chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
else {
/* Load the chunk into cache and lock it. */
if(H5D_chunk_cacheable(io_info, chunk_addr)) {
+ size_t tmp_src_accessed_bytes; /* Total accessed size in a chunk */
+
/* Compute # of bytes accessed in chunk */
- src_accessed_bytes = chunk_info->chunk_points * type_info->src_type_size;
+ tmp_src_accessed_bytes = chunk_info->chunk_points * type_info->src_type_size;
+ H5_ASSIGN_OVERFLOW(src_accessed_bytes, tmp_src_accessed_bytes, size_t, uint32_t);
/* Lock the chunk into the cache */
if(NULL == (chunk = H5D_istore_lock(io_info, &udata, FALSE, &idx_hint)))
@@ -1301,7 +1307,7 @@ H5D_chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
H5D_io_info_t cpt_io_info; /* Compact I/O info object */
H5D_storage_t cpt_store; /* Chunk storage information as compact dataset */
hbool_t cpt_dirty; /* Temporary placeholder for compact storage "dirty" flag */
- size_t dst_accessed_bytes = 0; /* Total accessed size in a chunk */
+ uint32_t dst_accessed_bytes = 0; /* Total accessed size in a chunk */
unsigned idx_hint = 0; /* Cache index hint */
herr_t ret_value = SUCCEED; /* Return value */
@@ -1319,7 +1325,7 @@ H5D_chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
ctg_io_info.layout_ops = *H5D_LOPS_CONTIG;
/* Initialize temporary contiguous storage info */
- ctg_store.contig.dset_size = (hsize_t)io_info->dset->shared->layout.u.chunk.size;
+ H5_ASSIGN_OVERFLOW(ctg_store.contig.dset_size, io_info->dset->shared->layout.u.chunk.size, uint32_t, hsize_t);
/* Set up compact I/O info object */
HDmemcpy(&cpt_io_info, io_info, sizeof(cpt_io_info));
@@ -1350,9 +1356,11 @@ H5D_chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info,
chunk_addr = H5D_istore_get_addr(io_info, &udata);
if(H5D_chunk_cacheable(io_info, chunk_addr)) {
hbool_t entire_chunk = TRUE; /* Whether whole chunk is selected */
+ size_t tmp_dst_accessed_bytes; /* Total accessed size in a chunk */
/* Compute # of bytes accessed in chunk */
- dst_accessed_bytes = chunk_info->chunk_points * type_info->dst_type_size;
+ tmp_dst_accessed_bytes = chunk_info->chunk_points * type_info->dst_type_size;
+ H5_ASSIGN_OVERFLOW(dst_accessed_bytes, tmp_dst_accessed_bytes, size_t, uint32_t);
/* Determine if we will access all the data in the chunk */
if(dst_accessed_bytes != ctg_store.contig.dset_size ||
diff --git a/src/H5Dint.c b/src/H5Dint.c
index acb95d2..f8577cc 100644
--- a/src/H5Dint.c
+++ b/src/H5Dint.c
@@ -1174,6 +1174,7 @@ H5D_create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id,
case H5D_CHUNKED:
{
hsize_t max_dim[H5O_LAYOUT_NDIMS]; /* Maximum size of data in elements */
+ uint64_t chunk_size; /* Size of chunk in bytes */
/* Set up layout information */
if((ndims = H5S_GET_EXTENT_NDIMS(new_dset->shared->space)) < 0)
@@ -1208,8 +1209,17 @@ H5D_create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id,
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "chunk size must be <= maximum dimension size for fixed-sized dimensions")
/* Compute the total size of a chunk */
- for(u = 1, new_dset->shared->layout.u.chunk.size = new_dset->shared->layout.u.chunk.dim[0]; u < new_dset->shared->layout.u.chunk.ndims; u++)
- new_dset->shared->layout.u.chunk.size *= new_dset->shared->layout.u.chunk.dim[u];
+ /* (Use 64-bit value to ensure that we can detect >4GB chunks) */
+ for(u = 1, chunk_size = (uint64_t)new_dset->shared->layout.u.chunk.dim[0]; u < new_dset->shared->layout.u.chunk.ndims; u++)
+ chunk_size *= (uint64_t)new_dset->shared->layout.u.chunk.dim[u];
+
+ /* Check for chunk larger than can be represented in 32-bits */
+ /* (Chunk size is encoded in 32-bit value in v1 B-tree records) */
+ if(chunk_size > (uint64_t)0xffffffff)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "chunk size must be < 4GB")
+
+ /* Retain computed chunk size */
+ H5_ASSIGN_OVERFLOW(new_dset->shared->layout.u.chunk.size, chunk_size, uint64_t, uint32_t);
/* Initialize the chunk cache for the dataset */
if(H5D_istore_init(file, new_dset) < 0)
diff --git a/src/H5Distore.c b/src/H5Distore.c
index efab271..aed5add 100644
--- a/src/H5Distore.c
+++ b/src/H5Distore.c
@@ -160,7 +160,7 @@ typedef struct H5D_istore_it_ud4_t {
H5T_path_t *tpath_mem_dst; /* Datatype conversion path from memory to dest. file */
void *reclaim_buf; /* Buffer for reclaiming data */
size_t reclaim_buf_size; /* Reclaim buffer size */
- size_t nelmts; /* Number of elements in buffer */
+ uint32_t nelmts; /* Number of elements in buffer */
H5S_t *buf_space; /* Dataspace describing buffer */
/* needed for compressed variable-length data */
@@ -431,7 +431,7 @@ H5D_istore_debug_key(FILE *stream, H5F_t UNUSED *f, hid_t UNUSED dxpl_id, int in
HDassert(key);
- HDfprintf(stream, "%*s%-*s %Zd bytes\n", indent, "", fwidth, "Chunk size:", key->nbytes);
+ HDfprintf(stream, "%*s%-*s %u bytes\n", indent, "", fwidth, "Chunk size:", (unsigned)key->nbytes);
HDfprintf(stream, "%*s%-*s 0x%08x\n", indent, "", fwidth, "Filter mask:", key->filter_mask);
HDfprintf(stream, "%*s%-*s {", indent, "", fwidth, "Logical offset:");
for(u = 0; u < udata->mesg->u.chunk.ndims; u++)
@@ -596,7 +596,7 @@ H5D_istore_new_node(H5F_t *f, hid_t dxpl_id, H5B_ins_t op,
/* Allocate new storage */
HDassert(udata->nbytes > 0);
- H5_CHECK_OVERFLOW(udata->nbytes, size_t, hsize_t);
+ H5_CHECK_OVERFLOW(udata->nbytes, uint32_t, hsize_t);
if(HADDR_UNDEF == (*addr_p = H5MF_alloc(f, H5FD_MEM_DRAW, dxpl_id, (hsize_t)udata->nbytes)))
HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "couldn't allocate new file storage")
udata->addr = *addr_p;
@@ -772,10 +772,10 @@ H5D_istore_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key,
(hsize_t)lt_key->nbytes, (hsize_t)udata->nbytes)))
HGOTO_ERROR(H5E_STORAGE, H5E_NOSPACE, H5B_INS_ERROR, "unable to reallocate chunk storage")
#else /* OLD_WAY */
- H5_CHECK_OVERFLOW( lt_key->nbytes ,size_t, hsize_t);
- if(H5MF_xfree(f, H5FD_MEM_DRAW, dxpl_id, addr, (hsize_t)lt_key->nbytes)<0)
+ H5_CHECK_OVERFLOW(lt_key->nbytes, uint32_t, hsize_t);
+ if(H5MF_xfree(f, H5FD_MEM_DRAW, dxpl_id, addr, (hsize_t)lt_key->nbytes) < 0)
HGOTO_ERROR(H5E_STORAGE, H5E_CANTFREE, H5B_INS_ERROR, "unable to free chunk")
- H5_CHECK_OVERFLOW(udata->nbytes ,size_t, hsize_t);
+ H5_CHECK_OVERFLOW(udata->nbytes, uint32_t, hsize_t);
if(HADDR_UNDEF == (*new_node_p = H5MF_alloc(f, H5FD_MEM_DRAW, dxpl_id, (hsize_t)udata->nbytes)))
HGOTO_ERROR(H5E_STORAGE, H5E_NOSPACE, H5B_INS_ERROR, "unable to reallocate chunk")
#endif /* OLD_WAY */
@@ -809,7 +809,7 @@ H5D_istore_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key,
/*
* Allocate storage for the new chunk
*/
- H5_CHECK_OVERFLOW(udata->nbytes, size_t, hsize_t);
+ H5_CHECK_OVERFLOW(udata->nbytes, uint32_t, hsize_t);
if(HADDR_UNDEF == (*new_node_p = H5MF_alloc(f, H5FD_MEM_DRAW, dxpl_id, (hsize_t)udata->nbytes)))
HGOTO_ERROR(H5E_STORAGE, H5E_NOSPACE, H5B_INS_ERROR, "file allocation failed")
udata->addr = *new_node_p;
@@ -874,14 +874,12 @@ H5D_istore_iter_chunkmap (H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const void *_lt
{
H5D_istore_it_ud5_t *udata = (H5D_istore_it_ud5_t *)_udata;
const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key;
- unsigned rank;
+ unsigned rank = udata->common.mesg->u.chunk.ndims - 1;
hsize_t chunk_index;
int ret_value = H5_ITER_CONT; /* Return value */
FUNC_ENTER_NOAPI_NOINIT(H5D_istore_iter_chunkmap)
- rank = udata->common.mesg->u.chunk.ndims - 1;
-
if(H5V_chunk_index(rank, lt_key->offset, udata->common.mesg->u.chunk.dim, udata->down_chunks, &chunk_index) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index")
@@ -968,13 +966,16 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key,
/* needed for commpressed variable length data */
hbool_t is_compressed = FALSE;
H5Z_EDC_t edc_read = H5Z_NO_EDC;
- size_t nbytes = lt_key->nbytes;
+ size_t nbytes; /* Size of chunk (in bytes) */
H5Z_cb_t cb_struct;
int ret_value = H5_ITER_CONT; /* Return value */
FUNC_ENTER_NOAPI_NOINIT(H5D_istore_iter_copy)
+ /* Get 'size_t' local value for number of bytes in chunk */
+ H5_ASSIGN_OVERFLOW(nbytes, lt_key->nbytes, uint32_t, size_t);
+
/* Check parameter for type conversion */
if(udata->do_convert) {
if(H5T_detect_class(udata->dt_src, H5T_VLEN) > 0)
@@ -1033,12 +1034,12 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key,
hid_t tid_src = udata->tid_src;
hid_t tid_dst = udata->tid_dst;
hid_t tid_mem = udata->tid_mem;
- size_t nelmts = udata->nelmts;
void *reclaim_buf = udata->reclaim_buf;
size_t reclaim_buf_size = udata->reclaim_buf_size;
/* Convert from source file to memory */
- if(H5T_convert(tpath_src_mem, tid_src, tid_mem, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0)
+ H5_CHECK_OVERFLOW(udata->nelmts, uint32_t, size_t);
+ if(H5T_convert(tpath_src_mem, tid_src, tid_mem, (size_t)udata->nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, H5_ITER_ERROR, "datatype conversion failed")
/* Copy into another buffer, to reclaim memory later */
@@ -1048,7 +1049,7 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key,
HDmemset(bkg, 0, buf_size);
/* Convert from memory to destination file */
- if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, bkg, dxpl_id) < 0)
+ if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, udata->nelmts, (size_t)0, (size_t)0, buf, bkg, dxpl_id) < 0)
HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, H5_ITER_ERROR, "datatype conversion failed")
/* Reclaim space from variable length data */
@@ -1085,7 +1086,7 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key,
if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), edc_read,
cb_struct, &nbytes, &buf_size, &buf) < 0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed")
- udata_dst.nbytes = nbytes;
+ H5_ASSIGN_OVERFLOW(udata_dst.nbytes, nbytes, size_t, uint32_t);
udata->buf = buf;
udata->buf_size = buf_size;
} /* end if */
@@ -1277,7 +1278,6 @@ static herr_t
H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_t reset)
{
void *buf = NULL; /*temporary buffer */
- size_t alloc; /*bytes allocated for BUF */
hbool_t point_of_no_return = FALSE;
herr_t ret_value = SUCCEED; /*return value */
@@ -1292,9 +1292,6 @@ H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_
if(ent->dirty) {
H5D_istore_ud1_t udata; /*pass through B-tree */
- /* Initial buffer size */
- alloc = ent->alloc_size;
-
/* Set up user data for B-tree callbacks */
udata.common.mesg = &io_info->dset->shared->layout;
udata.common.offset = ent->offset;
@@ -1304,16 +1301,19 @@ H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_
/* Should the chunk be filtered before writing it to disk? */
if(io_info->dset->shared->dcpl_cache.pline.nused) {
+ size_t alloc = ent->alloc_size; /* Bytes allocated for BUF */
+ size_t nbytes; /* Chunk size (in bytes) */
+
if(!reset) {
/*
* Copy the chunk to a new buffer before running it through
* the pipeline because we'll want to save the original buffer
* for later.
*/
- alloc = ent->chunk_size;
+ H5_ASSIGN_OVERFLOW(alloc, ent->chunk_size, uint32_t, size_t);
if(NULL == (buf = H5MM_malloc(alloc)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline")
- HDmemcpy(buf, ent->chunk, ent->chunk_size);
+ HDmemcpy(buf, ent->chunk, alloc);
} /* end if */
else {
/*
@@ -1326,9 +1326,11 @@ H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_
point_of_no_return = TRUE;
ent->chunk = NULL;
} /* end else */
+ H5_ASSIGN_OVERFLOW(nbytes, udata.nbytes, uint32_t, size_t);
if(H5Z_pipeline(&(io_info->dset->shared->dcpl_cache.pline), 0, &(udata.filter_mask), io_info->dxpl_cache->err_detect,
- io_info->dxpl_cache->filter_cb, &(udata.nbytes), &alloc, &buf) < 0)
+ io_info->dxpl_cache->filter_cb, &nbytes, &alloc, &buf) < 0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "output pipeline failed")
+ H5_ASSIGN_OVERFLOW(udata.nbytes, nbytes, size_t, uint32_t);
} /* end if */
/*
@@ -1337,7 +1339,8 @@ H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_
*/
if(H5B_insert(io_info->dset->oloc.file, io_info->dxpl_id, H5B_ISTORE, io_info->dset->shared->layout.u.chunk.addr, &udata)<0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to allocate chunk")
- if(H5F_block_write(io_info->dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.nbytes, io_info->dxpl_id, buf) < 0)
+ H5_CHECK_OVERFLOW(udata.nbytes, uint32_t, size_t);
+ if(H5F_block_write(io_info->dset->oloc.file, H5FD_MEM_DRAW, udata.addr, (size_t)udata.nbytes, io_info->dxpl_id, buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file")
/* Cache the chunk's info, in case it's accessed again shortly */
@@ -1688,20 +1691,20 @@ H5D_istore_prune (const H5D_io_info_t *io_info, size_t size)
while ((p[0] || p[1]) && rdcc->nbytes+size>total) {
/* Introduce new pointers */
- for (i=0; i<nmeth-1; i++)
- if (0==w[i])
- p[i+1] = rdcc->head;
+ for(i = 0; i < (nmeth - 1); i++)
+ if(0 == w[i])
+ p[i + 1] = rdcc->head;
/* Compute next value for each pointer */
- for (i=0; i<nmeth; i++)
+ for(i = 0; i < nmeth; i++)
n[i] = p[i] ? p[i]->next : NULL;
/* Give each method a chance */
- for (i=0; i<nmeth && rdcc->nbytes+size>total; i++) {
- if (0==i && p[0] && !p[0]->locked &&
- ((0==p[0]->rd_count && 0==p[0]->wr_count) ||
- (0==p[0]->rd_count && p[0]->chunk_size==p[0]->wr_count) ||
- (p[0]->chunk_size==p[0]->rd_count && 0==p[0]->wr_count))) {
+ for(i = 0; i < nmeth && (rdcc->nbytes + size) > total; i++) {
+ if(0 == i && p[0] && !p[0]->locked &&
+ ((0 == p[0]->rd_count && 0 == p[0]->wr_count) ||
+ (0 == p[0]->rd_count && p[0]->chunk_size == p[0]->wr_count) ||
+ (p[0]->chunk_size == p[0]->rd_count && 0 == p[0]->wr_count))) {
/*
* Method 0: Preempt entries that have been completely written
* and/or completely read but not entries that are partially
@@ -1813,7 +1816,7 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata,
/* Get the chunk's size */
HDassert(layout->u.chunk.size > 0);
- H5_ASSIGN_OVERFLOW(chunk_size, layout->u.chunk.size, hsize_t, size_t);
+ H5_ASSIGN_OVERFLOW(chunk_size, layout->u.chunk.size, uint32_t, size_t);
/* Search for the chunk in the cache */
if(rdcc->nslots > 0) {
@@ -1854,7 +1857,6 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata,
/* In the case that some dataset functions look through this data,
* clear it to all 0s. */
HDmemset(chunk, 0, chunk_size);
-
} /* end if */
else {
H5D_istore_ud1_t tmp_udata; /*B-tree pass-through */
@@ -1873,24 +1875,24 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata,
chunk_addr = H5D_istore_get_addr(io_info, udata);
} /* end else */
- if (H5F_addr_defined(chunk_addr)) {
- size_t chunk_alloc = 0; /*allocated chunk size */
+ /* Check if the chunk exists on disk */
+ if(H5F_addr_defined(chunk_addr)) {
+ size_t chunk_alloc; /* Allocated chunk size */
- /*
- * The chunk exists on disk.
- */
/* Chunk size on disk isn't [likely] the same size as the final chunk
* size in memory, so allocate memory big enough. */
- chunk_alloc = udata->nbytes;
- if(NULL == (chunk = H5D_istore_chunk_alloc (chunk_alloc, pline)))
+ H5_ASSIGN_OVERFLOW(chunk_alloc, udata->nbytes, uint32_t, size_t);
+ if(NULL == (chunk = H5D_istore_chunk_alloc(chunk_alloc, pline)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk")
- if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, chunk_addr, udata->nbytes, io_info->dxpl_id, chunk) < 0)
+ if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, chunk_addr, chunk_alloc, io_info->dxpl_id, chunk) < 0)
HGOTO_ERROR(H5E_IO, H5E_READERROR, NULL, "unable to read raw data chunk")
- if(pline->nused)
+ if(pline->nused) {
if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &(udata->filter_mask), io_info->dxpl_cache->err_detect,
- io_info->dxpl_cache->filter_cb, &(udata->nbytes), &chunk_alloc, &chunk) < 0)
+ io_info->dxpl_cache->filter_cb, &chunk_alloc, &chunk_alloc, &chunk) < 0)
HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, NULL, "data pipeline read failed")
+ H5_ASSIGN_OVERFLOW(udata->nbytes, chunk_alloc, size_t, uint32_t);
+ } /* end if */
#ifdef H5D_ISTORE_DEBUG
rdcc->nmisses++;
#endif /* H5D_ISTORE_DEBUG */
@@ -1904,7 +1906,7 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata,
/* Chunk size on disk isn't [likely] the same size as the final chunk
* size in memory, so allocate memory big enough. */
- if(NULL == (chunk = H5D_istore_chunk_alloc (chunk_size, pline)))
+ if(NULL == (chunk = H5D_istore_chunk_alloc(chunk_size, pline)))
HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk")
if(H5P_is_fill_value_defined(fill, &fill_status) < 0)
@@ -1947,31 +1949,31 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata,
* Add the chunk to the cache only if the slot is not already locked.
* Preempt enough things from the cache to make room.
*/
- if (ent) {
+ if(ent) {
#ifdef H5D_ISTORE_DEBUG
HDputc('#', stderr);
HDfflush(stderr);
#endif
- if (H5D_istore_preempt(io_info, ent, TRUE)<0)
+ if(H5D_istore_preempt(io_info, ent, TRUE) < 0)
HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk from cache")
- }
- if (H5D_istore_prune(io_info, chunk_size)<0)
+ } /* end if */
+ if(H5D_istore_prune(io_info, chunk_size) < 0)
HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk(s) from cache")
/* Create a new entry */
ent = H5FL_MALLOC(H5D_rdcc_ent_t);
ent->locked = 0;
ent->dirty = FALSE;
- ent->chunk_size = chunk_size;
+ H5_ASSIGN_OVERFLOW(ent->chunk_size, chunk_size, size_t, uint32_t);
ent->alloc_size = chunk_size;
- for (u=0; u<layout->u.chunk.ndims; u++)
+ for(u = 0; u < layout->u.chunk.ndims; u++)
ent->offset[u] = io_info->store->chunk.offset[u];
- ent->rd_count = chunk_size;
- ent->wr_count = chunk_size;
- ent->chunk = (uint8_t*)chunk;
+ H5_ASSIGN_OVERFLOW(ent->rd_count, chunk_size, size_t, uint32_t);
+ H5_ASSIGN_OVERFLOW(ent->wr_count, chunk_size, size_t, uint32_t);
+ ent->chunk = (uint8_t *)chunk;
/* Add it to the cache */
- assert(NULL==rdcc->slot[idx]);
+ HDassert(NULL == rdcc->slot[idx]);
rdcc->slot[idx] = ent;
ent->idx = idx;
rdcc->nbytes += chunk_size;
@@ -1979,16 +1981,17 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata,
/* Add it to the linked list */
ent->next = NULL;
- if (rdcc->tail) {
+ if(rdcc->tail) {
rdcc->tail->next = ent;
ent->prev = rdcc->tail;
rdcc->tail = ent;
- } else {
+ } /* end if */
+ else {
rdcc->head = rdcc->tail = ent;
ent->prev = NULL;
- }
+ } /* end else */
found = TRUE;
- } else if (!found) {
+ } else if(!found) {
/*
* The chunk is larger than the entire cache so we don't cache it.
* This is the reason all those arguments have to be repeated for the
@@ -1996,38 +1999,37 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata,
*/
ent = NULL;
idx = UINT_MAX;
-
} else {
/*
* The chunk is not at the beginning of the cache; move it backward
* by one slot. This is how we implement the LRU preemption
* algorithm.
*/
- assert(ent);
- if (ent->next) {
- if (ent->next->next)
+ HDassert(ent);
+ if(ent->next) {
+ if(ent->next->next)
ent->next->next->prev = ent;
else
rdcc->tail = ent;
ent->next->prev = ent->prev;
- if (ent->prev)
+ if(ent->prev)
ent->prev->next = ent->next;
else
rdcc->head = ent->next;
ent->prev = ent->next;
ent->next = ent->next->next;
ent->prev->next = ent;
- }
- }
+ } /* end if */
+ } /* end else */
/* Lock the chunk into the cache */
- if (ent) {
- assert (!ent->locked);
+ if(ent) {
+ HDassert(!ent->locked);
ent->locked = TRUE;
chunk = ent->chunk;
- }
+ } /* end if */
- if (idx_hint)
+ if(idx_hint)
*idx_hint = idx;
/* Set return value */
@@ -2071,19 +2073,18 @@ done:
*/
herr_t
H5D_istore_unlock(const H5D_io_info_t *io_info,
- hbool_t dirty, unsigned idx_hint, void *chunk, size_t naccessed)
+ hbool_t dirty, unsigned idx_hint, void *chunk, uint32_t naccessed)
{
const H5O_layout_t *layout=&(io_info->dset->shared->layout); /* Dataset layout */
const H5D_rdcc_t *rdcc = &(io_info->dset->shared->cache.chunk);
H5D_rdcc_ent_t *ent = NULL;
- unsigned u;
- herr_t ret_value=SUCCEED; /* Return value */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_NOAPI_NOINIT(H5D_istore_unlock)
assert(io_info);
- if (UINT_MAX==idx_hint) {
+ if(UINT_MAX == idx_hint) {
/*
* It's not in the cache, probably because it's too big. If it's
* dirty then flush it to disk. In any case, free the chunk.
@@ -2093,40 +2094,42 @@ H5D_istore_unlock(const H5D_io_info_t *io_info,
if (dirty) {
H5D_rdcc_ent_t x;
- HDmemset (&x, 0, sizeof x);
+ HDmemset(&x, 0, sizeof(x));
x.dirty = TRUE;
- for (u=0; u<layout->u.chunk.ndims; u++)
- x.offset[u] = io_info->store->chunk.offset[u];
- assert(layout->u.chunk.size>0);
- H5_ASSIGN_OVERFLOW(x.chunk_size,layout->u.chunk.size,hsize_t,size_t);
- x.alloc_size = x.chunk_size;
- x.chunk = (uint8_t*)chunk;
-
- if (H5D_istore_flush_entry(io_info, &x, TRUE)<0)
+ HDassert(sizeof(x.offset[0]) == sizeof(io_info->store->chunk.offset[0]));
+ HDmemcpy(x.offset, io_info->store->chunk.offset, layout->u.chunk.ndims * sizeof(x.offset[0]));
+ HDassert(layout->u.chunk.size > 0);
+ x.chunk_size = layout->u.chunk.size;
+ H5_ASSIGN_OVERFLOW(x.alloc_size, x.chunk_size, uint32_t, size_t);
+ x.chunk = (uint8_t *)chunk;
+
+ if(H5D_istore_flush_entry(io_info, &x, TRUE) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer")
- } else {
+ } /* end if */
+ else {
if(chunk)
- chunk=H5D_istore_chunk_xfree (chunk,&(io_info->dset->shared->dcpl_cache.pline));
- }
- } else {
+ chunk = H5D_istore_chunk_xfree(chunk, &(io_info->dset->shared->dcpl_cache.pline));
+ } /* end else */
+ } /* end if */
+ else {
/* Sanity check */
- assert(idx_hint<rdcc->nslots);
- assert(rdcc->slot[idx_hint]);
- assert(rdcc->slot[idx_hint]->chunk==chunk);
+ HDassert(idx_hint < rdcc->nslots);
+ HDassert(rdcc->slot[idx_hint]);
+ HDassert(rdcc->slot[idx_hint]->chunk == chunk);
/*
* It's in the cache so unlock it.
*/
ent = rdcc->slot[idx_hint];
- assert (ent->locked);
- if (dirty) {
+ HDassert(ent->locked);
+ if(dirty) {
ent->dirty = TRUE;
- ent->wr_count -= MIN (ent->wr_count, naccessed);
- } else {
- ent->rd_count -= MIN (ent->rd_count, naccessed);
- }
+ ent->wr_count -= MIN(ent->wr_count, naccessed);
+ } /* end if */
+ else
+ ent->rd_count -= MIN(ent->rd_count, naccessed);
ent->locked = FALSE;
- }
+ } /* end else */
done:
FUNC_LEAVE_NOAPI(ret_value)
@@ -2510,8 +2513,7 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite)
HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache")
/* Get original chunk size */
- H5_CHECK_OVERFLOW(layout->u.chunk.size, hsize_t, size_t);
- orig_chunk_size = (size_t)layout->u.chunk.size;
+ H5_ASSIGN_OVERFLOW(orig_chunk_size, layout->u.chunk.size, uint32_t, size_t);
/* Check the dataset's fill-value status */
if(H5P_is_fill_value_defined(fill, &fill_status) < 0)
@@ -2613,7 +2615,7 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite)
chunk_size = nbytes;
} /* end if */
else
- chunk_size = (size_t)layout->u.chunk.size;
+ H5_ASSIGN_OVERFLOW(chunk_size, layout->u.chunk.size, uint32_t, size_t);
} /* end if */
else
chunk_size = orig_chunk_size;
@@ -2621,7 +2623,7 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite)
/* Initialize the chunk information */
udata.common.mesg = layout;
udata.common.offset = chunk_offset;
- udata.nbytes = chunk_size;
+ H5_ASSIGN_OVERFLOW(udata.nbytes, chunk_size, size_t, uint32_t);
udata.filter_mask = filter_mask;
udata.addr = HADDR_UNDEF;
@@ -2639,16 +2641,19 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite)
if(using_mpi) {
/* Write the chunks out from only one process */
/* !! Use the internal "independent" DXPL!! -QAK */
- if(H5_PAR_META_WRITE == mpi_rank)
- if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.nbytes, data_dxpl_id, fb_info.fill_buf) < 0)
+ if(H5_PAR_META_WRITE == mpi_rank) {
+ HDassert(udata.nbytes == chunk_size);
+ if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, chunk_size, data_dxpl_id, fb_info.fill_buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file")
+ } /* end if */
/* Indicate that blocks are being written */
blocks_written = TRUE;
} /* end if */
else {
#endif /* H5_HAVE_PARALLEL */
- if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.nbytes, data_dxpl_id, fb_info.fill_buf) < 0)
+ HDassert(udata.nbytes == chunk_size);
+ if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, chunk_size, data_dxpl_id, fb_info.fill_buf) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file")
#ifdef H5_HAVE_PARALLEL
} /* end else */
@@ -3029,12 +3034,13 @@ H5D_istore_remove(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key /*in,out
hbool_t *rt_key_changed /*out */ )
{
H5D_istore_key_t *lt_key = (H5D_istore_key_t *)_lt_key;
- H5B_ins_t ret_value=H5B_INS_REMOVE; /* Return value */
+ H5B_ins_t ret_value = H5B_INS_REMOVE; /* Return value */
FUNC_ENTER_NOAPI_NOINIT(H5D_istore_remove)
/* Remove raw data chunk from file */
- if(H5MF_xfree(f, H5FD_MEM_DRAW, dxpl_id, addr, (hsize_t)lt_key->nbytes)<0)
+ H5_CHECK_OVERFLOW(lt_key->nbytes, uint32_t, hsize_t);
+ if(H5MF_xfree(f, H5FD_MEM_DRAW, dxpl_id, addr, (hsize_t)lt_key->nbytes) < 0)
HGOTO_ERROR(H5E_STORAGE, H5E_CANTFREE, H5B_INS_ERROR, "unable to free chunk")
/* Mark keys as unchanged */
@@ -3082,7 +3088,7 @@ H5D_istore_initialize_by_extent(H5D_io_info_t *io_info)
hsize_t hyper_start[H5O_LAYOUT_NDIMS]; /* Starting location of hyperslab */
hsize_t nchunks[H5O_LAYOUT_NDIMS]; /* Current number of chunks in each dimension */
hsize_t down_chunks[H5O_LAYOUT_NDIMS]; /* "down" size of number of elements in each dimension */
- hsize_t bytes_per_chunk; /* Bytes in chunk */
+ uint32_t elmts_per_chunk; /* Elements in a chunk */
int srank; /* # of chunk dimensions (signed) */
unsigned rank; /* # of chunk dimensions */
hbool_t carry; /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */
@@ -3105,11 +3111,11 @@ H5D_istore_initialize_by_extent(H5D_io_info_t *io_info)
/* Set size of lowest chunk dimension (the dataset element size) */
dset_dims[rank] = layout->u.chunk.dim[rank];
- /* Compute the number of chunks in dataset & the # of bytes in a chunk */
+ /* Compute the number of chunks in dataset & the # of elements in a chunk */
/* (round up to the next integer # of chunks, to accomodate partial chunks) */
- for(u = 0, bytes_per_chunk = layout->u.chunk.dim[rank]; u < rank; u++) {
+ for(u = 0, elmts_per_chunk = 1; u < rank; u++) {
nchunks[u] = ((dset_dims[u] - 1) / layout->u.chunk.dim[u]) + 1;
- bytes_per_chunk *= layout->u.chunk.dim[u];
+ elmts_per_chunk *= layout->u.chunk.dim[u];
} /* end for */
/* Get the "down" sizes for each dimension */
@@ -3159,9 +3165,10 @@ H5D_istore_initialize_by_extent(H5D_io_info_t *io_info)
/* Initialize the fill value buffer, if necessary */
if(!fb_info_init) {
+ H5_CHECK_OVERFLOW(elmts_per_chunk, uint32_t, size_t);
if(H5D_fill_init(&fb_info, NULL, FALSE, NULL, NULL, NULL, NULL,
&dset->shared->dcpl_cache.fill,
- dset->shared->type, dset->shared->type_id, (size_t)(bytes_per_chunk / layout->u.chunk.dim[rank]),
+ dset->shared->type, dset->shared->type_id, (size_t)elmts_per_chunk,
io_info->dxpl_cache->max_temp_buf, io_info->dxpl_id) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill buffer info")
fb_info_init = TRUE;
@@ -3217,7 +3224,8 @@ H5D_istore_initialize_by_extent(H5D_io_info_t *io_info)
HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't refill fill value buffer")
/* Create a selection iterator for scattering the elements to memory buffer */
- if(H5S_select_iter_init(&chunk_iter, space_chunk, layout->u.chunk.dim[rank]) < 0)
+ H5_CHECK_OVERFLOW(layout->u.chunk.dim[rank], uint32_t, size_t);
+ if(H5S_select_iter_init(&chunk_iter, space_chunk, (size_t)layout->u.chunk.dim[rank]) < 0)
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize chunk selection information")
/* Scatter the data into memory */
@@ -3236,8 +3244,8 @@ H5D_istore_initialize_by_extent(H5D_io_info_t *io_info)
bytes_accessed = nelmts * layout->u.chunk.dim[rank];
/* Release lock on chunk */
- H5_CHECK_OVERFLOW(bytes_accessed, hsize_t, size_t);
- if(H5D_istore_unlock(io_info, TRUE, idx_hint, chunk, (size_t)bytes_accessed) < 0)
+ H5_CHECK_OVERFLOW(bytes_accessed, hsize_t, uint32_t);
+ if(H5D_istore_unlock(io_info, TRUE, idx_hint, chunk, (uint32_t)bytes_accessed) < 0)
HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to unlock raw data chunk")
} /* end if */
@@ -3386,12 +3394,12 @@ H5D_istore_update_cache(H5D_t *dset, hid_t dxpl_id)
next = ent->next;
/* Calculate the index of this chunk */
- if(H5V_chunk_index(rank,ent->offset,dset->shared->layout.u.chunk.dim,down_chunks,&idx)<0)
+ if(H5V_chunk_index(rank, ent->offset, dset->shared->layout.u.chunk.dim, down_chunks, &idx) < 0)
HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index")
/* Compute the index for the chunk entry */
- old_idx=ent->idx; /* Save for later */
- ent->idx=H5D_CHUNK_HASH(dset->shared, idx);
+ old_idx = ent->idx; /* Save for later */
+ ent->idx = H5D_CHUNK_HASH(dset->shared, idx);
if(old_idx != ent->idx) {
/* Check if there is already a chunk at this chunk's new location */
@@ -3450,7 +3458,7 @@ H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst,
void *reclaim_buf = NULL; /* Buffer for reclaiming data */
H5S_t *buf_space = NULL; /* Dataspace describing buffer */
hid_t sid_buf = -1; /* ID for buffer dataspace */
- size_t nelmts = 0; /* Number of elements in buffer */
+ uint32_t nelmts = 0; /* Number of elements in buffer */
hbool_t do_convert = FALSE; /* Indicate that type conversions should be performed */
herr_t ret_value = SUCCEED; /* Return value */
@@ -3553,7 +3561,7 @@ H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst,
do_convert = TRUE;
} /* end if */
- buf_size = layout_src->u.chunk.size;
+ H5_ASSIGN_OVERFLOW(buf_size, layout_src->u.chunk.size, uint32_t, size_t);
reclaim_buf_size = 0;
} /* end else */
diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c
index 889fbce..dd0c602 100644
--- a/src/H5Dmpio.c
+++ b/src/H5Dmpio.c
@@ -1183,7 +1183,7 @@ if(H5DEBUG(D))
H5D_io_info_t *chk_io_info; /* Pointer to I/O info object for this chunk */
H5D_istore_ud1_t udata; /* B-tree pass-through */
void *chunk; /* Pointer to the data chunk in cache */
- size_t accessed_bytes; /* Total accessed size in a chunk */
+ uint32_t accessed_bytes; /* Total accessed size in a chunk */
unsigned idx_hint = 0; /* Cache index hint */
haddr_t caddr; /* Address of the cached chunk */
@@ -1203,9 +1203,11 @@ if(H5DEBUG(D))
/* Load the chunk into cache and lock it. */
if(H5D_chunk_cacheable(io_info, caddr)) {
hbool_t entire_chunk = TRUE; /* Whether whole chunk is selected */
+ size_t tmp_accessed_bytes; /* Total accessed size in a chunk */
/* Compute # of bytes accessed in chunk */
- accessed_bytes = chunk_info->chunk_points * type_info->src_type_size;
+ tmp_accessed_bytes = chunk_info->chunk_points * type_info->src_type_size;
+ H5_ASSIGN_OVERFLOW(accessed_bytes, tmp_accessed_bytes, size_t, uint32_t);
/* Determine if we will access all the data in the chunk */
if(((io_info->op_type == H5D_IO_OP_WRITE) && (accessed_bytes != ctg_store.contig.dset_size))
@@ -1419,7 +1421,7 @@ if(H5DEBUG(D)) {
if(make_ind) {
void *chunk; /* Pointer to the data chunk in cache */
H5D_io_info_t *chk_io_info; /* Pointer to I/O info object for this chunk */
- size_t accessed_bytes = 0; /* Total accessed size in a chunk */
+ uint32_t accessed_bytes = 0; /* Total accessed size in a chunk */
unsigned idx_hint = 0; /* Cache index hint */
/* Switch to independent I/O */
@@ -1429,9 +1431,11 @@ if(H5DEBUG(D)) {
/* Load the chunk into cache and lock it. */
if(H5D_chunk_cacheable(io_info, chunk_addr)) {
hbool_t entire_chunk = TRUE; /* Whether whole chunk is selected */
+ size_t tmp_accessed_bytes; /* Total accessed size in a chunk */
/* Compute # of bytes accessed in chunk */
- accessed_bytes = chunk_info->chunk_points * type_info->src_type_size;
+ tmp_accessed_bytes = chunk_info->chunk_points * type_info->src_type_size;
+ H5_ASSIGN_OVERFLOW(accessed_bytes, tmp_accessed_bytes, size_t, uint32_t);
/* Determine if we will access all the data in the chunk */
if(((io_info->op_type == H5D_IO_OP_WRITE) && (accessed_bytes != ctg_store.contig.dset_size))
diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h
index 97dcacf..1b20ad8 100644
--- a/src/H5Dpkg.h
+++ b/src/H5Dpkg.h
@@ -204,7 +204,7 @@ typedef struct H5D_io_info_t {
/* Structure holding information about a chunk's selection for mapping */
typedef struct H5D_chunk_info_t {
hsize_t index; /* "Index" of chunk in dataset */
- size_t chunk_points; /* Number of elements selected in chunk */
+ uint32_t chunk_points; /* Number of elements selected in chunk */
hsize_t coords[H5O_LAYOUT_NDIMS]; /* Coordinates of chunk in file dataset's dataspace */
H5S_t *fspace; /* Dataspace describing chunk & selection in it */
unsigned fspace_shared; /* Indicate that the file space for a chunk is shared and shouldn't be freed */
@@ -213,10 +213,10 @@ typedef struct H5D_chunk_info_t {
} H5D_chunk_info_t;
/* Cached information about a particular chunk */
-typedef struct {
+typedef struct H5D_chunk_cached_t{
hbool_t valid; /*whether cache info is valid*/
hsize_t offset[H5O_LAYOUT_NDIMS]; /*logical offset to start*/
- size_t nbytes; /*size of stored data */
+ uint32_t nbytes; /*size of stored data */
unsigned filter_mask; /*excluded filters */
haddr_t addr; /*file address of chunk */
} H5D_chunk_cached_t;
@@ -379,7 +379,7 @@ typedef struct {
* The chunk's file address is part of the B-tree and not part of the key.
*/
typedef struct H5D_istore_key_t {
- size_t nbytes; /*size of stored data */
+ uint32_t nbytes; /*size of stored data */
hsize_t offset[H5O_LAYOUT_NDIMS]; /*logical offset to start*/
unsigned filter_mask; /*excluded filters */
} H5D_istore_key_t;
@@ -401,7 +401,7 @@ typedef struct H5D_istore_ud1_t {
H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */
/* Upward */
- size_t nbytes; /*size of stored data */
+ uint32_t nbytes; /*size of stored data */
unsigned filter_mask; /*excluded filters */
haddr_t addr; /*file address of chunk */
} H5D_istore_ud1_t;
@@ -422,9 +422,9 @@ typedef struct H5D_rdcc_ent_t {
hbool_t locked; /*entry is locked in cache */
hbool_t dirty; /*needs to be written to disk? */
hsize_t offset[H5O_LAYOUT_NDIMS]; /*chunk name */
- size_t rd_count; /*bytes remaining to be read */
- size_t wr_count; /*bytes remaining to be written */
- size_t chunk_size; /*size of a chunk */
+ uint32_t rd_count; /*bytes remaining to be read */
+ uint32_t wr_count; /*bytes remaining to be written */
+ uint32_t chunk_size; /*size of a chunk */
size_t alloc_size; /*amount allocated for the chunk */
uint8_t *chunk; /*the unfiltered chunk data */
unsigned idx; /*index in hash table */
@@ -547,7 +547,7 @@ H5_DLL herr_t H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src,
H5_DLL void * H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata,
hbool_t relax, unsigned *idx_hint/*in,out*/);
H5_DLL herr_t H5D_istore_unlock(const H5D_io_info_t *io_info,
- hbool_t dirty, unsigned idx_hint, void *chunk, size_t naccessed);
+ hbool_t dirty, unsigned idx_hint, void *chunk, uint32_t naccessed);
/* Functions that perform fill value operations on datasets */
H5_DLL herr_t H5D_fill(const void *fill, const H5T_t *fill_type, void *buf,
diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h
index 4a55ac2..3c5204f 100644
--- a/src/H5Oprivate.h
+++ b/src/H5Oprivate.h
@@ -334,8 +334,8 @@ typedef struct H5O_layout_contig_t {
typedef struct H5O_layout_chunk_t {
haddr_t addr; /* File address of B-tree */
unsigned ndims; /* Num dimensions in chunk */
- size_t dim[H5O_LAYOUT_NDIMS]; /* Size of chunk in elements */
- size_t size; /* Size of chunk in bytes */
+ uint32_t dim[H5O_LAYOUT_NDIMS]; /* Size of chunk in elements */
+ uint32_t size; /* Size of chunk in bytes */
H5RC_t *btree_shared; /* Ref-counted info for B-tree nodes */
} H5O_layout_chunk_t;
diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c
index 8d6b9ab..9501019 100644
--- a/src/H5Pdcpl.c
+++ b/src/H5Pdcpl.c
@@ -55,7 +55,7 @@
#define H5D_CRT_CHUNK_DIM_SIZE sizeof(unsigned)
#define H5D_CRT_CHUNK_DIM_DEF 1
/* Definitions for chunk size */
-#define H5D_CRT_CHUNK_SIZE_SIZE sizeof(size_t[H5O_LAYOUT_NDIMS])
+#define H5D_CRT_CHUNK_SIZE_SIZE sizeof(uint32_t[H5O_LAYOUT_NDIMS])
#define H5D_CRT_CHUNK_SIZE_DEF {1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,\
1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}
/* Definitions for fill value. size=0 means fill value will be 0 as
@@ -148,7 +148,7 @@ H5P_dcrt_reg_prop(H5P_genclass_t *pclass)
{
H5D_layout_t layout = H5D_CRT_LAYOUT_DEF; /* Default storage layout */
unsigned chunk_ndims = H5D_CRT_CHUNK_DIM_DEF; /* Default rank for chunks */
- size_t chunk_size[H5O_LAYOUT_NDIMS] = H5D_CRT_CHUNK_SIZE_DEF; /* Default chunk size */
+ uint32_t chunk_size[H5O_LAYOUT_NDIMS] = H5D_CRT_CHUNK_SIZE_DEF; /* Default chunk size */
H5O_fill_t fill = H5D_CRT_FILL_VALUE_DEF; /* Default fill value */
unsigned alloc_time_state = H5D_CRT_ALLOC_TIME_STATE_DEF; /* Default allocation time state */
H5O_efl_t efl = H5D_CRT_EXT_FILE_LIST_DEF; /* Default external file list */
@@ -737,37 +737,43 @@ done:
herr_t
H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/])
{
- int i;
- size_t real_dims[H5O_LAYOUT_NDIMS]; /* Full-sized array to hold chunk dims */
H5P_genplist_t *plist; /* Property list pointer */
- herr_t ret_value=SUCCEED; /* return value */
+ uint32_t real_dims[H5O_LAYOUT_NDIMS]; /* Full-sized array to hold chunk dims */
+ uint64_t chunk_nelmts; /* Number of elements in chunk */
+ unsigned u; /* Local index variable */
+ herr_t ret_value = SUCCEED; /* Return value */
FUNC_ENTER_API(H5Pset_chunk, FAIL)
H5TRACE3("e", "iIs*[a1]h", plist_id, ndims, dim);
/* Check arguments */
- if (ndims <= 0)
+ if(ndims <= 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "chunk dimensionality must be positive")
- if (ndims > H5S_MAX_RANK)
+ if(ndims > H5S_MAX_RANK)
HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "chunk dimensionality is too large")
- if (!dim)
+ if(!dim)
HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no chunk dimensions specified")
- /* Get the plist structure */
- if(NULL == (plist = H5P_object_verify(plist_id,H5P_DATASET_CREATE)))
- HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
-
- /* Initialize chunk dims to 0s */
- HDmemset(real_dims,0,sizeof(real_dims));
- for (i=0; i<ndims; i++) {
- if (dim[i] == 0)
+ /* Verify & initialize internal chunk dims */
+ HDmemset(real_dims, 0, sizeof(real_dims));
+ chunk_nelmts = 1;
+ for(u = 0; u < (unsigned)ndims; u++) {
+ if(dim[u] == 0)
HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "all chunk dimensions must be positive")
- if (dim[i] != (dim[i]&0xffffffff))
+ if(dim[u] != (dim[u] & 0xffffffff))
HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "all chunk dimensions must be less than 2^32")
- real_dims[i]=(size_t)dim[i]; /* Store user's chunk dimensions */
+ chunk_nelmts *= dim[u];
+ if(chunk_nelmts > (uint64_t)0xffffffff)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "number of elements in chunk must be < 4GB")
+ real_dims[u] = (uint32_t)dim[u]; /* Store user's chunk dimensions */
} /* end for */
- if(H5P_set_layout (plist, H5D_CHUNKED) < 0)
+ /* Get the plist structure */
+ if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE)))
+ HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID")
+
+ /* Set chunk information in property list */
+ if(H5P_set_layout(plist, H5D_CHUNKED) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set layout")
if(H5P_set(plist, H5D_CRT_CHUNK_DIM_NAME, &ndims) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set chunk dimensionanlity")
@@ -776,7 +782,7 @@ H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/])
done:
FUNC_LEAVE_API(ret_value)
-}
+} /* end H5Pset_chunk() */
/*-------------------------------------------------------------------------
@@ -828,22 +834,22 @@ H5Pget_chunk(hid_t plist_id, int max_ndims, hsize_t dim[]/*out*/)
if(dim) {
int i;
- size_t chunk_size[H5O_LAYOUT_NDIMS];
+ uint32_t chunk_size[H5O_LAYOUT_NDIMS];
if(H5P_get(plist, H5D_CRT_CHUNK_SIZE_NAME, chunk_size) < 0)
HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get chunk size")
/* Get the dimension sizes */
- for (i=0; i<ndims && i<max_ndims; i++)
+ for(i = 0; i < ndims && i < max_ndims; i++)
dim[i] = chunk_size[i];
} /* end if */
/* Set the return value */
- ret_value=ndims;
+ ret_value = ndims;
done:
FUNC_LEAVE_API(ret_value)
-}
+} /* end H5Pget_chunk() */
/*-------------------------------------------------------------------------
diff --git a/src/H5V.c b/src/H5V.c
index 3e06636..34cf714 100644
--- a/src/H5V.c
+++ b/src/H5V.c
@@ -399,33 +399,34 @@ done:
*/
htri_t
H5V_hyper_disjointp(unsigned n,
- const hsize_t *offset1, const size_t *size1,
- const hsize_t *offset2, const size_t *size2)
+ const hsize_t *offset1, const uint32_t *size1,
+ const hsize_t *offset2, const uint32_t *size2)
{
unsigned u;
- htri_t ret_value=FALSE; /* Return value */
+ htri_t ret_value = FALSE; /* Return value */
/* Use FUNC_ENTER_NOAPI_NOINIT_NOFUNC here to avoid performance issues */
FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5V_hyper_disjointp)
- if (!n || !size1 || !size2) HGOTO_DONE(TRUE)
+ if(!n || !size1 || !size2)
+ HGOTO_DONE(TRUE)
- for (u=0; u<n; u++) {
- assert (size1[u]<HSIZET_MAX);
- assert (size2[u]<HSIZET_MAX);
+ for(u = 0; u < n; u++) {
+ HDassert(size1[u] < HSIZET_MAX);
+ HDassert(size2[u] < HSIZET_MAX);
- if (0==size1[u] || 0==size2[u])
+ if(0 == size1[u] || 0 == size2[u])
HGOTO_DONE(TRUE)
- if (((offset1?offset1[u]:0) < (offset2?offset2[u]:0) &&
- ((offset1?offset1[u]:0) + size1[u] <= (offset2?offset2[u]:0))) ||
- ((offset2?offset2[u]:0) < (offset1?offset1[u]:0) &&
- ((offset2?offset2[u]:0) + size2[u] <= (offset1?offset1[u]:0))))
+ if(((offset1 ? offset1[u] : 0) < (offset2 ? offset2[u] : 0) &&
+ ((offset1 ? offset1[u] : 0) + size1[u] <= (offset2 ? offset2[u] : 0))) ||
+ ((offset2 ? offset2[u] : 0) < (offset1 ? offset1[u] : 0) &&
+ ((offset2 ? offset2[u] : 0) + size2[u] <= (offset1 ? offset1[u] : 0))))
HGOTO_DONE(TRUE)
- }
+ } /* end for */
done:
FUNC_LEAVE_NOAPI(ret_value)
-}
+} /* end H5V_hyper_disjointp() */
/*-------------------------------------------------------------------------
@@ -1225,7 +1226,7 @@ H5V_array_calc(hsize_t offset, unsigned n, const hsize_t *total_size, hsize_t *c
*-------------------------------------------------------------------------
*/
herr_t
-H5V_chunk_index(unsigned ndims, const hsize_t *coord, const size_t *chunk,
+H5V_chunk_index(unsigned ndims, const hsize_t *coord, const uint32_t *chunk,
const hsize_t *down_nchunks, hsize_t *chunk_idx)
{
hsize_t scaled_coord[H5V_HYPER_NDIMS]; /* Scaled, coordinates, in terms of chunks */
diff --git a/src/H5Vprivate.h b/src/H5Vprivate.h
index b92266c..e562880 100644
--- a/src/H5Vprivate.h
+++ b/src/H5Vprivate.h
@@ -53,9 +53,7 @@ H5_DLL hsize_t H5V_hyper_stride(unsigned n, const hsize_t *size,
const hsize_t *offset,
hsize_t *stride);
H5_DLL htri_t H5V_hyper_disjointp(unsigned n, const hsize_t *offset1,
- const size_t *size1,
- const hsize_t *offset2,
- const size_t *size2);
+ const uint32_t *size1, const hsize_t *offset2, const uint32_t *size2);
H5_DLL htri_t H5V_hyper_eq(unsigned n, const hsize_t *offset1,
const hsize_t *size1, const hsize_t *offset2,
const hsize_t *size2);
@@ -88,7 +86,7 @@ H5_DLL hsize_t H5V_array_offset(unsigned n, const hsize_t *total_size,
H5_DLL herr_t H5V_array_calc(hsize_t offset, unsigned n,
const hsize_t *total_size, hsize_t *coords);
H5_DLL herr_t H5V_chunk_index(unsigned ndims, const hsize_t *coord,
- const size_t *chunk, const hsize_t *down_nchunks, hsize_t *chunk_idx);
+ const uint32_t *chunk, const hsize_t *down_nchunks, hsize_t *chunk_idx);
H5_DLL ssize_t H5V_memcpyvv(void *_dst,
size_t dst_max_nseq, size_t *dst_curr_seq, size_t dst_len_arr[], hsize_t dst_off_arr[],
const void *_src,
diff --git a/test/Makefile.am b/test/Makefile.am
index 9dbb0cf..a3f8660 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -104,7 +104,7 @@ flush2.chkexe_: flush1.chkexe_
# the temporary file name in ways that the makefile is not aware of.
CHECK_CLEANFILES+=cmpd_dset.h5 compact_dataset.h5 dataset.h5 dset_offset.h5 \
max_compact_dataset.h5 simple.h5 set_local.h5 random_chunks.h5 \
- extend.h5 istore.h5 \
+ huge_chunks.h5 extend.h5 istore.h5 \
tfile[1-4].h5 th5s[1-3].h5 lheap.h5 fheap.h5 ohdr.h5 stab.h5 \
extern_[1-3].h5 extern_[1-4][ab].raw gheap[0-4].h5 dt_arith[1-2] \
links.h5 links[0-6]*.h5 extlinks[0-15].h5 tmp \
diff --git a/test/dsets.c b/test/dsets.c
index e5fa5b5..8edc2d2 100644
--- a/test/dsets.c
+++ b/test/dsets.c
@@ -42,6 +42,7 @@ const char *FILENAME[] = {
"simple",
"set_local",
"random_chunks",
+ "huge_chunks",
NULL
};
#define FILENAME_BUF_SIZE 1024
@@ -145,6 +146,22 @@ const char *FILENAME[] = {
/* Names for random chunks test */
#define NPOINTS 50
+/* Parameters for huge chunks test */
+#define HUGE_DATASET "Dataset"
+#define HUGE_DIM ((hsize_t)16 * 1024 * 1024 * 1024)
+#define HUGE_CHUNK_DIM ((hsize_t)2 * 1024 * 1024 * 1024)
+#define TOO_HUGE_CHUNK_DIM ((hsize_t)4 * 1024 * 1024 * 1024)
+#define HUGE_DATASET2 "Dataset2"
+#define HUGE_DIM2_0 ((hsize_t)16 * 1024)
+#define HUGE_DIM2_1 ((hsize_t)16 * 1024)
+#define HUGE_DIM2_2 ((hsize_t)16 * 1024)
+#define HUGE_CHUNK_DIM2_0 ((hsize_t)2 * 1024)
+#define HUGE_CHUNK_DIM2_1 ((hsize_t)1024)
+#define HUGE_CHUNK_DIM2_2 ((hsize_t)1024)
+#define TOO_HUGE_CHUNK_DIM2_0 ((hsize_t)4 * 1024)
+#define TOO_HUGE_CHUNK_DIM2_1 ((hsize_t)1024)
+#define TOO_HUGE_CHUNK_DIM2_2 ((hsize_t)1024)
+
/* Shared global arrays */
#define DSET_DIM1 100
#define DSET_DIM2 200
@@ -1941,8 +1958,6 @@ UNUSED
data_corrupt[1] = 33;
data_corrupt[2] = 27;
- /* Temporarily disable this test because the changes in chunk caching conflicts with
- * the way this test is conducted. -slu 2007/7/20 */
if(H5Zregister (H5Z_CORRUPT) < 0) goto error;
if(H5Pset_filter(dc, H5Z_FILTER_CORRUPT, 0, (size_t)3, data_corrupt) < 0) goto error;
if(test_filter_internal(file,DSET_FLETCHER32_NAME_3,dc,DISABLE_FLETCHER32,DATA_CORRUPTED,&fletcher32_size) < 0) goto error;
@@ -6359,6 +6374,118 @@ test_deprec(hid_t file)
/*-------------------------------------------------------------------------
+ * Function: test_huge_chunks
+ *
+ * Purpose: Tests that datasets with chunks >4GB can't be created.
+ *
+ * Return: Success: 0
+ * Failure: -1
+ *
+ * Programmer: Quincey Koziol
+ * Thursday, May 1, 2008
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+test_huge_chunks(hid_t fapl)
+{
+ char filename[FILENAME_BUF_SIZE];
+ hid_t fid = -1; /* File ID */
+ hid_t dcpl = -1; /* Dataset creation property list ID */
+ hid_t sid = -1; /* Dataspace ID */
+ hid_t dsid = -1; /* Dataset ID */
+ hsize_t dim, chunk_dim; /* Dataset and chunk dimensions */
+ hsize_t dim2[3], chunk_dim2[3]; /* Dataset and chunk dimensions */
+ herr_t ret; /* Generic return value */
+
+ TESTING("creating dataset with >4GB chunks");
+
+ h5_fixname(FILENAME[7], fapl, filename, sizeof filename);
+
+ /* Create file */
+ if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR
+
+ /* Create dataset creation property list */
+ if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) FAIL_STACK_ERROR
+
+ /* Try to set too large of a chunk for 1-D dataset (# of elements) */
+ chunk_dim = TOO_HUGE_CHUNK_DIM;
+ H5E_BEGIN_TRY {
+ ret = H5Pset_chunk(dcpl, 1, &chunk_dim);
+ } H5E_END_TRY;
+ if(ret >= 0)
+ FAIL_PUTS_ERROR(" Set chunk size with too large of chunk dimensions.")
+
+ /* Try to set too large of a chunk for n-D dataset (# of elements) */
+ chunk_dim2[0] = TOO_HUGE_CHUNK_DIM2_0;
+ chunk_dim2[1] = TOO_HUGE_CHUNK_DIM2_1;
+ chunk_dim2[2] = TOO_HUGE_CHUNK_DIM2_2;
+ H5E_BEGIN_TRY {
+ ret = H5Pset_chunk(dcpl, 3, chunk_dim2);
+ } H5E_END_TRY;
+ if(ret >= 0)
+ FAIL_PUTS_ERROR(" Set chunk size with too large of chunk dimensions.")
+
+ /* Set 1-D chunk size */
+ chunk_dim = HUGE_CHUNK_DIM;
+ if(H5Pset_chunk(dcpl, 1, &chunk_dim) < 0) FAIL_STACK_ERROR
+
+ /* Create 1-D dataspace */
+ dim = HUGE_DIM;
+ if((sid = H5Screate_simple(1, &dim, NULL)) < 0) FAIL_STACK_ERROR
+
+ /* Try to create dataset */
+ H5E_BEGIN_TRY {
+ dsid = H5Dcreate2(fid, HUGE_DATASET, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT);
+ } H5E_END_TRY;
+ if(dsid >= 0)
+ FAIL_PUTS_ERROR(" 1-D Dataset with too large of chunk dimensions created.")
+
+ /* Close 1-D dataspace */
+ if(H5Sclose(sid) < 0) FAIL_STACK_ERROR
+
+
+ /* Set n-D chunk size */
+ chunk_dim2[0] = HUGE_CHUNK_DIM2_0;
+ chunk_dim2[1] = HUGE_CHUNK_DIM2_1;
+ chunk_dim2[2] = HUGE_CHUNK_DIM2_2;
+ if(H5Pset_chunk(dcpl, 3, chunk_dim2) < 0) FAIL_STACK_ERROR
+
+ /* Create n-D dataspace */
+ dim2[0] = HUGE_DIM2_0;
+ dim2[1] = HUGE_DIM2_1;
+ dim2[2] = HUGE_DIM2_2;
+ if((sid = H5Screate_simple(3, dim2, NULL)) < 0) FAIL_STACK_ERROR
+
+ /* Try to create dataset */
+ H5E_BEGIN_TRY {
+ dsid = H5Dcreate2(fid, HUGE_DATASET2, H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT);
+ } H5E_END_TRY;
+ if(dsid >= 0)
+ FAIL_PUTS_ERROR(" n-D Dataset with too large of chunk dimensions created.")
+
+ /* Close n-D dataspace */
+ if(H5Sclose(sid) < 0) FAIL_STACK_ERROR
+
+ /* Close everything else */
+ if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR
+ if(H5Fclose(fid) < 0) FAIL_STACK_ERROR
+
+ PASSED();
+ return 0;
+
+error:
+ H5E_BEGIN_TRY {
+ H5Pclose(dcpl);
+ H5Dclose(dsid);
+ H5Sclose(sid);
+ H5Fclose(fid);
+ } H5E_END_TRY;
+ return -1;
+} /* end test_huge_chunks() */
+
+
+/*-------------------------------------------------------------------------
* Function: main
*
* Purpose: Tests the dataset interface (H5D)
@@ -6393,12 +6520,19 @@ main(void)
size_t rdcc_nbytes;
double rdcc_w0;
+ /* Set the random # seed */
+ HDsrandom((unsigned long)HDtime(NULL));
+
/* Testing setup */
h5_reset();
fapl = h5_fileaccess();
- /* Set the random # seed */
- HDsrandom((unsigned long)HDtime(NULL));
+ /* Turn off the chunk cache, so all the chunks are immediately written to disk */
+ if(H5Pget_cache(fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0)
+ goto error;
+ rdcc_nbytes = 0;
+ if(H5Pset_cache(fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0)
+ goto error;
/* Copy the file access property list */
if((fapl2 = H5Pcopy(fapl)) < 0) TEST_ERROR
@@ -6422,13 +6556,6 @@ main(void)
my_fapl = fapl;
} /* end else */
- /* Turn off the chunk cache, so all the chunks are immediately written to disk */
- if(H5Pget_cache(my_fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0)
- goto error;
- rdcc_nbytes = 0;
- if(H5Pset_cache(my_fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0)
- goto error;
-
/* Create the file for this test */
if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0)
goto error;
@@ -6478,6 +6605,7 @@ main(void)
#ifndef H5_NO_DEPRECATED_SYMBOLS
nerrors += (test_deprec(file) < 0 ? 1 : 0);
#endif /* H5_NO_DEPRECATED_SYMBOLS */
+ nerrors += (test_huge_chunks(my_fapl) < 0 ? 1 : 0);
if(H5Fclose(file) < 0)
goto error;
diff --git a/test/istore.c b/test/istore.c
index dfe5c41..e0d2e82 100644
--- a/test/istore.c
+++ b/test/istore.c
@@ -171,19 +171,20 @@ test_create(hid_t f, const char *prefix)
{
hid_t dataset; /* Dataset ID */
hsize_t dims[H5O_LAYOUT_NDIMS+1]; /* Dimensions of dataset */
+ hsize_t my_chunk_dims[H5O_LAYOUT_NDIMS+1]; /* Dimensions of chunks */
char name[256]; /* Dataset name */
unsigned u; /* Local index variable */
TESTING("istore create");
- dims[0]=TEST_CHUNK_SIZE;
+ dims[0] = my_chunk_dims[0] = 1;
for (u = 1; u <= H5S_MAX_RANK; u++) {
/* Initialize the dimension size in this new dimension */
- dims[u]=TEST_CHUNK_SIZE;
+ dims[u] = my_chunk_dims[u] = 2;
/* Create chunked dataset of this dimensionality */
HDsnprintf(name, sizeof name, "%s_%02u", prefix, u);
- if ((dataset=new_object(f, name, (int)u, dims, chunk_dims)) < 0)
+ if ((dataset=new_object(f, name, (int)u, dims, my_chunk_dims)) < 0)
return FAIL;
/* Close dataset created */
diff --git a/test/tsohm.c b/test/tsohm.c
index abdbe77..0fc94f2 100644
--- a/test/tsohm.c
+++ b/test/tsohm.c
@@ -121,9 +121,9 @@ const int ENUM_VAL[] = {
20480, 10,
-1001, -10
};
-#define SIZE2_RANK1 10
-#define SIZE2_RANK2 20
-#define SIZE2_DIMS {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}
+#define SIZE2_RANK1 6
+#define SIZE2_RANK2 10
+#define SIZE2_DIMS {1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
#define LONG_STRING "00 index. A long string used for testing. To create new strings, set the first two characters to be some ASCII number other than 00, such as 01."
@@ -1376,7 +1376,7 @@ size2_helper(hid_t fcpl_id, int test_file_closing, size2_helper_struct *ret_size
/* Constants used in this function */
const int rank1 = SIZE2_RANK1;
const int rank2 = SIZE2_RANK2;
- const hsize_t dims[20] = SIZE2_DIMS;
+ const hsize_t dims[SIZE2_RANK2] = SIZE2_DIMS;
dtype1_struct fill1;
char fill2[DTYPE2_SIZE];
@@ -1745,8 +1745,8 @@ static void size2_verify(void)
char attr_correct_string[NAME_BUF_SIZE];
char attr_name[NAME_BUF_SIZE];
int ndims;
- hsize_t dims[20];
- hsize_t correct_dims[20] = SIZE2_DIMS;
+ hsize_t dims[SIZE2_RANK2];
+ hsize_t correct_dims[SIZE2_RANK2] = SIZE2_DIMS;
file_id = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT);
CHECK_I(file_id, "H5Fopen");
generated by cgit v0.12 at 2024-11-06 19:12:49 (GMT)