diff options
author | Quincey Koziol <koziol@hdfgroup.org> | 2008-05-16 03:27:13 (GMT) |
---|---|---|
committer | Quincey Koziol <koziol@hdfgroup.org> | 2008-05-16 03:27:13 (GMT) |
commit | c23c6b939229efde53e105008d29b8bf441f5d99 (patch) | |
tree | f38a5b13b938dcf88fa5d7c33b613adaa1f2396c /src | |
parent | 70cdaa12b2bceadd6a6f6d7b66eaad626e8523d6 (diff) | |
download | hdf5-c23c6b939229efde53e105008d29b8bf441f5d99.zip hdf5-c23c6b939229efde53e105008d29b8bf441f5d99.tar.gz hdf5-c23c6b939229efde53e105008d29b8bf441f5d99.tar.bz2 |
[svn-r15016] Description:
Port revision 15015 back to 1.8 branch:
> Detect chunks that are >4GB before dataset gets created and return error
> to application.
>
> Tweak lots of internal variables that hold the chunk size/dimensions to
> use an 'uint32_t', instead of a 'size_t', so that the integer size is constant.
>
> Correct a number of our tests which were creating datasets with chunks
> that were >4GB and add some specific tests for >4GB chunk size detection.
>
> Minor whitespace & other code cleanups.
Tested on:
Mac OS X/32 10.5.2 (amazon)
Forthcoming testing on other platforms...
Diffstat (limited to 'src')
-rw-r--r-- | src/H5Dchunk.c | 78 | ||||
-rw-r--r-- | src/H5Dint.c | 14 | ||||
-rw-r--r-- | src/H5Distore.c | 246 | ||||
-rw-r--r-- | src/H5Dmpio.c | 12 | ||||
-rw-r--r-- | src/H5Dpkg.h | 18 | ||||
-rw-r--r-- | src/H5Oprivate.h | 4 | ||||
-rw-r--r-- | src/H5Pdcpl.c | 54 | ||||
-rw-r--r-- | src/H5V.c | 31 | ||||
-rw-r--r-- | src/H5Vprivate.h | 6 |
9 files changed, 249 insertions, 214 deletions
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index baaa5f1..ba07725 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -189,16 +189,16 @@ H5D_chunk_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t *type_info HGOTO_ERROR(H5E_DATASET, H5E_BADSELECT, FAIL, "unable to normalize dataspace by offset") /* Decide the number of chunks in each dimension*/ - for(u=0; u<f_ndims; u++) { + for(u = 0; u < f_ndims; u++) { /* Keep the size of the chunk dimensions as hsize_t for various routines */ - fm->chunk_dim[u]=fm->layout->u.chunk.dim[u]; + fm->chunk_dim[u] = fm->layout->u.chunk.dim[u]; /* Round up to the next integer # of chunks, to accomodate partial chunks */ - fm->chunks[u] = ((fm->f_dims[u]+dataset->shared->layout.u.chunk.dim[u])-1) / dataset->shared->layout.u.chunk.dim[u]; + fm->chunks[u] = ((fm->f_dims[u] + dataset->shared->layout.u.chunk.dim[u]) - 1) / dataset->shared->layout.u.chunk.dim[u]; } /* end for */ /* Compute the "down" size of 'chunks' information */ - if(H5V_array_down(f_ndims,fm->chunks,fm->down_chunks) < 0) + if(H5V_array_down(f_ndims, fm->chunks, fm->down_chunks) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_BADVALUE, FAIL, "can't compute 'down' sizes") #ifdef H5_HAVE_PARALLEL @@ -591,16 +591,15 @@ H5D_create_chunk_file_map_hyper(H5D_chunk_map_t *fm, const H5D_io_info_t HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get file selection bound info") /* Set initial chunk location & hyperslab size */ - - for(u=0; u<fm->f_ndims; u++) { - start_coords[u]=(sel_start[u]/fm->layout->u.chunk.dim[u])*fm->layout->u.chunk.dim[u]; - coords[u]=start_coords[u]; - end[u]=(coords[u]+fm->chunk_dim[u])-1; + for(u = 0; u < fm->f_ndims; u++) { + start_coords[u] = (sel_start[u] / fm->layout->u.chunk.dim[u]) * fm->layout->u.chunk.dim[u]; + coords[u] = start_coords[u]; + end[u] = (coords[u] + fm->chunk_dim[u]) - 1; } /* end for */ /* Calculate the index of this chunk */ - if(H5V_chunk_index(fm->f_ndims,coords,fm->layout->u.chunk.dim,fm->down_chunks,&chunk_index) < 0) + if(H5V_chunk_index(fm->f_ndims, coords, fm->layout->u.chunk.dim, fm->down_chunks, &chunk_index) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index") /* Iterate through each chunk in the dataset */ @@ -679,17 +678,16 @@ H5D_create_chunk_file_map_hyper(H5D_chunk_map_t *fm, const H5D_io_info_t } /* end if */ /* Get number of elements selected in chunk */ - if((schunk_points=H5S_GET_SELECT_NPOINTS(tmp_fchunk)) < 0) + if((schunk_points = H5S_GET_SELECT_NPOINTS(tmp_fchunk)) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get file selection # of elements") - H5_ASSIGN_OVERFLOW(new_chunk_info->chunk_points,schunk_points,hssize_t,size_t); + H5_ASSIGN_OVERFLOW(new_chunk_info->chunk_points, schunk_points, hssize_t, uint32_t); /* Decrement # of points left in file selection */ - sel_points-=(hsize_t)schunk_points; + sel_points -= (hsize_t)schunk_points; /* Leave if we are done */ - if(sel_points==0) + if(sel_points == 0) HGOTO_DONE(SUCCEED) - assert(sel_points>0); } /* end if */ /* Increment chunk index */ @@ -719,7 +717,7 @@ H5D_create_chunk_file_map_hyper(H5D_chunk_map_t *fm, const H5D_io_info_t } while(coords[curr_dim]>sel_end[curr_dim]); /* Re-Calculate the index of this chunk */ - if(H5V_chunk_index(fm->f_ndims,coords,fm->layout->u.chunk.dim,fm->down_chunks,&chunk_index) < 0) + if(H5V_chunk_index(fm->f_ndims, coords, fm->layout->u.chunk.dim, fm->down_chunks, &chunk_index) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index") } /* end if */ } /* end while */ @@ -867,7 +865,7 @@ H5D_chunk_file_cb(void UNUSED *elem, hid_t UNUSED type_id, unsigned ndims, const FUNC_ENTER_NOAPI_NOINIT(H5D_chunk_file_cb) /* Calculate the index of this chunk */ - if(H5V_chunk_index(ndims,coords,fm->layout->u.chunk.dim,fm->down_chunks,&chunk_index) < 0) + if(H5V_chunk_index(ndims, coords, fm->layout->u.chunk.dim, fm->down_chunks, &chunk_index) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index") /* Find correct chunk in file & memory skip list */ @@ -912,18 +910,18 @@ H5D_chunk_file_cb(void UNUSED *elem, hid_t UNUSED type_id, unsigned ndims, const chunk_info->fspace_shared = FALSE; /* Set the memory chunk dataspace */ - chunk_info->mspace=NULL; + chunk_info->mspace = NULL; chunk_info->mspace_shared = FALSE; /* Set the number of selected elements in chunk to zero */ - chunk_info->chunk_points=0; + chunk_info->chunk_points = 0; /* Compute the chunk's coordinates */ - for(u=0; u<fm->f_ndims; u++) { - H5_CHECK_OVERFLOW(fm->layout->u.chunk.dim[u],hsize_t,hssize_t); - chunk_info->coords[u]=(coords[u]/(hssize_t)fm->layout->u.chunk.dim[u])*(hssize_t)fm->layout->u.chunk.dim[u]; + for(u = 0; u < fm->f_ndims; u++) { + H5_CHECK_OVERFLOW(fm->layout->u.chunk.dim[u], hsize_t, hssize_t); + chunk_info->coords[u] = (coords[u] / (hssize_t)fm->layout->u.chunk.dim[u]) * (hssize_t)fm->layout->u.chunk.dim[u]; } /* end for */ - chunk_info->coords[fm->f_ndims]=0; + chunk_info->coords[fm->f_ndims] = 0; /* Insert the new chunk into the skip list */ if(H5SL_insert(fm->sel_chunks,chunk_info,&chunk_info->index) < 0) { @@ -938,8 +936,8 @@ H5D_chunk_file_cb(void UNUSED *elem, hid_t UNUSED type_id, unsigned ndims, const } /* end else */ /* Get the coordinates of the element in the chunk */ - for(u=0; u<fm->f_ndims; u++) - coords_in_chunk[u]=coords[u]%fm->layout->u.chunk.dim[u]; + for(u = 0; u < fm->f_ndims; u++) + coords_in_chunk[u] = coords[u] % fm->layout->u.chunk.dim[u]; /* Add point to file selection for chunk */ if(H5S_select_elements(chunk_info->fspace, H5S_SELECT_APPEND, (size_t)1, coords_in_chunk) < 0) @@ -979,7 +977,7 @@ H5D_chunk_mem_cb(void UNUSED *elem, hid_t UNUSED type_id, unsigned ndims, const FUNC_ENTER_NOAPI_NOINIT(H5D_chunk_mem_cb) /* Calculate the index of this chunk */ - if(H5V_chunk_index(ndims,coords,fm->layout->u.chunk.dim,fm->down_chunks,&chunk_index) < 0) + if(H5V_chunk_index(ndims, coords, fm->layout->u.chunk.dim, fm->down_chunks, &chunk_index) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index") /* Find correct chunk in file & memory skip list */ @@ -1059,7 +1057,7 @@ H5D_chunk_cacheable(const H5D_io_info_t *io_info, haddr_t caddr) /* Must bring the whole chunk in if there are any filters */ if(dataset->shared->dcpl_cache.pline.nused > 0) ret_value = TRUE; - else + else { #ifdef H5_HAVE_PARALLEL /* If MPI based VFD is used and the file is opened for write access, must * bypass the chunk-cache scheme because other MPI processes could @@ -1068,17 +1066,22 @@ H5D_chunk_cacheable(const H5D_io_info_t *io_info, haddr_t caddr) */ if(io_info->using_mpi_vfd && (H5F_ACC_RDWR & H5F_INTENT(dataset->oloc.file))) ret_value = FALSE; - else + else { #endif /* H5_HAVE_PARALLEL */ /* If the chunk is too large to keep in the cache and if the address * for the chunk has been defined, then don't load the chunk into the * cache, just write the data to it directly. */ - if(dataset->shared->layout.u.chunk.size > dataset->shared->cache.chunk.nbytes + H5_CHECK_OVERFLOW(dataset->shared->layout.u.chunk.size, uint32_t, size_t); + if((size_t)dataset->shared->layout.u.chunk.size > dataset->shared->cache.chunk.nbytes && H5F_addr_defined(caddr)) ret_value = FALSE; else ret_value = TRUE; +#ifdef H5_HAVE_PARALLEL + } /* end else */ +#endif /* H5_HAVE_PARALLEL */ + } /* end else */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_chunk_cacheable() */ @@ -1152,7 +1155,7 @@ H5D_chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_io_info_t cpt_io_info; /* Compact I/O info object */ H5D_storage_t cpt_store; /* Chunk storage information as compact dataset */ hbool_t cpt_dirty; /* Temporary placeholder for compact storage "dirty" flag */ - size_t src_accessed_bytes = 0; /* Total accessed size in a chunk */ + uint32_t src_accessed_bytes = 0; /* Total accessed size in a chunk */ hbool_t skip_missing_chunks = FALSE; /* Whether to skip missing chunks */ unsigned idx_hint = 0; /* Cache index hint */ herr_t ret_value = SUCCEED; /*return value */ @@ -1175,7 +1178,7 @@ H5D_chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, ctg_io_info.layout_ops = *H5D_LOPS_CONTIG; /* Initialize temporary contiguous storage info */ - ctg_store.contig.dset_size = (hsize_t)io_info->dset->shared->layout.u.chunk.size; + H5_ASSIGN_OVERFLOW(ctg_store.contig.dset_size, io_info->dset->shared->layout.u.chunk.size, uint32_t, hsize_t); /* Set up compact I/O info object */ HDmemcpy(&cpt_io_info, io_info, sizeof(cpt_io_info)); @@ -1232,8 +1235,11 @@ H5D_chunk_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, else { /* Load the chunk into cache and lock it. */ if(H5D_chunk_cacheable(io_info, chunk_addr)) { + size_t tmp_src_accessed_bytes; /* Total accessed size in a chunk */ + /* Compute # of bytes accessed in chunk */ - src_accessed_bytes = chunk_info->chunk_points * type_info->src_type_size; + tmp_src_accessed_bytes = chunk_info->chunk_points * type_info->src_type_size; + H5_ASSIGN_OVERFLOW(src_accessed_bytes, tmp_src_accessed_bytes, size_t, uint32_t); /* Lock the chunk into the cache */ if(NULL == (chunk = H5D_istore_lock(io_info, &udata, FALSE, &idx_hint))) @@ -1301,7 +1307,7 @@ H5D_chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, H5D_io_info_t cpt_io_info; /* Compact I/O info object */ H5D_storage_t cpt_store; /* Chunk storage information as compact dataset */ hbool_t cpt_dirty; /* Temporary placeholder for compact storage "dirty" flag */ - size_t dst_accessed_bytes = 0; /* Total accessed size in a chunk */ + uint32_t dst_accessed_bytes = 0; /* Total accessed size in a chunk */ unsigned idx_hint = 0; /* Cache index hint */ herr_t ret_value = SUCCEED; /* Return value */ @@ -1319,7 +1325,7 @@ H5D_chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, ctg_io_info.layout_ops = *H5D_LOPS_CONTIG; /* Initialize temporary contiguous storage info */ - ctg_store.contig.dset_size = (hsize_t)io_info->dset->shared->layout.u.chunk.size; + H5_ASSIGN_OVERFLOW(ctg_store.contig.dset_size, io_info->dset->shared->layout.u.chunk.size, uint32_t, hsize_t); /* Set up compact I/O info object */ HDmemcpy(&cpt_io_info, io_info, sizeof(cpt_io_info)); @@ -1350,9 +1356,11 @@ H5D_chunk_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, chunk_addr = H5D_istore_get_addr(io_info, &udata); if(H5D_chunk_cacheable(io_info, chunk_addr)) { hbool_t entire_chunk = TRUE; /* Whether whole chunk is selected */ + size_t tmp_dst_accessed_bytes; /* Total accessed size in a chunk */ /* Compute # of bytes accessed in chunk */ - dst_accessed_bytes = chunk_info->chunk_points * type_info->dst_type_size; + tmp_dst_accessed_bytes = chunk_info->chunk_points * type_info->dst_type_size; + H5_ASSIGN_OVERFLOW(dst_accessed_bytes, tmp_dst_accessed_bytes, size_t, uint32_t); /* Determine if we will access all the data in the chunk */ if(dst_accessed_bytes != ctg_store.contig.dset_size || diff --git a/src/H5Dint.c b/src/H5Dint.c index acb95d2..f8577cc 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -1174,6 +1174,7 @@ H5D_create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, case H5D_CHUNKED: { hsize_t max_dim[H5O_LAYOUT_NDIMS]; /* Maximum size of data in elements */ + uint64_t chunk_size; /* Size of chunk in bytes */ /* Set up layout information */ if((ndims = H5S_GET_EXTENT_NDIMS(new_dset->shared->space)) < 0) @@ -1208,8 +1209,17 @@ H5D_create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "chunk size must be <= maximum dimension size for fixed-sized dimensions") /* Compute the total size of a chunk */ - for(u = 1, new_dset->shared->layout.u.chunk.size = new_dset->shared->layout.u.chunk.dim[0]; u < new_dset->shared->layout.u.chunk.ndims; u++) - new_dset->shared->layout.u.chunk.size *= new_dset->shared->layout.u.chunk.dim[u]; + /* (Use 64-bit value to ensure that we can detect >4GB chunks) */ + for(u = 1, chunk_size = (uint64_t)new_dset->shared->layout.u.chunk.dim[0]; u < new_dset->shared->layout.u.chunk.ndims; u++) + chunk_size *= (uint64_t)new_dset->shared->layout.u.chunk.dim[u]; + + /* Check for chunk larger than can be represented in 32-bits */ + /* (Chunk size is encoded in 32-bit value in v1 B-tree records) */ + if(chunk_size > (uint64_t)0xffffffff) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "chunk size must be < 4GB") + + /* Retain computed chunk size */ + H5_ASSIGN_OVERFLOW(new_dset->shared->layout.u.chunk.size, chunk_size, uint64_t, uint32_t); /* Initialize the chunk cache for the dataset */ if(H5D_istore_init(file, new_dset) < 0) diff --git a/src/H5Distore.c b/src/H5Distore.c index efab271..aed5add 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -160,7 +160,7 @@ typedef struct H5D_istore_it_ud4_t { H5T_path_t *tpath_mem_dst; /* Datatype conversion path from memory to dest. file */ void *reclaim_buf; /* Buffer for reclaiming data */ size_t reclaim_buf_size; /* Reclaim buffer size */ - size_t nelmts; /* Number of elements in buffer */ + uint32_t nelmts; /* Number of elements in buffer */ H5S_t *buf_space; /* Dataspace describing buffer */ /* needed for compressed variable-length data */ @@ -431,7 +431,7 @@ H5D_istore_debug_key(FILE *stream, H5F_t UNUSED *f, hid_t UNUSED dxpl_id, int in HDassert(key); - HDfprintf(stream, "%*s%-*s %Zd bytes\n", indent, "", fwidth, "Chunk size:", key->nbytes); + HDfprintf(stream, "%*s%-*s %u bytes\n", indent, "", fwidth, "Chunk size:", (unsigned)key->nbytes); HDfprintf(stream, "%*s%-*s 0x%08x\n", indent, "", fwidth, "Filter mask:", key->filter_mask); HDfprintf(stream, "%*s%-*s {", indent, "", fwidth, "Logical offset:"); for(u = 0; u < udata->mesg->u.chunk.ndims; u++) @@ -596,7 +596,7 @@ H5D_istore_new_node(H5F_t *f, hid_t dxpl_id, H5B_ins_t op, /* Allocate new storage */ HDassert(udata->nbytes > 0); - H5_CHECK_OVERFLOW(udata->nbytes, size_t, hsize_t); + H5_CHECK_OVERFLOW(udata->nbytes, uint32_t, hsize_t); if(HADDR_UNDEF == (*addr_p = H5MF_alloc(f, H5FD_MEM_DRAW, dxpl_id, (hsize_t)udata->nbytes))) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "couldn't allocate new file storage") udata->addr = *addr_p; @@ -772,10 +772,10 @@ H5D_istore_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key, (hsize_t)lt_key->nbytes, (hsize_t)udata->nbytes))) HGOTO_ERROR(H5E_STORAGE, H5E_NOSPACE, H5B_INS_ERROR, "unable to reallocate chunk storage") #else /* OLD_WAY */ - H5_CHECK_OVERFLOW( lt_key->nbytes ,size_t, hsize_t); - if(H5MF_xfree(f, H5FD_MEM_DRAW, dxpl_id, addr, (hsize_t)lt_key->nbytes)<0) + H5_CHECK_OVERFLOW(lt_key->nbytes, uint32_t, hsize_t); + if(H5MF_xfree(f, H5FD_MEM_DRAW, dxpl_id, addr, (hsize_t)lt_key->nbytes) < 0) HGOTO_ERROR(H5E_STORAGE, H5E_CANTFREE, H5B_INS_ERROR, "unable to free chunk") - H5_CHECK_OVERFLOW(udata->nbytes ,size_t, hsize_t); + H5_CHECK_OVERFLOW(udata->nbytes, uint32_t, hsize_t); if(HADDR_UNDEF == (*new_node_p = H5MF_alloc(f, H5FD_MEM_DRAW, dxpl_id, (hsize_t)udata->nbytes))) HGOTO_ERROR(H5E_STORAGE, H5E_NOSPACE, H5B_INS_ERROR, "unable to reallocate chunk") #endif /* OLD_WAY */ @@ -809,7 +809,7 @@ H5D_istore_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key, /* * Allocate storage for the new chunk */ - H5_CHECK_OVERFLOW(udata->nbytes, size_t, hsize_t); + H5_CHECK_OVERFLOW(udata->nbytes, uint32_t, hsize_t); if(HADDR_UNDEF == (*new_node_p = H5MF_alloc(f, H5FD_MEM_DRAW, dxpl_id, (hsize_t)udata->nbytes))) HGOTO_ERROR(H5E_STORAGE, H5E_NOSPACE, H5B_INS_ERROR, "file allocation failed") udata->addr = *new_node_p; @@ -874,14 +874,12 @@ H5D_istore_iter_chunkmap (H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const void *_lt { H5D_istore_it_ud5_t *udata = (H5D_istore_it_ud5_t *)_udata; const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; - unsigned rank; + unsigned rank = udata->common.mesg->u.chunk.ndims - 1; hsize_t chunk_index; int ret_value = H5_ITER_CONT; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_istore_iter_chunkmap) - rank = udata->common.mesg->u.chunk.ndims - 1; - if(H5V_chunk_index(rank, lt_key->offset, udata->common.mesg->u.chunk.dim, udata->down_chunks, &chunk_index) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index") @@ -968,13 +966,16 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, /* needed for commpressed variable length data */ hbool_t is_compressed = FALSE; H5Z_EDC_t edc_read = H5Z_NO_EDC; - size_t nbytes = lt_key->nbytes; + size_t nbytes; /* Size of chunk (in bytes) */ H5Z_cb_t cb_struct; int ret_value = H5_ITER_CONT; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_istore_iter_copy) + /* Get 'size_t' local value for number of bytes in chunk */ + H5_ASSIGN_OVERFLOW(nbytes, lt_key->nbytes, uint32_t, size_t); + /* Check parameter for type conversion */ if(udata->do_convert) { if(H5T_detect_class(udata->dt_src, H5T_VLEN) > 0) @@ -1033,12 +1034,12 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, hid_t tid_src = udata->tid_src; hid_t tid_dst = udata->tid_dst; hid_t tid_mem = udata->tid_mem; - size_t nelmts = udata->nelmts; void *reclaim_buf = udata->reclaim_buf; size_t reclaim_buf_size = udata->reclaim_buf_size; /* Convert from source file to memory */ - if(H5T_convert(tpath_src_mem, tid_src, tid_mem, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + H5_CHECK_OVERFLOW(udata->nelmts, uint32_t, size_t); + if(H5T_convert(tpath_src_mem, tid_src, tid_mem, (size_t)udata->nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, H5_ITER_ERROR, "datatype conversion failed") /* Copy into another buffer, to reclaim memory later */ @@ -1048,7 +1049,7 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, HDmemset(bkg, 0, buf_size); /* Convert from memory to destination file */ - if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, bkg, dxpl_id) < 0) + if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, udata->nelmts, (size_t)0, (size_t)0, buf, bkg, dxpl_id) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, H5_ITER_ERROR, "datatype conversion failed") /* Reclaim space from variable length data */ @@ -1085,7 +1086,7 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), edc_read, cb_struct, &nbytes, &buf_size, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed") - udata_dst.nbytes = nbytes; + H5_ASSIGN_OVERFLOW(udata_dst.nbytes, nbytes, size_t, uint32_t); udata->buf = buf; udata->buf_size = buf_size; } /* end if */ @@ -1277,7 +1278,6 @@ static herr_t H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_t reset) { void *buf = NULL; /*temporary buffer */ - size_t alloc; /*bytes allocated for BUF */ hbool_t point_of_no_return = FALSE; herr_t ret_value = SUCCEED; /*return value */ @@ -1292,9 +1292,6 @@ H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_ if(ent->dirty) { H5D_istore_ud1_t udata; /*pass through B-tree */ - /* Initial buffer size */ - alloc = ent->alloc_size; - /* Set up user data for B-tree callbacks */ udata.common.mesg = &io_info->dset->shared->layout; udata.common.offset = ent->offset; @@ -1304,16 +1301,19 @@ H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_ /* Should the chunk be filtered before writing it to disk? */ if(io_info->dset->shared->dcpl_cache.pline.nused) { + size_t alloc = ent->alloc_size; /* Bytes allocated for BUF */ + size_t nbytes; /* Chunk size (in bytes) */ + if(!reset) { /* * Copy the chunk to a new buffer before running it through * the pipeline because we'll want to save the original buffer * for later. */ - alloc = ent->chunk_size; + H5_ASSIGN_OVERFLOW(alloc, ent->chunk_size, uint32_t, size_t); if(NULL == (buf = H5MM_malloc(alloc))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline") - HDmemcpy(buf, ent->chunk, ent->chunk_size); + HDmemcpy(buf, ent->chunk, alloc); } /* end if */ else { /* @@ -1326,9 +1326,11 @@ H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_ point_of_no_return = TRUE; ent->chunk = NULL; } /* end else */ + H5_ASSIGN_OVERFLOW(nbytes, udata.nbytes, uint32_t, size_t); if(H5Z_pipeline(&(io_info->dset->shared->dcpl_cache.pline), 0, &(udata.filter_mask), io_info->dxpl_cache->err_detect, - io_info->dxpl_cache->filter_cb, &(udata.nbytes), &alloc, &buf) < 0) + io_info->dxpl_cache->filter_cb, &nbytes, &alloc, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "output pipeline failed") + H5_ASSIGN_OVERFLOW(udata.nbytes, nbytes, size_t, uint32_t); } /* end if */ /* @@ -1337,7 +1339,8 @@ H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_ */ if(H5B_insert(io_info->dset->oloc.file, io_info->dxpl_id, H5B_ISTORE, io_info->dset->shared->layout.u.chunk.addr, &udata)<0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to allocate chunk") - if(H5F_block_write(io_info->dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.nbytes, io_info->dxpl_id, buf) < 0) + H5_CHECK_OVERFLOW(udata.nbytes, uint32_t, size_t); + if(H5F_block_write(io_info->dset->oloc.file, H5FD_MEM_DRAW, udata.addr, (size_t)udata.nbytes, io_info->dxpl_id, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") /* Cache the chunk's info, in case it's accessed again shortly */ @@ -1688,20 +1691,20 @@ H5D_istore_prune (const H5D_io_info_t *io_info, size_t size) while ((p[0] || p[1]) && rdcc->nbytes+size>total) { /* Introduce new pointers */ - for (i=0; i<nmeth-1; i++) - if (0==w[i]) - p[i+1] = rdcc->head; + for(i = 0; i < (nmeth - 1); i++) + if(0 == w[i]) + p[i + 1] = rdcc->head; /* Compute next value for each pointer */ - for (i=0; i<nmeth; i++) + for(i = 0; i < nmeth; i++) n[i] = p[i] ? p[i]->next : NULL; /* Give each method a chance */ - for (i=0; i<nmeth && rdcc->nbytes+size>total; i++) { - if (0==i && p[0] && !p[0]->locked && - ((0==p[0]->rd_count && 0==p[0]->wr_count) || - (0==p[0]->rd_count && p[0]->chunk_size==p[0]->wr_count) || - (p[0]->chunk_size==p[0]->rd_count && 0==p[0]->wr_count))) { + for(i = 0; i < nmeth && (rdcc->nbytes + size) > total; i++) { + if(0 == i && p[0] && !p[0]->locked && + ((0 == p[0]->rd_count && 0 == p[0]->wr_count) || + (0 == p[0]->rd_count && p[0]->chunk_size == p[0]->wr_count) || + (p[0]->chunk_size == p[0]->rd_count && 0 == p[0]->wr_count))) { /* * Method 0: Preempt entries that have been completely written * and/or completely read but not entries that are partially @@ -1813,7 +1816,7 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, /* Get the chunk's size */ HDassert(layout->u.chunk.size > 0); - H5_ASSIGN_OVERFLOW(chunk_size, layout->u.chunk.size, hsize_t, size_t); + H5_ASSIGN_OVERFLOW(chunk_size, layout->u.chunk.size, uint32_t, size_t); /* Search for the chunk in the cache */ if(rdcc->nslots > 0) { @@ -1854,7 +1857,6 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, /* In the case that some dataset functions look through this data, * clear it to all 0s. */ HDmemset(chunk, 0, chunk_size); - } /* end if */ else { H5D_istore_ud1_t tmp_udata; /*B-tree pass-through */ @@ -1873,24 +1875,24 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, chunk_addr = H5D_istore_get_addr(io_info, udata); } /* end else */ - if (H5F_addr_defined(chunk_addr)) { - size_t chunk_alloc = 0; /*allocated chunk size */ + /* Check if the chunk exists on disk */ + if(H5F_addr_defined(chunk_addr)) { + size_t chunk_alloc; /* Allocated chunk size */ - /* - * The chunk exists on disk. - */ /* Chunk size on disk isn't [likely] the same size as the final chunk * size in memory, so allocate memory big enough. */ - chunk_alloc = udata->nbytes; - if(NULL == (chunk = H5D_istore_chunk_alloc (chunk_alloc, pline))) + H5_ASSIGN_OVERFLOW(chunk_alloc, udata->nbytes, uint32_t, size_t); + if(NULL == (chunk = H5D_istore_chunk_alloc(chunk_alloc, pline))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk") - if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, chunk_addr, udata->nbytes, io_info->dxpl_id, chunk) < 0) + if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, chunk_addr, chunk_alloc, io_info->dxpl_id, chunk) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, NULL, "unable to read raw data chunk") - if(pline->nused) + if(pline->nused) { if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &(udata->filter_mask), io_info->dxpl_cache->err_detect, - io_info->dxpl_cache->filter_cb, &(udata->nbytes), &chunk_alloc, &chunk) < 0) + io_info->dxpl_cache->filter_cb, &chunk_alloc, &chunk_alloc, &chunk) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, NULL, "data pipeline read failed") + H5_ASSIGN_OVERFLOW(udata->nbytes, chunk_alloc, size_t, uint32_t); + } /* end if */ #ifdef H5D_ISTORE_DEBUG rdcc->nmisses++; #endif /* H5D_ISTORE_DEBUG */ @@ -1904,7 +1906,7 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, /* Chunk size on disk isn't [likely] the same size as the final chunk * size in memory, so allocate memory big enough. */ - if(NULL == (chunk = H5D_istore_chunk_alloc (chunk_size, pline))) + if(NULL == (chunk = H5D_istore_chunk_alloc(chunk_size, pline))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for raw data chunk") if(H5P_is_fill_value_defined(fill, &fill_status) < 0) @@ -1947,31 +1949,31 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, * Add the chunk to the cache only if the slot is not already locked. * Preempt enough things from the cache to make room. */ - if (ent) { + if(ent) { #ifdef H5D_ISTORE_DEBUG HDputc('#', stderr); HDfflush(stderr); #endif - if (H5D_istore_preempt(io_info, ent, TRUE)<0) + if(H5D_istore_preempt(io_info, ent, TRUE) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk from cache") - } - if (H5D_istore_prune(io_info, chunk_size)<0) + } /* end if */ + if(H5D_istore_prune(io_info, chunk_size) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to preempt chunk(s) from cache") /* Create a new entry */ ent = H5FL_MALLOC(H5D_rdcc_ent_t); ent->locked = 0; ent->dirty = FALSE; - ent->chunk_size = chunk_size; + H5_ASSIGN_OVERFLOW(ent->chunk_size, chunk_size, size_t, uint32_t); ent->alloc_size = chunk_size; - for (u=0; u<layout->u.chunk.ndims; u++) + for(u = 0; u < layout->u.chunk.ndims; u++) ent->offset[u] = io_info->store->chunk.offset[u]; - ent->rd_count = chunk_size; - ent->wr_count = chunk_size; - ent->chunk = (uint8_t*)chunk; + H5_ASSIGN_OVERFLOW(ent->rd_count, chunk_size, size_t, uint32_t); + H5_ASSIGN_OVERFLOW(ent->wr_count, chunk_size, size_t, uint32_t); + ent->chunk = (uint8_t *)chunk; /* Add it to the cache */ - assert(NULL==rdcc->slot[idx]); + HDassert(NULL == rdcc->slot[idx]); rdcc->slot[idx] = ent; ent->idx = idx; rdcc->nbytes += chunk_size; @@ -1979,16 +1981,17 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, /* Add it to the linked list */ ent->next = NULL; - if (rdcc->tail) { + if(rdcc->tail) { rdcc->tail->next = ent; ent->prev = rdcc->tail; rdcc->tail = ent; - } else { + } /* end if */ + else { rdcc->head = rdcc->tail = ent; ent->prev = NULL; - } + } /* end else */ found = TRUE; - } else if (!found) { + } else if(!found) { /* * The chunk is larger than the entire cache so we don't cache it. * This is the reason all those arguments have to be repeated for the @@ -1996,38 +1999,37 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, */ ent = NULL; idx = UINT_MAX; - } else { /* * The chunk is not at the beginning of the cache; move it backward * by one slot. This is how we implement the LRU preemption * algorithm. */ - assert(ent); - if (ent->next) { - if (ent->next->next) + HDassert(ent); + if(ent->next) { + if(ent->next->next) ent->next->next->prev = ent; else rdcc->tail = ent; ent->next->prev = ent->prev; - if (ent->prev) + if(ent->prev) ent->prev->next = ent->next; else rdcc->head = ent->next; ent->prev = ent->next; ent->next = ent->next->next; ent->prev->next = ent; - } - } + } /* end if */ + } /* end else */ /* Lock the chunk into the cache */ - if (ent) { - assert (!ent->locked); + if(ent) { + HDassert(!ent->locked); ent->locked = TRUE; chunk = ent->chunk; - } + } /* end if */ - if (idx_hint) + if(idx_hint) *idx_hint = idx; /* Set return value */ @@ -2071,19 +2073,18 @@ done: */ herr_t H5D_istore_unlock(const H5D_io_info_t *io_info, - hbool_t dirty, unsigned idx_hint, void *chunk, size_t naccessed) + hbool_t dirty, unsigned idx_hint, void *chunk, uint32_t naccessed) { const H5O_layout_t *layout=&(io_info->dset->shared->layout); /* Dataset layout */ const H5D_rdcc_t *rdcc = &(io_info->dset->shared->cache.chunk); H5D_rdcc_ent_t *ent = NULL; - unsigned u; - herr_t ret_value=SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_istore_unlock) assert(io_info); - if (UINT_MAX==idx_hint) { + if(UINT_MAX == idx_hint) { /* * It's not in the cache, probably because it's too big. If it's * dirty then flush it to disk. In any case, free the chunk. @@ -2093,40 +2094,42 @@ H5D_istore_unlock(const H5D_io_info_t *io_info, if (dirty) { H5D_rdcc_ent_t x; - HDmemset (&x, 0, sizeof x); + HDmemset(&x, 0, sizeof(x)); x.dirty = TRUE; - for (u=0; u<layout->u.chunk.ndims; u++) - x.offset[u] = io_info->store->chunk.offset[u]; - assert(layout->u.chunk.size>0); - H5_ASSIGN_OVERFLOW(x.chunk_size,layout->u.chunk.size,hsize_t,size_t); - x.alloc_size = x.chunk_size; - x.chunk = (uint8_t*)chunk; - - if (H5D_istore_flush_entry(io_info, &x, TRUE)<0) + HDassert(sizeof(x.offset[0]) == sizeof(io_info->store->chunk.offset[0])); + HDmemcpy(x.offset, io_info->store->chunk.offset, layout->u.chunk.ndims * sizeof(x.offset[0])); + HDassert(layout->u.chunk.size > 0); + x.chunk_size = layout->u.chunk.size; + H5_ASSIGN_OVERFLOW(x.alloc_size, x.chunk_size, uint32_t, size_t); + x.chunk = (uint8_t *)chunk; + + if(H5D_istore_flush_entry(io_info, &x, TRUE) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") - } else { + } /* end if */ + else { if(chunk) - chunk=H5D_istore_chunk_xfree (chunk,&(io_info->dset->shared->dcpl_cache.pline)); - } - } else { + chunk = H5D_istore_chunk_xfree(chunk, &(io_info->dset->shared->dcpl_cache.pline)); + } /* end else */ + } /* end if */ + else { /* Sanity check */ - assert(idx_hint<rdcc->nslots); - assert(rdcc->slot[idx_hint]); - assert(rdcc->slot[idx_hint]->chunk==chunk); + HDassert(idx_hint < rdcc->nslots); + HDassert(rdcc->slot[idx_hint]); + HDassert(rdcc->slot[idx_hint]->chunk == chunk); /* * It's in the cache so unlock it. */ ent = rdcc->slot[idx_hint]; - assert (ent->locked); - if (dirty) { + HDassert(ent->locked); + if(dirty) { ent->dirty = TRUE; - ent->wr_count -= MIN (ent->wr_count, naccessed); - } else { - ent->rd_count -= MIN (ent->rd_count, naccessed); - } + ent->wr_count -= MIN(ent->wr_count, naccessed); + } /* end if */ + else + ent->rd_count -= MIN(ent->rd_count, naccessed); ent->locked = FALSE; - } + } /* end else */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -2510,8 +2513,7 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") /* Get original chunk size */ - H5_CHECK_OVERFLOW(layout->u.chunk.size, hsize_t, size_t); - orig_chunk_size = (size_t)layout->u.chunk.size; + H5_ASSIGN_OVERFLOW(orig_chunk_size, layout->u.chunk.size, uint32_t, size_t); /* Check the dataset's fill-value status */ if(H5P_is_fill_value_defined(fill, &fill_status) < 0) @@ -2613,7 +2615,7 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) chunk_size = nbytes; } /* end if */ else - chunk_size = (size_t)layout->u.chunk.size; + H5_ASSIGN_OVERFLOW(chunk_size, layout->u.chunk.size, uint32_t, size_t); } /* end if */ else chunk_size = orig_chunk_size; @@ -2621,7 +2623,7 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) /* Initialize the chunk information */ udata.common.mesg = layout; udata.common.offset = chunk_offset; - udata.nbytes = chunk_size; + H5_ASSIGN_OVERFLOW(udata.nbytes, chunk_size, size_t, uint32_t); udata.filter_mask = filter_mask; udata.addr = HADDR_UNDEF; @@ -2639,16 +2641,19 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) if(using_mpi) { /* Write the chunks out from only one process */ /* !! Use the internal "independent" DXPL!! -QAK */ - if(H5_PAR_META_WRITE == mpi_rank) - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.nbytes, data_dxpl_id, fb_info.fill_buf) < 0) + if(H5_PAR_META_WRITE == mpi_rank) { + HDassert(udata.nbytes == chunk_size); + if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, chunk_size, data_dxpl_id, fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") + } /* end if */ /* Indicate that blocks are being written */ blocks_written = TRUE; } /* end if */ else { #endif /* H5_HAVE_PARALLEL */ - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.nbytes, data_dxpl_id, fb_info.fill_buf) < 0) + HDassert(udata.nbytes == chunk_size); + if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, chunk_size, data_dxpl_id, fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") #ifdef H5_HAVE_PARALLEL } /* end else */ @@ -3029,12 +3034,13 @@ H5D_istore_remove(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key /*in,out hbool_t *rt_key_changed /*out */ ) { H5D_istore_key_t *lt_key = (H5D_istore_key_t *)_lt_key; - H5B_ins_t ret_value=H5B_INS_REMOVE; /* Return value */ + H5B_ins_t ret_value = H5B_INS_REMOVE; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_istore_remove) /* Remove raw data chunk from file */ - if(H5MF_xfree(f, H5FD_MEM_DRAW, dxpl_id, addr, (hsize_t)lt_key->nbytes)<0) + H5_CHECK_OVERFLOW(lt_key->nbytes, uint32_t, hsize_t); + if(H5MF_xfree(f, H5FD_MEM_DRAW, dxpl_id, addr, (hsize_t)lt_key->nbytes) < 0) HGOTO_ERROR(H5E_STORAGE, H5E_CANTFREE, H5B_INS_ERROR, "unable to free chunk") /* Mark keys as unchanged */ @@ -3082,7 +3088,7 @@ H5D_istore_initialize_by_extent(H5D_io_info_t *io_info) hsize_t hyper_start[H5O_LAYOUT_NDIMS]; /* Starting location of hyperslab */ hsize_t nchunks[H5O_LAYOUT_NDIMS]; /* Current number of chunks in each dimension */ hsize_t down_chunks[H5O_LAYOUT_NDIMS]; /* "down" size of number of elements in each dimension */ - hsize_t bytes_per_chunk; /* Bytes in chunk */ + uint32_t elmts_per_chunk; /* Elements in a chunk */ int srank; /* # of chunk dimensions (signed) */ unsigned rank; /* # of chunk dimensions */ hbool_t carry; /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */ @@ -3105,11 +3111,11 @@ H5D_istore_initialize_by_extent(H5D_io_info_t *io_info) /* Set size of lowest chunk dimension (the dataset element size) */ dset_dims[rank] = layout->u.chunk.dim[rank]; - /* Compute the number of chunks in dataset & the # of bytes in a chunk */ + /* Compute the number of chunks in dataset & the # of elements in a chunk */ /* (round up to the next integer # of chunks, to accomodate partial chunks) */ - for(u = 0, bytes_per_chunk = layout->u.chunk.dim[rank]; u < rank; u++) { + for(u = 0, elmts_per_chunk = 1; u < rank; u++) { nchunks[u] = ((dset_dims[u] - 1) / layout->u.chunk.dim[u]) + 1; - bytes_per_chunk *= layout->u.chunk.dim[u]; + elmts_per_chunk *= layout->u.chunk.dim[u]; } /* end for */ /* Get the "down" sizes for each dimension */ @@ -3159,9 +3165,10 @@ H5D_istore_initialize_by_extent(H5D_io_info_t *io_info) /* Initialize the fill value buffer, if necessary */ if(!fb_info_init) { + H5_CHECK_OVERFLOW(elmts_per_chunk, uint32_t, size_t); if(H5D_fill_init(&fb_info, NULL, FALSE, NULL, NULL, NULL, NULL, &dset->shared->dcpl_cache.fill, - dset->shared->type, dset->shared->type_id, (size_t)(bytes_per_chunk / layout->u.chunk.dim[rank]), + dset->shared->type, dset->shared->type_id, (size_t)elmts_per_chunk, io_info->dxpl_cache->max_temp_buf, io_info->dxpl_id) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill buffer info") fb_info_init = TRUE; @@ -3217,7 +3224,8 @@ H5D_istore_initialize_by_extent(H5D_io_info_t *io_info) HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't refill fill value buffer") /* Create a selection iterator for scattering the elements to memory buffer */ - if(H5S_select_iter_init(&chunk_iter, space_chunk, layout->u.chunk.dim[rank]) < 0) + H5_CHECK_OVERFLOW(layout->u.chunk.dim[rank], uint32_t, size_t); + if(H5S_select_iter_init(&chunk_iter, space_chunk, (size_t)layout->u.chunk.dim[rank]) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize chunk selection information") /* Scatter the data into memory */ @@ -3236,8 +3244,8 @@ H5D_istore_initialize_by_extent(H5D_io_info_t *io_info) bytes_accessed = nelmts * layout->u.chunk.dim[rank]; /* Release lock on chunk */ - H5_CHECK_OVERFLOW(bytes_accessed, hsize_t, size_t); - if(H5D_istore_unlock(io_info, TRUE, idx_hint, chunk, (size_t)bytes_accessed) < 0) + H5_CHECK_OVERFLOW(bytes_accessed, hsize_t, uint32_t); + if(H5D_istore_unlock(io_info, TRUE, idx_hint, chunk, (uint32_t)bytes_accessed) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to unlock raw data chunk") } /* end if */ @@ -3386,12 +3394,12 @@ H5D_istore_update_cache(H5D_t *dset, hid_t dxpl_id) next = ent->next; /* Calculate the index of this chunk */ - if(H5V_chunk_index(rank,ent->offset,dset->shared->layout.u.chunk.dim,down_chunks,&idx)<0) + if(H5V_chunk_index(rank, ent->offset, dset->shared->layout.u.chunk.dim, down_chunks, &idx) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_BADRANGE, FAIL, "can't get chunk index") /* Compute the index for the chunk entry */ - old_idx=ent->idx; /* Save for later */ - ent->idx=H5D_CHUNK_HASH(dset->shared, idx); + old_idx = ent->idx; /* Save for later */ + ent->idx = H5D_CHUNK_HASH(dset->shared, idx); if(old_idx != ent->idx) { /* Check if there is already a chunk at this chunk's new location */ @@ -3450,7 +3458,7 @@ H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, void *reclaim_buf = NULL; /* Buffer for reclaiming data */ H5S_t *buf_space = NULL; /* Dataspace describing buffer */ hid_t sid_buf = -1; /* ID for buffer dataspace */ - size_t nelmts = 0; /* Number of elements in buffer */ + uint32_t nelmts = 0; /* Number of elements in buffer */ hbool_t do_convert = FALSE; /* Indicate that type conversions should be performed */ herr_t ret_value = SUCCEED; /* Return value */ @@ -3553,7 +3561,7 @@ H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, do_convert = TRUE; } /* end if */ - buf_size = layout_src->u.chunk.size; + H5_ASSIGN_OVERFLOW(buf_size, layout_src->u.chunk.size, uint32_t, size_t); reclaim_buf_size = 0; } /* end else */ diff --git a/src/H5Dmpio.c b/src/H5Dmpio.c index 889fbce..dd0c602 100644 --- a/src/H5Dmpio.c +++ b/src/H5Dmpio.c @@ -1183,7 +1183,7 @@ if(H5DEBUG(D)) H5D_io_info_t *chk_io_info; /* Pointer to I/O info object for this chunk */ H5D_istore_ud1_t udata; /* B-tree pass-through */ void *chunk; /* Pointer to the data chunk in cache */ - size_t accessed_bytes; /* Total accessed size in a chunk */ + uint32_t accessed_bytes; /* Total accessed size in a chunk */ unsigned idx_hint = 0; /* Cache index hint */ haddr_t caddr; /* Address of the cached chunk */ @@ -1203,9 +1203,11 @@ if(H5DEBUG(D)) /* Load the chunk into cache and lock it. */ if(H5D_chunk_cacheable(io_info, caddr)) { hbool_t entire_chunk = TRUE; /* Whether whole chunk is selected */ + size_t tmp_accessed_bytes; /* Total accessed size in a chunk */ /* Compute # of bytes accessed in chunk */ - accessed_bytes = chunk_info->chunk_points * type_info->src_type_size; + tmp_accessed_bytes = chunk_info->chunk_points * type_info->src_type_size; + H5_ASSIGN_OVERFLOW(accessed_bytes, tmp_accessed_bytes, size_t, uint32_t); /* Determine if we will access all the data in the chunk */ if(((io_info->op_type == H5D_IO_OP_WRITE) && (accessed_bytes != ctg_store.contig.dset_size)) @@ -1419,7 +1421,7 @@ if(H5DEBUG(D)) { if(make_ind) { void *chunk; /* Pointer to the data chunk in cache */ H5D_io_info_t *chk_io_info; /* Pointer to I/O info object for this chunk */ - size_t accessed_bytes = 0; /* Total accessed size in a chunk */ + uint32_t accessed_bytes = 0; /* Total accessed size in a chunk */ unsigned idx_hint = 0; /* Cache index hint */ /* Switch to independent I/O */ @@ -1429,9 +1431,11 @@ if(H5DEBUG(D)) { /* Load the chunk into cache and lock it. */ if(H5D_chunk_cacheable(io_info, chunk_addr)) { hbool_t entire_chunk = TRUE; /* Whether whole chunk is selected */ + size_t tmp_accessed_bytes; /* Total accessed size in a chunk */ /* Compute # of bytes accessed in chunk */ - accessed_bytes = chunk_info->chunk_points * type_info->src_type_size; + tmp_accessed_bytes = chunk_info->chunk_points * type_info->src_type_size; + H5_ASSIGN_OVERFLOW(accessed_bytes, tmp_accessed_bytes, size_t, uint32_t); /* Determine if we will access all the data in the chunk */ if(((io_info->op_type == H5D_IO_OP_WRITE) && (accessed_bytes != ctg_store.contig.dset_size)) diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 97dcacf..1b20ad8 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -204,7 +204,7 @@ typedef struct H5D_io_info_t { /* Structure holding information about a chunk's selection for mapping */ typedef struct H5D_chunk_info_t { hsize_t index; /* "Index" of chunk in dataset */ - size_t chunk_points; /* Number of elements selected in chunk */ + uint32_t chunk_points; /* Number of elements selected in chunk */ hsize_t coords[H5O_LAYOUT_NDIMS]; /* Coordinates of chunk in file dataset's dataspace */ H5S_t *fspace; /* Dataspace describing chunk & selection in it */ unsigned fspace_shared; /* Indicate that the file space for a chunk is shared and shouldn't be freed */ @@ -213,10 +213,10 @@ typedef struct H5D_chunk_info_t { } H5D_chunk_info_t; /* Cached information about a particular chunk */ -typedef struct { +typedef struct H5D_chunk_cached_t{ hbool_t valid; /*whether cache info is valid*/ hsize_t offset[H5O_LAYOUT_NDIMS]; /*logical offset to start*/ - size_t nbytes; /*size of stored data */ + uint32_t nbytes; /*size of stored data */ unsigned filter_mask; /*excluded filters */ haddr_t addr; /*file address of chunk */ } H5D_chunk_cached_t; @@ -379,7 +379,7 @@ typedef struct { * The chunk's file address is part of the B-tree and not part of the key. */ typedef struct H5D_istore_key_t { - size_t nbytes; /*size of stored data */ + uint32_t nbytes; /*size of stored data */ hsize_t offset[H5O_LAYOUT_NDIMS]; /*logical offset to start*/ unsigned filter_mask; /*excluded filters */ } H5D_istore_key_t; @@ -401,7 +401,7 @@ typedef struct H5D_istore_ud1_t { H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ /* Upward */ - size_t nbytes; /*size of stored data */ + uint32_t nbytes; /*size of stored data */ unsigned filter_mask; /*excluded filters */ haddr_t addr; /*file address of chunk */ } H5D_istore_ud1_t; @@ -422,9 +422,9 @@ typedef struct H5D_rdcc_ent_t { hbool_t locked; /*entry is locked in cache */ hbool_t dirty; /*needs to be written to disk? */ hsize_t offset[H5O_LAYOUT_NDIMS]; /*chunk name */ - size_t rd_count; /*bytes remaining to be read */ - size_t wr_count; /*bytes remaining to be written */ - size_t chunk_size; /*size of a chunk */ + uint32_t rd_count; /*bytes remaining to be read */ + uint32_t wr_count; /*bytes remaining to be written */ + uint32_t chunk_size; /*size of a chunk */ size_t alloc_size; /*amount allocated for the chunk */ uint8_t *chunk; /*the unfiltered chunk data */ unsigned idx; /*index in hash table */ @@ -547,7 +547,7 @@ H5_DLL herr_t H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5_DLL void * H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, hbool_t relax, unsigned *idx_hint/*in,out*/); H5_DLL herr_t H5D_istore_unlock(const H5D_io_info_t *io_info, - hbool_t dirty, unsigned idx_hint, void *chunk, size_t naccessed); + hbool_t dirty, unsigned idx_hint, void *chunk, uint32_t naccessed); /* Functions that perform fill value operations on datasets */ H5_DLL herr_t H5D_fill(const void *fill, const H5T_t *fill_type, void *buf, diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 4a55ac2..3c5204f 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -334,8 +334,8 @@ typedef struct H5O_layout_contig_t { typedef struct H5O_layout_chunk_t { haddr_t addr; /* File address of B-tree */ unsigned ndims; /* Num dimensions in chunk */ - size_t dim[H5O_LAYOUT_NDIMS]; /* Size of chunk in elements */ - size_t size; /* Size of chunk in bytes */ + uint32_t dim[H5O_LAYOUT_NDIMS]; /* Size of chunk in elements */ + uint32_t size; /* Size of chunk in bytes */ H5RC_t *btree_shared; /* Ref-counted info for B-tree nodes */ } H5O_layout_chunk_t; diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 8d6b9ab..9501019 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -55,7 +55,7 @@ #define H5D_CRT_CHUNK_DIM_SIZE sizeof(unsigned) #define H5D_CRT_CHUNK_DIM_DEF 1 /* Definitions for chunk size */ -#define H5D_CRT_CHUNK_SIZE_SIZE sizeof(size_t[H5O_LAYOUT_NDIMS]) +#define H5D_CRT_CHUNK_SIZE_SIZE sizeof(uint32_t[H5O_LAYOUT_NDIMS]) #define H5D_CRT_CHUNK_SIZE_DEF {1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,\ 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1} /* Definitions for fill value. size=0 means fill value will be 0 as @@ -148,7 +148,7 @@ H5P_dcrt_reg_prop(H5P_genclass_t *pclass) { H5D_layout_t layout = H5D_CRT_LAYOUT_DEF; /* Default storage layout */ unsigned chunk_ndims = H5D_CRT_CHUNK_DIM_DEF; /* Default rank for chunks */ - size_t chunk_size[H5O_LAYOUT_NDIMS] = H5D_CRT_CHUNK_SIZE_DEF; /* Default chunk size */ + uint32_t chunk_size[H5O_LAYOUT_NDIMS] = H5D_CRT_CHUNK_SIZE_DEF; /* Default chunk size */ H5O_fill_t fill = H5D_CRT_FILL_VALUE_DEF; /* Default fill value */ unsigned alloc_time_state = H5D_CRT_ALLOC_TIME_STATE_DEF; /* Default allocation time state */ H5O_efl_t efl = H5D_CRT_EXT_FILE_LIST_DEF; /* Default external file list */ @@ -737,37 +737,43 @@ done: herr_t H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/]) { - int i; - size_t real_dims[H5O_LAYOUT_NDIMS]; /* Full-sized array to hold chunk dims */ H5P_genplist_t *plist; /* Property list pointer */ - herr_t ret_value=SUCCEED; /* return value */ + uint32_t real_dims[H5O_LAYOUT_NDIMS]; /* Full-sized array to hold chunk dims */ + uint64_t chunk_nelmts; /* Number of elements in chunk */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(H5Pset_chunk, FAIL) H5TRACE3("e", "iIs*[a1]h", plist_id, ndims, dim); /* Check arguments */ - if (ndims <= 0) + if(ndims <= 0) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "chunk dimensionality must be positive") - if (ndims > H5S_MAX_RANK) + if(ndims > H5S_MAX_RANK) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "chunk dimensionality is too large") - if (!dim) + if(!dim) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "no chunk dimensions specified") - /* Get the plist structure */ - if(NULL == (plist = H5P_object_verify(plist_id,H5P_DATASET_CREATE))) - HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") - - /* Initialize chunk dims to 0s */ - HDmemset(real_dims,0,sizeof(real_dims)); - for (i=0; i<ndims; i++) { - if (dim[i] == 0) + /* Verify & initialize internal chunk dims */ + HDmemset(real_dims, 0, sizeof(real_dims)); + chunk_nelmts = 1; + for(u = 0; u < (unsigned)ndims; u++) { + if(dim[u] == 0) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "all chunk dimensions must be positive") - if (dim[i] != (dim[i]&0xffffffff)) + if(dim[u] != (dim[u] & 0xffffffff)) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "all chunk dimensions must be less than 2^32") - real_dims[i]=(size_t)dim[i]; /* Store user's chunk dimensions */ + chunk_nelmts *= dim[u]; + if(chunk_nelmts > (uint64_t)0xffffffff) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "number of elements in chunk must be < 4GB") + real_dims[u] = (uint32_t)dim[u]; /* Store user's chunk dimensions */ } /* end for */ - if(H5P_set_layout (plist, H5D_CHUNKED) < 0) + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set chunk information in property list */ + if(H5P_set_layout(plist, H5D_CHUNKED) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set layout") if(H5P_set(plist, H5D_CRT_CHUNK_DIM_NAME, &ndims) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set chunk dimensionanlity") @@ -776,7 +782,7 @@ H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/]) done: FUNC_LEAVE_API(ret_value) -} +} /* end H5Pset_chunk() */ /*------------------------------------------------------------------------- @@ -828,22 +834,22 @@ H5Pget_chunk(hid_t plist_id, int max_ndims, hsize_t dim[]/*out*/) if(dim) { int i; - size_t chunk_size[H5O_LAYOUT_NDIMS]; + uint32_t chunk_size[H5O_LAYOUT_NDIMS]; if(H5P_get(plist, H5D_CRT_CHUNK_SIZE_NAME, chunk_size) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get chunk size") /* Get the dimension sizes */ - for (i=0; i<ndims && i<max_ndims; i++) + for(i = 0; i < ndims && i < max_ndims; i++) dim[i] = chunk_size[i]; } /* end if */ /* Set the return value */ - ret_value=ndims; + ret_value = ndims; done: FUNC_LEAVE_API(ret_value) -} +} /* end H5Pget_chunk() */ /*------------------------------------------------------------------------- @@ -399,33 +399,34 @@ done: */ htri_t H5V_hyper_disjointp(unsigned n, - const hsize_t *offset1, const size_t *size1, - const hsize_t *offset2, const size_t *size2) + const hsize_t *offset1, const uint32_t *size1, + const hsize_t *offset2, const uint32_t *size2) { unsigned u; - htri_t ret_value=FALSE; /* Return value */ + htri_t ret_value = FALSE; /* Return value */ /* Use FUNC_ENTER_NOAPI_NOINIT_NOFUNC here to avoid performance issues */ FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5V_hyper_disjointp) - if (!n || !size1 || !size2) HGOTO_DONE(TRUE) + if(!n || !size1 || !size2) + HGOTO_DONE(TRUE) - for (u=0; u<n; u++) { - assert (size1[u]<HSIZET_MAX); - assert (size2[u]<HSIZET_MAX); + for(u = 0; u < n; u++) { + HDassert(size1[u] < HSIZET_MAX); + HDassert(size2[u] < HSIZET_MAX); - if (0==size1[u] || 0==size2[u]) + if(0 == size1[u] || 0 == size2[u]) HGOTO_DONE(TRUE) - if (((offset1?offset1[u]:0) < (offset2?offset2[u]:0) && - ((offset1?offset1[u]:0) + size1[u] <= (offset2?offset2[u]:0))) || - ((offset2?offset2[u]:0) < (offset1?offset1[u]:0) && - ((offset2?offset2[u]:0) + size2[u] <= (offset1?offset1[u]:0)))) + if(((offset1 ? offset1[u] : 0) < (offset2 ? offset2[u] : 0) && + ((offset1 ? offset1[u] : 0) + size1[u] <= (offset2 ? offset2[u] : 0))) || + ((offset2 ? offset2[u] : 0) < (offset1 ? offset1[u] : 0) && + ((offset2 ? offset2[u] : 0) + size2[u] <= (offset1 ? offset1[u] : 0)))) HGOTO_DONE(TRUE) - } + } /* end for */ done: FUNC_LEAVE_NOAPI(ret_value) -} +} /* end H5V_hyper_disjointp() */ /*------------------------------------------------------------------------- @@ -1225,7 +1226,7 @@ H5V_array_calc(hsize_t offset, unsigned n, const hsize_t *total_size, hsize_t *c *------------------------------------------------------------------------- */ herr_t -H5V_chunk_index(unsigned ndims, const hsize_t *coord, const size_t *chunk, +H5V_chunk_index(unsigned ndims, const hsize_t *coord, const uint32_t *chunk, const hsize_t *down_nchunks, hsize_t *chunk_idx) { hsize_t scaled_coord[H5V_HYPER_NDIMS]; /* Scaled, coordinates, in terms of chunks */ diff --git a/src/H5Vprivate.h b/src/H5Vprivate.h index b92266c..e562880 100644 --- a/src/H5Vprivate.h +++ b/src/H5Vprivate.h @@ -53,9 +53,7 @@ H5_DLL hsize_t H5V_hyper_stride(unsigned n, const hsize_t *size, const hsize_t *offset, hsize_t *stride); H5_DLL htri_t H5V_hyper_disjointp(unsigned n, const hsize_t *offset1, - const size_t *size1, - const hsize_t *offset2, - const size_t *size2); + const uint32_t *size1, const hsize_t *offset2, const uint32_t *size2); H5_DLL htri_t H5V_hyper_eq(unsigned n, const hsize_t *offset1, const hsize_t *size1, const hsize_t *offset2, const hsize_t *size2); @@ -88,7 +86,7 @@ H5_DLL hsize_t H5V_array_offset(unsigned n, const hsize_t *total_size, H5_DLL herr_t H5V_array_calc(hsize_t offset, unsigned n, const hsize_t *total_size, hsize_t *coords); H5_DLL herr_t H5V_chunk_index(unsigned ndims, const hsize_t *coord, - const size_t *chunk, const hsize_t *down_nchunks, hsize_t *chunk_idx); + const uint32_t *chunk, const hsize_t *down_nchunks, hsize_t *chunk_idx); H5_DLL ssize_t H5V_memcpyvv(void *_dst, size_t dst_max_nseq, size_t *dst_curr_seq, size_t dst_len_arr[], hsize_t dst_off_arr[], const void *_src, |