summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorBinh-Minh Ribler <bmribler@hdfgroup.org>2018-10-29 14:52:50 (GMT)
committerBinh-Minh Ribler <bmribler@hdfgroup.org>2018-10-29 14:52:50 (GMT)
commit0b321904a3be4d3988f99fca158d7e576ddb6df2 (patch)
treec98a60e107e500fd36b9ac0d4b5fdec0f1dd69ed
parent202d7403282230e2071412237ac7ba86ccb7f3db (diff)
downloadhdf5-0b321904a3be4d3988f99fca158d7e576ddb6df2.zip
hdf5-0b321904a3be4d3988f99fca158d7e576ddb6df2.tar.gz
hdf5-0b321904a3be4d3988f99fca158d7e576ddb6df2.tar.bz2
New API functions
Description: Added functions to query chunk information: H5Dget_num_chunks(dset_id, fspace_id, *nchunks) Gets the number of written chunks that intersect with the given dataspace. However, in this version, the intersection is not yet completed. Thus, the number of all written chunks will be returned. H5Dget_chunk_info_by_coord(dset_id, *coord, *filter_mask, *addr, *size) Given a chunk's coordinates, returns the chunk's filter, address, and size. H5Dget_chunk_info(dset_id, fspace_id, index, *coord, *filter_mask, *addr, *size) Given a chunk's index, returns the chunk's coordinates, filter, address, and size. The chunk belongs to a set of chunks that have nonempty intersection with the specified dataspace. However, in this version, the intersection is not yet completed, and the index is of all the written chunks. Platforms tested: Linux/64 (jelly) Linux/64 (platypus) Darwin (osx1011test)
-rw-r--r--src/H5D.c153
-rw-r--r--src/H5Dchunk.c309
-rw-r--r--src/H5Dpkg.h3
-rw-r--r--src/H5Dpublic.h3
-rw-r--r--test/dsets.c281
5 files changed, 748 insertions, 1 deletions
diff --git a/src/H5D.c b/src/H5D.c
index 61ccb5a..286ab4e 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -1130,3 +1130,156 @@ done:
FUNC_LEAVE_API(ret_value);
} /* H5Dget_chunk_storage_size() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5Dget_num_chunks
+ *
+ * Purpose: Retrieves the number of chunks that have nonempty intersection
+ * with a specified selection.
+ *
+ * Note: Currently, this function only gets the number of all written
+ * chunks, regardless the dataspace.
+ *
+ * Parameters:
+ * hid_t dset_id; IN: Chunked dataset ID
+ * hid_t fspace_id; IN: File dataspace ID
+ * hsize_t *nchunks; OUT:: Number of non-empty chunks
+ *
+ * Return: Non-negative on success, negative on failure
+ *
+ * Programmer: Binh-Minh Ribler
+ * August 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks)
+{
+ H5D_t *dset = NULL;
+ const H5S_t *space; /* Dataspace for dataset */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE3("e", "ii*h", dset_id, fspace_id, nchunks);
+
+ /* Check arguments */
+ if(NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset")
+ if(NULL == (space = (const H5S_t *)H5I_object_verify(fspace_id, H5I_DATASPACE)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace ID")
+ if(NULL == nchunks)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)")
+
+ if(H5D_CHUNKED != dset->shared->layout.type)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset")
+
+ /* Get the number of written chunks */
+ if(H5D__get_num_chunks(dset, space, nchunks) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error getting number of chunks")
+
+done:
+ FUNC_LEAVE_API(ret_value);
+} /* H5Dget_num_chunks() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Dget_chunk_info
+ *
+ * Purpose: Retrieves information about a chunk specified by its index.
+ *
+ * Parameters:
+ * hid_t dset_id; IN: Chunked dataset ID
+ * hid_t fspace_id; IN: File dataspace ID
+ * hsize_t index; IN: Index of written chunk
+ * hsize_t *offset OUT: Offset coordinates of the chunk
+ * unsigned *filter_mask OUT: Filter mask
+ * haddr_t *addr OUT: Address of the chunk
+ * hsize_t *size OUT: Size of the chunk
+ *
+ * Return: Non-negative on success, negative on failure
+ *
+ * Programmer: Binh-Minh Ribler
+ * August 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Dget_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t index, hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size)
+{
+ H5D_t *dset = NULL;
+ const H5S_t *space; /* Dataspace for dataset */
+ hsize_t space_allocated = 0;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE7("e", "iih*h*Iu*a*h", dset_id, fspace_id, index, offset, filter_mask,
+ addr, size);
+
+ /* Check arguments */
+ if(NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset ID")
+ if(NULL == (space = (const H5S_t *)H5I_object_verify(fspace_id, H5I_DATASPACE)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace ID")
+ if(index < 0)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)")
+ if(NULL == offset && NULL == filter_mask && NULL == addr && NULL == size)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid arguments, must have at least one non-null output argument")
+
+ if(H5D_CHUNKED != dset->shared->layout.type)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset")
+
+ /* Call private function to get the chunk info given the chunk's index */
+ if(H5D__get_chunk_info(dset, space, index, offset, filter_mask, addr, size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't get chunk info")
+
+done:
+ FUNC_LEAVE_API(ret_value);
+} /* H5Dget_chunk_info() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Dget_chunk_info_by_coord
+ *
+ * Purpose: Retrieves information about a chunk specified by its offset
+ * coordinates.
+ *
+ * Parameters:
+ * hid_t dset_id IN: Chunked dataset ID
+ * hsize_t *offset IN: Coordinates of the chunk
+ * unsigned *filter_mask OUT: Filter mask
+ * haddr_t *addr OUT: Address of the chunk
+ * hsize_t *size OUT: Size of the chunk
+ *
+ * Return: Non-negative on success, negative on failure
+ *
+ * Programmer: Binh-Minh Ribler
+ * August 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size)
+{
+ H5D_t *dset = NULL;
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE5("e", "i*h*Iu*a*h", dset_id, offset, filter_mask, addr, size);
+
+ /* Check arguments */
+ if(NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset")
+ if(NULL == offset)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)")
+ if(NULL == filter_mask && NULL == addr && NULL == size)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid arguments, must have at least one non-null output argument")
+
+ if(H5D_CHUNKED != dset->shared->layout.type)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset")
+
+ /* Internal function to get the chunk info */
+ if (H5D__get_chunk_info_by_coord(dset, offset, filter_mask, addr, size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data")
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* end H5Dget_chunk_info_by_coord() */
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index c3f4a95..6bc89d1 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -216,6 +216,17 @@ typedef struct H5D_chunk_readvv_ud_t {
const H5D_t *dset; /* Dataset to operate on */
} H5D_chunk_readvv_ud_t;
+/* Typedef for chunk info iterator callback */
+typedef struct {
+ hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Logical offset of the chunk */
+ hsize_t ndims; /* Number of dimension in the dataset */
+ uint32_t nbytes; /* Size of stored data in the chunk */
+ unsigned filter_mask; /* Excluded filters */
+ haddr_t chunk_addr; /* Address of the chunk in file */
+ hsize_t chunk_idx; /* Chunk index, where the iteration needs to stop */
+ hsize_t curr_idx; /* Current index, where the iteration is */
+} H5D_chunk_info_iter_ud_t;
+
/* Callback info for file selection iteration */
typedef struct H5D_chunk_file_iter_ud_t {
H5D_chunk_map_t *fm; /* File->memory chunk mapping info */
@@ -6726,3 +6737,301 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5D__chunk_format_convert() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__get_num_chunks_cb
+ *
+ * Purpose: Callback function that increments the number of written
+ * chunks in the dataset.
+ *
+ * Return: Success: H5_ITER_CONT or H5_ITER_STOP
+ * Failure: Negative (H5_ITER_ERROR)
+ *
+ * Programmer: Binh-Minh Ribler
+ * September 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
+{
+ int ret_value = H5_ITER_CONT; /* Callback return value */
+
+ hsize_t *num_chunks = (hsize_t *)_udata;
+
+ FUNC_ENTER_STATIC_NOERR
+
+ (*num_chunks)++;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5D__get_num_chunks_cb() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__get_num_chunks
+ *
+ * Purpose: Gets the number of written chunks in a dataset.
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ * Programmer: Binh-Minh Ribler
+ * September 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5D__get_num_chunks(const H5D_t *dset, const H5S_t *space, hsize_t *nchunks)
+{
+ H5D_chk_idx_info_t idx_info; /* Chunked index info */
+ hsize_t num_chunks = 0; /* Number of written chunks */
+ H5D_rdcc_ent_t *ent; /* Cache entry */
+ const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk);/* Raw data chunk cache */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr)
+
+ HDassert(dset);
+ HDassert(dset->shared);
+
+ /* Search for cached chunks that haven't been written out */
+ for(ent = rdcc->head; ent; ent = ent->next)
+ /* Flush the chunk out to disk, to make certain the size is correct later */
+ if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer")
+
+ /* Compose chunked index info struct */
+ idx_info.f = dset->oloc.file;
+ idx_info.pline = &dset->shared->dcpl_cache.pline;
+ idx_info.layout = &dset->shared->layout.u.chunk;
+ idx_info.storage = &dset->shared->layout.storage.u.chunk;
+
+ /* Iterate over the allocated chunks */
+ if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_num_chunks_cb, &num_chunks) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index")
+ *nchunks = num_chunks;
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__get_num_chunks() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__get_chunk_info_cb
+ *
+ * Purpose: Get the chunk info of the desired chunk, given by its index.
+ *
+ * Return: Success: H5_ITER_CONT or H5_ITER_STOP
+ * Failure: Negative (H5_ITER_ERROR)
+ *
+ * Programmer: Binh-Minh Ribler
+ * September 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
+{
+ hsize_t ii;
+ int ret_value = H5_ITER_CONT; /* Callback return value */
+
+ H5D_chunk_info_iter_ud_t *chunk_info = (H5D_chunk_info_iter_ud_t *)_udata;
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /* If this is the desired chunk, retrieve its info and stop iterating */
+ if (chunk_info->curr_idx == chunk_info->chunk_idx)
+ {
+ chunk_info->filter_mask = chunk_rec->filter_mask;
+ chunk_info->chunk_addr = chunk_rec->chunk_addr;
+ chunk_info->nbytes = chunk_rec->nbytes;
+
+ for (ii = 0; ii < chunk_info->ndims; ii++)
+ chunk_info->scaled[ii] = chunk_rec->scaled[ii];
+
+ /* Stop iterating */
+ ret_value = H5_ITER_STOP;
+ }
+ /* Iterate the next chunk */
+ else
+ chunk_info->curr_idx++;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5D__get_chunk_info_cb() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__get_chunk_info
+ *
+ * Purpose: Iterate over the chunks in the dataset to get the info
+ * of the desired chunk.
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ * Programmer: Binh-Minh Ribler
+ * September 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_t index, hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size)
+{
+ H5D_chk_idx_info_t idx_info; /* Chunked index info */
+ H5D_chunk_info_iter_ud_t udata;
+ const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk);/* Raw data chunk cache */
+ H5D_rdcc_ent_t *ent; /* Cache entry */
+ hsize_t ii = 0; /* Dimension index */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr)
+
+ HDassert(dset);
+ HDassert(dset->shared);
+
+ /* Search for cached chunks that haven't been written out */
+ for(ent = rdcc->head; ent; ent = ent->next)
+ /* Flush the chunk out to disk, to make certain the size is correct later */
+ if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer")
+
+ /* Compose chunked index info struct */
+ idx_info.f = dset->oloc.file;
+ idx_info.pline = &dset->shared->dcpl_cache.pline;
+ idx_info.layout = &dset->shared->layout.u.chunk;
+ idx_info.storage = &dset->shared->layout.storage.u.chunk;
+
+ /* Initialize for iteration */
+ udata.chunk_idx = index;
+ udata.curr_idx = 0;
+ udata.ndims = dset->shared->ndims;
+
+ /* Iterate over the allocated chunks */
+ if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_cb, &udata) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index")
+
+ /* Obtain requested info */
+ if (filter_mask)
+ *filter_mask = udata.filter_mask;
+ if (addr)
+ *addr = udata.chunk_addr;
+ if (size)
+ *size = udata.nbytes;
+ if (offset)
+ for (ii = 0; ii < udata.ndims; ii++)
+ offset[ii] = udata.scaled[ii] * dset->shared->layout.u.chunk.dim[ii];
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__get_chunk_info() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__get_chunk_info_by_coord_cb
+ *
+ * Purpose: Get the chunk info of the desired chunk, given its offset
+ * coordinates.
+ *
+ * Return: Success: H5_ITER_CONT or H5_ITER_STOP
+ * Failure: Negative (H5_ITER_ERROR)
+ *
+ * Programmer: Binh-Minh Ribler
+ * September 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
+{
+ hbool_t different = FALSE;
+int ii;
+ int ret_value = H5_ITER_CONT; /* Callback return value */
+
+ H5D_chunk_info_iter_ud_t *chunk_info = (H5D_chunk_info_iter_ud_t *)_udata;
+
+ FUNC_ENTER_STATIC_NOERR
+
+ for (ii = 0; ii < chunk_info->ndims && !different; ii++)
+ {
+ if (chunk_info->scaled[ii] != chunk_rec->scaled[ii])
+ different = TRUE;
+ }
+ if (!different)
+ {
+ chunk_info->nbytes = chunk_rec->nbytes;
+ chunk_info->filter_mask = chunk_rec->filter_mask;
+ chunk_info->chunk_addr = chunk_rec->chunk_addr;
+
+ /* Stop iterating */
+ ret_value = H5_ITER_STOP;
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5D__get_chunk_info_by_coord_cb() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__get_chunk_info_by_coord
+ *
+ * Purpose: Iterate over the chunks in the dataset to get the info
+ * of the desired chunk, given by its offset coordinates.
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ * Programmer: Binh-Minh Ribler
+ * September 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned* filter_mask, haddr_t *addr, hsize_t *size)
+{
+ const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */
+ const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /* Raw data chunk cache */
+ H5D_rdcc_ent_t *ent; /* Cache entry */
+ H5D_chunk_info_iter_ud_t udata;
+ H5D_chk_idx_info_t idx_info; /* Chunked index info */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr)
+
+ /* Check args */
+ HDassert(dset && H5D_CHUNKED == layout->type);
+ HDassert(offset);
+ HDassert(filter_mask);
+ HDassert(addr); /* Question: should some OUT args be allowed to be NULL? */
+ HDassert(size);
+
+ /* Search for cached chunks that haven't been written out */
+ for(ent = rdcc->head; ent; ent = ent->next)
+ /* Flush the chunk out to disk, to make certain the size is correct later */
+ if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer")
+
+ /* Calculate the scaled of this chunk */
+ H5VM_chunk_scaled(dset->shared->ndims, offset, layout->u.chunk.dim, udata.scaled);
+ udata.scaled[dset->shared->ndims] = 0;
+
+ /* Get the number of dimensions for use in callback function */
+ udata.ndims = dset->shared->ndims;
+
+ /* Compose chunked index info struct */
+ idx_info.f = dset->oloc.file;
+ idx_info.pline = &dset->shared->dcpl_cache.pline;
+ idx_info.layout = &dset->shared->layout.u.chunk;
+ idx_info.storage = &dset->shared->layout.storage.u.chunk;
+
+ /* Iterate over the allocated chunks */
+ if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_by_coord_cb, &udata) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from scaled")
+
+ /* Return the filter mask and chunk address and size */
+ *filter_mask = udata.filter_mask;
+ *addr = udata.chunk_addr;
+ *size = udata.nbytes;
+
+done:
+ FUNC_LEAVE_NOAPI_TAG(ret_value)
+} /* end H5D__get_chunk_info_by_coord() */
+
diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h
index b887b87..7d550b5 100644
--- a/src/H5Dpkg.h
+++ b/src/H5Dpkg.h
@@ -566,6 +566,9 @@ H5_DLL herr_t H5D__alloc_storage(const H5D_io_info_t *io_info, H5D_time_alloc_t
H5_DLL herr_t H5D__get_storage_size(const H5D_t *dset, hsize_t *storage_size);
H5_DLL herr_t H5D__get_chunk_storage_size(H5D_t *dset, const hsize_t *offset,
hsize_t *storage_size);
+H5_DLL herr_t H5D__get_num_chunks(const H5D_t *dset, const H5S_t *space, hsize_t *nchunks);
+H5_DLL herr_t H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_t index, hsize_t *coord, unsigned *filter_mask, haddr_t *offset, hsize_t *size);
+H5_DLL herr_t H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *coord, unsigned* filter_mask, haddr_t *addr, hsize_t *size);
H5_DLL haddr_t H5D__get_offset(const H5D_t *dset);
H5_DLL void *H5D__vlen_get_buf_size_alloc(size_t size, void *info);
H5_DLL herr_t H5D__vlen_get_buf_size(void *elem, hid_t type_id, unsigned ndim,
diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h
index a1ccda0..fcc76ee 100644
--- a/src/H5Dpublic.h
+++ b/src/H5Dpublic.h
@@ -141,6 +141,9 @@ H5_DLL hid_t H5Dget_create_plist(hid_t dset_id);
H5_DLL hid_t H5Dget_access_plist(hid_t dset_id);
H5_DLL hsize_t H5Dget_storage_size(hid_t dset_id);
H5_DLL herr_t H5Dget_chunk_storage_size(hid_t dset_id, const hsize_t *offset, hsize_t *chunk_bytes);
+H5_DLL herr_t H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks);
+H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size);
+H5_DLL herr_t H5Dget_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t index, hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size);
H5_DLL haddr_t H5Dget_offset(hid_t dset_id);
H5_DLL herr_t H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id,
hid_t file_space_id, hid_t plist_id, void *buf/*out*/);
diff --git a/test/dsets.c b/test/dsets.c
index d23f438..e8d86d1 100644
--- a/test/dsets.c
+++ b/test/dsets.c
@@ -84,6 +84,7 @@ const char *FILENAME[] = {
"dls_01_strings", /* 23 */
"power2up", /* 24 */
"version_bounds", /* 25 */
+ "chunk_info", /* 26 */
NULL
};
@@ -268,6 +269,17 @@ const char *FILENAME[] = {
#define STORAGE_SIZE_CHUNK_DIM1 5
#define STORAGE_SIZE_CHUNK_DIM2 5
+/* Parameters for testing chunk querying */
+#define DSET_SIMPLE_CHUNKED "Chunked Dataset"
+#define RANK 2
+#define NX 16
+#define NY 16
+#define CHUNK_NX 4
+#define CHUNK_NY 4
+#define CHUNK_SIZE 64
+#define NUM_CHUNKS 16
+#define NUM_CHUNKS_WRITTEN 4
+
/* Shared global arrays */
#define DSET_DIM1 100
#define DSET_DIM2 200
@@ -13034,6 +13046,270 @@ test_versionbounds()
return -1;
} /* test_versionbounds() */
+/*-------------------------------------------------------------------------
+ * Function: read_each_chunk (helper function)
+ *
+ * Purpose: Reads the chunk specified by its offset and verifies that
+ * it contains the same data as what was written. This function
+ * is used in test_get_chunk_info.
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ * Date: September 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t read_each_chunk(hid_t dset_id, hsize_t offset1, hsize_t offset2, unsigned check_filter_mask, void *direct_buf)
+{
+ size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int);
+ int read_buf[CHUNK_NX][CHUNK_NY];
+ hsize_t offset[2] = {offset1, offset2};
+ unsigned read_filter_mask = 0;
+
+ /* Read the raw chunk back */
+ HDmemset(&read_buf, 0, sizeof(read_buf));
+
+ /* Read the chunk specified by its offset */
+ if (H5Dread_chunk(dset_id, H5P_DEFAULT, offset, &read_filter_mask, read_buf) < 0)
+ return(FAIL);
+
+ /* Verify that read chunk is the same as the corresponding written one */
+ if (HDmemcmp(direct_buf, read_buf, CHUNK_NX*CHUNK_NY) != 0)
+ {
+ fprintf(stderr, "Read chunk differs than written chunk at offset (%d,%d)\n", offset1, offset2);
+ return(FAIL);
+ }
+
+ return(SUCCEED);
+}
+
+/*-------------------------------------------------------------------------
+ * Function: reinit_vars (helper function)
+ *
+ * Purpose: Helper function to wipe out variables for the next use,
+ * used in test_get_chunk_info.
+ *
+ * Return: Won't fail
+ *
+ * Date: September 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+void reinit_vars(unsigned *read_filter_mask, hsize_t *addr, hsize_t *size)
+{
+ if (read_filter_mask) *read_filter_mask = 0;
+ if (addr) *addr = 0;
+ if (size) *size = 0;
+}
+
+/*-------------------------------------------------------------------------
+ * Function: test_get_chunk_info
+ *
+ * Purpose: Test getting various chunk information
+ *
+ * Return: Success: 0
+ * Failure: 1
+ *
+ * Date: September 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+test_get_chunk_info()
+{
+ hid_t chunkfile = -1, fapl = -1;
+ hid_t fspace = -1, dset = -1;
+ hid_t mspace = -1;
+ hid_t cparms = -1, dxpl = -1;
+ hsize_t dims[2] = {NX, NY};
+ hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
+ hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY};
+ int fillvalue = -1;
+ char filename[FILENAME_BUF_SIZE];
+ unsigned filter_mask = 0;
+ int direct_buf[16][CHUNK_NX][CHUNK_NY];
+ int out_buf[NX][NY];
+ size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int);
+ int aggression = 9; /* Compression aggression setting */
+ unsigned read_filter_mask = 0; /* filter mask after direct read */
+ H5F_libver_t low, high; /* File format bounds */
+ hsize_t start[2]; /* Start of hyperslab */
+ hsize_t stride[2]; /* Stride of hyperslab */
+ hsize_t count[2]; /* Block count */
+ hsize_t block[2]; /* Block sizes */
+ hsize_t offset[2];
+ hsize_t coord[H5S_MAX_RANK];
+ hsize_t out_offset[2] = {-1, -1};
+ hsize_t size = 0;
+ hsize_t nchunks = 0;
+ hsize_t addr = 0;
+ hsize_t index = 0;
+ hsize_t i, j, n;
+
+ TESTING("getting chunk information");
+
+ /* Create a copy of file access property list */
+ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR
+
+ /* Set high bound to V18 */
+ low = H5F_LIBVER_EARLIEST;
+ high = H5F_LIBVER_V18;
+ if (H5Pset_libver_bounds(fapl, low, high) < 0)
+ TEST_ERROR;
+
+ /* Create a file */
+ h5_fixname(FILENAME[26], fapl, filename, sizeof filename);
+ if((chunkfile = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ TEST_ERROR
+
+ /* Create the data space */
+ if((fspace = H5Screate_simple(RANK, dims, maxdims)) < 0)
+ TEST_ERROR
+
+ if((mspace = H5Screate_simple(RANK, chunk_dims, NULL)) < 0)
+ TEST_ERROR
+
+ /* Modify dataset creation properties, i.e. enable chunking and compression */
+ if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ TEST_ERROR
+
+ if(H5Pset_chunk(cparms, RANK, chunk_dims) < 0)
+ TEST_ERROR
+
+ if(H5Pset_deflate(cparms, (unsigned ) aggression) < 0)
+ TEST_ERROR
+
+ if (H5Pset_fill_value(cparms, H5T_NATIVE_INT, &fillvalue) < 0)
+ TEST_ERROR;
+
+ /* Create a new dataset using cparms creation properties */
+ if((dset = H5Dcreate2(chunkfile, DSET_SIMPLE_CHUNKED, H5T_NATIVE_INT, fspace,
+ H5P_DEFAULT, cparms, H5P_DEFAULT)) < 0) TEST_ERROR
+
+ if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0)
+ TEST_ERROR
+
+ /* Indicate skipping the compression filter. */
+ filter_mask = 0x00000001;
+
+ /* Initialize the array of chunk data, that is for all 16 chunks */
+ for(n = 0; n < 16; n++)
+ for(i = 0; i < CHUNK_NX; i++)
+ for(j = 0; j < CHUNK_NY; j++)
+ direct_buf[n][i][j] = n + 1;
+
+ /* Write NUM_CHUNKS_WRITTEN chunks: (0,2) (0,3) (1,2) (1,3) */
+ n = 0;
+ for (i = 0; i < 2; i++)
+ for (j = 2; j < 4; j++, n++)
+ {
+ offset[0] = i * CHUNK_NX;
+ offset[1] = j * CHUNK_NY;
+ if (H5Dwrite_chunk(dset, dxpl, filter_mask, offset, buf_size, (void*)direct_buf[n]) < 0)
+ TEST_ERROR
+ }
+
+ /* Close the dataset then... */
+ if(H5Dclose(dset) < 0) TEST_ERROR
+
+ /* ...open it again to test the chunk query functions */
+ if((dset = H5Dopen2(chunkfile, DSET_SIMPLE_CHUNKED, H5P_DEFAULT)) < 0) TEST_ERROR
+
+ /* Get and verify the number of chunks written */
+ if (H5Dget_num_chunks(dset, mspace, &nchunks) < 0) TEST_ERROR;
+ if (nchunks != NUM_CHUNKS_WRITTEN) TEST_ERROR;
+
+ /* Read the entire dataset back */
+ start[0] = 0; start[1] = 0;
+ stride[0] = 1; stride[1] = 1;
+ count[0] = 1; count[1] = 1;
+ block[0] = CHUNK_NX; block[1] = CHUNK_NY;
+ if(H5Dread(dset, H5T_NATIVE_INT, fspace, fspace, H5P_DEFAULT, out_buf) < 0)
+ TEST_ERROR
+
+ /* Get and verify info of the first chunk */
+ index = 0;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+ if (out_offset[0] != 0 || out_offset[1] != 8) TEST_ERROR;
+
+ /* Get and verify info of the second chunk */
+ index = 1;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+ if (out_offset[0] != 0 || out_offset[1] != 12) TEST_ERROR;
+
+ /* Get and verify info of the third chunk */
+ index = 2;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+ if (out_offset[0] != 4 || out_offset[1] != 8) TEST_ERROR;
+
+ /* Get and verify info of the last chunk */
+ index = 3;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+ if (out_offset[0] != 4 || out_offset[1] != 12) TEST_ERROR;
+
+ /* Get info of the chunk at logical coordinates (0,2) */
+ offset[0] = 0;
+ offset[1] = 2 * CHUNK_NY;
+ if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR;
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+
+ /* Get info of the chunk at logical coordinates (1,3) */
+ offset[0] = 1 * CHUNK_NX;
+ offset[1] = 3 * CHUNK_NY;
+ if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR;
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+
+ /* Read each chunk and print the values */
+ n = 0;
+ for (i = 0; i < 2; i++)
+ for (j = 2; j < 4; j++, n++)
+ if (read_each_chunk(dset, i*CHUNK_NX, j*CHUNK_NY, filter_mask, (void*)direct_buf[n]) < 0)
+ TEST_ERROR
+
+ /* Close/release resources. */
+ H5Dclose(dset);
+ H5Sclose(mspace);
+ H5Sclose(fspace);
+ H5Pclose(cparms);
+ H5Pclose(dxpl);
+ H5Fclose(chunkfile);
+
+ PASSED();
+ return 0;
+
+error:
+ H5E_BEGIN_TRY {
+ H5Dclose(dset);
+ H5Sclose(mspace);
+ H5Sclose(fspace);
+ H5Pclose(cparms);
+ H5Pclose(dxpl);
+ } H5E_END_TRY;
+
+ H5_FAILED();
+ return 1;
+} /* test_get_chunk_info() */
+
/*-------------------------------------------------------------------------
* Function: main
@@ -13232,7 +13508,10 @@ main(void)
nerrors += (test_gather_error() < 0 ? 1 : 0);
/* Tests version bounds using its own file */
- nerrors += (test_versionbounds() < 0 ? 1 : 0);
+ nerrors += (test_versionbounds() < 0 ? 1 : 0);
+
+ /* Tests getting chunk information */
+ nerrors += (test_get_chunk_info() < 0 ? 1 : 0);
/* Run misc tests */
nerrors += dls_01_main();