summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAllen Byrne <byrn@hdfgroup.org>2018-11-07 15:04:02 (GMT)
committerAllen Byrne <byrn@hdfgroup.org>2018-11-07 15:04:02 (GMT)
commit25e3b32d125a58785bd2f8026c45f44a2c861134 (patch)
treebcc06dd059990f31cdb56e4bde513a23ecac56ef
parent3044a61ac5cf6c806a3ab576a1437077f0f8b824 (diff)
parent91771bdf4d6c2ddcdbc127ee2f9d29525d490757 (diff)
downloadhdf5-25e3b32d125a58785bd2f8026c45f44a2c861134.zip
hdf5-25e3b32d125a58785bd2f8026c45f44a2c861134.tar.gz
hdf5-25e3b32d125a58785bd2f8026c45f44a2c861134.tar.bz2
Merging in latest from upstream (HDFFV/hdf5:refs/heads/hdf5_1_10)
* commit '91771bdf4d6c2ddcdbc127ee2f9d29525d490757': Updated new API functions Description: - Addressed additional review comments - Moved the test function test_get_chunk_info from dsets.c to chunk_info.c because chunk_info.c was created for the purpose of testing chunk information. Platforms tested: Linux/64 (jelly) Linux/64 (platypus) Darwin (osx1011test) Updated new API functions Description: - Per Vailin's review, revised H5Dget_chunk_info_by_coord to handle non-existing chunk and H5Dget_num_chunks and H5Dget_chunk_info to handle dataset with no data. - Addressed other review comments - Note that additional tests will be added as we need to send users these functions asap for feedback. Platforms tested: Linux/64 (jelly) Linux/64 (platypus) Darwin (osx1011test) New API functions Description: Added functions to query chunk information:
-rw-r--r--src/H5D.c153
-rw-r--r--src/H5Dchunk.c390
-rw-r--r--src/H5Dpkg.h3
-rw-r--r--src/H5Dpublic.h3
-rw-r--r--test/CMakeLists.txt1
-rw-r--r--test/Makefile.am4
-rw-r--r--test/chunk_info.c444
-rw-r--r--test/dsets.c349
8 files changed, 1307 insertions, 40 deletions
diff --git a/src/H5D.c b/src/H5D.c
index 61ccb5a..32b2453 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -1130,3 +1130,156 @@ done:
FUNC_LEAVE_API(ret_value);
} /* H5Dget_chunk_storage_size() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5Dget_num_chunks
+ *
+ * Purpose: Retrieves the number of chunks that have nonempty intersection
+ * with a specified selection.
+ *
+ * Note: Currently, this function only gets the number of all written
+ * chunks, regardless the dataspace.
+ *
+ * Parameters:
+ * hid_t dset_id; IN: Chunked dataset ID
+ * hid_t fspace_id; IN: File dataspace ID
+ * hsize_t *nchunks; OUT:: Number of non-empty chunks
+ *
+ * Return: Non-negative on success, negative on failure
+ *
+ * Programmer: Binh-Minh Ribler
+ * August 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks)
+{
+ H5D_t *dset = NULL;
+ const H5S_t *space; /* Dataspace for dataset */
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE3("e", "ii*h", dset_id, fspace_id, nchunks);
+
+ /* Check arguments */
+ if(NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset")
+ if(NULL == (space = (const H5S_t *)H5I_object_verify(fspace_id, H5I_DATASPACE)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace ID")
+ if(NULL == nchunks)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)")
+
+ if(H5D_CHUNKED != dset->shared->layout.type)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset")
+
+ /* Get the number of written chunks */
+ if(H5D__get_num_chunks(dset, space, nchunks) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error getting number of chunks")
+
+done:
+ FUNC_LEAVE_API(ret_value);
+} /* H5Dget_num_chunks() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Dget_chunk_info
+ *
+ * Purpose: Retrieves information about a chunk specified by its index.
+ *
+ * Parameters:
+ * hid_t dset_id; IN: Chunked dataset ID
+ * hid_t fspace_id; IN: File dataspace ID
+ * hsize_t index; IN: Index of written chunk
+ * hsize_t *offset OUT: Offset coordinates of the chunk
+ * unsigned *filter_mask OUT: Filter mask
+ * haddr_t *addr OUT: Address of the chunk
+ * hsize_t *size OUT: Size of the chunk
+ *
+ * Return: Non-negative on success, negative on failure
+ *
+ * Programmer: Binh-Minh Ribler
+ * August 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Dget_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t index, hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size)
+{
+ H5D_t *dset = NULL;
+ const H5S_t *space; /* Dataspace for dataset */
+ hsize_t space_allocated = 0;
+ herr_t ret_value = SUCCEED;
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE7("e", "iih*h*Iu*a*h", dset_id, fspace_id, index, offset, filter_mask,
+ addr, size);
+
+ /* Check arguments */
+ if(NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset ID")
+ if(NULL == (space = (const H5S_t *)H5I_object_verify(fspace_id, H5I_DATASPACE)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace ID")
+ if(index < 0)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)")
+ if(NULL == offset && NULL == filter_mask && NULL == addr && NULL == size)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid arguments, must have at least one non-null output argument")
+
+ if(H5D_CHUNKED != dset->shared->layout.type)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset")
+
+ /* Call private function to get the chunk info given the chunk's index */
+ if(H5D__get_chunk_info(dset, space, index, offset, filter_mask, addr, size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info")
+
+done:
+ FUNC_LEAVE_API(ret_value);
+} /* H5Dget_chunk_info() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Dget_chunk_info_by_coord
+ *
+ * Purpose: Retrieves information about a chunk specified by its offset
+ * coordinates.
+ *
+ * Parameters:
+ * hid_t dset_id IN: Chunked dataset ID
+ * hsize_t *offset IN: Coordinates of the chunk
+ * unsigned *filter_mask OUT: Filter mask
+ * haddr_t *addr OUT: Address of the chunk
+ * hsize_t *size OUT: Size of the chunk
+ *
+ * Return: Non-negative on success, negative on failure
+ *
+ * Programmer: Binh-Minh Ribler
+ * August 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size)
+{
+ H5D_t *dset = NULL;
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_API(FAIL)
+ H5TRACE5("e", "i*h*Iu*a*h", dset_id, offset, filter_mask, addr, size);
+
+ /* Check arguments */
+ if(NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET)))
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset")
+ if(NULL == offset)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)")
+ if(NULL == filter_mask && NULL == addr && NULL == size)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid arguments, must have at least one non-null output argument")
+
+ if(H5D_CHUNKED != dset->shared->layout.type)
+ HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset")
+
+ /* Internal function to get the chunk info */
+ if (H5D__get_chunk_info_by_coord(dset, offset, filter_mask, addr, size) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't get chunk info")
+done:
+ FUNC_LEAVE_API(ret_value)
+} /* end H5Dget_chunk_info_by_coord() */
diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c
index c3f4a95..7aaea74 100644
--- a/src/H5Dchunk.c
+++ b/src/H5Dchunk.c
@@ -216,6 +216,18 @@ typedef struct H5D_chunk_readvv_ud_t {
const H5D_t *dset; /* Dataset to operate on */
} H5D_chunk_readvv_ud_t;
+/* Typedef for chunk info iterator callback */
+typedef struct H5D_chunk_info_iter_ud_t {
+ hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Logical offset of the chunk */
+ hsize_t ndims; /* Number of dimensions in the dataset */
+ uint32_t nbytes; /* Size of stored data in the chunk */
+ unsigned filter_mask; /* Excluded filters */
+ haddr_t chunk_addr; /* Address of the chunk in file */
+ hsize_t chunk_idx; /* Chunk index, where the iteration needs to stop */
+ hsize_t curr_idx; /* Current index, where the iteration is */
+ hbool_t found; /* Whether the chunk was found */
+} H5D_chunk_info_iter_ud_t;
+
/* Callback info for file selection iteration */
typedef struct H5D_chunk_file_iter_ud_t {
H5D_chunk_map_t *fm; /* File->memory chunk mapping info */
@@ -252,6 +264,12 @@ static herr_t H5D__chunk_flush(H5D_t *dset);
static herr_t H5D__chunk_io_term(const H5D_chunk_map_t *fm);
static herr_t H5D__chunk_dest(H5D_t *dset);
+/* Chunk query operation callbacks */
+static int H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata);
+static int H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata);
+static int H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata);
+
+
/* "Nonexistent" layout operation callback */
static ssize_t
H5D__nonexistent_readvv(const H5D_io_info_t *io_info,
@@ -6726,3 +6744,375 @@ done:
FUNC_LEAVE_NOAPI(ret_value)
} /* end H5D__chunk_format_convert() */
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__get_num_chunks_cb
+ *
+ * Purpose: Callback function that increments the number of written
+ * chunks in the dataset.
+ *
+ * Note: Currently, this function only gets the number of all written
+ * chunks, regardless the dataspace.
+ *
+ * Return: Success: H5_ITER_CONT or H5_ITER_STOP
+ * Failure: Negative (H5_ITER_ERROR)
+ *
+ * Programmer: Binh-Minh Ribler
+ * September 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5D__get_num_chunks_cb(const H5D_chunk_rec_t H5_ATTR_UNUSED *chunk_rec, void *_udata)
+{
+ hsize_t *num_chunks = (hsize_t *)_udata;
+ int ret_value = H5_ITER_CONT; /* Callback return value */
+
+ FUNC_ENTER_STATIC_NOERR
+
+ HDassert(num_chunks);
+
+ (*num_chunks)++;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5D__get_num_chunks_cb() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__get_num_chunks
+ *
+ * Purpose: Gets the number of written chunks in a dataset.
+ *
+ * Note: Currently, this function only gets the number of all written
+ * chunks, regardless the dataspace.
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ * Programmer: Binh-Minh Ribler
+ * September 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5D__get_num_chunks(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t *nchunks)
+{
+ H5D_chk_idx_info_t idx_info; /* Chunked index info */
+ hsize_t num_chunks = 0; /* Number of written chunks */
+ H5D_rdcc_ent_t *ent; /* Cache entry */
+ const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr)
+
+ HDassert(dset);
+ HDassert(dset->shared);
+ HDassert(space);
+ HDassert(nchunks);
+
+ /* Get the raw data chunk cache */
+ rdcc = &(dset->shared->cache.chunk);
+ HDassert(rdcc);
+
+ /* Search for cached chunks that haven't been written out */
+ for(ent = rdcc->head; ent; ent = ent->next)
+ /* Flush the chunk out to disk, to make certain the size is correct later */
+ if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer")
+
+ /* Compose chunked index info struct */
+ idx_info.f = dset->oloc.file;
+ idx_info.pline = &dset->shared->dcpl_cache.pline;
+ idx_info.layout = &dset->shared->layout.u.chunk;
+ idx_info.storage = &dset->shared->layout.storage.u.chunk;
+
+ /* If the dataset is not written, number of chunks will be 0 */
+ if(!H5F_addr_defined(idx_info.storage->idx_addr)) {
+ *nchunks = 0;
+ HGOTO_DONE(SUCCEED);
+ }
+ else {
+ /* Iterate over the allocated chunks */
+ if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_num_chunks_cb, &num_chunks) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index")
+ *nchunks = num_chunks;
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__get_num_chunks() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__get_chunk_info_cb
+ *
+ * Purpose: Get the chunk info of the desired chunk, given by its index.
+ *
+ * Return: Success: H5_ITER_CONT or H5_ITER_STOP
+ * Failure: Negative (H5_ITER_ERROR)
+ *
+ * Programmer: Binh-Minh Ribler
+ * September 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
+{
+ H5D_chunk_info_iter_ud_t *chunk_info = (H5D_chunk_info_iter_ud_t *)_udata;
+ hsize_t ii = 0; /* Dimension index */
+ int ret_value = H5_ITER_CONT; /* Callback return value */
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /* Check args */
+ HDassert(chunk_rec);
+ HDassert(chunk_info);
+
+ /* If this is the desired chunk, retrieve its info and stop iterating */
+ if (chunk_info->curr_idx == chunk_info->chunk_idx) {
+ chunk_info->filter_mask = chunk_rec->filter_mask;
+ chunk_info->chunk_addr = chunk_rec->chunk_addr;
+ chunk_info->nbytes = chunk_rec->nbytes;
+ for (ii = 0; ii < chunk_info->ndims; ii++)
+ chunk_info->scaled[ii] = chunk_rec->scaled[ii];
+ chunk_info->found = TRUE;
+
+ /* Stop iterating */
+ ret_value = H5_ITER_STOP;
+ }
+ /* Iterate the next chunk */
+ else
+ chunk_info->curr_idx++;
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5D__get_chunk_info_cb() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__get_chunk_info
+ *
+ * Purpose: Iterate over the chunks in the dataset to get the info
+ * of the desired chunk.
+ *
+ * Note: Currently, this function only gets the number of all written
+ * chunks, regardless the dataspace.
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ * Programmer: Binh-Minh Ribler
+ * September 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5D__get_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t index, hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size)
+{
+ H5D_chk_idx_info_t idx_info; /* Chunked index info */
+ H5D_chunk_info_iter_ud_t udata; /* User data for callback */
+ const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */
+ H5D_rdcc_ent_t *ent; /* Cache entry index */
+ hsize_t ii = 0; /* Dimension index */
+ herr_t ret_value = SUCCEED;/* Return value */
+
+ FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr)
+
+ HDassert(dset);
+ HDassert(dset->shared);
+ HDassert(space);
+
+ /* Get the raw data chunk cache */
+ rdcc = &(dset->shared->cache.chunk);
+ HDassert(rdcc);
+
+ /* Search for cached chunks that haven't been written out */
+ for(ent = rdcc->head; ent; ent = ent->next)
+ /* Flush the chunk out to disk, to make certain the size is correct later */
+ if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer")
+
+ /* Compose chunked index info struct */
+ idx_info.f = dset->oloc.file;
+ idx_info.pline = &dset->shared->dcpl_cache.pline;
+ idx_info.layout = &dset->shared->layout.u.chunk;
+ idx_info.storage = &dset->shared->layout.storage.u.chunk;
+
+ /* Set addr & size for when dset is not written or queried chunk is not found */
+ if (addr)
+ *addr = HADDR_UNDEF;
+ if (size)
+ *size = 0;
+
+ /* If the dataset is not written, return without error */
+ if(!H5F_addr_defined(idx_info.storage->idx_addr))
+ HGOTO_DONE(SUCCEED);
+
+ /* Initialize before iteration */
+ udata.chunk_idx = index;
+ udata.curr_idx = 0;
+ udata.ndims = dset->shared->ndims;
+ udata.nbytes = 0;
+ udata.filter_mask = 0;
+ udata.chunk_addr = HADDR_UNDEF;
+ udata.found = FALSE;
+
+ /* Iterate over the allocated chunks */
+ if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_cb, &udata) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index")
+
+ /* Obtain requested info if the chunk is found */
+ if (udata.found) {
+ if (filter_mask)
+ *filter_mask = udata.filter_mask;
+ if (addr)
+ *addr = udata.chunk_addr;
+ if (size)
+ *size = udata.nbytes;
+ if (offset)
+ for (ii = 0; ii < udata.ndims; ii++)
+ offset[ii] = udata.scaled[ii] * dset->shared->layout.u.chunk.dim[ii];
+ }
+
+done:
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* end H5D__get_chunk_info() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__get_chunk_info_by_coord_cb
+ *
+ * Purpose: Get the chunk info of the desired chunk, given its offset
+ * coordinates.
+ *
+ * Return: Success: H5_ITER_CONT or H5_ITER_STOP
+ * Failure: Negative (H5_ITER_ERROR)
+ *
+ * Programmer: Binh-Minh Ribler
+ * September 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata)
+{
+ hsize_t ii;
+ H5D_chunk_info_iter_ud_t *chunk_info = (H5D_chunk_info_iter_ud_t *)_udata;
+ hbool_t different = FALSE; /* TRUE when a scaled value pair mismatch */
+ int ret_value = H5_ITER_CONT; /* Callback return value */
+
+ FUNC_ENTER_STATIC_NOERR
+
+ /* Check args */
+ HDassert(chunk_rec);
+ HDassert(chunk_info);
+
+ /* Going through the scaled, stop when a mismatch is found */
+ for (ii = 0; ii < chunk_info->ndims && !different; ii++)
+ if (chunk_info->scaled[ii] != chunk_rec->scaled[ii])
+ different = TRUE;
+
+ /* Same scaled coords means the chunk is found, copy the chunk info */
+ if (!different) {
+ chunk_info->nbytes = chunk_rec->nbytes;
+ chunk_info->filter_mask = chunk_rec->filter_mask;
+ chunk_info->chunk_addr = chunk_rec->chunk_addr;
+ chunk_info->found = TRUE;
+
+ /* Stop iterating */
+ ret_value = H5_ITER_STOP;
+ }
+
+ FUNC_LEAVE_NOAPI(ret_value)
+} /* H5D__get_chunk_info_by_coord_cb() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5D__get_chunk_info_by_coord
+ *
+ * Purpose: Iterate over the chunks in the dataset to get the info
+ * of the desired chunk, given by its offset coordinates.
+ *
+ * Return: Success: Non-negative
+ * Failure: Negative
+ *
+ * Programmer: Binh-Minh Ribler
+ * September 2018 (EED-343)
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned* filter_mask, haddr_t *addr, hsize_t *size)
+{
+ const H5O_layout_t *layout = NULL; /* Dataset layout */
+ const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */
+ H5D_rdcc_ent_t *ent; /* Cache entry index */
+ H5D_chk_idx_info_t idx_info; /* Chunked index info */
+ H5D_chunk_info_iter_ud_t udata; /* User data for callback */
+ herr_t ret_value = SUCCEED; /* Return value */
+
+ FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr)
+
+ /* Check args */
+ HDassert(dset);
+ HDassert(dset->shared);
+ HDassert(offset);
+
+ /* Get dataset layout and raw data chunk cache */
+ layout = &(dset->shared->layout);
+ rdcc = &(dset->shared->cache.chunk);
+ HDassert(layout);
+ HDassert(rdcc);
+ HDassert(H5D_CHUNKED == layout->type);
+
+ /* Search for cached chunks that haven't been written out */
+ for(ent = rdcc->head; ent; ent = ent->next)
+ /* Flush the chunk out to disk, to make certain the size is correct later */
+ if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0)
+ HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer")
+
+ /* Compose chunked index info struct */
+ idx_info.f = dset->oloc.file;
+ idx_info.pline = &dset->shared->dcpl_cache.pline;
+ idx_info.layout = &dset->shared->layout.u.chunk;
+ idx_info.storage = &dset->shared->layout.storage.u.chunk;
+
+ /* Set addr & size for when dset is not written or queried chunk is not found */
+ if (addr)
+ *addr = HADDR_UNDEF;
+ if (size)
+ *size = 0;
+
+ /* If the dataset is not written, return without errors */
+ if(!H5F_addr_defined(idx_info.storage->idx_addr)) {
+ HGOTO_DONE(SUCCEED);
+ }
+
+ /* Calculate the scaled of this chunk */
+ H5VM_chunk_scaled(dset->shared->ndims, offset, layout->u.chunk.dim, udata.scaled);
+ udata.scaled[dset->shared->ndims] = 0;
+
+ /* Initialize before iteration */
+ udata.ndims = dset->shared->ndims;
+ udata.nbytes = 0;
+ udata.filter_mask = 0;
+ udata.chunk_addr = HADDR_UNDEF;
+ udata.found = FALSE;
+
+ /* Iterate over the allocated chunks to find the requested chunk */
+ if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_by_coord_cb, &udata) < 0)
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve information of the chunk by its scaled coordinates")
+
+ /* If the chunk is found, return the filter mask and chunk address/size */
+ if (udata.found) {
+ if (filter_mask)
+ *filter_mask = udata.filter_mask;
+ if (addr)
+ *addr = udata.chunk_addr;
+ if (size)
+ *size = udata.nbytes;
+ }
+done:
+ FUNC_LEAVE_NOAPI_TAG(ret_value)
+} /* end H5D__get_chunk_info_by_coord() */
+
diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h
index b887b87..ee70bae 100644
--- a/src/H5Dpkg.h
+++ b/src/H5Dpkg.h
@@ -566,6 +566,9 @@ H5_DLL herr_t H5D__alloc_storage(const H5D_io_info_t *io_info, H5D_time_alloc_t
H5_DLL herr_t H5D__get_storage_size(const H5D_t *dset, hsize_t *storage_size);
H5_DLL herr_t H5D__get_chunk_storage_size(H5D_t *dset, const hsize_t *offset,
hsize_t *storage_size);
+H5_DLL herr_t H5D__get_num_chunks(const H5D_t *dset, const H5S_t *space, hsize_t *nchunks);
+H5_DLL herr_t H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_t index, hsize_t *coord, unsigned *filter_mask, haddr_t *offset, hsize_t *size);
+H5_DLL herr_t H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size);
H5_DLL haddr_t H5D__get_offset(const H5D_t *dset);
H5_DLL void *H5D__vlen_get_buf_size_alloc(size_t size, void *info);
H5_DLL herr_t H5D__vlen_get_buf_size(void *elem, hid_t type_id, unsigned ndim,
diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h
index a1ccda0..fcc76ee 100644
--- a/src/H5Dpublic.h
+++ b/src/H5Dpublic.h
@@ -141,6 +141,9 @@ H5_DLL hid_t H5Dget_create_plist(hid_t dset_id);
H5_DLL hid_t H5Dget_access_plist(hid_t dset_id);
H5_DLL hsize_t H5Dget_storage_size(hid_t dset_id);
H5_DLL herr_t H5Dget_chunk_storage_size(hid_t dset_id, const hsize_t *offset, hsize_t *chunk_bytes);
+H5_DLL herr_t H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks);
+H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size);
+H5_DLL herr_t H5Dget_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t index, hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size);
H5_DLL haddr_t H5Dget_offset(hid_t dset_id);
H5_DLL herr_t H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id,
hid_t file_space_id, hid_t plist_id, void *buf/*out*/);
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index c8119fe..2aedc82 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -197,6 +197,7 @@ set (H5_TESTS
page_buffer
dtypes
dsets
+ chunk_info
cmpd_dset
filter_fail
extend
diff --git a/test/Makefile.am b/test/Makefile.am
index d4db6d0..5b210a9 100644
--- a/test/Makefile.am
+++ b/test/Makefile.am
@@ -55,8 +55,8 @@ TEST_PROG= testhdf5 \
cache cache_api cache_image cache_tagging lheap ohdr stab gheap \
evict_on_close farray earray btree2 fheap \
pool accum hyperslab istore bittests dt_arith page_buffer \
- dtypes dsets cmpd_dset filter_fail extend direct_chunk external efc \
- objcopy links unlink twriteorder big mtime fillval mount \
+ dtypes dsets chunk_info cmpd_dset filter_fail extend direct_chunk \
+ external efc objcopy links unlink twriteorder big mtime fillval mount \
flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \
enc_dec_plist_cross_platform getname vfd ntypes dangle dtransform \
reserved cross_read freespace mf vds file_image unregister \
diff --git a/test/chunk_info.c b/test/chunk_info.c
index e3006cc..8a569d6 100644
--- a/test/chunk_info.c
+++ b/test/chunk_info.c
@@ -15,23 +15,374 @@
* Programmer: Pedro Vicente <pvn@hdfgroup.edu>
* April 7, 2008
*
- * Purpose: Tests the H5Dget_chunk_info API function
- * This program writes a 4x4 dataset by iterating on 2x2 chunks
- * at a time
+ * Purpose: Tests chunk query API functions
*/
-
#include "h5test.h"
-#define PRINT_DATA
+ /* #define PRINT_DATA
+ */
#define H5FILE_NAME "chunk_info.h5"
#define DATASETNAME "2d"
#define RANK 2
+#define FILENAME_BUF_SIZE 1024
+
+/* Parameters for testing chunk querying */
+#define FILENAME "tchunk_info"
+#define DSET_SIMPLE_CHUNKED "Chunked Dataset"
+#define DSET_EMPTY "Empty Dataset"
+#define NX 16
+#define NY 16
+#define CHUNK_NX 4
+#define CHUNK_NY 4
+#define CHUNK_SIZE 64
+#define NUM_CHUNKS_WRITTEN 4
+
+void reinit_vars(unsigned *read_filter_mask, haddr_t *addr, hsize_t *size);
+
+/*-------------------------------------------------------------------------
+ * Function: read_each_chunk (helper function)
+ *
+ * Purpose: Reads the chunk specified by its offset and verifies that
+ * it contains the same data as what was written. This function
+ * is used in test_get_chunk_info.
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ * Date: September 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t read_each_chunk(hid_t dset_id, hsize_t offset1, hsize_t offset2, void *direct_buf)
+{
+ int read_buf[CHUNK_NX][CHUNK_NY];
+ hsize_t offset[2] = {offset1, offset2};
+ unsigned read_filter_mask = 0;
+
+ /* Read the raw chunk back */
+ HDmemset(&read_buf, 0, sizeof(read_buf));
+
+ /* Read the chunk specified by its offset */
+ if (H5Dread_chunk(dset_id, H5P_DEFAULT, offset, &read_filter_mask, read_buf) < 0)
+ return(FAIL);
+
+ /* Verify that read chunk is the same as the corresponding written one */
+ if (HDmemcmp(direct_buf, read_buf, CHUNK_NX*CHUNK_NY) != 0)
+ {
+ HDfprintf(stderr, "Read chunk differs from written chunk at offset (%d,%d)\n", offset1, offset2);
+ return(FAIL);
+ }
+ return(SUCCEED);
+}
-int main( void )
+/*-------------------------------------------------------------------------
+ * Function: reinit_vars (helper function)
+ *
+ * Purpose: Helper function to wipe out variables for the next use,
+ * used in test_get_chunk_info.
+ *
+ * Return: Won't fail
+ *
+ * Date: September 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+void reinit_vars(unsigned *read_filter_mask, haddr_t *addr, hsize_t *size)
{
+ if (read_filter_mask)
+ *read_filter_mask = 0;
+ if (addr)
+ *addr = 0;
+ if (size)
+ *size = 0;
+}
+/*-------------------------------------------------------------------------
+ * Function: test_get_chunk_info
+ *
+ * Purpose: Test getting various chunk information
+ *
+ * Return: Success: 0
+ * Failure: 1
+ *
+ * Description:
+ * This function tests the new API functions added for EED-343:
+ * H5Dget_num_chunks, H5Dget_chunk_info, and H5Dget_chunk_info_by_coord.
+ *
+ * Date: September 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+test_get_chunk_info(void)
+{
+ hid_t chunkfile = -1, fapl = -1;
+ hid_t fspace = -1, dset = -1;
+ hid_t mspace = -1;
+ hid_t cparms = -1, dxpl = -1;
+ hsize_t dims[2] = {NX, NY};
+ hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
+ hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY};
+ int fillvalue = -1;
+ char filename[FILENAME_BUF_SIZE];
+ unsigned filter_mask = 0;
+ int direct_buf[16][CHUNK_NX][CHUNK_NY];
+ int out_buf[NX][NY];
+ size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int);
+ int aggression = 9; /* Compression aggression setting */
+ unsigned read_filter_mask = 0; /* filter mask after direct read */
+ H5F_libver_t low, high; /* File format bounds */
+ hsize_t offset[2];
+ hsize_t out_offset[2] = {0, 0};
+ hsize_t size = 0;
+ hsize_t nchunks = 0;
+ haddr_t addr = 0;
+ hsize_t index = 0;
+ hsize_t i, j;
+ int n; /* for use on buffer, to avoid conversion warning */
+
+ TESTING("getting chunk information");
+
+ /* Create a copy of file access property list */
+ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR
+
+ /* Set high bound to V18 */
+ low = H5F_LIBVER_EARLIEST;
+ high = H5F_LIBVER_V18;
+ if (H5Pset_libver_bounds(fapl, low, high) < 0)
+ TEST_ERROR;
+
+ /* Create a file */
+ h5_fixname(FILENAME, fapl, filename, sizeof filename);
+ if((chunkfile = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ TEST_ERROR
+
+ /* Create the data space */
+ if((fspace = H5Screate_simple(RANK, dims, maxdims)) < 0)
+ TEST_ERROR
+
+ if((mspace = H5Screate_simple(RANK, chunk_dims, NULL)) < 0)
+ TEST_ERROR
+
+ /* Modify dataset creation properties, i.e. enable chunking and compression */
+ if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ TEST_ERROR
+
+ if(H5Pset_chunk(cparms, RANK, chunk_dims) < 0)
+ TEST_ERROR
+
+ if(H5Pset_deflate(cparms, (unsigned ) aggression) < 0)
+ TEST_ERROR
+
+ if (H5Pset_fill_value(cparms, H5T_NATIVE_INT, &fillvalue) < 0)
+ TEST_ERROR;
+
+ /* Create a new dataset using cparms creation properties */
+ if((dset = H5Dcreate2(chunkfile, DSET_SIMPLE_CHUNKED, H5T_NATIVE_INT, fspace,
+ H5P_DEFAULT, cparms, H5P_DEFAULT)) < 0) TEST_ERROR
+
+ if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0)
+ TEST_ERROR
+
+ /* Indicate skipping the compression filter. */
+ filter_mask = 0x00000001;
+
+ /* Initialize the array of chunk data, that is for all 16 chunks */
+ for(n = 0; n < 16; n++)
+ for(i = 0; i < CHUNK_NX; i++)
+ for(j = 0; j < CHUNK_NY; j++)
+ direct_buf[n][i][j] = n + 1;
+
+ /* Write NUM_CHUNKS_WRITTEN chunks: (0,2) (0,3) (1,2) (1,3) */
+ n = 0;
+ for (i = 0; i < 2; i++)
+ for (j = 2; j < 4; j++, n++)
+ {
+ offset[0] = i * CHUNK_NX;
+ offset[1] = j * CHUNK_NY;
+ if (H5Dwrite_chunk(dset, dxpl, filter_mask, offset, buf_size, (void*)direct_buf[n]) < 0)
+ TEST_ERROR
+ }
+
+ /* Close the dataset then... */
+ if(H5Dclose(dset) < 0) TEST_ERROR
+
+ /* ...open it again to test the chunk query functions */
+ if((dset = H5Dopen2(chunkfile, DSET_SIMPLE_CHUNKED, H5P_DEFAULT)) < 0) TEST_ERROR
+
+ /* Get and verify the number of chunks written */
+ if (H5Dget_num_chunks(dset, mspace, &nchunks) < 0) TEST_ERROR;
+ if (nchunks != NUM_CHUNKS_WRITTEN) TEST_ERROR;
+
+ /* Read the entire dataset back */
+ if(H5Dread(dset, H5T_NATIVE_INT, fspace, fspace, H5P_DEFAULT, out_buf) < 0)
+ TEST_ERROR
+
+ /* Get and verify info of the first chunk */
+ index = 0;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+ if (out_offset[0] != 0 || out_offset[1] != 8) TEST_ERROR;
+
+ /* Get and verify info of the second chunk */
+ index = 1;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+ if (out_offset[0] != 0 || out_offset[1] != 12) TEST_ERROR;
+
+ /* Get and verify info of the third chunk */
+ index = 2;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+ if (out_offset[0] != 4 || out_offset[1] != 8) TEST_ERROR;
+
+ /* Get and verify info of the last chunk */
+ index = 3;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+ if (out_offset[0] != 4 || out_offset[1] != 12) TEST_ERROR;
+
+ /* Attempt to get info of empty chunk and verify the returned address and size */
+ index = 5;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (addr != HADDR_UNDEF) TEST_ERROR;
+ if (size != 0) TEST_ERROR;
+
+ /* Get info of the chunk at logical coordinates (0,2) */
+ offset[0] = 0;
+ offset[1] = 2 * CHUNK_NY;
+ if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR;
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+
+ /* Get info of the chunk at logical coordinates (1,3) */
+ offset[0] = 1 * CHUNK_NX;
+ offset[1] = 3 * CHUNK_NY;
+ if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR;
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+
+ /* Attempt to get info of empty chunks, verify the returned address and size */
+ offset[0] = 0;
+ offset[1] = 0;
+ if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR;
+ if (addr != HADDR_UNDEF) TEST_ERROR;
+ if (size != 0) TEST_ERROR;
+
+ offset[0] = 3 * CHUNK_NX;
+ offset[1] = 3 * CHUNK_NY;
+ if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR;
+ if (addr != HADDR_UNDEF) TEST_ERROR;
+ if (size != 0) TEST_ERROR;
+
+ /* Read each chunk and verify the values */
+ n = 0;
+ for (i = 0; i < 2; i++)
+ for (j = 2; j < 4; j++, n++)
+ if (read_each_chunk(dset, i*CHUNK_NX, j*CHUNK_NY, (void*)direct_buf[n]) < 0)
+ TEST_ERROR
+
+ /* Close the first dataset */
+ if (H5Dclose(dset) < 0) TEST_ERROR
+
+ /* Create an empty dataset */
+ if((dset = H5Dcreate2(chunkfile, DSET_EMPTY, H5T_NATIVE_INT, fspace,
+ H5P_DEFAULT, cparms, H5P_DEFAULT)) < 0) TEST_ERROR
+ if (H5Dclose(dset) < 0) TEST_ERROR
+
+ /* Reopen the empty dataset to verify the chunk query functions on it */
+ if((dset = H5Dopen2(chunkfile, DSET_EMPTY, H5P_DEFAULT)) < 0) TEST_ERROR
+
+ /* Verify that the number of chunks is 0 */
+ if (H5Dget_num_chunks(dset, mspace, &nchunks) < 0) TEST_ERROR;
+ if (nchunks != 0) TEST_ERROR;
+
+ /* Attempt to get info of a chunk from an empty dataset, verify the
+ returned address and size */
+ index = 0;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (addr != HADDR_UNDEF) TEST_ERROR;
+ if (size != 0) TEST_ERROR;
+
+ /* Attempt to get info of a chunk given its coords from an empty dataset,
+ verify the returned address and size */
+ offset[0] = 0;
+ offset[1] = 0;
+ if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR;
+ if (addr != HADDR_UNDEF) TEST_ERROR;
+ if (size != 0) TEST_ERROR;
+
+ /* Close/release resources. */
+ if (H5Sclose(mspace) < 0) TEST_ERROR
+ if (H5Sclose(fspace) < 0) TEST_ERROR
+ if (H5Pclose(cparms) < 0) TEST_ERROR
+ if (H5Pclose(dxpl) < 0) TEST_ERROR
+ if (H5Fclose(chunkfile) < 0) TEST_ERROR
+
+ PASSED();
+ return SUCCEED;
+
+error:
+ H5E_BEGIN_TRY {
+ H5Dclose(dset);
+ H5Sclose(mspace);
+ H5Sclose(fspace);
+ H5Pclose(cparms);
+ H5Pclose(dxpl);
+ H5Fclose(chunkfile);
+ } H5E_END_TRY;
+
+ H5_FAILED();
+ return FAIL;
+} /* test_get_chunk_info() */
+
+/*-------------------------------------------------------------------------
+ * Function: create_4x4_dset
+ *
+ * Purpose: Tests functions related to chunk information
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ * Description:
+ * The code in this create_4x4_dset function was originally written by
+ * Pedro in main() that writes a 4x4 dataset by iterating on
+ * 2x2 chunks at a time, with the intention of making a frame
+ * work to test H5Dget_chunk_info, which was not in the library,
+ * until now. For the work in EED-343, the test function
+ * test_get_chunk_info was added to test the new query chunk
+ * API functions: H5Dget_num_chunk, H5Dget_chunk_info, and
+ * H5Dget_chunk_info_by_coord. This code can be used at a
+ * later time, so it is kept here.
+ * -BMR, November 5, 2018
+ *
+ * Programmer: Pedro Vicente <pvn@hdfgroup.edu>
+ * April 7, 2008
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+create_4x4_dset(void)
+{
hid_t fid; /* file ID */
hid_t did; /* dataset ID */
hid_t f_sid; /* file space ID */
@@ -43,20 +394,20 @@ int main( void )
int chunk_data[2][2] = { {1, 1}, {1, 1} };
int buf[4][4];
int fillvalue = 0;
- int i, j, ii, jj;
+ hsize_t i, j;
/* create a new file using default properties. */
if ((fid = H5Fcreate(H5FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR;
+
/* create the file space */
if ((f_sid = H5Screate_simple(RANK, dims, dims)) < 0) TEST_ERROR;
+
/* create the memory space with chunk dimensions */
if ((m_sid = H5Screate_simple(RANK, chunk_dims, chunk_dims)) < 0) TEST_ERROR;
start[0] = 0;
start[1] = 0;
if (H5Sselect_hyperslab(m_sid, H5S_SELECT_SET, start, NULL, chunk_dims, NULL) < 0) TEST_ERROR;
- TESTING("chunk info");
-
/*-------------------------------------------------------------------------
* create a dataset
*-------------------------------------------------------------------------
@@ -77,16 +428,12 @@ int main( void )
*/
/* iterate in dim 0 */
- for (j = 0; j < chunk_dims[0]; j++)
- {
-
+ for (j = 0; j < chunk_dims[0]; j++) {
/* reset start in dim 1 */
start[1] = 0;
/* iterate in dim 1 */
- for (i = 0; i < chunk_dims[1]; i++)
- {
-
+ for (i = 0; i < chunk_dims[1]; i++) {
/* select file hyperslab to save a 2x2 chunk */
if (H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, chunk_dims, NULL) < 0) TEST_ERROR;
@@ -96,7 +443,6 @@ int main( void )
/* read back and display complete dataset 4x4 */
if (H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) TEST_ERROR;
-
#if defined (PRINT_DATA)
printf("\n");
printf("dataset: \n");
@@ -105,25 +451,17 @@ int main( void )
printf("\n");
}
#endif
-
-
/* increment start in dim 1 */
start[1] += 2;
-
-
}
-
/* increment start in dim 0 */
start[0] += 2;
}
-
-
/*-------------------------------------------------------------------------
* close
*-------------------------------------------------------------------------
*/
-
if (H5Dclose(did) < 0) TEST_ERROR
if (H5Sclose(f_sid) < 0) TEST_ERROR
if (H5Sclose(m_sid) < 0) TEST_ERROR
@@ -132,18 +470,58 @@ int main( void )
PASSED();
- puts("All chunk info tests passed.");
- return 0;
+ return SUCCEED;
+
+/* this will be removed once the existing code is moved out of main */
+error:
+ H5Dclose(did);
+ H5Sclose(f_sid);
+ H5Sclose(m_sid);
+ H5Pclose(pid);
+ H5Fclose(fid);
+ return FAIL;
+} /* end create_4x4_dset */
+
+/*-------------------------------------------------------------------------
+ * Function: main
+ *
+ * Purpose: Tests functions related to chunk information
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ * Programmer: Binh-Minh Ribler
+ * November 5, 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+int
+main(void)
+{
+ int nerrors = 0;
+
+ /* Tests getting chunk information */
+ nerrors += (test_get_chunk_info() < 0 ? 1 : 0);
+
+ /* Create a 4x4 dataset (using the existing code to avoid compilation
+ warnings for now) */
+ nerrors += (create_4x4_dset() < 0 ? 1 : 0);
+
+ if(nerrors) {
+ goto error;
+
+ } /* end if */
+
+ HDprintf("All chunk query tests passed.\n");
+
+ return SUCCEED;
error:
- H5Dclose( did );
- H5Sclose( f_sid );
- H5Sclose( m_sid );
- H5Pclose( pid );
- H5Fclose( fid );
- H5_FAILED();
- return 1;
+ nerrors = MAX(1, nerrors);
+ HDprintf("***** %d QUERY CHUNK INFO TEST%s FAILED! *****\n",
+ nerrors, 1 == nerrors ? "" : "S");
+ return FAIL;
}
diff --git a/test/dsets.c b/test/dsets.c
index d23f438..b16459b 100644
--- a/test/dsets.c
+++ b/test/dsets.c
@@ -84,6 +84,7 @@ const char *FILENAME[] = {
"dls_01_strings", /* 23 */
"power2up", /* 24 */
"version_bounds", /* 25 */
+ "chunk_info", /* 26 */
NULL
};
@@ -268,6 +269,18 @@ const char *FILENAME[] = {
#define STORAGE_SIZE_CHUNK_DIM1 5
#define STORAGE_SIZE_CHUNK_DIM2 5
+/* Parameters for testing chunk querying */
+#define DSET_SIMPLE_CHUNKED "Chunked Dataset"
+#define DSET_EMPTY "Empty Dataset"
+#define RANK 2
+#define NX 16
+#define NY 16
+#define CHUNK_NX 4
+#define CHUNK_NY 4
+#define CHUNK_SIZE 64
+#define NUM_CHUNKS 16
+#define NUM_CHUNKS_WRITTEN 4
+
/* Shared global arrays */
#define DSET_DIM1 100
#define DSET_DIM2 200
@@ -12888,8 +12901,8 @@ error:
* Purpose: Tests various format versions.
* (Currently, only virtual dataset feature)
*
- * Return: Success: 0
- * Failure: -1
+ * Return: Success: SUCCEED
+ * Failure: FAIL
* Description:
* This function attempts to create a virtual dataset in all
* valid combinations of low/high library format bounds. Creation
@@ -13018,7 +13031,7 @@ test_versionbounds()
TEST_ERROR
dcpl = -1;
PASSED();
- return 0;
+ return SUCCEED;
error:
H5E_BEGIN_TRY {
@@ -13031,9 +13044,332 @@ test_versionbounds()
H5Fclose(srcfile);
H5Fclose(vfile);
} H5E_END_TRY;
- return -1;
+ return FAIL;
} /* test_versionbounds() */
+/*-------------------------------------------------------------------------
+ * Function: read_each_chunk (helper function)
+ *
+ * Purpose: Reads the chunk specified by its offset and verifies that
+ * it contains the same data as what was written. This function
+ * is used in test_get_chunk_info.
+ *
+ * Return: Success: SUCCEED
+ * Failure: FAIL
+ *
+ * Date: September 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t read_each_chunk(hid_t dset_id, hsize_t offset1, hsize_t offset2, unsigned check_filter_mask, void *direct_buf)
+{
+ size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int);
+ int read_buf[CHUNK_NX][CHUNK_NY];
+ hsize_t offset[2] = {offset1, offset2};
+ unsigned read_filter_mask = 0;
+
+ /* Read the raw chunk back */
+ HDmemset(&read_buf, 0, sizeof(read_buf));
+
+ /* Read the chunk specified by its offset */
+ if (H5Dread_chunk(dset_id, H5P_DEFAULT, offset, &read_filter_mask, read_buf) < 0)
+ return(FAIL);
+
+ /* Verify that read chunk is the same as the corresponding written one */
+ if (HDmemcmp(direct_buf, read_buf, CHUNK_NX*CHUNK_NY) != 0)
+ {
+ HDfprintf(stderr, "Read chunk differs from written chunk at offset (%d,%d)\n", offset1, offset2);
+ return(FAIL);
+ }
+
+ return(SUCCEED);
+}
+
+/*-------------------------------------------------------------------------
+ * Function: reinit_vars (helper function)
+ *
+ * Purpose: Helper function to wipe out variables for the next use,
+ * used in test_get_chunk_info.
+ *
+ * Return: Won't fail
+ *
+ * Date: September 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+void reinit_vars(unsigned *read_filter_mask, hsize_t *addr, hsize_t *size)
+{
+ if (read_filter_mask)
+ *read_filter_mask = 0;
+ if (addr)
+ *addr = 0;
+ if (size)
+ *size = 0;
+}
+
+/*-------------------------------------------------------------------------
+ * Function: test_get_chunk_info
+ *
+ * Purpose: Test getting various chunk information
+ *
+ * Return: Success: 0
+ * Failure: 1
+ *
+ * Description:
+ * This function tests the new API functions added for EED-343:
+ * H5Dget_num_chunks, H5Dget_chunk_info, and H5Dget_chunk_info_by_coord.
+ *
+ * Date: September 2018
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+test_get_chunk_info()
+{
+ hid_t chunkfile = -1, fapl = -1;
+ hid_t fspace = -1, dset = -1;
+ hid_t mspace = -1;
+ hid_t cparms = -1, dxpl = -1;
+ hsize_t dims[2] = {NX, NY};
+ hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED};
+ hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY};
+ int fillvalue = -1;
+ char filename[FILENAME_BUF_SIZE];
+ unsigned filter_mask = 0;
+ int direct_buf[16][CHUNK_NX][CHUNK_NY];
+ int out_buf[NX][NY];
+ size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int);
+ int aggression = 9; /* Compression aggression setting */
+ unsigned read_filter_mask = 0; /* filter mask after direct read */
+ H5F_libver_t low, high; /* File format bounds */
+ hsize_t start[2]; /* Start of hyperslab */
+ hsize_t stride[2]; /* Stride of hyperslab */
+ hsize_t count[2]; /* Block count */
+ hsize_t block[2]; /* Block sizes */
+ hsize_t offset[2];
+ hsize_t coord[H5S_MAX_RANK];
+ hsize_t out_offset[2] = {-1, -1};
+ hsize_t size = 0;
+ hsize_t nchunks = 0;
+ hsize_t addr = 0;
+ hsize_t index = 0;
+ hsize_t i, j, n;
+
+ TESTING("getting chunk information");
+
+ /* Create a copy of file access property list */
+ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR
+
+ /* Set high bound to V18 */
+ low = H5F_LIBVER_EARLIEST;
+ high = H5F_LIBVER_V18;
+ if (H5Pset_libver_bounds(fapl, low, high) < 0)
+ TEST_ERROR;
+
+ /* Create a file */
+ h5_fixname(FILENAME[26], fapl, filename, sizeof filename);
+ if((chunkfile = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ TEST_ERROR
+
+ /* Create the data space */
+ if((fspace = H5Screate_simple(RANK, dims, maxdims)) < 0)
+ TEST_ERROR
+
+ if((mspace = H5Screate_simple(RANK, chunk_dims, NULL)) < 0)
+ TEST_ERROR
+
+ /* Modify dataset creation properties, i.e. enable chunking and compression */
+ if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ TEST_ERROR
+
+ if(H5Pset_chunk(cparms, RANK, chunk_dims) < 0)
+ TEST_ERROR
+
+ if(H5Pset_deflate(cparms, (unsigned ) aggression) < 0)
+ TEST_ERROR
+
+ if (H5Pset_fill_value(cparms, H5T_NATIVE_INT, &fillvalue) < 0)
+ TEST_ERROR;
+
+ /* Create a new dataset using cparms creation properties */
+ if((dset = H5Dcreate2(chunkfile, DSET_SIMPLE_CHUNKED, H5T_NATIVE_INT, fspace,
+ H5P_DEFAULT, cparms, H5P_DEFAULT)) < 0) TEST_ERROR
+
+ if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0)
+ TEST_ERROR
+
+ /* Indicate skipping the compression filter. */
+ filter_mask = 0x00000001;
+
+ /* Initialize the array of chunk data, that is for all 16 chunks */
+ for(n = 0; n < 16; n++)
+ for(i = 0; i < CHUNK_NX; i++)
+ for(j = 0; j < CHUNK_NY; j++)
+ direct_buf[n][i][j] = n + 1;
+
+ /* Write NUM_CHUNKS_WRITTEN chunks: (0,2) (0,3) (1,2) (1,3) */
+ n = 0;
+ for (i = 0; i < 2; i++)
+ for (j = 2; j < 4; j++, n++)
+ {
+ offset[0] = i * CHUNK_NX;
+ offset[1] = j * CHUNK_NY;
+ if (H5Dwrite_chunk(dset, dxpl, filter_mask, offset, buf_size, (void*)direct_buf[n]) < 0)
+ TEST_ERROR
+ }
+
+ /* Close the dataset then... */
+ if(H5Dclose(dset) < 0) TEST_ERROR
+
+ /* ...open it again to test the chunk query functions */
+ if((dset = H5Dopen2(chunkfile, DSET_SIMPLE_CHUNKED, H5P_DEFAULT)) < 0) TEST_ERROR
+
+ /* Get and verify the number of chunks written */
+ if (H5Dget_num_chunks(dset, mspace, &nchunks) < 0) TEST_ERROR;
+ if (nchunks != NUM_CHUNKS_WRITTEN) TEST_ERROR;
+
+ /* Read the entire dataset back */
+ start[0] = 0; start[1] = 0;
+ stride[0] = 1; stride[1] = 1;
+ count[0] = 1; count[1] = 1;
+ block[0] = CHUNK_NX; block[1] = CHUNK_NY;
+ if(H5Dread(dset, H5T_NATIVE_INT, fspace, fspace, H5P_DEFAULT, out_buf) < 0)
+ TEST_ERROR
+
+ /* Get and verify info of the first chunk */
+ index = 0;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+ if (out_offset[0] != 0 || out_offset[1] != 8) TEST_ERROR;
+
+ /* Get and verify info of the second chunk */
+ index = 1;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+ if (out_offset[0] != 0 || out_offset[1] != 12) TEST_ERROR;
+
+ /* Get and verify info of the third chunk */
+ index = 2;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+ if (out_offset[0] != 4 || out_offset[1] != 8) TEST_ERROR;
+
+ /* Get and verify info of the last chunk */
+ index = 3;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+ if (out_offset[0] != 4 || out_offset[1] != 12) TEST_ERROR;
+
+ /* Attempt to get info of empty chunk and verify the returned address and size */
+ index = 5;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (addr != HADDR_UNDEF) TEST_ERROR;
+ if (size != 0) TEST_ERROR;
+
+ /* Get info of the chunk at logical coordinates (0,2) */
+ offset[0] = 0;
+ offset[1] = 2 * CHUNK_NY;
+ if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR;
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+
+ /* Get info of the chunk at logical coordinates (1,3) */
+ offset[0] = 1 * CHUNK_NX;
+ offset[1] = 3 * CHUNK_NY;
+ if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR;
+ if (read_filter_mask != filter_mask) TEST_ERROR;
+ if (size != CHUNK_SIZE) TEST_ERROR;
+
+ /* Attempt to get info of empty chunks, verify the returned address and size */
+ offset[0] = 0;
+ offset[1] = 0;
+ if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR;
+ if (addr != HADDR_UNDEF) TEST_ERROR;
+ if (size != 0) TEST_ERROR;
+
+ offset[0] = 3 * CHUNK_NX;
+ offset[1] = 3 * CHUNK_NY;
+ if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR;
+ if (addr != HADDR_UNDEF) TEST_ERROR;
+ if (size != 0) TEST_ERROR;
+
+ /* Read each chunk and verify the values */
+ n = 0;
+ for (i = 0; i < 2; i++)
+ for (j = 2; j < 4; j++, n++)
+ if (read_each_chunk(dset, i*CHUNK_NX, j*CHUNK_NY, filter_mask, (void*)direct_buf[n]) < 0)
+ TEST_ERROR
+
+ /* Close the first dataset */
+ if (H5Dclose(dset) < 0) TEST_ERROR
+
+ /* Create an empty dataset */
+ if((dset = H5Dcreate2(chunkfile, DSET_EMPTY, H5T_NATIVE_INT, fspace,
+ H5P_DEFAULT, cparms, H5P_DEFAULT)) < 0) TEST_ERROR
+ if (H5Dclose(dset) < 0) TEST_ERROR
+
+ /* Reopen the empty dataset to verify the chunk query functions on it */
+ if((dset = H5Dopen2(chunkfile, DSET_EMPTY, H5P_DEFAULT)) < 0) TEST_ERROR
+
+ /* Verify that the number of chunks is 0 */
+ if (H5Dget_num_chunks(dset, mspace, &nchunks) < 0) TEST_ERROR;
+ if (nchunks != 0) TEST_ERROR;
+
+ /* Attempt to get info of a chunk from an empty dataset, verify the
+ returned address and size */
+ index = 0;
+ reinit_vars(&read_filter_mask, &addr, &size);
+ if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0)
+ TEST_ERROR
+ if (addr != HADDR_UNDEF) TEST_ERROR;
+ if (size != 0) TEST_ERROR;
+
+ /* Attempt to get info of a chunk given its coords from an empty dataset,
+ verify the returned address and size */
+ offset[0] = 0;
+ offset[1] = 0;
+ if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR;
+ if (addr != HADDR_UNDEF) TEST_ERROR;
+ if (size != 0) TEST_ERROR;
+
+ /* Close/release resources. */
+ if (H5Sclose(mspace) < 0) TEST_ERROR
+ if (H5Sclose(fspace) < 0) TEST_ERROR
+ if (H5Pclose(cparms) < 0) TEST_ERROR
+ if (H5Pclose(dxpl) < 0) TEST_ERROR
+ if (H5Fclose(chunkfile) < 0) TEST_ERROR
+
+ PASSED();
+ return SUCCEED;
+
+error:
+ H5E_BEGIN_TRY {
+ H5Dclose(dset);
+ H5Sclose(mspace);
+ H5Sclose(fspace);
+ H5Pclose(cparms);
+ H5Pclose(dxpl);
+ } H5E_END_TRY;
+
+ H5_FAILED();
+ return FAIL;
+} /* test_get_chunk_info() */
+
/*-------------------------------------------------------------------------
* Function: main
@@ -13232,7 +13568,10 @@ main(void)
nerrors += (test_gather_error() < 0 ? 1 : 0);
/* Tests version bounds using its own file */
- nerrors += (test_versionbounds() < 0 ? 1 : 0);
+ nerrors += (test_versionbounds() < 0 ? 1 : 0);
+
+ /* Tests getting chunk information */
+ nerrors += (test_get_chunk_info() < 0 ? 1 : 0);
/* Run misc tests */
nerrors += dls_01_main();