From 0b321904a3be4d3988f99fca158d7e576ddb6df2 Mon Sep 17 00:00:00 2001 From: Binh-Minh Ribler Date: Mon, 29 Oct 2018 09:52:50 -0500 Subject: New API functions Description: Added functions to query chunk information: H5Dget_num_chunks(dset_id, fspace_id, *nchunks) Gets the number of written chunks that intersect with the given dataspace. However, in this version, the intersection is not yet completed. Thus, the number of all written chunks will be returned. H5Dget_chunk_info_by_coord(dset_id, *coord, *filter_mask, *addr, *size) Given a chunk's coordinates, returns the chunk's filter, address, and size. H5Dget_chunk_info(dset_id, fspace_id, index, *coord, *filter_mask, *addr, *size) Given a chunk's index, returns the chunk's coordinates, filter, address, and size. The chunk belongs to a set of chunks that have nonempty intersection with the specified dataspace. However, in this version, the intersection is not yet completed, and the index is of all the written chunks. Platforms tested: Linux/64 (jelly) Linux/64 (platypus) Darwin (osx1011test) --- src/H5D.c | 153 ++++++++++++++++++++++++++++ src/H5Dchunk.c | 309 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ src/H5Dpkg.h | 3 + src/H5Dpublic.h | 3 + test/dsets.c | 281 ++++++++++++++++++++++++++++++++++++++++++++++++++- 5 files changed, 748 insertions(+), 1 deletion(-) diff --git a/src/H5D.c b/src/H5D.c index 61ccb5a..286ab4e 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -1130,3 +1130,156 @@ done: FUNC_LEAVE_API(ret_value); } /* H5Dget_chunk_storage_size() */ + +/*------------------------------------------------------------------------- + * Function: H5Dget_num_chunks + * + * Purpose: Retrieves the number of chunks that have nonempty intersection + * with a specified selection. + * + * Note: Currently, this function only gets the number of all written + * chunks, regardless the dataspace. + * + * Parameters: + * hid_t dset_id; IN: Chunked dataset ID + * hid_t fspace_id; IN: File dataspace ID + * hsize_t *nchunks; OUT:: Number of non-empty chunks + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Binh-Minh Ribler + * August 2018 (EED-343) + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks) +{ + H5D_t *dset = NULL; + const H5S_t *space; /* Dataspace for dataset */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ii*h", dset_id, fspace_id, nchunks); + + /* Check arguments */ + if(NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") + if(NULL == (space = (const H5S_t *)H5I_object_verify(fspace_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace ID") + if(NULL == nchunks) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)") + + if(H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + + /* Get the number of written chunks */ + if(H5D__get_num_chunks(dset, space, nchunks) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "error getting number of chunks") + +done: + FUNC_LEAVE_API(ret_value); +} /* H5Dget_num_chunks() */ + + +/*------------------------------------------------------------------------- + * Function: H5Dget_chunk_info + * + * Purpose: Retrieves information about a chunk specified by its index. + * + * Parameters: + * hid_t dset_id; IN: Chunked dataset ID + * hid_t fspace_id; IN: File dataspace ID + * hsize_t index; IN: Index of written chunk + * hsize_t *offset OUT: Offset coordinates of the chunk + * unsigned *filter_mask OUT: Filter mask + * haddr_t *addr OUT: Address of the chunk + * hsize_t *size OUT: Size of the chunk + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Binh-Minh Ribler + * August 2018 (EED-343) + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t index, hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size) +{ + H5D_t *dset = NULL; + const H5S_t *space; /* Dataspace for dataset */ + hsize_t space_allocated = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE7("e", "iih*h*Iu*a*h", dset_id, fspace_id, index, offset, filter_mask, + addr, size); + + /* Check arguments */ + if(NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset ID") + if(NULL == (space = (const H5S_t *)H5I_object_verify(fspace_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace ID") + if(index < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)") + if(NULL == offset && NULL == filter_mask && NULL == addr && NULL == size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid arguments, must have at least one non-null output argument") + + if(H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + + /* Call private function to get the chunk info given the chunk's index */ + if(H5D__get_chunk_info(dset, space, index, offset, filter_mask, addr, size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't get chunk info") + +done: + FUNC_LEAVE_API(ret_value); +} /* H5Dget_chunk_info() */ + + +/*------------------------------------------------------------------------- + * Function: H5Dget_chunk_info_by_coord + * + * Purpose: Retrieves information about a chunk specified by its offset + * coordinates. + * + * Parameters: + * hid_t dset_id IN: Chunked dataset ID + * hsize_t *offset IN: Coordinates of the chunk + * unsigned *filter_mask OUT: Filter mask + * haddr_t *addr OUT: Address of the chunk + * hsize_t *size OUT: Size of the chunk + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Binh-Minh Ribler + * August 2018 (EED-343) + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size) +{ + H5D_t *dset = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE5("e", "i*h*Iu*a*h", dset_id, offset, filter_mask, addr, size); + + /* Check arguments */ + if(NULL == (dset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") + if(NULL == offset) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)") + if(NULL == filter_mask && NULL == addr && NULL == size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid arguments, must have at least one non-null output argument") + + if(H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + + /* Internal function to get the chunk info */ + if (H5D__get_chunk_info_by_coord(dset, offset, filter_mask, addr, size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data") +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dget_chunk_info_by_coord() */ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index c3f4a95..6bc89d1 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -216,6 +216,17 @@ typedef struct H5D_chunk_readvv_ud_t { const H5D_t *dset; /* Dataset to operate on */ } H5D_chunk_readvv_ud_t; +/* Typedef for chunk info iterator callback */ +typedef struct { + hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Logical offset of the chunk */ + hsize_t ndims; /* Number of dimension in the dataset */ + uint32_t nbytes; /* Size of stored data in the chunk */ + unsigned filter_mask; /* Excluded filters */ + haddr_t chunk_addr; /* Address of the chunk in file */ + hsize_t chunk_idx; /* Chunk index, where the iteration needs to stop */ + hsize_t curr_idx; /* Current index, where the iteration is */ +} H5D_chunk_info_iter_ud_t; + /* Callback info for file selection iteration */ typedef struct H5D_chunk_file_iter_ud_t { H5D_chunk_map_t *fm; /* File->memory chunk mapping info */ @@ -6726,3 +6737,301 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_format_convert() */ + +/*------------------------------------------------------------------------- + * Function: H5D__get_num_chunks_cb + * + * Purpose: Callback function that increments the number of written + * chunks in the dataset. + * + * Return: Success: H5_ITER_CONT or H5_ITER_STOP + * Failure: Negative (H5_ITER_ERROR) + * + * Programmer: Binh-Minh Ribler + * September 2018 (EED-343) + * + *------------------------------------------------------------------------- + */ +static int +H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) +{ + int ret_value = H5_ITER_CONT; /* Callback return value */ + + hsize_t *num_chunks = (hsize_t *)_udata; + + FUNC_ENTER_STATIC_NOERR + + (*num_chunks)++; + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__get_num_chunks_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_num_chunks + * + * Purpose: Gets the number of written chunks in a dataset. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Binh-Minh Ribler + * September 2018 (EED-343) + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_num_chunks(const H5D_t *dset, const H5S_t *space, hsize_t *nchunks) +{ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + hsize_t num_chunks = 0; /* Number of written chunks */ + H5D_rdcc_ent_t *ent; /* Cache entry */ + const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk);/* Raw data chunk cache */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) + + HDassert(dset); + HDassert(dset->shared); + + /* Search for cached chunks that haven't been written out */ + for(ent = rdcc->head; ent; ent = ent->next) + /* Flush the chunk out to disk, to make certain the size is correct later */ + if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.pline = &dset->shared->dcpl_cache.pline; + idx_info.layout = &dset->shared->layout.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; + + /* Iterate over the allocated chunks */ + if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_num_chunks_cb, &num_chunks) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index") + *nchunks = num_chunks; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__get_num_chunks() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_chunk_info_cb + * + * Purpose: Get the chunk info of the desired chunk, given by its index. + * + * Return: Success: H5_ITER_CONT or H5_ITER_STOP + * Failure: Negative (H5_ITER_ERROR) + * + * Programmer: Binh-Minh Ribler + * September 2018 (EED-343) + * + *------------------------------------------------------------------------- + */ +static int +H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) +{ + hsize_t ii; + int ret_value = H5_ITER_CONT; /* Callback return value */ + + H5D_chunk_info_iter_ud_t *chunk_info = (H5D_chunk_info_iter_ud_t *)_udata; + + FUNC_ENTER_STATIC_NOERR + + /* If this is the desired chunk, retrieve its info and stop iterating */ + if (chunk_info->curr_idx == chunk_info->chunk_idx) + { + chunk_info->filter_mask = chunk_rec->filter_mask; + chunk_info->chunk_addr = chunk_rec->chunk_addr; + chunk_info->nbytes = chunk_rec->nbytes; + + for (ii = 0; ii < chunk_info->ndims; ii++) + chunk_info->scaled[ii] = chunk_rec->scaled[ii]; + + /* Stop iterating */ + ret_value = H5_ITER_STOP; + } + /* Iterate the next chunk */ + else + chunk_info->curr_idx++; + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__get_chunk_info_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_chunk_info + * + * Purpose: Iterate over the chunks in the dataset to get the info + * of the desired chunk. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Binh-Minh Ribler + * September 2018 (EED-343) + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_t index, hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size) +{ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + H5D_chunk_info_iter_ud_t udata; + const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk);/* Raw data chunk cache */ + H5D_rdcc_ent_t *ent; /* Cache entry */ + hsize_t ii = 0; /* Dimension index */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) + + HDassert(dset); + HDassert(dset->shared); + + /* Search for cached chunks that haven't been written out */ + for(ent = rdcc->head; ent; ent = ent->next) + /* Flush the chunk out to disk, to make certain the size is correct later */ + if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.pline = &dset->shared->dcpl_cache.pline; + idx_info.layout = &dset->shared->layout.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; + + /* Initialize for iteration */ + udata.chunk_idx = index; + udata.curr_idx = 0; + udata.ndims = dset->shared->ndims; + + /* Iterate over the allocated chunks */ + if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_cb, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index") + + /* Obtain requested info */ + if (filter_mask) + *filter_mask = udata.filter_mask; + if (addr) + *addr = udata.chunk_addr; + if (size) + *size = udata.nbytes; + if (offset) + for (ii = 0; ii < udata.ndims; ii++) + offset[ii] = udata.scaled[ii] * dset->shared->layout.u.chunk.dim[ii]; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__get_chunk_info() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_chunk_info_by_coord_cb + * + * Purpose: Get the chunk info of the desired chunk, given its offset + * coordinates. + * + * Return: Success: H5_ITER_CONT or H5_ITER_STOP + * Failure: Negative (H5_ITER_ERROR) + * + * Programmer: Binh-Minh Ribler + * September 2018 (EED-343) + * + *------------------------------------------------------------------------- + */ +static int +H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) +{ + hbool_t different = FALSE; +int ii; + int ret_value = H5_ITER_CONT; /* Callback return value */ + + H5D_chunk_info_iter_ud_t *chunk_info = (H5D_chunk_info_iter_ud_t *)_udata; + + FUNC_ENTER_STATIC_NOERR + + for (ii = 0; ii < chunk_info->ndims && !different; ii++) + { + if (chunk_info->scaled[ii] != chunk_rec->scaled[ii]) + different = TRUE; + } + if (!different) + { + chunk_info->nbytes = chunk_rec->nbytes; + chunk_info->filter_mask = chunk_rec->filter_mask; + chunk_info->chunk_addr = chunk_rec->chunk_addr; + + /* Stop iterating */ + ret_value = H5_ITER_STOP; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__get_chunk_info_by_coord_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_chunk_info_by_coord + * + * Purpose: Iterate over the chunks in the dataset to get the info + * of the desired chunk, given by its offset coordinates. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Binh-Minh Ribler + * September 2018 (EED-343) + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned* filter_mask, haddr_t *addr, hsize_t *size) +{ + const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ + const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /* Raw data chunk cache */ + H5D_rdcc_ent_t *ent; /* Cache entry */ + H5D_chunk_info_iter_ud_t udata; + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) + + /* Check args */ + HDassert(dset && H5D_CHUNKED == layout->type); + HDassert(offset); + HDassert(filter_mask); + HDassert(addr); /* Question: should some OUT args be allowed to be NULL? */ + HDassert(size); + + /* Search for cached chunks that haven't been written out */ + for(ent = rdcc->head; ent; ent = ent->next) + /* Flush the chunk out to disk, to make certain the size is correct later */ + if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") + + /* Calculate the scaled of this chunk */ + H5VM_chunk_scaled(dset->shared->ndims, offset, layout->u.chunk.dim, udata.scaled); + udata.scaled[dset->shared->ndims] = 0; + + /* Get the number of dimensions for use in callback function */ + udata.ndims = dset->shared->ndims; + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.pline = &dset->shared->dcpl_cache.pline; + idx_info.layout = &dset->shared->layout.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; + + /* Iterate over the allocated chunks */ + if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_by_coord_cb, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from scaled") + + /* Return the filter mask and chunk address and size */ + *filter_mask = udata.filter_mask; + *addr = udata.chunk_addr; + *size = udata.nbytes; + +done: + FUNC_LEAVE_NOAPI_TAG(ret_value) +} /* end H5D__get_chunk_info_by_coord() */ + diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index b887b87..7d550b5 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -566,6 +566,9 @@ H5_DLL herr_t H5D__alloc_storage(const H5D_io_info_t *io_info, H5D_time_alloc_t H5_DLL herr_t H5D__get_storage_size(const H5D_t *dset, hsize_t *storage_size); H5_DLL herr_t H5D__get_chunk_storage_size(H5D_t *dset, const hsize_t *offset, hsize_t *storage_size); +H5_DLL herr_t H5D__get_num_chunks(const H5D_t *dset, const H5S_t *space, hsize_t *nchunks); +H5_DLL herr_t H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_t index, hsize_t *coord, unsigned *filter_mask, haddr_t *offset, hsize_t *size); +H5_DLL herr_t H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *coord, unsigned* filter_mask, haddr_t *addr, hsize_t *size); H5_DLL haddr_t H5D__get_offset(const H5D_t *dset); H5_DLL void *H5D__vlen_get_buf_size_alloc(size_t size, void *info); H5_DLL herr_t H5D__vlen_get_buf_size(void *elem, hid_t type_id, unsigned ndim, diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index a1ccda0..fcc76ee 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -141,6 +141,9 @@ H5_DLL hid_t H5Dget_create_plist(hid_t dset_id); H5_DLL hid_t H5Dget_access_plist(hid_t dset_id); H5_DLL hsize_t H5Dget_storage_size(hid_t dset_id); H5_DLL herr_t H5Dget_chunk_storage_size(hid_t dset_id, const hsize_t *offset, hsize_t *chunk_bytes); +H5_DLL herr_t H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks); +H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size); +H5_DLL herr_t H5Dget_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t index, hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size); H5_DLL haddr_t H5Dget_offset(hid_t dset_id); H5_DLL herr_t H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, void *buf/*out*/); diff --git a/test/dsets.c b/test/dsets.c index d23f438..e8d86d1 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -84,6 +84,7 @@ const char *FILENAME[] = { "dls_01_strings", /* 23 */ "power2up", /* 24 */ "version_bounds", /* 25 */ + "chunk_info", /* 26 */ NULL }; @@ -268,6 +269,17 @@ const char *FILENAME[] = { #define STORAGE_SIZE_CHUNK_DIM1 5 #define STORAGE_SIZE_CHUNK_DIM2 5 +/* Parameters for testing chunk querying */ +#define DSET_SIMPLE_CHUNKED "Chunked Dataset" +#define RANK 2 +#define NX 16 +#define NY 16 +#define CHUNK_NX 4 +#define CHUNK_NY 4 +#define CHUNK_SIZE 64 +#define NUM_CHUNKS 16 +#define NUM_CHUNKS_WRITTEN 4 + /* Shared global arrays */ #define DSET_DIM1 100 #define DSET_DIM2 200 @@ -13034,6 +13046,270 @@ test_versionbounds() return -1; } /* test_versionbounds() */ +/*------------------------------------------------------------------------- + * Function: read_each_chunk (helper function) + * + * Purpose: Reads the chunk specified by its offset and verifies that + * it contains the same data as what was written. This function + * is used in test_get_chunk_info. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Date: September 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t read_each_chunk(hid_t dset_id, hsize_t offset1, hsize_t offset2, unsigned check_filter_mask, void *direct_buf) +{ + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + int read_buf[CHUNK_NX][CHUNK_NY]; + hsize_t offset[2] = {offset1, offset2}; + unsigned read_filter_mask = 0; + + /* Read the raw chunk back */ + HDmemset(&read_buf, 0, sizeof(read_buf)); + + /* Read the chunk specified by its offset */ + if (H5Dread_chunk(dset_id, H5P_DEFAULT, offset, &read_filter_mask, read_buf) < 0) + return(FAIL); + + /* Verify that read chunk is the same as the corresponding written one */ + if (HDmemcmp(direct_buf, read_buf, CHUNK_NX*CHUNK_NY) != 0) + { + fprintf(stderr, "Read chunk differs than written chunk at offset (%d,%d)\n", offset1, offset2); + return(FAIL); + } + + return(SUCCEED); +} + +/*------------------------------------------------------------------------- + * Function: reinit_vars (helper function) + * + * Purpose: Helper function to wipe out variables for the next use, + * used in test_get_chunk_info. + * + * Return: Won't fail + * + * Date: September 2018 + * + *------------------------------------------------------------------------- + */ +void reinit_vars(unsigned *read_filter_mask, hsize_t *addr, hsize_t *size) +{ + if (read_filter_mask) *read_filter_mask = 0; + if (addr) *addr = 0; + if (size) *size = 0; +} + +/*------------------------------------------------------------------------- + * Function: test_get_chunk_info + * + * Purpose: Test getting various chunk information + * + * Return: Success: 0 + * Failure: 1 + * + * Date: September 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_get_chunk_info() +{ + hid_t chunkfile = -1, fapl = -1; + hid_t fspace = -1, dset = -1; + hid_t mspace = -1; + hid_t cparms = -1, dxpl = -1; + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; + int fillvalue = -1; + char filename[FILENAME_BUF_SIZE]; + unsigned filter_mask = 0; + int direct_buf[16][CHUNK_NX][CHUNK_NY]; + int out_buf[NX][NY]; + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + int aggression = 9; /* Compression aggression setting */ + unsigned read_filter_mask = 0; /* filter mask after direct read */ + H5F_libver_t low, high; /* File format bounds */ + hsize_t start[2]; /* Start of hyperslab */ + hsize_t stride[2]; /* Stride of hyperslab */ + hsize_t count[2]; /* Block count */ + hsize_t block[2]; /* Block sizes */ + hsize_t offset[2]; + hsize_t coord[H5S_MAX_RANK]; + hsize_t out_offset[2] = {-1, -1}; + hsize_t size = 0; + hsize_t nchunks = 0; + hsize_t addr = 0; + hsize_t index = 0; + hsize_t i, j, n; + + TESTING("getting chunk information"); + + /* Create a copy of file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR + + /* Set high bound to V18 */ + low = H5F_LIBVER_EARLIEST; + high = H5F_LIBVER_V18; + if (H5Pset_libver_bounds(fapl, low, high) < 0) + TEST_ERROR; + + /* Create a file */ + h5_fixname(FILENAME[26], fapl, filename, sizeof filename); + if((chunkfile = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Create the data space */ + if((fspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + TEST_ERROR + + if((mspace = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + TEST_ERROR + + /* Modify dataset creation properties, i.e. enable chunking and compression */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR + + if(H5Pset_chunk(cparms, RANK, chunk_dims) < 0) + TEST_ERROR + + if(H5Pset_deflate(cparms, (unsigned ) aggression) < 0) + TEST_ERROR + + if (H5Pset_fill_value(cparms, H5T_NATIVE_INT, &fillvalue) < 0) + TEST_ERROR; + + /* Create a new dataset using cparms creation properties */ + if((dset = H5Dcreate2(chunkfile, DSET_SIMPLE_CHUNKED, H5T_NATIVE_INT, fspace, + H5P_DEFAULT, cparms, H5P_DEFAULT)) < 0) TEST_ERROR + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + TEST_ERROR + + /* Indicate skipping the compression filter. */ + filter_mask = 0x00000001; + + /* Initialize the array of chunk data, that is for all 16 chunks */ + for(n = 0; n < 16; n++) + for(i = 0; i < CHUNK_NX; i++) + for(j = 0; j < CHUNK_NY; j++) + direct_buf[n][i][j] = n + 1; + + /* Write NUM_CHUNKS_WRITTEN chunks: (0,2) (0,3) (1,2) (1,3) */ + n = 0; + for (i = 0; i < 2; i++) + for (j = 2; j < 4; j++, n++) + { + offset[0] = i * CHUNK_NX; + offset[1] = j * CHUNK_NY; + if (H5Dwrite_chunk(dset, dxpl, filter_mask, offset, buf_size, (void*)direct_buf[n]) < 0) + TEST_ERROR + } + + /* Close the dataset then... */ + if(H5Dclose(dset) < 0) TEST_ERROR + + /* ...open it again to test the chunk query functions */ + if((dset = H5Dopen2(chunkfile, DSET_SIMPLE_CHUNKED, H5P_DEFAULT)) < 0) TEST_ERROR + + /* Get and verify the number of chunks written */ + if (H5Dget_num_chunks(dset, mspace, &nchunks) < 0) TEST_ERROR; + if (nchunks != NUM_CHUNKS_WRITTEN) TEST_ERROR; + + /* Read the entire dataset back */ + start[0] = 0; start[1] = 0; + stride[0] = 1; stride[1] = 1; + count[0] = 1; count[1] = 1; + block[0] = CHUNK_NX; block[1] = CHUNK_NY; + if(H5Dread(dset, H5T_NATIVE_INT, fspace, fspace, H5P_DEFAULT, out_buf) < 0) + TEST_ERROR + + /* Get and verify info of the first chunk */ + index = 0; + reinit_vars(&read_filter_mask, &addr, &size); + if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0) + TEST_ERROR + if (read_filter_mask != filter_mask) TEST_ERROR; + if (size != CHUNK_SIZE) TEST_ERROR; + if (out_offset[0] != 0 || out_offset[1] != 8) TEST_ERROR; + + /* Get and verify info of the second chunk */ + index = 1; + reinit_vars(&read_filter_mask, &addr, &size); + if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0) + TEST_ERROR + if (read_filter_mask != filter_mask) TEST_ERROR; + if (size != CHUNK_SIZE) TEST_ERROR; + if (out_offset[0] != 0 || out_offset[1] != 12) TEST_ERROR; + + /* Get and verify info of the third chunk */ + index = 2; + reinit_vars(&read_filter_mask, &addr, &size); + if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0) + TEST_ERROR + + if (read_filter_mask != filter_mask) TEST_ERROR; + if (size != CHUNK_SIZE) TEST_ERROR; + if (out_offset[0] != 4 || out_offset[1] != 8) TEST_ERROR; + + /* Get and verify info of the last chunk */ + index = 3; + reinit_vars(&read_filter_mask, &addr, &size); + if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0) + TEST_ERROR + if (read_filter_mask != filter_mask) TEST_ERROR; + if (size != CHUNK_SIZE) TEST_ERROR; + if (out_offset[0] != 4 || out_offset[1] != 12) TEST_ERROR; + + /* Get info of the chunk at logical coordinates (0,2) */ + offset[0] = 0; + offset[1] = 2 * CHUNK_NY; + if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR; + if (read_filter_mask != filter_mask) TEST_ERROR; + if (size != CHUNK_SIZE) TEST_ERROR; + + /* Get info of the chunk at logical coordinates (1,3) */ + offset[0] = 1 * CHUNK_NX; + offset[1] = 3 * CHUNK_NY; + if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR; + if (read_filter_mask != filter_mask) TEST_ERROR; + if (size != CHUNK_SIZE) TEST_ERROR; + + /* Read each chunk and print the values */ + n = 0; + for (i = 0; i < 2; i++) + for (j = 2; j < 4; j++, n++) + if (read_each_chunk(dset, i*CHUNK_NX, j*CHUNK_NY, filter_mask, (void*)direct_buf[n]) < 0) + TEST_ERROR + + /* Close/release resources. */ + H5Dclose(dset); + H5Sclose(mspace); + H5Sclose(fspace); + H5Pclose(cparms); + H5Pclose(dxpl); + H5Fclose(chunkfile); + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Sclose(mspace); + H5Sclose(fspace); + H5Pclose(cparms); + H5Pclose(dxpl); + } H5E_END_TRY; + + H5_FAILED(); + return 1; +} /* test_get_chunk_info() */ + /*------------------------------------------------------------------------- * Function: main @@ -13232,7 +13508,10 @@ main(void) nerrors += (test_gather_error() < 0 ? 1 : 0); /* Tests version bounds using its own file */ - nerrors += (test_versionbounds() < 0 ? 1 : 0); + nerrors += (test_versionbounds() < 0 ? 1 : 0); + + /* Tests getting chunk information */ + nerrors += (test_get_chunk_info() < 0 ? 1 : 0); /* Run misc tests */ nerrors += dls_01_main(); -- cgit v0.12 From 471917c6384587c5de7ff4473faaae5d3fa248a6 Mon Sep 17 00:00:00 2001 From: Binh-Minh Ribler Date: Sun, 4 Nov 2018 19:54:41 -0600 Subject: Updated new API functions Description: - Per Vailin's review, revised H5Dget_chunk_info_by_coord to handle non-existing chunk and H5Dget_num_chunks and H5Dget_chunk_info to handle dataset with no data. - Addressed other review comments - Note that additional tests will be added as we need to send users these functions asap for feedback. Platforms tested: Linux/64 (jelly) Linux/64 (platypus) Darwin (osx1011test) --- src/H5D.c | 4 +- src/H5Dchunk.c | 226 ++++++++++++++++++++++++++++++++++++++++----------------- src/H5Dpkg.h | 2 +- test/dsets.c | 94 +++++++++++++++++++----- 4 files changed, 239 insertions(+), 87 deletions(-) diff --git a/src/H5D.c b/src/H5D.c index 286ab4e..32b2453 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -1230,7 +1230,7 @@ H5Dget_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t index, hsize_t *offset /* Call private function to get the chunk info given the chunk's index */ if(H5D__get_chunk_info(dset, space, index, offset, filter_mask, addr, size) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't get chunk info") + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info") done: FUNC_LEAVE_API(ret_value); @@ -1279,7 +1279,7 @@ H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filte /* Internal function to get the chunk info */ if (H5D__get_chunk_info_by_coord(dset, offset, filter_mask, addr, size) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read unprocessed chunk data") + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't get chunk info") done: FUNC_LEAVE_API(ret_value) } /* end H5Dget_chunk_info_by_coord() */ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 6bc89d1..7e7a94c 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -217,14 +217,15 @@ typedef struct H5D_chunk_readvv_ud_t { } H5D_chunk_readvv_ud_t; /* Typedef for chunk info iterator callback */ -typedef struct { +typedef struct H5D_chunk_info_iter_ud_t { hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Logical offset of the chunk */ - hsize_t ndims; /* Number of dimension in the dataset */ + hsize_t ndims; /* Number of dimensions in the dataset */ uint32_t nbytes; /* Size of stored data in the chunk */ unsigned filter_mask; /* Excluded filters */ haddr_t chunk_addr; /* Address of the chunk in file */ hsize_t chunk_idx; /* Chunk index, where the iteration needs to stop */ hsize_t curr_idx; /* Current index, where the iteration is */ + hbool_t found; /* Whether the chunk was found */ } H5D_chunk_info_iter_ud_t; /* Callback info for file selection iteration */ @@ -263,6 +264,12 @@ static herr_t H5D__chunk_flush(H5D_t *dset); static herr_t H5D__chunk_io_term(const H5D_chunk_map_t *fm); static herr_t H5D__chunk_dest(H5D_t *dset); +/* Chunk query operation callbacks */ +static int H5D__get_num_chunks_cb(const H5D_chunk_rec_t H5_ATTR_UNUSED *chunk_rec, void *_udata); +static int H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); +static int H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); + + /* "Nonexistent" layout operation callback */ static ssize_t H5D__nonexistent_readvv(const H5D_io_info_t *io_info, @@ -6744,6 +6751,9 @@ done: * Purpose: Callback function that increments the number of written * chunks in the dataset. * + * Note: Currently, this function only gets the number of all written + * chunks, regardless the dataspace. + * * Return: Success: H5_ITER_CONT or H5_ITER_STOP * Failure: Negative (H5_ITER_ERROR) * @@ -6753,14 +6763,15 @@ done: *------------------------------------------------------------------------- */ static int -H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) +H5D__get_num_chunks_cb(const H5D_chunk_rec_t H5_ATTR_UNUSED *chunk_rec, void *_udata) { - int ret_value = H5_ITER_CONT; /* Callback return value */ - hsize_t *num_chunks = (hsize_t *)_udata; + int ret_value = H5_ITER_CONT; /* Callback return value */ FUNC_ENTER_STATIC_NOERR + HDassert(num_chunks); + (*num_chunks)++; FUNC_LEAVE_NOAPI(ret_value) @@ -6772,6 +6783,9 @@ H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) * * Purpose: Gets the number of written chunks in a dataset. * + * Note: Currently, this function only gets the number of all written + * chunks, regardless the dataspace. + * * Return: Success: Non-negative * Failure: Negative * @@ -6781,18 +6795,24 @@ H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) *------------------------------------------------------------------------- */ herr_t -H5D__get_num_chunks(const H5D_t *dset, const H5S_t *space, hsize_t *nchunks) +H5D__get_num_chunks(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t *nchunks) { - H5D_chk_idx_info_t idx_info; /* Chunked index info */ - hsize_t num_chunks = 0; /* Number of written chunks */ - H5D_rdcc_ent_t *ent; /* Cache entry */ - const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk);/* Raw data chunk cache */ - herr_t ret_value = SUCCEED; /* Return value */ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + hsize_t num_chunks = 0; /* Number of written chunks */ + H5D_rdcc_ent_t *ent; /* Cache entry */ + const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) HDassert(dset); HDassert(dset->shared); + HDassert(space); + HDassert(nchunks); + + /* Get the raw data chunk cache */ + rdcc = &(dset->shared->cache.chunk); + HDassert(rdcc); /* Search for cached chunks that haven't been written out */ for(ent = rdcc->head; ent; ent = ent->next) @@ -6806,10 +6826,17 @@ H5D__get_num_chunks(const H5D_t *dset, const H5S_t *space, hsize_t *nchunks) idx_info.layout = &dset->shared->layout.u.chunk; idx_info.storage = &dset->shared->layout.storage.u.chunk; + /* If the dataset is not written, number of chunks will be 0 */ + if (idx_info.storage->idx_addr == HADDR_UNDEF) { + *nchunks = 0; + HGOTO_DONE(SUCCEED); + } + else { /* Iterate over the allocated chunks */ if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_num_chunks_cb, &num_chunks) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index") *nchunks = num_chunks; + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -6832,22 +6859,24 @@ done: static int H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) { - hsize_t ii; - int ret_value = H5_ITER_CONT; /* Callback return value */ - H5D_chunk_info_iter_ud_t *chunk_info = (H5D_chunk_info_iter_ud_t *)_udata; + hsize_t ii = 0; /* Dimension index */ + int ret_value = H5_ITER_CONT; /* Callback return value */ FUNC_ENTER_STATIC_NOERR + /* Check args */ + HDassert(chunk_rec); + HDassert(chunk_info); + /* If this is the desired chunk, retrieve its info and stop iterating */ - if (chunk_info->curr_idx == chunk_info->chunk_idx) - { + if (chunk_info->curr_idx == chunk_info->chunk_idx) { chunk_info->filter_mask = chunk_rec->filter_mask; chunk_info->chunk_addr = chunk_rec->chunk_addr; chunk_info->nbytes = chunk_rec->nbytes; - for (ii = 0; ii < chunk_info->ndims; ii++) chunk_info->scaled[ii] = chunk_rec->scaled[ii]; + chunk_info->found = TRUE; /* Stop iterating */ ret_value = H5_ITER_STOP; @@ -6866,6 +6895,9 @@ H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) * Purpose: Iterate over the chunks in the dataset to get the info * of the desired chunk. * + * Note: Currently, this function only gets the number of all written + * chunks, regardless the dataspace. + * * Return: Success: SUCCEED * Failure: FAIL * @@ -6875,19 +6907,24 @@ H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) *------------------------------------------------------------------------- */ herr_t -H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_t index, hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size) +H5D__get_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t index, hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size) { - H5D_chk_idx_info_t idx_info; /* Chunked index info */ - H5D_chunk_info_iter_ud_t udata; - const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk);/* Raw data chunk cache */ - H5D_rdcc_ent_t *ent; /* Cache entry */ - hsize_t ii = 0; /* Dimension index */ - herr_t ret_value = SUCCEED; /* Return value */ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + H5D_chunk_info_iter_ud_t udata; /* User data for callback */ + const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */ + H5D_rdcc_ent_t *ent; /* Cache entry index */ + hsize_t ii = 0; /* Dimension index */ + herr_t ret_value = SUCCEED;/* Return value */ FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) HDassert(dset); HDassert(dset->shared); + HDassert(space); + + /* Get the raw data chunk cache */ + rdcc = &(dset->shared->cache.chunk); + HDassert(rdcc); /* Search for cached chunks that haven't been written out */ for(ent = rdcc->head; ent; ent = ent->next) @@ -6901,25 +6938,47 @@ H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_t index, hsize_ idx_info.layout = &dset->shared->layout.u.chunk; idx_info.storage = &dset->shared->layout.storage.u.chunk; - /* Initialize for iteration */ + /* If the dataset is not written, return the address as undefined */ + if (idx_info.storage->idx_addr == HADDR_UNDEF) { + if (addr) + *addr = HADDR_UNDEF; + if (size) + *size = 0; + HGOTO_DONE(SUCCEED); + } + + /* Initialize before iteration */ udata.chunk_idx = index; udata.curr_idx = 0; udata.ndims = dset->shared->ndims; + udata.nbytes = 0; + udata.filter_mask = 0; + udata.chunk_addr = HADDR_UNDEF; + udata.found = FALSE; /* Iterate over the allocated chunks */ if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_cb, &udata) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index") - /* Obtain requested info */ - if (filter_mask) - *filter_mask = udata.filter_mask; - if (addr) - *addr = udata.chunk_addr; - if (size) - *size = udata.nbytes; - if (offset) - for (ii = 0; ii < udata.ndims; ii++) - offset[ii] = udata.scaled[ii] * dset->shared->layout.u.chunk.dim[ii]; + /* Obtain requested info if the chunk is found */ + if (udata.found) { + if (filter_mask) + *filter_mask = udata.filter_mask; + if (addr) + *addr = udata.chunk_addr; + if (size) + *size = udata.nbytes; + if (offset) + for (ii = 0; ii < udata.ndims; ii++) + offset[ii] = udata.scaled[ii] * dset->shared->layout.u.chunk.dim[ii]; + } + /* otherwise, return HADDR_UNDEF for address and 0 for size */ + else { + if (addr) + *addr = HADDR_UNDEF; + if (size) + *size = 0; + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -6943,24 +7002,28 @@ done: static int H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) { - hbool_t different = FALSE; -int ii; - int ret_value = H5_ITER_CONT; /* Callback return value */ - + hsize_t ii; H5D_chunk_info_iter_ud_t *chunk_info = (H5D_chunk_info_iter_ud_t *)_udata; + hbool_t different = FALSE; /* TRUE when a scaled value pair mismatch */ + int ret_value = H5_ITER_CONT; /* Callback return value */ FUNC_ENTER_STATIC_NOERR + /* Check args */ + HDassert(chunk_rec); + HDassert(chunk_info); + + /* Going through the scaled, stop when a mismatch is found */ for (ii = 0; ii < chunk_info->ndims && !different; ii++) - { if (chunk_info->scaled[ii] != chunk_rec->scaled[ii]) different = TRUE; - } - if (!different) - { + + /* Same scaled coords means the chunk is found, copy the chunk info */ + if (!different) { chunk_info->nbytes = chunk_rec->nbytes; chunk_info->filter_mask = chunk_rec->filter_mask; chunk_info->chunk_addr = chunk_rec->chunk_addr; + chunk_info->found = TRUE; /* Stop iterating */ ret_value = H5_ITER_STOP; @@ -6987,21 +7050,26 @@ int ii; herr_t H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned* filter_mask, haddr_t *addr, hsize_t *size) { - const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ - const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /* Raw data chunk cache */ - H5D_rdcc_ent_t *ent; /* Cache entry */ - H5D_chunk_info_iter_ud_t udata; - H5D_chk_idx_info_t idx_info; /* Chunked index info */ - herr_t ret_value = SUCCEED; /* Return value */ + const H5O_layout_t *layout = NULL; /* Dataset layout */ + const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */ + H5D_rdcc_ent_t *ent; /* Cache entry index */ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + H5D_chunk_info_iter_ud_t udata; /* User data for callback */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) /* Check args */ - HDassert(dset && H5D_CHUNKED == layout->type); + HDassert(dset); + HDassert(dset->shared); HDassert(offset); - HDassert(filter_mask); - HDassert(addr); /* Question: should some OUT args be allowed to be NULL? */ - HDassert(size); + + /* Get dataset layout and raw data chunk cache */ + layout = &(dset->shared->layout); + rdcc = &(dset->shared->cache.chunk); + HDassert(layout); + HDassert(rdcc); + HDassert(H5D_CHUNKED == layout->type); /* Search for cached chunks that haven't been written out */ for(ent = rdcc->head; ent; ent = ent->next) @@ -7009,28 +7077,52 @@ H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned* if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") - /* Calculate the scaled of this chunk */ - H5VM_chunk_scaled(dset->shared->ndims, offset, layout->u.chunk.dim, udata.scaled); - udata.scaled[dset->shared->ndims] = 0; - - /* Get the number of dimensions for use in callback function */ - udata.ndims = dset->shared->ndims; - /* Compose chunked index info struct */ idx_info.f = dset->oloc.file; idx_info.pline = &dset->shared->dcpl_cache.pline; idx_info.layout = &dset->shared->layout.u.chunk; idx_info.storage = &dset->shared->layout.storage.u.chunk; - /* Iterate over the allocated chunks */ - if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_by_coord_cb, &udata) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from scaled") + /* If the dataset is not written, return the address as undefined */ + if (idx_info.storage->idx_addr == HADDR_UNDEF) { + if (addr) + *addr = HADDR_UNDEF; + if (size) + *size = 0; + HGOTO_DONE(SUCCEED); + } - /* Return the filter mask and chunk address and size */ - *filter_mask = udata.filter_mask; - *addr = udata.chunk_addr; - *size = udata.nbytes; + /* Calculate the scaled of this chunk */ + H5VM_chunk_scaled(dset->shared->ndims, offset, layout->u.chunk.dim, udata.scaled); + udata.scaled[dset->shared->ndims] = 0; + + /* Initialize before iteration */ + udata.ndims = dset->shared->ndims; + udata.nbytes = 0; + udata.filter_mask = 0; + udata.chunk_addr = HADDR_UNDEF; + udata.found = FALSE; + /* Iterate over the allocated chunks to find the requested chunk */ + if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_by_coord_cb, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve information of the chunk by its scaled coordinates") + + /* If the chunk is found, return the filter mask and chunk address/size */ + if (udata.found) { + if (filter_mask) + *filter_mask = udata.filter_mask; + if (addr) + *addr = udata.chunk_addr; + if (size) + *size = udata.nbytes; + } else { + /* Otherwise, return the address as undefined */ + if (addr) + *addr = HADDR_UNDEF; + if (size) + *size = 0; + HGOTO_DONE(SUCCEED); + } done: FUNC_LEAVE_NOAPI_TAG(ret_value) } /* end H5D__get_chunk_info_by_coord() */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 7d550b5..ee70bae 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -568,7 +568,7 @@ H5_DLL herr_t H5D__get_chunk_storage_size(H5D_t *dset, const hsize_t *offset, hsize_t *storage_size); H5_DLL herr_t H5D__get_num_chunks(const H5D_t *dset, const H5S_t *space, hsize_t *nchunks); H5_DLL herr_t H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_t index, hsize_t *coord, unsigned *filter_mask, haddr_t *offset, hsize_t *size); -H5_DLL herr_t H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *coord, unsigned* filter_mask, haddr_t *addr, hsize_t *size); +H5_DLL herr_t H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size); H5_DLL haddr_t H5D__get_offset(const H5D_t *dset); H5_DLL void *H5D__vlen_get_buf_size_alloc(size_t size, void *info); H5_DLL herr_t H5D__vlen_get_buf_size(void *elem, hid_t type_id, unsigned ndim, diff --git a/test/dsets.c b/test/dsets.c index e8d86d1..b16459b 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -271,6 +271,7 @@ const char *FILENAME[] = { /* Parameters for testing chunk querying */ #define DSET_SIMPLE_CHUNKED "Chunked Dataset" +#define DSET_EMPTY "Empty Dataset" #define RANK 2 #define NX 16 #define NY 16 @@ -12900,8 +12901,8 @@ error: * Purpose: Tests various format versions. * (Currently, only virtual dataset feature) * - * Return: Success: 0 - * Failure: -1 + * Return: Success: SUCCEED + * Failure: FAIL * Description: * This function attempts to create a virtual dataset in all * valid combinations of low/high library format bounds. Creation @@ -13030,7 +13031,7 @@ test_versionbounds() TEST_ERROR dcpl = -1; PASSED(); - return 0; + return SUCCEED; error: H5E_BEGIN_TRY { @@ -13043,7 +13044,7 @@ test_versionbounds() H5Fclose(srcfile); H5Fclose(vfile); } H5E_END_TRY; - return -1; + return FAIL; } /* test_versionbounds() */ /*------------------------------------------------------------------------- @@ -13077,7 +13078,7 @@ static herr_t read_each_chunk(hid_t dset_id, hsize_t offset1, hsize_t offset2, u /* Verify that read chunk is the same as the corresponding written one */ if (HDmemcmp(direct_buf, read_buf, CHUNK_NX*CHUNK_NY) != 0) { - fprintf(stderr, "Read chunk differs than written chunk at offset (%d,%d)\n", offset1, offset2); + HDfprintf(stderr, "Read chunk differs from written chunk at offset (%d,%d)\n", offset1, offset2); return(FAIL); } @@ -13098,9 +13099,12 @@ static herr_t read_each_chunk(hid_t dset_id, hsize_t offset1, hsize_t offset2, u */ void reinit_vars(unsigned *read_filter_mask, hsize_t *addr, hsize_t *size) { - if (read_filter_mask) *read_filter_mask = 0; - if (addr) *addr = 0; - if (size) *size = 0; + if (read_filter_mask) + *read_filter_mask = 0; + if (addr) + *addr = 0; + if (size) + *size = 0; } /*------------------------------------------------------------------------- @@ -13111,6 +13115,10 @@ void reinit_vars(unsigned *read_filter_mask, hsize_t *addr, hsize_t *size) * Return: Success: 0 * Failure: 1 * + * Description: + * This function tests the new API functions added for EED-343: + * H5Dget_num_chunks, H5Dget_chunk_info, and H5Dget_chunk_info_by_coord. + * * Date: September 2018 * *------------------------------------------------------------------------- @@ -13265,6 +13273,14 @@ test_get_chunk_info() if (size != CHUNK_SIZE) TEST_ERROR; if (out_offset[0] != 4 || out_offset[1] != 12) TEST_ERROR; + /* Attempt to get info of empty chunk and verify the returned address and size */ + index = 5; + reinit_vars(&read_filter_mask, &addr, &size); + if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0) + TEST_ERROR + if (addr != HADDR_UNDEF) TEST_ERROR; + if (size != 0) TEST_ERROR; + /* Get info of the chunk at logical coordinates (0,2) */ offset[0] = 0; offset[1] = 2 * CHUNK_NY; @@ -13279,23 +13295,67 @@ test_get_chunk_info() if (read_filter_mask != filter_mask) TEST_ERROR; if (size != CHUNK_SIZE) TEST_ERROR; - /* Read each chunk and print the values */ + /* Attempt to get info of empty chunks, verify the returned address and size */ + offset[0] = 0; + offset[1] = 0; + if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR; + if (addr != HADDR_UNDEF) TEST_ERROR; + if (size != 0) TEST_ERROR; + + offset[0] = 3 * CHUNK_NX; + offset[1] = 3 * CHUNK_NY; + if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR; + if (addr != HADDR_UNDEF) TEST_ERROR; + if (size != 0) TEST_ERROR; + + /* Read each chunk and verify the values */ n = 0; for (i = 0; i < 2; i++) for (j = 2; j < 4; j++, n++) if (read_each_chunk(dset, i*CHUNK_NX, j*CHUNK_NY, filter_mask, (void*)direct_buf[n]) < 0) TEST_ERROR + /* Close the first dataset */ + if (H5Dclose(dset) < 0) TEST_ERROR + + /* Create an empty dataset */ + if((dset = H5Dcreate2(chunkfile, DSET_EMPTY, H5T_NATIVE_INT, fspace, + H5P_DEFAULT, cparms, H5P_DEFAULT)) < 0) TEST_ERROR + if (H5Dclose(dset) < 0) TEST_ERROR + + /* Reopen the empty dataset to verify the chunk query functions on it */ + if((dset = H5Dopen2(chunkfile, DSET_EMPTY, H5P_DEFAULT)) < 0) TEST_ERROR + + /* Verify that the number of chunks is 0 */ + if (H5Dget_num_chunks(dset, mspace, &nchunks) < 0) TEST_ERROR; + if (nchunks != 0) TEST_ERROR; + + /* Attempt to get info of a chunk from an empty dataset, verify the + returned address and size */ + index = 0; + reinit_vars(&read_filter_mask, &addr, &size); + if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0) + TEST_ERROR + if (addr != HADDR_UNDEF) TEST_ERROR; + if (size != 0) TEST_ERROR; + + /* Attempt to get info of a chunk given its coords from an empty dataset, + verify the returned address and size */ + offset[0] = 0; + offset[1] = 0; + if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR; + if (addr != HADDR_UNDEF) TEST_ERROR; + if (size != 0) TEST_ERROR; + /* Close/release resources. */ - H5Dclose(dset); - H5Sclose(mspace); - H5Sclose(fspace); - H5Pclose(cparms); - H5Pclose(dxpl); - H5Fclose(chunkfile); + if (H5Sclose(mspace) < 0) TEST_ERROR + if (H5Sclose(fspace) < 0) TEST_ERROR + if (H5Pclose(cparms) < 0) TEST_ERROR + if (H5Pclose(dxpl) < 0) TEST_ERROR + if (H5Fclose(chunkfile) < 0) TEST_ERROR PASSED(); - return 0; + return SUCCEED; error: H5E_BEGIN_TRY { @@ -13307,7 +13367,7 @@ error: } H5E_END_TRY; H5_FAILED(); - return 1; + return FAIL; } /* test_get_chunk_info() */ -- cgit v0.12 From a1a79203b8027631061cb626a5efdd37268752c2 Mon Sep 17 00:00:00 2001 From: Binh-Minh Ribler Date: Tue, 6 Nov 2018 13:55:05 -0600 Subject: Updated new API functions Description: - Addressed additional review comments - Moved the test function test_get_chunk_info from dsets.c to chunk_info.c because chunk_info.c was created for the purpose of testing chunk information. Platforms tested: Linux/64 (jelly) Linux/64 (platypus) Darwin (osx1011test) --- src/H5Dchunk.c | 47 +++--- test/CMakeLists.txt | 1 + test/Makefile.am | 4 +- test/chunk_info.c | 444 ++++++++++++++++++++++++++++++++++++++++++++++++---- 4 files changed, 432 insertions(+), 64 deletions(-) diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 7e7a94c..7aaea74 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -265,7 +265,7 @@ static herr_t H5D__chunk_io_term(const H5D_chunk_map_t *fm); static herr_t H5D__chunk_dest(H5D_t *dset); /* Chunk query operation callbacks */ -static int H5D__get_num_chunks_cb(const H5D_chunk_rec_t H5_ATTR_UNUSED *chunk_rec, void *_udata); +static int H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); static int H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); static int H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); @@ -6827,7 +6827,7 @@ H5D__get_num_chunks(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_ idx_info.storage = &dset->shared->layout.storage.u.chunk; /* If the dataset is not written, number of chunks will be 0 */ - if (idx_info.storage->idx_addr == HADDR_UNDEF) { + if(!H5F_addr_defined(idx_info.storage->idx_addr)) { *nchunks = 0; HGOTO_DONE(SUCCEED); } @@ -6938,14 +6938,15 @@ H5D__get_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_ idx_info.layout = &dset->shared->layout.u.chunk; idx_info.storage = &dset->shared->layout.storage.u.chunk; - /* If the dataset is not written, return the address as undefined */ - if (idx_info.storage->idx_addr == HADDR_UNDEF) { - if (addr) - *addr = HADDR_UNDEF; - if (size) - *size = 0; + /* Set addr & size for when dset is not written or queried chunk is not found */ + if (addr) + *addr = HADDR_UNDEF; + if (size) + *size = 0; + + /* If the dataset is not written, return without error */ + if(!H5F_addr_defined(idx_info.storage->idx_addr)) HGOTO_DONE(SUCCEED); - } /* Initialize before iteration */ udata.chunk_idx = index; @@ -6972,13 +6973,6 @@ H5D__get_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_ for (ii = 0; ii < udata.ndims; ii++) offset[ii] = udata.scaled[ii] * dset->shared->layout.u.chunk.dim[ii]; } - /* otherwise, return HADDR_UNDEF for address and 0 for size */ - else { - if (addr) - *addr = HADDR_UNDEF; - if (size) - *size = 0; - } done: FUNC_LEAVE_NOAPI(ret_value) @@ -7083,12 +7077,14 @@ H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned* idx_info.layout = &dset->shared->layout.u.chunk; idx_info.storage = &dset->shared->layout.storage.u.chunk; - /* If the dataset is not written, return the address as undefined */ - if (idx_info.storage->idx_addr == HADDR_UNDEF) { - if (addr) - *addr = HADDR_UNDEF; - if (size) - *size = 0; + /* Set addr & size for when dset is not written or queried chunk is not found */ + if (addr) + *addr = HADDR_UNDEF; + if (size) + *size = 0; + + /* If the dataset is not written, return without errors */ + if(!H5F_addr_defined(idx_info.storage->idx_addr)) { HGOTO_DONE(SUCCEED); } @@ -7115,13 +7111,6 @@ H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned* *addr = udata.chunk_addr; if (size) *size = udata.nbytes; - } else { - /* Otherwise, return the address as undefined */ - if (addr) - *addr = HADDR_UNDEF; - if (size) - *size = 0; - HGOTO_DONE(SUCCEED); } done: FUNC_LEAVE_NOAPI_TAG(ret_value) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index c8119fe..2aedc82 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -197,6 +197,7 @@ set (H5_TESTS page_buffer dtypes dsets + chunk_info cmpd_dset filter_fail extend diff --git a/test/Makefile.am b/test/Makefile.am index d4db6d0..5b210a9 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -55,8 +55,8 @@ TEST_PROG= testhdf5 \ cache cache_api cache_image cache_tagging lheap ohdr stab gheap \ evict_on_close farray earray btree2 fheap \ pool accum hyperslab istore bittests dt_arith page_buffer \ - dtypes dsets cmpd_dset filter_fail extend direct_chunk external efc \ - objcopy links unlink twriteorder big mtime fillval mount \ + dtypes dsets chunk_info cmpd_dset filter_fail extend direct_chunk \ + external efc objcopy links unlink twriteorder big mtime fillval mount \ flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \ enc_dec_plist_cross_platform getname vfd ntypes dangle dtransform \ reserved cross_read freespace mf vds file_image unregister \ diff --git a/test/chunk_info.c b/test/chunk_info.c index e3006cc..8a569d6 100644 --- a/test/chunk_info.c +++ b/test/chunk_info.c @@ -15,23 +15,374 @@ * Programmer: Pedro Vicente * April 7, 2008 * - * Purpose: Tests the H5Dget_chunk_info API function - * This program writes a 4x4 dataset by iterating on 2x2 chunks - * at a time + * Purpose: Tests chunk query API functions */ - #include "h5test.h" -#define PRINT_DATA + /* #define PRINT_DATA + */ #define H5FILE_NAME "chunk_info.h5" #define DATASETNAME "2d" #define RANK 2 +#define FILENAME_BUF_SIZE 1024 + +/* Parameters for testing chunk querying */ +#define FILENAME "tchunk_info" +#define DSET_SIMPLE_CHUNKED "Chunked Dataset" +#define DSET_EMPTY "Empty Dataset" +#define NX 16 +#define NY 16 +#define CHUNK_NX 4 +#define CHUNK_NY 4 +#define CHUNK_SIZE 64 +#define NUM_CHUNKS_WRITTEN 4 + +void reinit_vars(unsigned *read_filter_mask, haddr_t *addr, hsize_t *size); + +/*------------------------------------------------------------------------- + * Function: read_each_chunk (helper function) + * + * Purpose: Reads the chunk specified by its offset and verifies that + * it contains the same data as what was written. This function + * is used in test_get_chunk_info. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Date: September 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t read_each_chunk(hid_t dset_id, hsize_t offset1, hsize_t offset2, void *direct_buf) +{ + int read_buf[CHUNK_NX][CHUNK_NY]; + hsize_t offset[2] = {offset1, offset2}; + unsigned read_filter_mask = 0; + + /* Read the raw chunk back */ + HDmemset(&read_buf, 0, sizeof(read_buf)); + + /* Read the chunk specified by its offset */ + if (H5Dread_chunk(dset_id, H5P_DEFAULT, offset, &read_filter_mask, read_buf) < 0) + return(FAIL); + + /* Verify that read chunk is the same as the corresponding written one */ + if (HDmemcmp(direct_buf, read_buf, CHUNK_NX*CHUNK_NY) != 0) + { + HDfprintf(stderr, "Read chunk differs from written chunk at offset (%d,%d)\n", offset1, offset2); + return(FAIL); + } + return(SUCCEED); +} -int main( void ) +/*------------------------------------------------------------------------- + * Function: reinit_vars (helper function) + * + * Purpose: Helper function to wipe out variables for the next use, + * used in test_get_chunk_info. + * + * Return: Won't fail + * + * Date: September 2018 + * + *------------------------------------------------------------------------- + */ +void reinit_vars(unsigned *read_filter_mask, haddr_t *addr, hsize_t *size) { + if (read_filter_mask) + *read_filter_mask = 0; + if (addr) + *addr = 0; + if (size) + *size = 0; +} +/*------------------------------------------------------------------------- + * Function: test_get_chunk_info + * + * Purpose: Test getting various chunk information + * + * Return: Success: 0 + * Failure: 1 + * + * Description: + * This function tests the new API functions added for EED-343: + * H5Dget_num_chunks, H5Dget_chunk_info, and H5Dget_chunk_info_by_coord. + * + * Date: September 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_get_chunk_info(void) +{ + hid_t chunkfile = -1, fapl = -1; + hid_t fspace = -1, dset = -1; + hid_t mspace = -1; + hid_t cparms = -1, dxpl = -1; + hsize_t dims[2] = {NX, NY}; + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; + hsize_t chunk_dims[2] ={CHUNK_NX, CHUNK_NY}; + int fillvalue = -1; + char filename[FILENAME_BUF_SIZE]; + unsigned filter_mask = 0; + int direct_buf[16][CHUNK_NX][CHUNK_NY]; + int out_buf[NX][NY]; + size_t buf_size = CHUNK_NX*CHUNK_NY*sizeof(int); + int aggression = 9; /* Compression aggression setting */ + unsigned read_filter_mask = 0; /* filter mask after direct read */ + H5F_libver_t low, high; /* File format bounds */ + hsize_t offset[2]; + hsize_t out_offset[2] = {0, 0}; + hsize_t size = 0; + hsize_t nchunks = 0; + haddr_t addr = 0; + hsize_t index = 0; + hsize_t i, j; + int n; /* for use on buffer, to avoid conversion warning */ + + TESTING("getting chunk information"); + + /* Create a copy of file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR + + /* Set high bound to V18 */ + low = H5F_LIBVER_EARLIEST; + high = H5F_LIBVER_V18; + if (H5Pset_libver_bounds(fapl, low, high) < 0) + TEST_ERROR; + + /* Create a file */ + h5_fixname(FILENAME, fapl, filename, sizeof filename); + if((chunkfile = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Create the data space */ + if((fspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + TEST_ERROR + + if((mspace = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + TEST_ERROR + + /* Modify dataset creation properties, i.e. enable chunking and compression */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR + + if(H5Pset_chunk(cparms, RANK, chunk_dims) < 0) + TEST_ERROR + + if(H5Pset_deflate(cparms, (unsigned ) aggression) < 0) + TEST_ERROR + + if (H5Pset_fill_value(cparms, H5T_NATIVE_INT, &fillvalue) < 0) + TEST_ERROR; + + /* Create a new dataset using cparms creation properties */ + if((dset = H5Dcreate2(chunkfile, DSET_SIMPLE_CHUNKED, H5T_NATIVE_INT, fspace, + H5P_DEFAULT, cparms, H5P_DEFAULT)) < 0) TEST_ERROR + + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + TEST_ERROR + + /* Indicate skipping the compression filter. */ + filter_mask = 0x00000001; + + /* Initialize the array of chunk data, that is for all 16 chunks */ + for(n = 0; n < 16; n++) + for(i = 0; i < CHUNK_NX; i++) + for(j = 0; j < CHUNK_NY; j++) + direct_buf[n][i][j] = n + 1; + + /* Write NUM_CHUNKS_WRITTEN chunks: (0,2) (0,3) (1,2) (1,3) */ + n = 0; + for (i = 0; i < 2; i++) + for (j = 2; j < 4; j++, n++) + { + offset[0] = i * CHUNK_NX; + offset[1] = j * CHUNK_NY; + if (H5Dwrite_chunk(dset, dxpl, filter_mask, offset, buf_size, (void*)direct_buf[n]) < 0) + TEST_ERROR + } + + /* Close the dataset then... */ + if(H5Dclose(dset) < 0) TEST_ERROR + + /* ...open it again to test the chunk query functions */ + if((dset = H5Dopen2(chunkfile, DSET_SIMPLE_CHUNKED, H5P_DEFAULT)) < 0) TEST_ERROR + + /* Get and verify the number of chunks written */ + if (H5Dget_num_chunks(dset, mspace, &nchunks) < 0) TEST_ERROR; + if (nchunks != NUM_CHUNKS_WRITTEN) TEST_ERROR; + + /* Read the entire dataset back */ + if(H5Dread(dset, H5T_NATIVE_INT, fspace, fspace, H5P_DEFAULT, out_buf) < 0) + TEST_ERROR + + /* Get and verify info of the first chunk */ + index = 0; + reinit_vars(&read_filter_mask, &addr, &size); + if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0) + TEST_ERROR + if (read_filter_mask != filter_mask) TEST_ERROR; + if (size != CHUNK_SIZE) TEST_ERROR; + if (out_offset[0] != 0 || out_offset[1] != 8) TEST_ERROR; + + /* Get and verify info of the second chunk */ + index = 1; + reinit_vars(&read_filter_mask, &addr, &size); + if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0) + TEST_ERROR + if (read_filter_mask != filter_mask) TEST_ERROR; + if (size != CHUNK_SIZE) TEST_ERROR; + if (out_offset[0] != 0 || out_offset[1] != 12) TEST_ERROR; + + /* Get and verify info of the third chunk */ + index = 2; + reinit_vars(&read_filter_mask, &addr, &size); + if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0) + TEST_ERROR + + if (read_filter_mask != filter_mask) TEST_ERROR; + if (size != CHUNK_SIZE) TEST_ERROR; + if (out_offset[0] != 4 || out_offset[1] != 8) TEST_ERROR; + + /* Get and verify info of the last chunk */ + index = 3; + reinit_vars(&read_filter_mask, &addr, &size); + if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0) + TEST_ERROR + if (read_filter_mask != filter_mask) TEST_ERROR; + if (size != CHUNK_SIZE) TEST_ERROR; + if (out_offset[0] != 4 || out_offset[1] != 12) TEST_ERROR; + + /* Attempt to get info of empty chunk and verify the returned address and size */ + index = 5; + reinit_vars(&read_filter_mask, &addr, &size); + if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0) + TEST_ERROR + if (addr != HADDR_UNDEF) TEST_ERROR; + if (size != 0) TEST_ERROR; + + /* Get info of the chunk at logical coordinates (0,2) */ + offset[0] = 0; + offset[1] = 2 * CHUNK_NY; + if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR; + if (read_filter_mask != filter_mask) TEST_ERROR; + if (size != CHUNK_SIZE) TEST_ERROR; + + /* Get info of the chunk at logical coordinates (1,3) */ + offset[0] = 1 * CHUNK_NX; + offset[1] = 3 * CHUNK_NY; + if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR; + if (read_filter_mask != filter_mask) TEST_ERROR; + if (size != CHUNK_SIZE) TEST_ERROR; + + /* Attempt to get info of empty chunks, verify the returned address and size */ + offset[0] = 0; + offset[1] = 0; + if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR; + if (addr != HADDR_UNDEF) TEST_ERROR; + if (size != 0) TEST_ERROR; + + offset[0] = 3 * CHUNK_NX; + offset[1] = 3 * CHUNK_NY; + if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR; + if (addr != HADDR_UNDEF) TEST_ERROR; + if (size != 0) TEST_ERROR; + + /* Read each chunk and verify the values */ + n = 0; + for (i = 0; i < 2; i++) + for (j = 2; j < 4; j++, n++) + if (read_each_chunk(dset, i*CHUNK_NX, j*CHUNK_NY, (void*)direct_buf[n]) < 0) + TEST_ERROR + + /* Close the first dataset */ + if (H5Dclose(dset) < 0) TEST_ERROR + + /* Create an empty dataset */ + if((dset = H5Dcreate2(chunkfile, DSET_EMPTY, H5T_NATIVE_INT, fspace, + H5P_DEFAULT, cparms, H5P_DEFAULT)) < 0) TEST_ERROR + if (H5Dclose(dset) < 0) TEST_ERROR + + /* Reopen the empty dataset to verify the chunk query functions on it */ + if((dset = H5Dopen2(chunkfile, DSET_EMPTY, H5P_DEFAULT)) < 0) TEST_ERROR + + /* Verify that the number of chunks is 0 */ + if (H5Dget_num_chunks(dset, mspace, &nchunks) < 0) TEST_ERROR; + if (nchunks != 0) TEST_ERROR; + + /* Attempt to get info of a chunk from an empty dataset, verify the + returned address and size */ + index = 0; + reinit_vars(&read_filter_mask, &addr, &size); + if (H5Dget_chunk_info(dset, fspace, index, out_offset, &read_filter_mask, &addr, &size) < 0) + TEST_ERROR + if (addr != HADDR_UNDEF) TEST_ERROR; + if (size != 0) TEST_ERROR; + + /* Attempt to get info of a chunk given its coords from an empty dataset, + verify the returned address and size */ + offset[0] = 0; + offset[1] = 0; + if (H5Dget_chunk_info_by_coord(dset, offset, &read_filter_mask, &addr, &size) < 0) TEST_ERROR; + if (addr != HADDR_UNDEF) TEST_ERROR; + if (size != 0) TEST_ERROR; + + /* Close/release resources. */ + if (H5Sclose(mspace) < 0) TEST_ERROR + if (H5Sclose(fspace) < 0) TEST_ERROR + if (H5Pclose(cparms) < 0) TEST_ERROR + if (H5Pclose(dxpl) < 0) TEST_ERROR + if (H5Fclose(chunkfile) < 0) TEST_ERROR + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Sclose(mspace); + H5Sclose(fspace); + H5Pclose(cparms); + H5Pclose(dxpl); + H5Fclose(chunkfile); + } H5E_END_TRY; + + H5_FAILED(); + return FAIL; +} /* test_get_chunk_info() */ + +/*------------------------------------------------------------------------- + * Function: create_4x4_dset + * + * Purpose: Tests functions related to chunk information + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Description: + * The code in this create_4x4_dset function was originally written by + * Pedro in main() that writes a 4x4 dataset by iterating on + * 2x2 chunks at a time, with the intention of making a frame + * work to test H5Dget_chunk_info, which was not in the library, + * until now. For the work in EED-343, the test function + * test_get_chunk_info was added to test the new query chunk + * API functions: H5Dget_num_chunk, H5Dget_chunk_info, and + * H5Dget_chunk_info_by_coord. This code can be used at a + * later time, so it is kept here. + * -BMR, November 5, 2018 + * + * Programmer: Pedro Vicente + * April 7, 2008 + * + *------------------------------------------------------------------------- + */ +int +create_4x4_dset(void) +{ hid_t fid; /* file ID */ hid_t did; /* dataset ID */ hid_t f_sid; /* file space ID */ @@ -43,20 +394,20 @@ int main( void ) int chunk_data[2][2] = { {1, 1}, {1, 1} }; int buf[4][4]; int fillvalue = 0; - int i, j, ii, jj; + hsize_t i, j; /* create a new file using default properties. */ if ((fid = H5Fcreate(H5FILE_NAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) TEST_ERROR; + /* create the file space */ if ((f_sid = H5Screate_simple(RANK, dims, dims)) < 0) TEST_ERROR; + /* create the memory space with chunk dimensions */ if ((m_sid = H5Screate_simple(RANK, chunk_dims, chunk_dims)) < 0) TEST_ERROR; start[0] = 0; start[1] = 0; if (H5Sselect_hyperslab(m_sid, H5S_SELECT_SET, start, NULL, chunk_dims, NULL) < 0) TEST_ERROR; - TESTING("chunk info"); - /*------------------------------------------------------------------------- * create a dataset *------------------------------------------------------------------------- @@ -77,16 +428,12 @@ int main( void ) */ /* iterate in dim 0 */ - for (j = 0; j < chunk_dims[0]; j++) - { - + for (j = 0; j < chunk_dims[0]; j++) { /* reset start in dim 1 */ start[1] = 0; /* iterate in dim 1 */ - for (i = 0; i < chunk_dims[1]; i++) - { - + for (i = 0; i < chunk_dims[1]; i++) { /* select file hyperslab to save a 2x2 chunk */ if (H5Sselect_hyperslab(f_sid, H5S_SELECT_SET, start, NULL, chunk_dims, NULL) < 0) TEST_ERROR; @@ -96,7 +443,6 @@ int main( void ) /* read back and display complete dataset 4x4 */ if (H5Dread(did, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) TEST_ERROR; - #if defined (PRINT_DATA) printf("\n"); printf("dataset: \n"); @@ -105,25 +451,17 @@ int main( void ) printf("\n"); } #endif - - /* increment start in dim 1 */ start[1] += 2; - - } - /* increment start in dim 0 */ start[0] += 2; } - - /*------------------------------------------------------------------------- * close *------------------------------------------------------------------------- */ - if (H5Dclose(did) < 0) TEST_ERROR if (H5Sclose(f_sid) < 0) TEST_ERROR if (H5Sclose(m_sid) < 0) TEST_ERROR @@ -132,18 +470,58 @@ int main( void ) PASSED(); - puts("All chunk info tests passed."); - return 0; + return SUCCEED; + +/* this will be removed once the existing code is moved out of main */ +error: + H5Dclose(did); + H5Sclose(f_sid); + H5Sclose(m_sid); + H5Pclose(pid); + H5Fclose(fid); + return FAIL; +} /* end create_4x4_dset */ + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Tests functions related to chunk information + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Binh-Minh Ribler + * November 5, 2018 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + int nerrors = 0; + + /* Tests getting chunk information */ + nerrors += (test_get_chunk_info() < 0 ? 1 : 0); + + /* Create a 4x4 dataset (using the existing code to avoid compilation + warnings for now) */ + nerrors += (create_4x4_dset() < 0 ? 1 : 0); + + if(nerrors) { + goto error; + + } /* end if */ + + HDprintf("All chunk query tests passed.\n"); + + return SUCCEED; error: - H5Dclose( did ); - H5Sclose( f_sid ); - H5Sclose( m_sid ); - H5Pclose( pid ); - H5Fclose( fid ); - H5_FAILED(); - return 1; + nerrors = MAX(1, nerrors); + HDprintf("***** %d QUERY CHUNK INFO TEST%s FAILED! *****\n", + nerrors, 1 == nerrors ? "" : "S"); + return FAIL; } -- cgit v0.12