diff options
| -rw-r--r-- | MANIFEST | 1 | ||||
| -rw-r--r-- | release_docs/RELEASE.txt | 80 | ||||
| -rw-r--r-- | src/H5D.c | 179 | ||||
| -rw-r--r-- | src/H5Dchunk.c | 387 | ||||
| -rw-r--r-- | src/H5Dnone.c | 57 | ||||
| -rw-r--r-- | src/H5Dpkg.h | 3 | ||||
| -rw-r--r-- | src/H5Dpublic.h | 3 | ||||
| -rw-r--r-- | src/H5EA.c | 14 | ||||
| -rw-r--r-- | src/H5FA.c | 14 | ||||
| -rw-r--r-- | src/H5VLnative.h | 7 | ||||
| -rw-r--r-- | src/H5VLnative_dataset.c | 99 | ||||
| -rw-r--r-- | test/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | test/Makefile.am | 6 | ||||
| -rw-r--r-- | test/chunk_info.c | 1964 |
14 files changed, 2719 insertions, 96 deletions
@@ -990,6 +990,7 @@ ./test/cache_image.c ./test/cache_logging.c ./test/cache_tagging.c +./test/chunk_info.c ./test/cmpd_dset.c ./test/cork.c ./test/corrupt_stab_msg.h5 diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 202d39d..37dec3e 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -191,10 +191,15 @@ New Features Library: -------- - - Add new API H5M for map objects. Currently not supported by native - library, can be supported by VOL connectors. + - Added new chunk query functions - (NAF - 2019/03/01) + The following public functions were added to discover information about + the chunks in an HDF5 file. + herr_t H5Dget_num_chunks(dset_id, fspace_id, *nchunks) + herr_t H5Dget_chunk_info_by_coord(dset_id, *coord, *filter_mask, *addr, *size) + herr_t H5Dget_chunk_info(dset_id, fspace_id, index, *coord, *filter_mask, *addr, *size) + + (BMR - 2019/06/11, HDFFV-10677) - Improved the performance of virtual dataset I/O @@ -204,12 +209,19 @@ New Features (NAF - 2019/05/31, HDFFV-10693) - - Allow pre-generated H5Tinit.c and H5make_libsettings.c to be used. + - Added the ability to open files with UTF-8 file names on Windows. - Rather than always running H5detect and generating H5Tinit.c and - H5make_libsettings.c, supply a location for those files. + The POSIX open(2) API call on Windows is limited to ASCII + file names. The library has been updated to convert incoming file + names to UTF-16 (via MultiByteToWideChar(CP_UTF8, ...) and use + _wopen() instead. - (ADB - 2018/09/18, HDFFV-10332) + (DER - 2019/03/15, HDFFV-2714, HDFFV-3914, HDFFV-3895, HDFFV-8237, HDFFV-10413, HDFFV-10691) + + - Add new API H5M for map objects. Currently not supported by native + library, can be supported by VOL connectors. + + (NAF - 2019/03/01) - Remove H5I_REFERENCE from the library @@ -217,14 +229,13 @@ New Features (DER - 2018/12/08, HDFFV-10252) - - Added the ability to open files with UTF-8 file names on Windows. + - Allow pre-generated H5Tinit.c and H5make_libsettings.c to be used. - The POSIX open(2) API call on Windows is limited to ASCII - file names. The library has been updated to convert incoming file - names to UTF-16 (via MultiByteToWideChar(CP_UTF8, ...) and use - _wopen() instead. + Rather than always running H5detect and generating H5Tinit.c and + H5make_libsettings.c, supply a location for those files. + + (ADB - 2018/09/18, HDFFV-10332) - (DER - 2019/03/15, HDFFV-2714, HDFFV-3914, HDFFV-3895, HDFFV-8237, HDFFV-10413, HDFFV-10691) Parallel Library: ----------------- @@ -257,17 +268,17 @@ New Features C++ Library: ------------ - - Added new wrapper for H5Ovisit2() - H5Object::visit() - - (BMR - 2019/02/14, HDFFV-10532) - - Added new wrappers for H5Pset/get_create_intermediate_group() LinkCreatPropList::setCreateIntermediateGroup() LinkCreatPropList::getCreateIntermediateGroup() (BMR - 2019/04/22, HDFFV-10622) + - Added new wrapper for H5Ovisit2() + H5Object::visit() + + (BMR - 2019/02/14, HDFFV-10532) + Java Library: ---------------- @@ -373,6 +384,18 @@ Bug Fixes since HDF5-1.10.3 release (VC - 2019/6/25, HDFFV-10808) + - When iterating over an old-style group (i.e., when not using the latest + file format) of size 0, a NULL pointer representing the empty links + table would be sent to qsort(3) for sorting, which is undefined behavior. + + Iterating over an empty group is explicitly tested in the links test. + This has not caused any failures to date and was flagged by gcc's + -fsanitize=undefined. + + The library no longer attempts to sort an empty array. + + (DER - 2019/06/18, HDFFV-10829) + - Fixed an issue where copying a version 1.8 dataset between files using H5Ocopy fails due to an incompatible fill version @@ -386,6 +409,16 @@ Bug Fixes since HDF5-1.10.3 release (VC - 2019/6/14, HDFFV-10800) + - Some oversights in the index iterating area of the library caused + a callback function to continue iterating even though it's supposed + to stop. + + Added the returned value check to the for loop's conditions in + H5EA_iterate(), H5FA_iterate(), and H5D__none_idx_iterate(). The + iteration now stops when it should. + + (BMR - 2019/06/11, HDFFV-10661) + - Fixed a bug that would cause an error or cause fill values to be incorrectly read from a chunked dataset using the "single chunk" index if the data was held in cache and there was no data on disk. @@ -480,17 +513,6 @@ Bug Fixes since HDF5-1.10.3 release (JTH - 2018/08/25, HDFFV-10501) - - When iterating over an old-style group (i.e., when not using the latest - file format) of size 0, a NULL pointer representing the empty links - table would be sent to qsort(3) for sorting, which is undefined behavior. - - Iterating over an empty group is explicitly tested in the links test. - This has not caused any failures to date and was flagged by gcc's - -fsanitize=undefined. - - The library no longer attempts to sort an empty array. - - (DER - 2019/06/18, HDFFV-10829) Java Library: ---------------- @@ -457,8 +457,8 @@ done: * * Failure: H5I_INVALID_HID * - * Programmer: Robb Matzke - * Tuesday, February 3, 1998 + * Programmer: Robb Matzke + * Tuesday, February 3, 1998 * *------------------------------------------------------------------------- */ @@ -512,8 +512,8 @@ done: * * Failure: H5I_INVALID_HID * - * Programmer: Neil Fortner - * Wednesday, October 29, 2008 + * Programmer: Neil Fortner + * Wednesday, October 29, 2008 * *------------------------------------------------------------------------- */ @@ -610,9 +610,9 @@ done: /*------------------------------------------------------------------------- - * Function: H5Diterate + * Function: H5Diterate * - * Purpose: This routine iterates over all the elements selected in a memory + * Purpose: This routine iterates over all the elements selected in a memory * buffer. The callback function is called once for each element selected * in the dataspace. The selection in the dataspace is modified so * that any elements already iterated over are removed from the selection @@ -659,11 +659,11 @@ done: * indicating failure. The iterator can be restarted at the next * element. * - * Return: Returns the return value of the last operator if it was non-zero, + * Return: Returns the return value of the last operator if it was non-zero, * or zero if all elements were processed. Otherwise returns a * negative value. * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Friday, June 11, 1999 * *------------------------------------------------------------------------- @@ -706,16 +706,16 @@ done: /*------------------------------------------------------------------------- - * Function: H5Dvlen_reclaim + * Function: H5Dvlen_reclaim * - * Purpose: Frees the buffers allocated for storing variable-length data + * Purpose: Frees the buffers allocated for storing variable-length data * in memory. Only frees the VL data in the selection defined in the * dataspace. The dataset transfer property list is required to find the * correct allocation/free methods for the VL data in the buffer. * - * Return: Non-negative on success, negative on failure + * Return: Non-negative on success, negative on failure * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Thursday, June 10, 1999 * *------------------------------------------------------------------------- @@ -756,9 +756,9 @@ done: /*------------------------------------------------------------------------- - * Function: H5Dvlen_get_buf_size + * Function: H5Dvlen_get_buf_size * - * Purpose: This routine checks the number of bytes required to store the VL + * Purpose: This routine checks the number of bytes required to store the VL * data from the dataset, using the space_id for the selection in the * dataset on disk and the type_id for the memory representation of the * VL data, in memory. The *size value is modified according to how many @@ -772,9 +772,9 @@ done: * Kinda kludgy, but easier than the other method of trying to figure out * the sizes without actually reading the data in... - QAK * - * Return: Non-negative on success, negative on failure + * Return: Non-negative on success, negative on failure * - * Programmer: Quincey Koziol + * Programmer: Quincey Koziol * Wednesday, August 11, 1999 * *------------------------------------------------------------------------- @@ -1095,3 +1095,150 @@ done: FUNC_LEAVE_API(ret_value); } /* H5Dget_chunk_storage_size() */ + +/*------------------------------------------------------------------------- + * Function: H5Dget_num_chunks + * + * Purpose: Retrieves the number of chunks that have non-empty intersection + * with a specified selection. + * + * Note: Currently, this function only gets the number of all written + * chunks, regardless the dataspace. + * + * Parameters: + * hid_t dset_id; IN: Chunked dataset ID + * hid_t fspace_id; IN: File dataspace ID + * hsize_t *nchunks; OUT: Number of non-empty chunks + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Binh-Minh Ribler + * May 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE3("e", "ii*h", dset_id, fspace_id, nchunks); + + /* Check arguments */ + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier") + if(NULL == nchunks) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)") + + /* Get the number of written chunks */ + if(H5VL_dataset_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_NATIVE_DATASET_GET_NUM_CHUNKS, fspace_id, nchunks) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't get number of chunks") + +done: + FUNC_LEAVE_API(ret_value); +} /* H5Dget_num_chunks() */ + + +/*------------------------------------------------------------------------- + * Function: H5Dget_chunk_info + * + * Purpose: Retrieves information about a chunk specified by its index. + * + * Parameters: + * hid_t dset_id; IN: Chunked dataset ID + * hid_t fspace_id; IN: File dataspace ID + * hsize_t index; IN: Index of written chunk + * hsize_t *offset OUT: Logical position of the chunk’s + * first element in the dataspace + * unsigned *filter_mask OUT: Mask for identifying the filters in use + * haddr_t *addr OUT: Address of the chunk + * hsize_t *size OUT: Size of the chunk + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Binh-Minh Ribler + * May 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chk_index, hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + hsize_t nchunks = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE7("e", "iih*h*Iu*a*h", dset_id, fspace_id, chk_index, offset, + filter_mask, addr, size); + + /* Check arguments */ + if(NULL == offset && NULL == filter_mask && NULL == addr && NULL == size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid arguments, must have at least one non-null output argument") + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier") + + /* Get the number of written chunks to check range */ + if(H5VL_dataset_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_NATIVE_DATASET_GET_NUM_CHUNKS, fspace_id, &nchunks) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't get number of chunks") + + /* Check range for chunk index */ + if(chk_index >= nchunks) + HGOTO_ERROR(H5E_DATASET, H5E_BADRANGE, FAIL, "chunk index is out of range") + + /* Call private function to get the chunk info given the chunk's index */ + if(H5VL_dataset_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX, fspace_id, chk_index, offset, filter_mask, addr, size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't get chunk info by index") + +done: + FUNC_LEAVE_API(ret_value); +} /* H5Dget_chunk_info() */ + + +/*------------------------------------------------------------------------- + * Function: H5Dget_chunk_info_by_coord + * + * Purpose: Retrieves information about a chunk specified by its logical + * coordinates. + * + * Parameters: + * hid_t dset_id; IN: Chunked dataset ID + * hsize_t *offset IN: Logical position of the chunk’s + * first element in the dataspace + * unsigned *filter_mask OUT: Mask for identifying the filters in use + * haddr_t *addr OUT: Address of the chunk + * hsize_t *size OUT: Size of the chunk + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Binh-Minh Ribler + * May 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +herr_t +H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size) +{ + H5VL_object_t *vol_obj = NULL; /* Dataset for this operation */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_API(FAIL) + H5TRACE5("e", "i*h*Iu*a*h", dset_id, offset, filter_mask, addr, size); + + /* Check arguments */ + if(NULL == (vol_obj = (H5VL_object_t *)H5I_object_verify(dset_id, H5I_DATASET))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "invalid dataset identifier") + if(NULL == filter_mask && NULL == addr && NULL == size) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid arguments, must have at least one non-null output argument") + if(NULL == offset) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid argument (null)") + + /* Call private function to get the chunk info given the chunk's index */ + if(H5VL_dataset_optional(vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL, H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD, offset, filter_mask, addr, size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "Can't get chunk info by its logical coordinates") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Dget_chunk_info_by_coord() */ diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 082aac3..e3bbd59 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -216,6 +216,18 @@ typedef struct H5D_chunk_readvv_ud_t { const H5D_t *dset; /* Dataset to operate on */ } H5D_chunk_readvv_ud_t; +typedef struct H5D_chunk_info_iter_ud_t { + hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Logical offset of the chunk */ + hsize_t ndims; /* Number of dimensions in the dataset */ + uint32_t nbytes; /* Size of stored data in the chunk */ + unsigned filter_mask; /* Excluded filters */ + haddr_t chunk_addr; /* Address of the chunk in file */ + hsize_t chunk_idx; /* Chunk index, where the iteration needs to stop */ + hsize_t curr_idx; /* Current index, where the iteration is */ + unsigned idx_hint; /* Index of chunk in cache, if present */ + hbool_t found; /* Whether the chunk was found */ +} H5D_chunk_info_iter_ud_t; + /* Callback info for file selection iteration */ typedef struct H5D_chunk_file_iter_ud_t { H5D_chunk_map_t *fm; /* File->memory chunk mapping info */ @@ -252,6 +264,11 @@ static herr_t H5D__chunk_flush(H5D_t *dset); static herr_t H5D__chunk_io_term(const H5D_chunk_map_t *fm); static herr_t H5D__chunk_dest(H5D_t *dset); +/* Chunk query operation callbacks */ +static int H5D__get_num_chunks_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); +static int H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); +static int H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata); + /* "Nonexistent" layout operation callback */ static ssize_t H5D__nonexistent_readvv(const H5D_io_info_t *io_info, @@ -6956,3 +6973,373 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__chunk_format_convert() */ + +/*------------------------------------------------------------------------- + * Function: H5D__get_num_chunks_cb + * + * Purpose: Callback function that increments the number of written + * chunks in the dataset. + * + * Note: Currently, this function only gets the number of all written + * chunks, regardless the dataspace. + * + * Return: H5_ITER_CONT + * + * Programmer: Binh-Minh Ribler + * June 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +static int +H5D__get_num_chunks_cb(const H5D_chunk_rec_t H5_ATTR_UNUSED *chunk_rec, void *_udata) +{ + hsize_t *num_chunks = (hsize_t *)_udata; + int ret_value = H5_ITER_CONT; /* Callback return value */ + + FUNC_ENTER_STATIC_NOERR + + HDassert(num_chunks); + + (*num_chunks)++; + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__get_num_chunks_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_num_chunks + * + * Purpose: Gets the number of written chunks in a dataset. + * + * Note: Currently, this function only gets the number of all written + * chunks, regardless the dataspace. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Binh-Minh Ribler + * June 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_num_chunks(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t *nchunks) +{ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + hsize_t num_chunks = 0; /* Number of written chunks */ + H5D_rdcc_ent_t *ent; /* Cache entry */ + const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */ + const H5O_layout_t *layout; /* Dataset layout */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) + + HDassert(dset); + HDassert(dset->shared); + HDassert(space); + HDassert(nchunks); + + layout = &(dset->shared->layout); /* Dataset layout */ + rdcc = &(dset->shared->cache.chunk); /* raw data chunk cache */ + HDassert(rdcc); + + /* Search for cached chunks that haven't been written out */ + for(ent = rdcc->head; ent; ent = ent->next) + /* Flush the chunk out to disk, to make certain the size is correct later */ + if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.pline = &dset->shared->dcpl_cache.pline; + idx_info.layout = &dset->shared->layout.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; + + /* If the dataset is not written, number of chunks will be 0 */ + if(!H5F_addr_defined(idx_info.storage->idx_addr)) { + *nchunks = 0; + } + else { + /* Iterate over the allocated chunks */ + if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_num_chunks_cb, &num_chunks) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index") + *nchunks = num_chunks; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__get_num_chunks() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_chunk_info_cb + * + * Purpose: Get the chunk info of the queried chunk, given by its index. + * + * Return: Success: H5_ITER_CONT or H5_ITER_STOP + * H5_ITER_STOP indicates the queried chunk is found + * Failure: Negative (H5_ITER_ERROR) + * + * Programmer: Binh-Minh Ribler + * June 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +static int +H5D__get_chunk_info_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) +{ + H5D_chunk_info_iter_ud_t *chunk_info = (H5D_chunk_info_iter_ud_t *)_udata; + hsize_t ii = 0; /* Dimension index */ + int ret_value = H5_ITER_CONT; /* Callback return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check args */ + HDassert(chunk_rec); + HDassert(chunk_info); + + /* If this is the queried chunk, retrieve its info and stop iterating */ + if (chunk_info->curr_idx == chunk_info->chunk_idx) { + chunk_info->filter_mask = chunk_rec->filter_mask; + chunk_info->chunk_addr = chunk_rec->chunk_addr; + chunk_info->nbytes = chunk_rec->nbytes; + for (ii = 0; ii < chunk_info->ndims; ii++) + chunk_info->scaled[ii] = chunk_rec->scaled[ii]; + chunk_info->found = TRUE; + + /* Stop iterating */ + ret_value = H5_ITER_STOP; + } + /* Go to the next chunk */ + else + chunk_info->curr_idx++; + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__get_chunk_info_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_chunk_info + * + * Purpose: Iterate over the chunks in the dataset to get the info + * of the desired chunk. + * + * Note: Currently, the domain of the index in this function is of all + * the written chunks, regardless the dataspace. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Binh-Minh Ribler + * June 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_chunk_info(const H5D_t *dset, const H5S_t H5_ATTR_UNUSED *space, hsize_t chk_index, hsize_t *offset, unsigned *filter_mask, haddr_t *addr, hsize_t *size) +{ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + H5D_chunk_info_iter_ud_t udata; /* User data for callback */ + const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */ + H5D_rdcc_ent_t *ent; /* Cache entry index */ + hsize_t ii = 0; /* Dimension index */ + herr_t ret_value = SUCCEED;/* Return value */ + + FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) + + HDassert(dset); + HDassert(dset->shared); + HDassert(space); + + /* Get the raw data chunk cache */ + rdcc = &(dset->shared->cache.chunk); + HDassert(rdcc); + + /* Search for cached chunks that haven't been written out */ + for(ent = rdcc->head; ent; ent = ent->next) + /* Flush the chunk out to disk, to make certain the size is correct later */ + if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.pline = &dset->shared->dcpl_cache.pline; + idx_info.layout = &dset->shared->layout.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; + + /* Set addr & size for when dset is not written or queried chunk is not found */ + if (addr) + *addr = HADDR_UNDEF; + if (size) + *size = 0; + + /* If the chunk is written, get its info, otherwise, return without error */ + if(H5F_addr_defined(idx_info.storage->idx_addr)) { + /* Initialize before iteration */ + udata.chunk_idx = chk_index; + udata.curr_idx = 0; + udata.ndims = dset->shared->ndims; + udata.nbytes = 0; + udata.filter_mask = 0; + udata.chunk_addr = HADDR_UNDEF; + udata.found = FALSE; + + /* Iterate over the allocated chunks */ + if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_cb, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index") + + /* Obtain requested info if the chunk is found */ + if(udata.found) { + if(filter_mask) + *filter_mask = udata.filter_mask; + if(addr) + *addr = udata.chunk_addr; + if(size) + *size = udata.nbytes; + if(offset) + for(ii = 0; ii < udata.ndims; ii++) + offset[ii] = udata.scaled[ii] * dset->shared->layout.u.chunk.dim[ii]; + } + } /* end if H5F_addr_defined */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__get_chunk_info() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_chunk_info_by_coord_cb + * + * Purpose: Get the chunk info of the desired chunk, given its offset + * coordinates. + * + * Return: Success: H5_ITER_CONT or H5_ITER_STOP + * Failure: Negative (H5_ITER_ERROR) + * + * Programmer: Binh-Minh Ribler + * June 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +static int +H5D__get_chunk_info_by_coord_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) +{ + hsize_t ii; + H5D_chunk_info_iter_ud_t *chunk_info = (H5D_chunk_info_iter_ud_t *)_udata; + hbool_t different = FALSE; /* TRUE when a scaled value pair mismatch */ + int ret_value = H5_ITER_CONT; /* Callback return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check args */ + HDassert(chunk_rec); + HDassert(chunk_info); + + /* Going through the scaled, stop when a mismatch is found */ + for (ii = 0; ii < chunk_info->ndims && !different; ii++) + if (chunk_info->scaled[ii] != chunk_rec->scaled[ii]) + different = TRUE; + + /* Same scaled coords means the chunk is found, copy the chunk info */ + if (!different) { + chunk_info->nbytes = chunk_rec->nbytes; + chunk_info->filter_mask = chunk_rec->filter_mask; + chunk_info->chunk_addr = chunk_rec->chunk_addr; + chunk_info->found = TRUE; + + /* Stop iterating */ + ret_value = H5_ITER_STOP; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D__get_chunk_info_by_coord_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__get_chunk_info_by_coord + * + * Purpose: Iterate over the chunks in the dataset to get the info + * of the desired chunk, given by its offset coordinates. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Binh-Minh Ribler + * June 2019 (HDFFV-10677) + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *offset, unsigned* filter_mask, haddr_t *addr, hsize_t *size) +{ + const H5O_layout_t *layout = NULL; /* Dataset layout */ + const H5D_rdcc_t *rdcc = NULL; /* Raw data chunk cache */ + H5D_rdcc_ent_t *ent; /* Cache entry index */ + H5D_chk_idx_info_t idx_info; /* Chunked index info */ + H5D_chunk_info_iter_ud_t udata; /* User data for callback */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_TAG(dset->oloc.addr) + + /* Check args */ + HDassert(dset); + HDassert(dset->shared); + HDassert(offset); + + /* Get dataset layout and raw data chunk cache */ + layout = &(dset->shared->layout); + rdcc = &(dset->shared->cache.chunk); + HDassert(layout); + HDassert(rdcc); + HDassert(H5D_CHUNKED == layout->type); + + /* Search for cached chunks that haven't been written out */ + for(ent = rdcc->head; ent; ent = ent->next) + /* Flush the chunk out to disk, to make certain the size is correct later */ + if(H5D__chunk_flush_entry(dset, ent, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "cannot flush indexed storage buffer") + + /* Set addr & size for when dset is not written or queried chunk is not found */ + if (addr) + *addr = HADDR_UNDEF; + if (size) + *size = 0; + + /* Compose chunked index info struct */ + idx_info.f = dset->oloc.file; + idx_info.pline = &dset->shared->dcpl_cache.pline; + idx_info.layout = &dset->shared->layout.u.chunk; + idx_info.storage = &dset->shared->layout.storage.u.chunk; + + /* If the dataset is not written, return without errors */ + if(H5F_addr_defined(idx_info.storage->idx_addr)) { + /* Calculate the scaled of this chunk */ + H5VM_chunk_scaled(dset->shared->ndims, offset, layout->u.chunk.dim, udata.scaled); + udata.scaled[dset->shared->ndims] = 0; + + /* Initialize before iteration */ + udata.ndims = dset->shared->ndims; + udata.nbytes = 0; + udata.filter_mask = 0; + udata.chunk_addr = HADDR_UNDEF; + udata.found = FALSE; + + /* Iterate over the allocated chunks to find the requested chunk */ + if((dset->shared->layout.storage.u.chunk.ops->iterate)(&idx_info, H5D__get_chunk_info_by_coord_cb, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve information of the chunk by its scaled coordinates") + + /* Obtain requested info if the chunk is found */ + if (udata.found) { + if(filter_mask) + *filter_mask = udata.filter_mask; + if(addr) + *addr = udata.chunk_addr; + if(size) + *size = udata.nbytes; + } + } /* end if H5F_addr_defined */ + +done: + FUNC_LEAVE_NOAPI_TAG(ret_value) +} /* end H5D__get_chunk_info_by_coord() */ + diff --git a/src/H5Dnone.c b/src/H5Dnone.c index be421b6..9346220 100644 --- a/src/H5Dnone.c +++ b/src/H5Dnone.c @@ -242,9 +242,9 @@ H5D__none_idx_iterate(const H5D_chk_idx_info_t *idx_info, unsigned u; /* Local index variable */ int curr_dim; /* Current rank */ hsize_t idx; /* Array index of chunk */ - int ret_value = -1; /* Return value */ + int ret_value = H5_ITER_CONT; /* Return value */ - FUNC_ENTER_STATIC_NOERR + FUNC_ENTER_STATIC /* Sanity checks */ HDassert(idx_info); @@ -266,34 +266,35 @@ H5D__none_idx_iterate(const H5D_chk_idx_info_t *idx_info, HDassert(ndims > 0); /* Iterate over all the chunks in the dataset's dataspace */ - for(u = 0; u < idx_info->layout->nchunks; u++) { - /* Calculate the index of this chunk */ - idx = H5VM_array_offset_pre(ndims, idx_info->layout->max_down_chunks, chunk_rec.scaled); - - /* Calculate the address of the chunk */ - chunk_rec.chunk_addr = idx_info->storage->idx_addr + idx * idx_info->layout->size; - - /* Make "generic chunk" callback */ - if((ret_value = (*chunk_cb)(&chunk_rec, chunk_udata)) < 0) - HERROR(H5E_DATASET, H5E_CALLBACK, "failure in generic chunk iterator callback"); - - /* Update coordinates of chunk in dataset */ - curr_dim = (int)(ndims - 1); - while(curr_dim >= 0) { - /* Increment coordinate in current dimension */ - chunk_rec.scaled[curr_dim]++; - - /* Check if we went off the end of the current dimension */ - if(chunk_rec.scaled[curr_dim] >= idx_info->layout->chunks[curr_dim]) { - /* Reset coordinate & move to next faster dimension */ - chunk_rec.scaled[curr_dim] = 0; - curr_dim--; - } /* end if */ - else - break; - } /* end while */ + for(u = 0; u < idx_info->layout->nchunks && ret_value == H5_ITER_CONT; u++) { + /* Calculate the index of this chunk */ + idx = H5VM_array_offset_pre(ndims, idx_info->layout->max_down_chunks, chunk_rec.scaled); + + /* Calculate the address of the chunk */ + chunk_rec.chunk_addr = idx_info->storage->idx_addr + idx * idx_info->layout->size; + + /* Make "generic chunk" callback */ + if((ret_value = (*chunk_cb)(&chunk_rec, chunk_udata)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CALLBACK, H5_ITER_ERROR, "failure in generic chunk iterator callback") + + /* Update coordinates of chunk in dataset */ + curr_dim = (int)(ndims - 1); + while(curr_dim >= 0) { + /* Increment coordinate in current dimension */ + chunk_rec.scaled[curr_dim]++; + + /* Check if we went off the end of the current dimension */ + if(chunk_rec.scaled[curr_dim] >= idx_info->layout->chunks[curr_dim]) { + /* Reset coordinate & move to next faster dimension */ + chunk_rec.scaled[curr_dim] = 0; + curr_dim--; + } /* end if */ + else + break; + } /* end while */ } /* end for */ +done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__none_idx_iterate() */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index ed3bc12..723acf9 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -569,6 +569,9 @@ H5_DLL herr_t H5D__get_space_status(const H5D_t *dset, H5D_space_status_t *alloc H5_DLL herr_t H5D__alloc_storage(const H5D_io_info_t *io_info, H5D_time_alloc_t time_alloc, hbool_t full_overwrite, hsize_t old_dim[]); H5_DLL herr_t H5D__get_storage_size(const H5D_t *dset, hsize_t *storage_size); H5_DLL herr_t H5D__get_chunk_storage_size(H5D_t *dset, const hsize_t *offset, hsize_t *storage_size); +H5_DLL herr_t H5D__get_num_chunks(const H5D_t *dset, const H5S_t *space, hsize_t *nchunks); +H5_DLL herr_t H5D__get_chunk_info(const H5D_t *dset, const H5S_t *space, hsize_t chk_idx, hsize_t *coord, unsigned *filter_mask, haddr_t *offset, hsize_t *size); +H5_DLL herr_t H5D__get_chunk_info_by_coord(const H5D_t *dset, const hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size); H5_DLL haddr_t H5D__get_offset(const H5D_t *dset); H5_DLL void *H5D__vlen_get_buf_size_alloc(size_t size, void *info); H5_DLL herr_t H5D__vlen_get_buf_size(void *elem, hid_t type_id, unsigned ndim, const hsize_t *point, void *op_data); diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 63f1a24..7234d16 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -142,6 +142,9 @@ H5_DLL hid_t H5Dget_create_plist(hid_t dset_id); H5_DLL hid_t H5Dget_access_plist(hid_t dset_id); H5_DLL hsize_t H5Dget_storage_size(hid_t dset_id); H5_DLL herr_t H5Dget_chunk_storage_size(hid_t dset_id, const hsize_t *offset, hsize_t *chunk_bytes); +H5_DLL herr_t H5Dget_num_chunks(hid_t dset_id, hid_t fspace_id, hsize_t *nchunks); +H5_DLL herr_t H5Dget_chunk_info_by_coord(hid_t dset_id, const hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size); +H5_DLL herr_t H5Dget_chunk_info(hid_t dset_id, hid_t fspace_id, hsize_t chk_idx, hsize_t *coord, unsigned *filter_mask, haddr_t *addr, hsize_t *size); H5_DLL haddr_t H5Dget_offset(hid_t dset_id); H5_DLL herr_t H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, void *buf/*out*/); @@ -987,19 +987,25 @@ END_FUNC(PRIV) /* end H5EA_delete() */ * Purpose: Iterate over the elements of an extensible array * (copied and modified from FA_iterate() in H5FA.c) * - * Return: SUCCEED/FAIL + * Return: H5_ITER_CONT/H5_ITER_ERROR * * Programmer: Vailin Choi; Feb 2015 * + * Modification: + * Prototype changed (HDFFV-10661) + * - herr_t to int + * - SUCCEED/FAIL to H5_ITER_CONT/H5_ITER_ERROR + * June 6, 2019 -BMR *------------------------------------------------------------------------- */ BEGIN_FUNC(PRIV, ERR, -herr_t, SUCCEED, FAIL, +int, H5_ITER_CONT, H5_ITER_ERROR, H5EA_iterate(H5EA_t *ea, H5EA_operator_t op, void *udata)) /* Local variables */ uint8_t *elmt = NULL; hsize_t u; + int cb_ret = H5_ITER_CONT; /* Return value from callback */ /* * Check arguments. @@ -1013,9 +1019,7 @@ H5EA_iterate(H5EA_t *ea, H5EA_operator_t op, void *udata)) H5E_THROW(H5E_CANTALLOC, "memory allocation failed for extensible array element") /* Iterate over all elements in array */ - for(u = 0; u < ea->hdr->stats.stored.max_idx_set; u++) { - int cb_ret; /* Return value from callback */ - + for(u = 0; u < ea->hdr->stats.stored.max_idx_set && cb_ret == H5_ITER_CONT; u++) { /* Get array element */ if(H5EA_get(ea, u, elmt) < 0) H5E_THROW(H5E_CANTGET, "unable to delete fixed array") @@ -686,20 +686,26 @@ END_FUNC(PRIV) /* end H5FA_delete() */ * Note: This is not very efficient, we should be iterating directly * over the fixed array's direct block [pages]. * - * Return: SUCCEED/FAIL + * Return: H5_ITER_CONT/H5_ITER_ERROR * * Programmer: Vailin Choi * Thursday, April 30, 2009 * + * Modification: + * Prototype changed (HDFFV-10661) + * - herr_t to int + * - SUCCEED/FAIL to H5_ITER_CONT/H5_ITER_ERROR + * June 6, 2019 -BMR *------------------------------------------------------------------------- */ BEGIN_FUNC(PRIV, ERR, -herr_t, SUCCEED, FAIL, +int, H5_ITER_CONT, H5_ITER_ERROR, H5FA_iterate(H5FA_t *fa, H5FA_operator_t op, void *udata)) /* Local variables */ uint8_t *elmt = NULL; hsize_t u; + int cb_ret = H5_ITER_CONT; /* Return value from callback */ /* * Check arguments. @@ -713,9 +719,7 @@ H5FA_iterate(H5FA_t *fa, H5FA_operator_t op, void *udata)) H5E_THROW(H5E_CANTALLOC, "memory allocation failed for fixed array element") /* Iterate over all elements in array */ - for(u = 0; u < fa->hdr->stats.nelmts; u++) { - int cb_ret; /* Return value from callback */ - + for(u = 0; u < fa->hdr->stats.nelmts && cb_ret == H5_ITER_CONT; u++) { /* Get array element */ if(H5FA_get(fa, u, elmt) < 0) H5E_THROW(H5E_CANTGET, "unable to delete fixed array") diff --git a/src/H5VLnative.h b/src/H5VLnative.h index 1a3007a..a8d5720 100644 --- a/src/H5VLnative.h +++ b/src/H5VLnative.h @@ -36,8 +36,11 @@ typedef int H5VL_native_dataset_optional_t; #define H5VL_NATIVE_DATASET_FORMAT_CONVERT 0 /* H5Dformat_convert (internal) */ #define H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE 1 /* H5Dget_chunk_index_type */ #define H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE 2 /* H5Dget_chunk_storage_size */ -#define H5VL_NATIVE_DATASET_CHUNK_READ 3 /* H5Dchunk_read */ -#define H5VL_NATIVE_DATASET_CHUNK_WRITE 4 /* H5Dchunk_write */ +#define H5VL_NATIVE_DATASET_GET_NUM_CHUNKS 3 /* H5Dget_num_chunks */ +#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX 4 /* H5Dget_chunk_info */ +#define H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD 5 /* H5Dget_chunk_info_by_coord */ +#define H5VL_NATIVE_DATASET_CHUNK_READ 6 /* H5Dchunk_read */ +#define H5VL_NATIVE_DATASET_CHUNK_WRITE 7 /* H5Dchunk_write */ /* Typedef and values for native VOL connector file optional VOL operations */ typedef int H5VL_native_file_optional_t; diff --git a/src/H5VLnative_dataset.c b/src/H5VLnative_dataset.c index 1e3d263..676d859 100644 --- a/src/H5VLnative_dataset.c +++ b/src/H5VLnative_dataset.c @@ -339,7 +339,7 @@ H5VL__native_dataset_specific(void *obj, H5VL_dataset_specific_t specific_type, switch(specific_type) { /* H5Dspecific_space */ case H5VL_DATASET_SET_EXTENT: - { + { /* H5Dset_extent (H5Dextend - deprecated) */ const hsize_t *size = HDva_arg(arguments, const hsize_t *); if(H5D__set_extent(dset, size) < 0) @@ -348,7 +348,7 @@ H5VL__native_dataset_specific(void *obj, H5VL_dataset_specific_t specific_type, } case H5VL_DATASET_FLUSH: - { + { /* H5Dflush */ hid_t dset_id = HDva_arg(arguments, hid_t); /* Flush the dataset */ @@ -359,7 +359,7 @@ H5VL__native_dataset_specific(void *obj, H5VL_dataset_specific_t specific_type, } case H5VL_DATASET_REFRESH: - { + { /* H5Drefresh */ hid_t dset_id = HDva_arg(arguments, hid_t); /* Refresh the dataset */ @@ -399,7 +399,7 @@ H5VL__native_dataset_optional(void *obj, hid_t H5_ATTR_UNUSED dxpl_id, switch(optional_type) { case H5VL_NATIVE_DATASET_FORMAT_CONVERT: - { + { /* H5Dformat_convert */ dset = (H5D_t *)obj; switch(dset->shared->layout.type) { @@ -434,7 +434,7 @@ H5VL__native_dataset_optional(void *obj, hid_t H5_ATTR_UNUSED dxpl_id, } case H5VL_NATIVE_DATASET_GET_CHUNK_INDEX_TYPE: - { + { /* H5Dget_chunk_index_type */ H5D_chunk_index_t *idx_type = HDva_arg(arguments, H5D_chunk_index_t *); dset = (H5D_t *)obj; @@ -450,7 +450,7 @@ H5VL__native_dataset_optional(void *obj, hid_t H5_ATTR_UNUSED dxpl_id, } case H5VL_NATIVE_DATASET_GET_CHUNK_STORAGE_SIZE: - { + { /* H5Dget_chunk_storage_size */ hsize_t *offset = HDva_arg(arguments, hsize_t *); hsize_t *chunk_nbytes = HDva_arg(arguments, hsize_t *); @@ -467,8 +467,91 @@ H5VL__native_dataset_optional(void *obj, hid_t H5_ATTR_UNUSED dxpl_id, break; } + case H5VL_NATIVE_DATASET_GET_NUM_CHUNKS: + { /* H5Dget_num_chunks */ + const H5S_t *space = NULL; + hid_t space_id = HDva_arg(arguments, hid_t); + hsize_t *nchunks = HDva_arg(arguments, hsize_t *); + + dset = (H5D_t *)obj; + HDassert(dset); + HDassert(dset->shared); + HDassert(dset->shared->space); + + /* When default dataspace is given, use the dataset's dataspace */ + if(space_id == H5S_ALL) + space = dset->shared->space; + else /* otherwise, use the given space ID */ + if(NULL == (space = (const H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID") + + /* Make sure the dataset is chunked */ + if(H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + + /* Call private function */ + if(H5D__get_num_chunks(dset, space, nchunks) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get number of chunks") + + break; + } + + case H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_IDX: + { /* H5Dget_chunk_info */ + const H5S_t *space = NULL; + hid_t space_id = HDva_arg(arguments, hid_t); + hsize_t chk_index = HDva_arg(arguments, hsize_t); + hsize_t *offset = HDva_arg(arguments, hsize_t *); + unsigned *filter_mask = HDva_arg(arguments, unsigned *); + haddr_t *addr = HDva_arg(arguments, haddr_t *); + hsize_t *size = HDva_arg(arguments, hsize_t *); + + dset = (H5D_t *)obj; + HDassert(dset); + HDassert(dset->shared); + HDassert(dset->shared->space); + + /* When default dataspace is given, use the dataset's dataspace */ + if(space_id == H5S_ALL) + space = dset->shared->space; + else /* otherwise, use the given space ID */ + if(NULL == (space = (const H5S_t *)H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid dataspace ID") + + /* Make sure the dataset is chunked */ + if(H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + + /* Call private function */ + if(H5D__get_chunk_info(dset, space, chk_index, offset, filter_mask, addr, size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info by index") + break; + } + + case H5VL_NATIVE_DATASET_GET_CHUNK_INFO_BY_COORD: + { /* H5Dget_chunk_info_by_coord */ + hsize_t *offset = HDva_arg(arguments, hsize_t *); + unsigned *filter_mask = HDva_arg(arguments, unsigned *); + haddr_t *addr = HDva_arg(arguments, haddr_t *); + hsize_t *size = HDva_arg(arguments, hsize_t *); + + dset = (H5D_t *)obj; + HDassert(dset); + HDassert(dset->shared); + + /* Make sure the dataset is chunked */ + if(H5D_CHUNKED != dset->shared->layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a chunked dataset") + + /* Call private function */ + if(H5D__get_chunk_info_by_coord(dset, offset, filter_mask, addr, size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk info by its logical coordinates") + + break; + } + case H5VL_NATIVE_DATASET_CHUNK_READ: - { + { /* H5Dread_chunk */ const hsize_t *offset = HDva_arg(arguments, hsize_t *); uint32_t *filters = HDva_arg(arguments, uint32_t *); void *buf = HDva_arg(arguments, void *); @@ -496,7 +579,7 @@ H5VL__native_dataset_optional(void *obj, hid_t H5_ATTR_UNUSED dxpl_id, } case H5VL_NATIVE_DATASET_CHUNK_WRITE: - { + { /* H5Dwrite_chunk */ uint32_t filters = HDva_arg(arguments, uint32_t); const hsize_t *offset = HDva_arg(arguments, const hsize_t *); uint32_t data_size_32 = HDva_arg(arguments, uint32_t); diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index cbd1901..c1ac44c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -242,6 +242,7 @@ set (H5_TESTS page_buffer dtypes dsets + chunk_info cmpd_dset filter_fail extend diff --git a/test/Makefile.am b/test/Makefile.am index ff6d1c6..a19079d 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -58,9 +58,9 @@ TEST_PROG= testhdf5 \ cache cache_api cache_image cache_tagging lheap ohdr \ stab gheap evict_on_close farray earray btree2 fheap \ pool accum hyperslab istore bittests dt_arith page_buffer \ - dtypes dsets cmpd_dset filter_fail extend direct_chunk external efc \ - objcopy links unlink twriteorder big mtime fillval mount flush1 \ - flush2 app_ref enum set_extent ttsafe enc_dec_plist \ + dtypes dsets chunk_info cmpd_dset filter_fail extend direct_chunk \ + external efc objcopy links unlink twriteorder big mtime fillval mount \ + flush1 flush2 app_ref enum set_extent ttsafe enc_dec_plist \ enc_dec_plist_cross_platform getname vfd ros3 s3comms hdfs ntypes \ dangle dtransform reserved cross_read freespace mf vds file_image \ unregister cache_logging cork swmr vol diff --git a/test/chunk_info.c b/test/chunk_info.c new file mode 100644 index 0000000..a28ca84 --- /dev/null +++ b/test/chunk_info.c @@ -0,0 +1,1964 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +/* + * + * Purpose: Tests chunk query API functions + * + * Modification: + * Many tests were added for HDFFV-10677. -BMR, August 2019 + * + * Test structure: + * main() + * test_basic_query() + * test_get_chunk_info_highest_v18() + * test_get_chunk_info_v110() + * test_chunk_info_single_chunk() + * test_chunk_info_implicit() + * test_chunk_info_fixed_array() + * test_chunk_info_extensible_array() + * test_chunk_info_version2_btrees() + * test_failed_attempts() + * test_flt_msk_with_skip_compress() + * + * Helper functions: + * read_each_chunk() + * verify_and_write() + * verify_get_chunk_info() + * verify_get_chunk_info_by_coord() + * verify_empty_chunk_info() + * index_type_str() + * + */ +#define H5D_FRIEND +#define H5D_TESTING /* to use H5D__ functions */ +#include "H5Dpkg.h" + +#include "testhdf5.h" +#include "zlib.h" + +/* Test file names, using H5F_libver_t as indices */ +const char *FILENAME[] = { + "tchunk_info_earliest", + "tchunk_info_v18", + "tchunk_info_v110", + "tchunk_info_v112", + NULL +}; + +/* File to be used in test_failed_attempts */ +#define FILTERMASK_FILE "tflt_msk" +#define BASIC_FILE "basic_query" + +/* Parameters for testing chunk querying */ +#define SIMPLE_CHUNKED_DSET_NAME "Chunked Dataset" +#define CONTIGUOUS_DSET_NAME "Contiguous Dataset" +#define EMPTY_DSET_NAME "Empty Dataset" +#define EMPTY_EARLY_ALLOC_DSET_NAME "Empty Dataset with ALLOC_TIME_EARLY" +#define SINGLE_CHUNK_DSET_NAME "Single Chunk Index Dataset" +#define IMPLICIT_INDEX_DSET_NAME "Implicit Index Dataset" +#define FIXED_ARR_INDEX_DSET_NAME "Fixed Array Index Dataset" +#define EXT_ARR_INDEX_DSET_NAME "Extensible Array Index Dataset" +#define V2_BTREE_INDEX_DSET_NAME "Version 2 B-Tree Index Dataset" +#define SKIP_FILTER_DSET_NAME "Dataset with Skipping One Filter" +#define FILENAME_BUF_SIZE 256 /* Size for file names */ +#define RANK 2 /* Rank for datasets */ + +/* Dimension of the dataset */ +#define NX 24 +#define NY 16 + +/* Dimension of the chunk */ +#define CHUNK_NX 6 +#define CHUNK_NY 4 + +/* X/Y coords of first chunk written */ +#define START_CHK_X 0 +#define START_CHK_Y 2 + +/* X/Y coord of last chunk written */ +#define END_CHK_X 2 +#define END_CHK_Y 4 + +/* X and Y coords of an empty chunk */ +#define EMPTY_CHK_X 0 +#define EMPTY_CHK_Y 0 + +/* Size of a chunk when the entire dataset is a one single chunk */ +#define SINGLE_CHK_SIZE (NX*NY*sizeof(int)) + +/* Size of a chunk */ +#define CHK_SIZE (CHUNK_NX*CHUNK_NY*sizeof(int)) + +/* Size of an empty chunk */ +#define EMPTY_CHK_SIZE 0 + +/* Number of maximum chunks without extending */ +#define NUM_CHUNKS ((NX/CHUNK_NX)*(NY/CHUNK_NY)) + +/* Number of chunks that have been written */ +#define NUM_CHUNKS_WRITTEN 4 +#define ONE_CHUNK_WRITTEN 1 +#define TWO_CHUNKS_WRITTEN 2 +#define NO_CHUNK_WRITTEN 0 + +/* For testing invalid arguments */ +#define NONEXIST_CHK_INDEX 3 +#define OUTOFRANGE_CHK_INDEX 5 +#define INVALID_CHK_INDEX 5 + +/* For compressed data */ +#define DEFLATE_SIZE_ADJUST(s) (ceil(((double)(s))*1.001)+12) + +/* For use in error reporting */ +#define MSG_CHK_ADDR "Chunk address should not be HADDR_UNDEF because of H5D_ALLOC_TIME_EARLY." +#define MSG_CHK_SIZE "Chunk size should not be 0 because of H5D_ALLOC_TIME_EARLY." + +/* Utility function to initialize arguments */ +void reinit_vars(unsigned *read_flt_msk, haddr_t *addr, hsize_t *size); + +/* Helper function containing common code that verifies indexing type + and number of chunks */ +static int verify_and_write(hid_t chunkfile, const char* dset_name, hid_t dspace, H5D_chunk_index_t exp_idx_type, hsize_t exp_num_chunks, unsigned flt_msk); +static int verify_get_chunk_info(hid_t dset, hid_t dspace, hsize_t chk_index, hsize_t exp_chk_size, hsize_t *exp_offset, unsigned exp_flt_msk); +static int verify_get_chunk_info_by_coord(hid_t dset, hsize_t *offset, hsize_t exp_chk_size, unsigned exp_flt_msk); +static int verify_empty_chunk_info(hid_t dset, hsize_t *offset); +static const char* index_type_str(H5D_chunk_index_t idx_type); + +/*------------------------------------------------------------------------- + * Function: read_each_chunk (helper function) + * + * Purpose: Reads the chunk specified by its offset and verifies that + * it contains the same data as what was written. This function + * is used in various test_get_chunk_info... functions. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Date: September 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t read_each_chunk(hid_t dset_id, hsize_t *offset, void *direct_buf) +{ + int read_buf[CHUNK_NX][CHUNK_NY]; + unsigned read_flt_msk = 0; + + HDmemset(&read_buf, 0, sizeof(read_buf)); + + /* Read the chunk specified by its offset */ + if(H5Dread_chunk(dset_id, H5P_DEFAULT, offset, &read_flt_msk, read_buf) < 0) + return FAIL; + + /* Verify that read chunk is the same as the corresponding written one */ + if(HDmemcmp(direct_buf, read_buf, CHUNK_NX*CHUNK_NY) != 0) + { + HDfprintf(stderr, "Read chunk differs from written chunk at offset (%d,%d)\n", offset[0], offset[1]); + return FAIL; + } + + return SUCCEED; +} + +/*------------------------------------------------------------------------- + * Function: reinit_vars (helper function) + * + * Purpose: Wipes out variables for the next use, used in various tests. + * + * Return: Won't fail + * + * Date: September 2018 + * + *------------------------------------------------------------------------- + */ +void reinit_vars(unsigned *read_flt_msk, haddr_t *addr, hsize_t *size) +{ + if(read_flt_msk) + *read_flt_msk = 0; + if(addr) + *addr = 0; + if(size) + *size = 0; +} + +/*------------------------------------------------------------------------- + * Function: verify_get_chunk_info (helper function) + * + * Purpose: Verifies that H5Dget_chunk_info returns correct + * values for a chunk. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Date: August 2019 + * + *------------------------------------------------------------------------- + */ +static int +verify_get_chunk_info(hid_t dset, hid_t dspace, hsize_t chk_index, hsize_t exp_chk_size, hsize_t *exp_offset, unsigned exp_flt_msk) +{ + unsigned read_flt_msk = 0; /* Read filter mask */ + hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ + hsize_t size = 0; /* Size of an allocated/written chunk */ + haddr_t addr = 0; /* Address of an allocated/written chunk */ + + if(H5Dget_chunk_info(dset, dspace, chk_index, out_offset, &read_flt_msk, &addr, &size) < 0) + TEST_ERROR + CHECK(addr, HADDR_UNDEF, "H5Dget_chunk_info"); + VERIFY(size, exp_chk_size, "H5Dget_chunk_info, chunk size"); + VERIFY(read_flt_msk, exp_flt_msk, "H5Dget_chunk_info, filter mask"); + VERIFY(out_offset[0], exp_offset[0], "H5Dget_chunk_info, offset[0]"); + VERIFY(out_offset[1], exp_offset[1], "H5Dget_chunk_info, offset[1]"); + return SUCCEED; + +error: + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: verify_get_chunk_info_by_coord (helper function) + * + * Purpose: Verifies that H5Dget_chunk_info_by_coord returns correct + * values for a chunk. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Date: August 2019 + * + *------------------------------------------------------------------------- + */ +static int +verify_get_chunk_info_by_coord(hid_t dset, hsize_t *offset, hsize_t exp_chk_size, unsigned exp_flt_msk) +{ + unsigned read_flt_msk = 0; /* Read filter mask */ + hsize_t size = 0; /* Size of an allocated/written chunk */ + haddr_t addr = 0; /* Address of an allocated/written chunk */ + + /* Get info of the chunk at logical coordinates specified by offset */ + if(H5Dget_chunk_info_by_coord(dset, offset, &read_flt_msk, &addr, &size) < 0) + TEST_ERROR + CHECK(addr, HADDR_UNDEF, "H5Dget_chunk_info_by_coord"); + VERIFY(size, exp_chk_size, "H5Dget_chunk_info_by_coord, chunk size"); + VERIFY(read_flt_msk, exp_flt_msk, "H5Dget_chunk_info_by_coord, filter mask"); + return SUCCEED; + +error: + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: verify_empty_chunk_info (helper function) + * + * Purpose: Verifies that H5Dget_chunk_info_by_coord returns correct + * values for an empty chunk. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Date: August 2018 + * + *------------------------------------------------------------------------- + */ +static int +verify_empty_chunk_info(hid_t dset, hsize_t *offset) +{ + unsigned read_flt_msk = 0; /* Read filter mask */ + hsize_t size = 0; /* Size of an allocated/written chunk */ + haddr_t addr = 0; /* Address of an allocated/written chunk */ + + /* Get info of the chunk at logical coordinates specified by offset */ + if(H5Dget_chunk_info_by_coord(dset, offset, &read_flt_msk, &addr, &size) < 0) + TEST_ERROR + VERIFY(addr, HADDR_UNDEF, "H5Dget_chunk_info_by_coord, chunk address"); + VERIFY(size, EMPTY_CHK_SIZE, "H5Dget_chunk_info_by_coord, chunk size"); + return SUCCEED; + +error: + return FAIL; +} + +/*------------------------------------------------------------------------- + * Function: index_type_str (helper function) + * + * Purpose: Returns the string containing the text associated with the + * given indexing scheme. For use in error messages. + * + * Return: Success: a valid indexing scheme string + * Failure: a note indicating the indexing type is invalid + * + * Date: August 2019 + * + *------------------------------------------------------------------------- + */ +static const char* +index_type_str(H5D_chunk_index_t idx_type) +{ + switch (idx_type) { + case H5D_CHUNK_IDX_SINGLE: + return("Single Chunk index type"); + case H5D_CHUNK_IDX_NONE: + return("Implicit index type"); + case H5D_CHUNK_IDX_FARRAY: + return("Fixed Array index type"); + case H5D_CHUNK_IDX_EARRAY: + return("Extensible Array index type"); + case H5D_CHUNK_IDX_BT2: + return("Version 2 B-tree index type"); + case H5D_CHUNK_IDX_BTREE: + return("Version 1 B-tree index type (default)"); + case H5D_CHUNK_IDX_NTYPES: + default: + return("invalid index type"); + } +} /* index_type_str */ + +/*------------------------------------------------------------------------- + * Function: verify_and_write (helper function) + * + * Purpose: Verifies that chunk indexing scheme and number of chunks of + * the dataset matches the expected values, then write data to + * a subset of chunks. This function opens the dataset then + * closes it after writing. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Date: August 2019 + * + *------------------------------------------------------------------------- + */ +static int +verify_and_write(hid_t chunkfile, const char* dset_name, hid_t dspace, H5D_chunk_index_t exp_idx_type, hsize_t exp_num_chunks, unsigned flt_msk) +{ + hid_t dset = H5I_INVALID_HID; /* Dataset ID */ + H5D_chunk_index_t idx_type; /* Dataset chunk index type */ + hsize_t offset[2] = {0, 0}; /* Offset coordinates of a chunk */ + hsize_t nchunks = 0; /* Number of chunks */ + hsize_t ii, jj; /* Array indices */ + int n; /* Used as chunk index, but int to avoid conversion warning */ + int direct_buf[NUM_CHUNKS][CHUNK_NX][CHUNK_NY];/* Data in chunks */ + + /* Open the dataset */ + if((dset = H5Dopen2(chunkfile, dset_name, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Get the chunk indexing type of the dataset */ + if(H5Dget_chunk_index_type(dset, &idx_type) < 0) + TEST_ERROR + + /* Ensure the correct chunk indexing scheme is used */ + if(idx_type != exp_idx_type) + { + char msg[256]; + sprintf(msg, "Should be using %s.\n", index_type_str(idx_type)); + FAIL_PUTS_ERROR(msg); + } + + /* Get and verify the number of chunks */ + if(H5Dget_num_chunks(dset, dspace, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, exp_num_chunks, "H5Dget_num_chunks, number of chunks"); + + /* Get and verify the number of chunks again, passing in H5S_ALL */ + if(H5Dget_num_chunks(dset, H5S_ALL, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, exp_num_chunks, "H5Dget_num_chunks, number of chunks"); + + /* Initialize the array of chunk data for all NUM_CHUNKS chunks */ + for(n = 0; n < NUM_CHUNKS; n++) + for(ii = 0; ii < CHUNK_NX; ii++) + for(jj = 0; jj < CHUNK_NY; jj++) + direct_buf[n][ii][jj] = n + 1; + + /* Write only NUM_CHUNKS_WRITTEN chunks at the following logical coords: + (0,2) (0,3) (1,2) (1,3) */ + n = 0; + for(ii = START_CHK_X; ii < END_CHK_X; ii++) + for(jj = START_CHK_Y; jj < END_CHK_Y; jj++, n++) { + offset[0] = ii * CHUNK_NX; + offset[1] = jj * CHUNK_NY; + if(H5Dwrite_chunk(dset, H5P_DEFAULT, flt_msk, offset, CHK_SIZE, (void*)direct_buf[n]) < 0) + TEST_ERROR + } + + /* Close the dataset */ + if(H5Dclose(dset) < 0) TEST_ERROR + + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + } H5E_END_TRY; + return FAIL; +} /* verify_and_write */ + +/*------------------------------------------------------------------------- + * Function: test_get_chunk_info_highest_v18 + * + * Purpose: Test getting various chunk information + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Note: Note that the dataspace argument in these new functions are + * currently not used. The functionality involved the dataspace + * will be implemented in the next version. + * + * Description: + * This function tests the new API functions added for EED-343: + * H5Dget_num_chunks, H5Dget_chunk_info, and + * H5Dget_chunk_info_by_coord for high bound up to 1.8. + * + * Date: September 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_get_chunk_info_highest_v18(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t chunkfile = H5I_INVALID_HID; /* File ID */ + hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dset = H5I_INVALID_HID; /* Dataset ID */ + hid_t cparms = H5I_INVALID_HID; /* Creation plist */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + int direct_buf[NUM_CHUNKS][CHUNK_NX][CHUNK_NY]; /* Data in chunks */ + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* 2 unlimited dims */ + hsize_t out_offset[2]; /* Buffer to get offset coordinates */ + hsize_t size = 0; /* Size of an allocated/written chunk */ + hsize_t nchunks = 0; /* Number of chunks */ + haddr_t addr = 0; /* Address of an allocated/written chunk */ + hsize_t chk_index = 0; /* Index of a chunk */ + hsize_t dims[2] = {NX, NY};/* Dataset dimensions */ + unsigned flt_msk = 0; /* Filter mask */ + unsigned read_flt_msk = 0; /* Filter mask after direct read */ + int fillvalue = -1; /* Fill value */ + int aggression = 9; /* Compression aggression setting */ + hsize_t offset[2] = {0, 0}; /* Offset coordinates of a chunk */ + const Bytef *z_src = (const Bytef*)(direct_buf); + Bytef *z_dst; /*destination buffer */ + uLongf z_dst_nbytes = (uLongf)DEFLATE_SIZE_ADJUST(CHK_SIZE); + uLong z_src_nbytes = (uLong)CHK_SIZE; + void *outbuf = NULL; /* Pointer to new buffer */ + hsize_t ii, jj; /* Array indices */ + int n; /* Used as chunk index, but int to avoid conversion warning */ + herr_t ret; /* Temporary returned value for verifying failure */ + + TESTING("getting chunk information in file with version prior to 1.10"); + + /* Create the file */ + h5_fixname(FILENAME[H5F_LIBVER_V18], fapl, filename, sizeof filename); + + /* Set version bounds for creating the file. High bound to V18 to test + chunked dataset that use B-tree v1 structures to index chunks. */ + if(H5Pset_libver_bounds(fapl, H5F_LIBVER_EARLIEST, H5F_LIBVER_V18) < 0) + TEST_ERROR + + chunkfile = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl); + if(chunkfile < 0) + TEST_ERROR + + /* Create the file and memory dataspace */ + if((dspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + TEST_ERROR + + /* Set dset creation properties with chunking, compression, and fillvalue */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR + if(H5Pset_chunk(cparms, RANK, chunk_dims) < 0) TEST_ERROR + +#ifdef H5_HAVE_FILTER_DEFLATE + if(H5Pset_deflate(cparms, (unsigned)aggression) < 0) TEST_ERROR +#endif /* end H5_HAVE_FILTER_DEFLATE */ + + /* Set fill value */ + if(H5Pset_fill_value(cparms, H5T_NATIVE_INT, &fillvalue) < 0) TEST_ERROR + + /* Create a new dataset using cparms creation properties */ + dset = H5Dcreate2(chunkfile, SIMPLE_CHUNKED_DSET_NAME, H5T_NATIVE_INT, dspace, H5P_DEFAULT, cparms, H5P_DEFAULT); + if(dset < 0) TEST_ERROR + + /* Initialize the array of chunk data for all NUM_CHUNKS chunks */ + for(n = 0; n < NUM_CHUNKS; n++) + for(ii = 0; ii < CHUNK_NX; ii++) + for(jj = 0; jj < CHUNK_NY; jj++) + direct_buf[n][ii][jj] = n + 1; + + /* Allocate output (compressed) buffer */ + outbuf = malloc(z_dst_nbytes); + z_dst = (Bytef *)outbuf; + + /* Perform compression from the source to the destination buffer */ + ret = compress2(z_dst, &z_dst_nbytes, z_src, z_src_nbytes, aggression); + + /* Check for various zlib errors */ + if(Z_BUF_ERROR == ret) { + fprintf(stderr, "overflow"); + TEST_ERROR + } else if(Z_MEM_ERROR == ret) { + fprintf(stderr, "deflate memory error"); + TEST_ERROR + } else if(Z_OK != ret) { + fprintf(stderr, "other deflate error"); + TEST_ERROR + } + + /* Write only NUM_CHUNKS_WRITTEN chunks at the following logical coords: + (0,2) (0,3) (1,2) (1,3) */ + n = 0; + for(ii = START_CHK_X; ii < END_CHK_X; ii++) + for(jj = START_CHK_Y; jj < END_CHK_Y; jj++, n++) { + offset[0] = ii * CHUNK_NX; + offset[1] = jj * CHUNK_NY; + ret = H5Dwrite_chunk(dset, H5P_DEFAULT, flt_msk, offset, CHK_SIZE, (void*)direct_buf[n]); + if(ret < 0) TEST_ERROR + } + + /* Read each chunk in the subset of chunks and verify the values */ + /* if(read_each_chunk(dset, offset, (void*)direct_buf[chk_index]) < 0) + if(read_each_chunk(dset) == FAIL) + TEST_ERROR + */ + + /* Free the read buffer */ + if(outbuf) + HDfree(outbuf); + + if(H5Fflush(dset, H5F_SCOPE_LOCAL) < 0) + TEST_ERROR + + /* Close the dataset */ + if(H5Dclose(dset) < 0) TEST_ERROR + + /* ...open it again to test the chunk query functions */ + if((dset = H5Dopen2(chunkfile, SIMPLE_CHUNKED_DSET_NAME, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Get and verify the number of chunks written */ + if(H5Dget_num_chunks(dset, dspace, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, NUM_CHUNKS_WRITTEN, "H5Dget_num_chunks, number of chunks"); + + /* Get and verify info of the last written chunk again, passing in H5S_ALL + this time */ + offset[0] = 6; + offset[1] = 12; + if(verify_get_chunk_info(dset, H5S_ALL, NUM_CHUNKS_WRITTEN-1, CHK_SIZE, offset, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info failed\n"); + + /* Attempt to get info of a non-existing chunk, should fail */ + chk_index = OUTOFRANGE_CHK_INDEX; + H5E_BEGIN_TRY { + ret = H5Dget_chunk_info(dset, H5S_ALL, chk_index, out_offset, &read_flt_msk, &addr, &size); + } H5E_END_TRY; + if(ret != FAIL) + FAIL_PUTS_ERROR(" Attempt to get info of a non-existing chunk."); + + /* Attempt to get info of empty chunks, verify the returned addr and size */ + offset[0] = 0; + offset[1] = 0; + if(verify_empty_chunk_info(dset, offset) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord on empty chunk failed\n"); + + offset[0] = 3 * CHUNK_NX; + offset[1] = 3 * CHUNK_NY; + if(verify_empty_chunk_info(dset, offset) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord on empty chunk failed\n"); + + /* Go through all written chunks, get their info and verify the values */ + chk_index = 0; + for(ii = START_CHK_X; ii < END_CHK_X; ii++) + for(jj = START_CHK_Y; jj < END_CHK_Y; jj++, chk_index++) { + offset[0] = ii * CHUNK_NX; + offset[1] = jj * CHUNK_NY; + + if(verify_get_chunk_info(dset, dspace, chk_index, CHK_SIZE, offset, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info failed\n"); + + /* Use the same offset to pass into the next ...by_coord function */ + if(verify_get_chunk_info_by_coord(dset, offset, CHK_SIZE, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord failed\n"); + } + + /* Close the first dataset */ + if(H5Dclose(dset) < 0) TEST_ERROR + + /* Create an empty dataset and close it */ + dset = H5Dcreate2(chunkfile, EMPTY_DSET_NAME, H5T_NATIVE_INT, dspace, H5P_DEFAULT, cparms, H5P_DEFAULT); + if(dset < 0) TEST_ERROR + if(H5Dclose(dset) < 0) TEST_ERROR + + /* Reopen the empty dataset to verify the chunk query functions on it */ + if((dset = H5Dopen2(chunkfile, EMPTY_DSET_NAME, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Verify that the number of chunks is 0 */ + if(H5Dget_num_chunks(dset, dspace, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, NO_CHUNK_WRITTEN, "H5Dget_num_chunks, number of chunks"); + + /* Attempt to get info of a chunk from an empty dataset, should fail */ + chk_index = OUTOFRANGE_CHK_INDEX; + H5E_BEGIN_TRY { + ret = H5Dget_chunk_info(dset, dspace, chk_index, out_offset, &read_flt_msk, &addr, &size); + } H5E_END_TRY; + if(ret != FAIL) + FAIL_PUTS_ERROR(" Attempt to get info of a non-existing chunk."); + + /* Attempt to get info of a chunk given its coords from an empty dataset, + should succeed with the returned address as HADDR_UNDEF and size as 0 */ + offset[0] = EMPTY_CHK_X; + offset[1] = EMPTY_CHK_Y; + if(verify_empty_chunk_info(dset, offset) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord on empty chunk failed\n"); + + if(H5Dclose(dset) < 0) TEST_ERROR + + /************************************************************************ + * Test empty dataset with H5D_ALLOC_TIME_EARLY * + ************************************************************************/ + + /* Set space allocation to early so that chunk query functions will + retrieve chunk information even though the dataset is empty */ + if(H5Pset_alloc_time(cparms, H5D_ALLOC_TIME_EARLY) < 0) + TEST_ERROR + + /* Create an empty dataset and close it */ + dset = H5Dcreate2(chunkfile, EMPTY_EARLY_ALLOC_DSET_NAME, H5T_NATIVE_INT, dspace, H5P_DEFAULT, cparms, H5P_DEFAULT); + if(dset < 0) TEST_ERROR + if(H5Dclose(dset) < 0) TEST_ERROR + + /* Reopen the empty dataset to verify the chunk query functions on it */ + if((dset = H5Dopen2(chunkfile, EMPTY_EARLY_ALLOC_DSET_NAME, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Verify that the number of chunks is NUM_CHUNKS */ + if(H5Dget_num_chunks(dset, dspace, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, NUM_CHUNKS, "H5Dget_num_chunks, number of chunks"); + + /* Attempt to get info of a chunk from an empty dataset, verify the + returned address and size in the case of H5D_ALLOC_TIME_EARLY */ + chk_index = NONEXIST_CHK_INDEX; + reinit_vars(&read_flt_msk, &addr, &size); + ret = H5Dget_chunk_info(dset, dspace, chk_index, out_offset, &read_flt_msk, &addr, &size); + if(ret < 0) TEST_ERROR + /* Because of H5D_ALLOC_TIME_EARLY, addr cannot be HADDR_UNDEF and size not 0 */ + if(addr == HADDR_UNDEF) + FAIL_PUTS_ERROR(MSG_CHK_ADDR); + if(size == EMPTY_CHK_SIZE) + FAIL_PUTS_ERROR(MSG_CHK_SIZE); + + chk_index = 10; + reinit_vars(&read_flt_msk, &addr, &size); + ret = H5Dget_chunk_info(dset, dspace, chk_index, out_offset, &read_flt_msk, &addr, &size); + if(ret < 0) TEST_ERROR + /* Because of H5D_ALLOC_TIME_EARLY, addr cannot be HADDR_UNDEF and size not 0 */ + if(addr == HADDR_UNDEF) + FAIL_PUTS_ERROR(MSG_CHK_ADDR); + if(size == EMPTY_CHK_SIZE) + FAIL_PUTS_ERROR(MSG_CHK_SIZE); + + /* Attempt to get info of a chunk given its coords from an empty dataset, + verify the returned address and size */ + offset[0] = 0; + offset[1] = 0; + if(H5Dget_chunk_info_by_coord(dset, offset, &read_flt_msk, &addr, &size) < 0) + TEST_ERROR + /* Because of H5D_ALLOC_TIME_EARLY, addr cannot be HADDR_UNDEF and size not 0 */ + if(addr == HADDR_UNDEF) + FAIL_PUTS_ERROR(MSG_CHK_ADDR); + if(size == 0) + FAIL_PUTS_ERROR(MSG_CHK_SIZE); + + if(H5Dclose(dset) < 0) TEST_ERROR + + /* Close/release resources. */ + if(H5Sclose(dspace) < 0) TEST_ERROR + if(H5Pclose(cparms) < 0) TEST_ERROR + if(H5Fclose(chunkfile) < 0) TEST_ERROR + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Sclose(dspace); + H5Pclose(cparms); + H5Fclose(chunkfile); + } H5E_END_TRY; + + H5_FAILED(); + return FAIL; +} /* test_get_chunk_info_highest_v18() */ + +/*------------------------------------------------------------------------- + * Function: test_chunk_info_single_chunk + * + * Purpose: Test getting various chunk information when Single Chunk + * index type is used + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Note: Note that the dataspace argument in these new functions are + * currently not used. The functionality involved the dataspace + * will be implemented in the next version. + * + * Date: November 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_chunk_info_single_chunk(const char *filename, hid_t fapl) +{ + hid_t chunkfile = H5I_INVALID_HID; /* File ID */ + hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dset = H5I_INVALID_HID; /* Dataset ID */ + hid_t cparms = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {NX, NY}; /* Chunk dimensions */ + int data_buf[NX][NY]; /* Input buffer */ + H5D_chunk_index_t idx_type; /* Dataset chunk index type */ + unsigned flt_msk = 0; /* Filter mask */ + unsigned read_flt_msk = 0; /* Filter mask after direct read */ + hsize_t offset[2]; /* Offset coordinates of a chunk */ + hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ + hsize_t size = 0; /* Size of an allocated/written chunk */ + hsize_t nchunks = 0; /* Number of chunks */ + haddr_t addr = 0; /* Address of an allocated/written chunk */ + hsize_t chk_index = 0; /* Index of a chunk */ + hsize_t ii, jj; /* Array indices */ + herr_t ret; /* Temporary returned value for verifying failure */ + + TESTING(" Single Chunk index"); + + /* Open the file for reading/writing */ + if((chunkfile = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + TEST_ERROR + + /* Create dataspace */ + if((dspace = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR + + /* Enable chunking */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR + + if(H5Pset_chunk(cparms, RANK, chunk_dims) < 0) + TEST_ERROR + + /* Create a new dataset using cparms creation properties */ + dset = H5Dcreate2(chunkfile, SINGLE_CHUNK_DSET_NAME, H5T_NATIVE_INT, dspace, H5P_DEFAULT, cparms, H5P_DEFAULT); + if(dset < 0) TEST_ERROR + + /* Close the dataset */ + if(H5Dclose(dset) < 0) TEST_ERROR + + /* ...open it again to test the chunk query functions on a single empty + chunk */ + if((dset = H5Dopen2(chunkfile, SINGLE_CHUNK_DSET_NAME, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Ensure the correct chunk indexing scheme is used */ + if(H5Dget_chunk_index_type(dset, &idx_type) < 0) + TEST_ERROR + if(idx_type != H5D_CHUNK_IDX_SINGLE) + FAIL_PUTS_ERROR("Should be using Single Chunk index type"); + + /* Get the number of chunks and verify that no chunk has been written */ + if(H5Dget_num_chunks(dset, dspace, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, NO_CHUNK_WRITTEN, "H5Dget_num_chunks, number of chunks"); + + /* Initialize the array of chunk data for the single chunk */ + for(ii = 0; ii < NX; ii++) + for(jj = 0; jj < NY; jj++) + data_buf[ii][jj] = (int)(ii*jj); + + /* Write the chunk */ + if(H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_buf) < 0) + TEST_ERROR + + /* Get and verify that one chunk had been written */ + if(H5Dget_num_chunks(dset, dspace, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, ONE_CHUNK_WRITTEN, "H5Dget_num_chunks, number of chunks"); + + /* Offset of the only chunk */ + offset[0] = 0; + offset[1] = 0; + + /* Get and verify info of the first and only chunk */ + if(verify_get_chunk_info(dset, H5S_ALL, 0, SINGLE_CHK_SIZE, offset, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification H5Dget_chunk_info failed\n"); + + /* Get and verify info of the chunk at logical coordinates (0,0) */ + if(verify_get_chunk_info_by_coord(dset, offset, SINGLE_CHK_SIZE, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord failed\n"); + + /* Attempt to get chunk info given an invalid chunk index and verify + * that failure occurs */ + chk_index = INVALID_CHK_INDEX; + reinit_vars(&read_flt_msk, &addr, &size); + H5E_BEGIN_TRY { + ret = H5Dget_chunk_info(dset, dspace, chk_index, out_offset, &read_flt_msk, &addr, &size); + } H5E_END_TRY; + if(ret != FAIL) + TEST_ERROR + + /* Release resourse */ + if(H5Dclose(dset) < 0) TEST_ERROR + if(H5Sclose(dspace) < 0) TEST_ERROR + if(H5Fclose(chunkfile) < 0) TEST_ERROR + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Sclose(dspace); + H5Pclose(cparms); + H5Fclose(chunkfile); + } H5E_END_TRY; + + H5_FAILED(); + return FAIL; +} /* test_chunk_info_single_chunk() */ + +/*------------------------------------------------------------------------- + * Function: test_chunk_info_implicit + * + * Purpose: Test getting various chunk information when Implicit + * index type is used + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Note: Note that the dataspace argument in these new functions are + * currently not used. The functionality involved the dataspace + * will be implemented in the next version. + * + * Date: November 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_chunk_info_implicit(char *filename, hid_t fapl) +{ + hid_t chunkfile = H5I_INVALID_HID; /* File ID */ + hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dset = H5I_INVALID_HID; /* Dataset ID */ + hid_t cparms = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + unsigned flt_msk = 0; /* Filter mask */ + hsize_t chk_index = 0; /* Index of a chunk */ + hsize_t ii, jj; /* Array indices */ + + TESTING(" Implicit index"); + + /* Open the file for reading/writing */ + if((chunkfile = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + TEST_ERROR + + /* Create dataspace */ + if((dspace = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR + + /* Enable chunking */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR + + if(H5Pset_chunk(cparms, RANK, chunk_dims) < 0) + TEST_ERROR + + /* Set allocation time to early */ + if(H5Pset_alloc_time(cparms, H5D_ALLOC_TIME_EARLY) < 0) + TEST_ERROR + + /* Create a new dataset using cparms creation properties */ + dset = H5Dcreate2(chunkfile, IMPLICIT_INDEX_DSET_NAME, H5T_NATIVE_INT, dspace, H5P_DEFAULT, cparms, H5P_DEFAULT); + if(dset < 0) TEST_ERROR + + /* Close the dataset */ + if(H5Dclose(dset) < 0) TEST_ERROR + + /* Verify chunk indexing scheme and number of chunks, and write data to a + subset of chunks. */ + if(verify_and_write(chunkfile, IMPLICIT_INDEX_DSET_NAME, dspace, H5D_CHUNK_IDX_NONE, NUM_CHUNKS, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification and write failed\n"); + + /* Open the dataset again to test getting chunk info */ + if((dset = H5Dopen2(chunkfile, IMPLICIT_INDEX_DSET_NAME, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Go through all chunks, and get their info and verify the values */ + chk_index = 0; + for(ii = 0; ii < NX/CHUNK_NX; ii++) + for(jj = 0; jj < NY/CHUNK_NY; jj++, chk_index++) { + hsize_t offset[2] = {ii * CHUNK_NX, jj * CHUNK_NY}; + + if(verify_get_chunk_info(dset, H5S_ALL, chk_index, CHK_SIZE, offset, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info failed\n"); + + /* Get info of a chunk and verify its information. Note that + all chunks in this dataset are allocated because of the property + H5D_ALLOC_TIME_EARLY */ + if(verify_get_chunk_info_by_coord(dset, offset, CHK_SIZE, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord failed\n"); + } + + /* Release resourse */ + if(H5Dclose(dset) < 0) TEST_ERROR + if(H5Sclose(dspace) < 0) TEST_ERROR + if(H5Pclose(cparms) < 0) TEST_ERROR + if(H5Fclose(chunkfile) < 0) TEST_ERROR + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Sclose(dspace); + H5Pclose(cparms); + H5Fclose(chunkfile); + } H5E_END_TRY; + + H5_FAILED(); + return FAIL; +} /* test_chunk_info_implicit() */ + +/*------------------------------------------------------------------------- + * Function: test_chunk_info_fixed_array + * + * Purpose: Test getting various chunk information when Fixed Array + * index type is used + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Note: Note that the dataspace argument in these new functions are + * currently not used. The functionality involved the dataspace + * will be implemented in the next version. + * + * Date: November 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_chunk_info_fixed_array(const char *filename, hid_t fapl) +{ + hid_t chunkfile = H5I_INVALID_HID; /* File ID */ + hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dset = H5I_INVALID_HID; /* Dataset ID */ + hid_t cparms = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + int direct_buf[NUM_CHUNKS][CHUNK_NX][CHUNK_NY];/* Data in chunks */ + unsigned flt_msk = 0; /* Filter mask */ + unsigned read_flt_msk = 0; /* Filter mask after direct read */ + hsize_t offset[2]; /* Offset coordinates of a chunk */ + hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ + hsize_t size = 0; /* Size of an allocated/written chunk */ + hsize_t nchunks = 0; /* Number of chunks */ + haddr_t addr = 0; /* Address of an allocated/written chunk */ + hsize_t chk_index = 0; /* Index of a chunk */ + hsize_t ii, jj; /* Array indices */ + int n; /* Used as chunk index, but int to avoid conversion warning */ + herr_t ret; /* Temporary returned value for verifying failure */ + + TESTING(" Fixed Array index"); + + /* Open the file for reading/writing */ + if((chunkfile = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + TEST_ERROR + + /* Create dataspace */ + if((dspace = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR + + /* Enable chunking */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR + + if(H5Pset_chunk(cparms, RANK, chunk_dims) < 0) + TEST_ERROR + + /* Create a new dataset using cparms creation properties */ + dset = H5Dcreate2(chunkfile, FIXED_ARR_INDEX_DSET_NAME, H5T_NATIVE_INT, dspace, H5P_DEFAULT, cparms, H5P_DEFAULT); + if(dset < 0) TEST_ERROR + + /* Close the dataset */ + if(H5Dclose(dset) < 0) TEST_ERROR + + /* Verify chunk indexing scheme and number of chunks, and write data + to a subset of chunks */ + if(verify_and_write(chunkfile, FIXED_ARR_INDEX_DSET_NAME, dspace, H5D_CHUNK_IDX_FARRAY, NO_CHUNK_WRITTEN, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification and write failed\n"); + + /* Open the dataset again to test getting chunk info */ + if((dset = H5Dopen2(chunkfile, FIXED_ARR_INDEX_DSET_NAME, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Get and verify the number of chunks written */ + if(H5Dget_num_chunks(dset, dspace, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, NUM_CHUNKS_WRITTEN, "H5Dget_num_chunks, number of chunks"); + + /* Get and verify info of each written chunk */ + chk_index = 0; + for(ii = START_CHK_X; ii < END_CHK_X; ii++) + for(jj = START_CHK_Y; jj < END_CHK_Y; jj++, chk_index++) { + offset[0] = ii * CHUNK_NX; + offset[1] = jj * CHUNK_NY; + if(verify_get_chunk_info(dset, dspace, chk_index, CHK_SIZE, offset, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info failed\n"); + } + + /* Attempt to get info using an out-of-range index, chk_index is now > NUM_CHUNKS_WRITTEN. should fail */ + H5E_BEGIN_TRY { + ret = H5Dget_chunk_info(dset, dspace, chk_index, out_offset, &read_flt_msk, &addr, &size); + } H5E_END_TRY; + if(ret != FAIL) + FAIL_PUTS_ERROR(" Attempted to get info of a chunk using an out-of-range index."); + + /* Attempt to get info of empty chunks, verify the returned address and size */ + offset[0] = 0; + offset[1] = 0; + if(verify_empty_chunk_info(dset, offset) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord on empty chunk failed\n"); + + offset[0] = 3 * CHUNK_NX; + offset[1] = 3 * CHUNK_NY; + if(verify_empty_chunk_info(dset, offset) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord on empty chunk failed\n"); + + /* Initialize the array of chunk data for all NUM_CHUNKS chunks */ + for(n = 0; n < NUM_CHUNKS; n++) + for(ii = 0; ii < CHUNK_NX; ii++) + for(jj = 0; jj < CHUNK_NY; jj++) + direct_buf[n][ii][jj] = n + 1; + + /* Read each chunk and verify the values */ + chk_index = 0; + for(ii = START_CHK_X; ii < END_CHK_X; ii++) + for(jj = START_CHK_Y; jj < END_CHK_Y; jj++, chk_index++) { + offset[0] = ii * CHUNK_NX; + offset[1] = jj * CHUNK_NY; + + if(read_each_chunk(dset, offset, (void*)direct_buf[chk_index]) < 0) + TEST_ERROR + } + + /* Release resourse */ + if(H5Dclose(dset) < 0) TEST_ERROR + if(H5Sclose(dspace) < 0) TEST_ERROR + if(H5Fclose(chunkfile) < 0) TEST_ERROR + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Sclose(dspace); + H5Pclose(cparms); + H5Fclose(chunkfile); + } H5E_END_TRY; + + H5_FAILED(); + return FAIL; +} /* test_chunk_info_fixed_array() */ + +/*------------------------------------------------------------------------- + * Function: test_chunk_info_extensible_array + * + * Purpose: Test getting various chunk information when Extensible Array + * index type is used + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Note: Note that the dataspace argument in these new functions are + * currently not used. The functionality involved the dataspace + * will be implemented in the next version. + * + * Date: November 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_chunk_info_extensible_array(const char *filename, hid_t fapl) +{ + hid_t chunkfile = H5I_INVALID_HID; /* File ID */ + hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dset = H5I_INVALID_HID; /* Dataset ID */ + hid_t cparms = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + hsize_t maxdims[2] = {H5S_UNLIMITED, NY}; /* One unlimited dimension */ + int direct_buf[NUM_CHUNKS][CHUNK_NX][CHUNK_NY];/* Data in chunks */ + unsigned flt_msk = 0; /* Filter mask */ + unsigned read_flt_msk = 0; /* Filter mask after direct read */ + hsize_t offset[2]; /* Offset coordinates of a chunk */ + hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ + hsize_t size = 0; /* Size of an allocated/written chunk */ + hsize_t nchunks = 0; /* Number of chunks */ + haddr_t addr = 0; /* Address of an allocated/written chunk */ + hsize_t chk_index = 0; /* Index of a chunk */ + hsize_t ii, jj; /* Array indices */ + int n; /* Used as chunk index, but int to avoid conversion warning */ + herr_t ret; /* Temporary returned value for verifying failure */ + + TESTING(" Extensible Array index"); + + /* Open the file for reading/writing */ + if((chunkfile = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + TEST_ERROR + + /* Create dataspace */ + if((dspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + TEST_ERROR + + /* Enable chunking */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR + + if(H5Pset_chunk(cparms, RANK, chunk_dims) < 0) + TEST_ERROR + + /* Create a new dataset using cparms creation properties */ + dset = H5Dcreate2(chunkfile, EXT_ARR_INDEX_DSET_NAME, H5T_NATIVE_INT, dspace, H5P_DEFAULT, cparms, H5P_DEFAULT); + if(dset < 0) TEST_ERROR + + /* Close the dataset */ + if(H5Dclose(dset) < 0) TEST_ERROR + + /* Verify chunk indexing scheme and number of chunks, and write data + to a subset of chunks */ + if(verify_and_write(chunkfile, EXT_ARR_INDEX_DSET_NAME, dspace, H5D_CHUNK_IDX_EARRAY, NO_CHUNK_WRITTEN, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification and write failed\n"); + + /* Open the dataset again to test getting chunk info */ + if((dset = H5Dopen2(chunkfile, EXT_ARR_INDEX_DSET_NAME, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Get and verify the number of chunks written */ + if(H5Dget_num_chunks(dset, dspace, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, NUM_CHUNKS_WRITTEN, "H5Dget_num_chunks, number of chunks"); + + /* Get and verify info of each written chunk */ + chk_index = 0; + for(ii = START_CHK_X; ii < END_CHK_X; ii++) + for(jj = START_CHK_Y; jj < END_CHK_Y; jj++, chk_index++) { + offset[0] = ii * CHUNK_NX; + offset[1] = jj * CHUNK_NY; + + if(verify_get_chunk_info(dset, dspace, chk_index, CHK_SIZE, offset, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info failed\n"); + + if(verify_get_chunk_info_by_coord(dset, offset, CHK_SIZE, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord failed\n"); + } + + /* Attempt to get info using an out-of-range index, should fail */ + chk_index = OUTOFRANGE_CHK_INDEX; + H5E_BEGIN_TRY { + ret = H5Dget_chunk_info(dset, dspace, chk_index, out_offset, &read_flt_msk, &addr, &size); + } H5E_END_TRY; + if(ret != FAIL) + FAIL_PUTS_ERROR(" Attempted to get info of a chunk using an out-of-range index."); + + /* Attempt to get info of empty chunks, verify the returned address and size */ + offset[0] = 0; + offset[1] = 0; + if(verify_empty_chunk_info(dset, offset) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord on empty chunk failed\n"); + + offset[0] = 3 * CHUNK_NX; + offset[1] = 3 * CHUNK_NY; + if(verify_empty_chunk_info(dset, offset) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord on empty chunk failed\n"); + + /* Initialize the array of chunk data for all NUM_CHUNKS chunks */ + for(n = 0; n < NUM_CHUNKS; n++) + for(ii = 0; ii < CHUNK_NX; ii++) + for(jj = 0; jj < CHUNK_NY; jj++) + direct_buf[n][ii][jj] = n + 1; + + /* Read each chunk and verify the values */ + chk_index = 0; + for(ii = START_CHK_X; ii < END_CHK_X; ii++) + for(jj = START_CHK_Y; jj < END_CHK_Y; jj++, chk_index++) { + offset[0] = ii * CHUNK_NX; + offset[1] = jj * CHUNK_NY; + + if(read_each_chunk(dset, offset, (void*)direct_buf[chk_index]) < 0) + TEST_ERROR + } + + /* Release resourse */ + if(H5Dclose(dset) < 0) TEST_ERROR + if(H5Sclose(dspace) < 0) TEST_ERROR + if(H5Fclose(chunkfile) < 0) TEST_ERROR + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Sclose(dspace); + H5Pclose(cparms); + H5Fclose(chunkfile); + } H5E_END_TRY; + + H5_FAILED(); + return FAIL; +} /* test_chunk_info_extensible_array() */ + +/*------------------------------------------------------------------------- + * Function: test_chunk_info_version2_btrees + * + * Purpose: Test getting various chunk information when Version 2 B-trees + * index type is used + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Note: Note that the dataspace argument in these new functions are + * currently not used. The functionality involved the dataspace + * will be implemented in the next version. + * + * Date: November 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_chunk_info_version2_btrees(const char *filename, hid_t fapl) +{ + hid_t chunkfile = H5I_INVALID_HID; /* File ID */ + hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dset = H5I_INVALID_HID; /* Dataset ID */ + hid_t cparms = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY};/* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* Two unlimited dims */ + int direct_buf[NUM_CHUNKS][CHUNK_NX][CHUNK_NY];/* Data in chunks */ + unsigned flt_msk = 0; /* Filter mask */ + unsigned read_flt_msk = 0; /* Filter mask after direct read */ + hsize_t offset[2]; /* Offset coordinates of a chunk */ + hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ + hsize_t size = 0; /* Size of an allocated/written chunk */ + hsize_t nchunks = 0; /* Number of chunks */ + haddr_t addr = 0; /* Address of an allocated/written chunk */ + hsize_t chk_index = 0; /* Index of a chunk */ + hsize_t ii, jj; /* Array indices */ + herr_t ret; /* Temporary returned value for verifying failure */ + + TESTING(" Version 2 B-trees index"); + + /* Open the file for reading/writing */ + if((chunkfile = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + TEST_ERROR + + /* Create dataspace */ + if((dspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + TEST_ERROR + + /* Enable chunking */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR + + if(H5Pset_chunk(cparms, RANK, chunk_dims) < 0) + TEST_ERROR + + /* Create a new dataset using cparms creation properties */ + dset = H5Dcreate2(chunkfile, V2_BTREE_INDEX_DSET_NAME, H5T_NATIVE_INT, dspace, H5P_DEFAULT, cparms, H5P_DEFAULT); + if(dset < 0) TEST_ERROR + + /* Close the dataset */ + if(H5Dclose(dset) < 0) TEST_ERROR + + /* Verify chunk indexing scheme and number of chunks, and write data + to a subset of chunks */ + if(verify_and_write(chunkfile, V2_BTREE_INDEX_DSET_NAME, dspace, H5D_CHUNK_IDX_BT2, NO_CHUNK_WRITTEN, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification and write failed\n"); + + /* Open the dataset again to test getting chunk info */ + if((dset = H5Dopen2(chunkfile, V2_BTREE_INDEX_DSET_NAME, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Get and verify the number of chunks written */ + if(H5Dget_num_chunks(dset, dspace, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, NUM_CHUNKS_WRITTEN, "H5Dget_num_chunks, number of chunks"); + + /* Go through all written chunks, get their info and verify the values */ + chk_index = 0; + for(ii = START_CHK_X; ii < END_CHK_X; ii++) + for(jj = START_CHK_Y; jj < END_CHK_Y; jj++, chk_index++) { + offset[0] = ii * CHUNK_NX; + offset[1] = jj * CHUNK_NY; + + if(verify_get_chunk_info(dset, dspace, chk_index, CHK_SIZE, offset, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info failed\n"); + + if(verify_get_chunk_info_by_coord(dset, offset, CHK_SIZE, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord failed\n"); + } + + /* Attempt to provide out-of-range offsets, should fail */ + chk_index = OUTOFRANGE_CHK_INDEX; + H5E_BEGIN_TRY { + ret = H5Dget_chunk_info(dset, dspace, chk_index, out_offset, &read_flt_msk, &addr, &size); + } H5E_END_TRY; + if(ret != FAIL) + FAIL_PUTS_ERROR(" Attempted to get info of a chunk using an out-of-range index."); + + /* Attempt to get info of empty chunks, verify the returned address and size */ + offset[0] = 0; + offset[1] = 0; + if(verify_empty_chunk_info(dset, offset) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord on empty chunk failed\n"); + + offset[0] = 3 * CHUNK_NX; + offset[1] = 3 * CHUNK_NY; + if(verify_empty_chunk_info(dset, offset) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord on empty chunk failed\n"); + + /* Read each chunk and verify the values */ + chk_index = 0; + for(ii = START_CHK_X; ii < END_CHK_X; ii++) + for(jj = START_CHK_Y; jj < END_CHK_Y; jj++, chk_index++) { + offset[0] = ii * CHUNK_NX; + offset[1] = jj * CHUNK_NY; + + if(read_each_chunk(dset, offset, (void*)direct_buf[chk_index]) < 0) + TEST_ERROR + } + + /* Release resourse */ + if(H5Dclose(dset) < 0) TEST_ERROR + if(H5Sclose(dspace) < 0) TEST_ERROR + if(H5Fclose(chunkfile) < 0) TEST_ERROR + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Sclose(dspace); + H5Pclose(cparms); + H5Fclose(chunkfile); + } H5E_END_TRY; + + H5_FAILED(); + return FAIL; +} /* test_chunk_info_version2_btrees() */ + +/*------------------------------------------------------------------------- + * Function: test_basic_query + * + * Purpose: Tests basic operations to ensure the chunk query functions + * work properly. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Note: Note that the dataspace argument in these new functions are + * currently not used. The functionality involved the dataspace + * will be implemented in the next version. + * + * Date: August 2019 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_basic_query(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t basicfile = H5I_INVALID_HID; /* File ID */ + hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dset = H5I_INVALID_HID; /* Dataset ID */ + hid_t cparms = H5I_INVALID_HID; /* Creation plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + int direct_buf[CHUNK_NX][CHUNK_NY];/* Data in chunks */ + unsigned flt_msk = 0; /* Filter mask */ + unsigned read_flt_msk = 0; /* Filter mask after direct read */ + hsize_t offset[2]; /* Offset coordinates of a chunk */ + hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ + hsize_t size = 0; /* Size of an allocated/written chunk */ + hsize_t nchunks = 0; /* Number of chunks */ + haddr_t addr = 0; /* Address of an allocated/written chunk */ + hsize_t chk_index = 0; /* Index of a chunk */ + hsize_t ii, jj; /* Array indices */ + herr_t ret; /* Temporary returned value for verifying failure */ + + TESTING("basic operations"); + + /* Create the file */ + h5_fixname(BASIC_FILE, fapl, filename, sizeof filename); + + /* Create a new file. */ + if((basicfile = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create dataspace */ + if((dspace = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR + + /* Enable chunking */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR + + if(H5Pset_chunk(cparms, RANK, chunk_dims) < 0) + TEST_ERROR + + /* Create a new dataset using cparms creation properties */ + dset = H5Dcreate2(basicfile, SIMPLE_CHUNKED_DSET_NAME, H5T_NATIVE_INT, dspace, H5P_DEFAULT, cparms, H5P_DEFAULT); + if(dset < 0) TEST_ERROR + + /* Get the number of chunks and verify that no chunk has been written */ + if(H5Dget_num_chunks(dset, dspace, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, NO_CHUNK_WRITTEN, "H5Dget_num_chunks, number of chunks"); + + /* Initialize the array of chunk data for the single chunk */ + for(ii = 0; ii < CHUNK_NX; ii++) + for(jj = 0; jj < CHUNK_NY; jj++) + direct_buf[ii][jj] = (int)(ii*jj); + + /* Write the chunk of data */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + if(H5Dwrite_chunk(dset, H5P_DEFAULT, flt_msk, offset, CHK_SIZE, direct_buf) < 0) + TEST_ERROR; + + /* Get and verify that one chunk had been written */ + if(H5Dget_num_chunks(dset, dspace, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, ONE_CHUNK_WRITTEN, "H5Dget_num_chunks, number of chunks"); + + /* Get and verify info of the first and only chunk */ + if(verify_get_chunk_info(dset, H5S_ALL, 0, CHK_SIZE, offset, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification H5Dget_chunk_info failed\n"); + + /* Get and verify info of the chunk at the offset (CHUNK_NX,CHUNK_NY) */ + if(verify_get_chunk_info_by_coord(dset, offset, CHK_SIZE, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord failed\n"); + + /* Attempt to get chunk info given an invalid chunk index and verify + * that failure occurs */ + chk_index = INVALID_CHK_INDEX; + reinit_vars(&read_flt_msk, &addr, &size); + H5E_BEGIN_TRY { + ret = H5Dget_chunk_info(dset, dspace, chk_index, out_offset, &read_flt_msk, &addr, &size); + } H5E_END_TRY; + if(ret != FAIL) + TEST_ERROR + + /* Write the chunk of data to another location */ + offset[0] = 0; + offset[1] = 0; + if(H5Dwrite_chunk(dset, H5P_DEFAULT, flt_msk, offset, CHK_SIZE, direct_buf) < 0) + TEST_ERROR; + + /* Get and verify that two chunks had been written */ + if(H5Dget_num_chunks(dset, dspace, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, TWO_CHUNKS_WRITTEN, "H5Dget_num_chunks, number of chunks"); + + /* Get and verify info of the first written chunk in the dataset, its + offset should be (0,0) */ + if(verify_get_chunk_info(dset, H5S_ALL, 0, CHK_SIZE, offset, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification H5Dget_chunk_info failed\n"); + + /* Get and verify info of the chunk at the offset (0,0) */ + if(verify_get_chunk_info_by_coord(dset, offset, CHK_SIZE, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord failed\n"); + + /* Get and verify info of the second written chunk in the dataset, its + offset should be (CHUNK_NX, CHUNK_NY) */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + if(verify_get_chunk_info(dset, H5S_ALL, 1, CHK_SIZE, offset, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification H5Dget_chunk_info failed\n"); + + /* Get and verify info of the chunk at the offset (CHUNK_NX, CHUNK_NY) */ + if(verify_get_chunk_info_by_coord(dset, offset, CHK_SIZE, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord failed\n"); + + /* Get and verify info of an empty chunk, at offset + (2*CHUNK_NX, 2*CHUNK_NY) */ + offset[0] = 2*CHUNK_NX; + offset[1] = 2*CHUNK_NY; + /* Get and verify info of the chunk at the offset (CHUNK_NX, CHUNK_NY) */ + if(verify_empty_chunk_info(dset, offset) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord on empty chunk failed\n"); + + /* Release resourse */ + if(H5Dclose(dset) < 0) TEST_ERROR + if(H5Sclose(dspace) < 0) TEST_ERROR + if(H5Pclose(cparms) < 0) TEST_ERROR + if(H5Fclose(basicfile) < 0) TEST_ERROR + + /* Remove the test file */ + remove(filename); + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Sclose(dspace); + H5Pclose(cparms); + H5Fclose(basicfile); + } H5E_END_TRY; + + H5_FAILED(); + return FAIL; +} /* test_basic_query() */ + +/*------------------------------------------------------------------------- + * Function: test_failed_attempts + * + * Purpose: Test attempting to use chunk query functions incorrectly. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Note: Note that the dataspace argument in these new functions are + * currently not used. The functionality involved the dataspace + * will be implemented in the next version. + * + * Date: August 2019 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_failed_attempts(const char *filename, hid_t fapl) +{ + hid_t chunkfile = H5I_INVALID_HID; /* File ID */ + hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dset = H5I_INVALID_HID; /* Dataset ID */ + hsize_t dims[2] = {NX, NY};/* Dataset dimensions */ + int data_buf[NX][NY]; /* Input buffer */ + unsigned read_flt_msk = 0; /* Filter mask after direct read */ + hsize_t offset[2]; /* Offset coordinates of a chunk */ + hsize_t out_offset[2] = {0, 0}; /* Buffer to get offset coordinates */ + hsize_t size = 0; /* Size of an allocated/written chunk */ + hsize_t nchunks = 0; /* Number of chunks */ + haddr_t addr = 0; /* Address of an allocated/written chunk */ + hsize_t chk_index = 0; /* Index of a chunk */ + hsize_t ii, jj; /* Array indices */ + herr_t ret; /* Temporary returned value for verifying failure */ + + TESTING(" Invalid Operations"); + + /* Open the file for reading/writing */ + if((chunkfile = H5Fopen(filename, H5F_ACC_RDWR, fapl)) < 0) + TEST_ERROR + + /* Create dataspace */ + if((dspace = H5Screate_simple(RANK, dims, NULL)) < 0) + TEST_ERROR + + /* Create a contiguous dataset */ + dset = H5Dcreate2(chunkfile, CONTIGUOUS_DSET_NAME, H5T_NATIVE_INT, dspace, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if(dset < 0) TEST_ERROR + + /* Initialize the array of data */ + for(ii = 0; ii < NX; ii++) + for(jj = 0; jj < NY; jj++) + data_buf[ii][jj] = (int)(ii*jj); + + /* Write the data */ + if(H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data_buf) < 0) + TEST_ERROR + + /* Close the dataset */ + if(H5Dclose(dset) < 0) TEST_ERROR + + /* Open it again to test the chunk query functions on contiguous dataset */ + if((dset = H5Dopen2(chunkfile, CONTIGUOUS_DSET_NAME, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Attempt to get the number of chunks on contiguous dataset, should fail */ + H5E_BEGIN_TRY { + ret = H5Dget_num_chunks(dset, dspace, &nchunks); + } H5E_END_TRY; + if(ret != FAIL) + FAIL_PUTS_ERROR(" Attempt a chunk query function on a contiguous dataset.") + + /* Attempt to get chunk info on contiguous data, should fail */ + chk_index = 0; + reinit_vars(&read_flt_msk, &addr, &size); + H5E_BEGIN_TRY { + ret = H5Dget_chunk_info(dset, dspace, chk_index, out_offset, &read_flt_msk, &addr, &size); + } H5E_END_TRY; + if(ret != FAIL) + FAIL_PUTS_ERROR(" Attempt a chunk query function on a contiguous dataset.") + + /* Attempt to get chunk info at logical coordinates (0,0) on contiguous + * dataset, should fail */ + offset[0] = 0; + offset[1] = 0; + H5E_BEGIN_TRY { + ret = H5Dget_chunk_info_by_coord(dset, offset, &read_flt_msk, &addr, &size); + } H5E_END_TRY; + if(ret != FAIL) + FAIL_PUTS_ERROR(" Attempt a chunk query function on a contiguous dataset.") + + /* Release resourse */ + if(H5Dclose(dset) < 0) TEST_ERROR + if(H5Sclose(dspace) < 0) TEST_ERROR + if(H5Fclose(chunkfile) < 0) TEST_ERROR + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Sclose(dspace); + H5Fclose(chunkfile); + } H5E_END_TRY; + + H5_FAILED(); + return FAIL; +} /* test_failed_attempts() */ + +/*------------------------------------------------------------------------- + * Function: test_get_chunk_info_v110 + * + * Purpose: Test getting various chunk information in version 1.10. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Note: Note that the dataspace argument in these new functions are + * currently not used. The functionality involved the dataspace + * will be implemented in the next version. + * + * Description: + * This function tests the new API functions added for HDFFV-10677: + * H5Dget_num_chunks, H5Dget_chunk_info, and + * H5Dget_chunk_info_by_coord for low bound beyond 1.8. + * + * Date: October 2018 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_get_chunk_info_v110(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t chunkfile = H5I_INVALID_HID; /* File ID */ + H5F_libver_t low, high; /* File format bounds */ + + TESTING("getting chunk information in file with versions 1.10 and later"); + HDprintf("\n"); /* to list sub-tests */ + + /* Set high bound to the current latest version */ + high = H5F_LIBVER_LATEST; + + /* Test getting info of chunked datasets in version combo up to 1.10 */ + for(low = H5F_LIBVER_V110; low <= H5F_LIBVER_LATEST; low++) { + /* Set version bounds for creating file */ + if(H5Pset_libver_bounds(fapl, low, high) < 0) + TEST_ERROR + + /* Create the file */ + h5_fixname(FILENAME[low], fapl, filename, sizeof filename); + chunkfile = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if(chunkfile < 0) TEST_ERROR + + /* Close the file, individual tests will re-open the file with different + libvers via the fapl */ + if(H5Fclose(chunkfile) < 0) TEST_ERROR + + /* Test getting chunk info when Single Chunk index type is used */ + if(test_chunk_info_single_chunk(filename, fapl) < 0) + TEST_ERROR + + /* Test getting chunk info when Implicit index type is used */ + if(test_chunk_info_implicit(filename, fapl) < 0) + TEST_ERROR + + /* Test getting chunk info when Fixed Array index type is used */ + if(test_chunk_info_fixed_array(filename, fapl) < 0) + TEST_ERROR + + /* Test getting chunk info when Extensible Array index type is used */ + if(test_chunk_info_extensible_array(filename, fapl) < 0) + TEST_ERROR + + /* Test getting chunk info when Version 2 B-trees index type is used */ + if(test_chunk_info_version2_btrees(filename, fapl) < 0) + TEST_ERROR + + /* Test various attempts to use the functions incorrectly */ + if(test_failed_attempts(filename, fapl) < 0) + TEST_ERROR + + } /* for low libver bound */ + + return SUCCEED; + +error: + H5_FAILED(); + return FAIL; +} /* test_get_chunk_info_v110() */ + +/*------------------------------------------------------------------------- + * Function: test_flt_msk_with_skip_compress + * + * Purpose: Test getting chunk info when compression filter is skipped. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Date: August 2019 (based on direct_chunk.c/test_skip_compress_write1) + * + *------------------------------------------------------------------------- + */ +static herr_t +test_flt_msk_with_skip_compress(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; /* File name */ + hid_t filter_file = H5I_INVALID_HID; /* File ID for filter mask */ + hid_t dspace = H5I_INVALID_HID; /* Dataspace ID */ + hid_t mem_space = H5I_INVALID_HID; /* Dataspace ID */ + hid_t dset = H5I_INVALID_HID; /* Dataset ID */ + hid_t cparms = H5I_INVALID_HID; /* Creation plist */ + hid_t dxpl = H5I_INVALID_HID; /* Transfer plist */ + hsize_t dims[2] = {NX, NY}; /* Dataset dimensions */ + hsize_t maxdims[2] = {H5S_UNLIMITED, H5S_UNLIMITED}; /* 2 unlimited dims */ + hsize_t chunk_dims[2] = {CHUNK_NX, CHUNK_NY}; /* Chunk dimensions */ + int direct_buf[CHUNK_NX][CHUNK_NY]; /* One chunk of data */ + int check_chunk[CHUNK_NX][CHUNK_NY]; /* Buffer to read data in */ + int read_direct_buf[CHUNK_NX][CHUNK_NY]; /* Buffer to read a chunk */ + hsize_t read_buf_size = 0; /* buf size */ + unsigned flt_msk = 0; /* Filter mask */ + unsigned read_flt_msk = 0; /* Filter mask after direct read */ + hsize_t offset[2] = {0, 0}; /* Offset coordinates of a chunk */ + hsize_t nchunks = 0; /* Number of chunks */ + hsize_t chk_index = 0; /* Index of a chunk */ + int aggression = 9; /* Compression aggression setting */ + hsize_t start[2]; /* Start of hyperslab */ + hsize_t stride[2]; /* Stride of hyperslab */ + hsize_t count[2]; /* Block count */ + hsize_t block[2]; /* Block sizes */ + int ii, jj; /* Array indices */ + + TESTING("getting filter mask when compression filter is skipped"); + + /* Create the file */ + h5_fixname(FILTERMASK_FILE, fapl, filename, sizeof filename); + + /* Create a new file. */ + if((filter_file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create file data space with unlimited dimensions. */ + if((dspace = H5Screate_simple(RANK, dims, maxdims)) < 0) + TEST_ERROR; + + /* Create memory data space. */ + if((mem_space = H5Screate_simple(RANK, chunk_dims, NULL)) < 0) + TEST_ERROR; + + /* Create dataset create property list with chunking and compression + enabled. */ + if((cparms = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR; + + if(H5Pset_chunk( cparms, RANK, chunk_dims) < 0) + TEST_ERROR; + + if(H5Pset_deflate( cparms, (unsigned ) aggression) < 0) + TEST_ERROR; + + /* Create a new dataset using cparms creation properties. */ + if((dset = H5Dcreate2(filter_file, SKIP_FILTER_DSET_NAME, H5T_NATIVE_INT, dspace, H5P_DEFAULT, cparms, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Create transfer property list for writing */ + if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) + TEST_ERROR; + + /* Initialize data for one chunk */ + for(ii = 0; ii < CHUNK_NX; ii++) + for(jj = 0; jj < CHUNK_NY; jj++) + direct_buf[ii][jj] = (int)(ii*jj); + + /* Indicate the compression filter is to be skipped. */ + flt_msk = 0x00000001; + + /* Write a chunk of uncompressed data */ + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + if(H5Dwrite_chunk(dset, H5P_DEFAULT, flt_msk, offset, CHK_SIZE, direct_buf) < 0) + TEST_ERROR; + + if(H5Fflush(dset, H5F_SCOPE_LOCAL) < 0) + TEST_ERROR; + + /* Close and re-open the dataset */ + if(H5Dclose(dset) < 0) + TEST_ERROR; + if((dset = H5Dopen2(filter_file, SKIP_FILTER_DSET_NAME, H5P_DEFAULT)) < 0) + TEST_ERROR; + + /* Select hyperslab for the chunk just written in the file */ + start[0] = CHUNK_NX; start[1] = CHUNK_NY; + stride[0] = 1; stride[1] = 1; + count[0] = 1; count[1] = 1; + block[0] = CHUNK_NX; block[1] = CHUNK_NY; + if(H5Sselect_hyperslab(dspace, H5S_SELECT_SET, start, stride, count, block) < 0) + TEST_ERROR; + + /* Read the chunk back */ + if(H5Dread(dset, H5T_NATIVE_INT, mem_space, dspace, H5P_DEFAULT, check_chunk) < 0) + TEST_ERROR; + + /* Check that the values read are the same as the values written */ + for(ii = 0; ii < CHUNK_NX; ii++) + for(jj = 0; jj < CHUNK_NY; jj++) + if(direct_buf[ii][jj] != check_chunk[ii][jj]) { + HDprintf(" 1. Read different values than written."); + HDprintf(" At index %d,%d\n", ii, jj); + HDprintf(" direct_buf=%d, check_chunk=%d\n", direct_buf[ii][jj], check_chunk[ii][jj]); + TEST_ERROR; + } + + /* Query chunk storage size */ + if(H5Dget_chunk_storage_size(dset, offset, &read_buf_size) < 0) + TEST_ERROR; + if(read_buf_size != CHK_SIZE) + TEST_ERROR; + + /* Read the raw chunk back with H5Dread_chunk */ + HDmemset(&read_direct_buf, 0, sizeof(read_direct_buf)); + if(H5Dread_chunk(dset, H5P_DEFAULT, offset, &read_flt_msk, read_direct_buf) < 0) + TEST_ERROR; + if(read_flt_msk != flt_msk) + TEST_ERROR; + + /* Check that the direct chunk read is the same as the chunk written */ + for(ii = 0; ii < CHUNK_NX; ii++) + for(jj = 0; jj < CHUNK_NY; jj++) + if(direct_buf[ii][jj] != read_direct_buf[ii][jj]) { + HDprintf(" 1. Read different values than written."); + HDprintf(" At index %d,%d\n", ii, jj); + HDprintf(" direct_buf=%d, read_direct_buf=%d\n", direct_buf[ii][jj], read_direct_buf[ii][jj]); + TEST_ERROR; + } + + /* Get and verify the number of chunks written */ + if(H5Dget_num_chunks(dset, H5S_ALL, &nchunks) < 0) TEST_ERROR + VERIFY(nchunks, ONE_CHUNK_WRITTEN, "H5Dget_num_chunks, number of chunks"); + + /* Get and verify info of the first and only chunk */ + chk_index = 0; + offset[0] = CHUNK_NX; + offset[1] = CHUNK_NY; + if(verify_get_chunk_info(dset, H5S_ALL, chk_index, CHK_SIZE, offset, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info failed\n"); + + /* Get info of the chunk at the specified offsets and verify its info */ + if(verify_get_chunk_info_by_coord(dset, offset, CHK_SIZE, flt_msk) == FAIL) + FAIL_PUTS_ERROR("Verification of H5Dget_chunk_info_by_coord failed\n"); + + /* Release resourse */ + if(H5Dclose(dset) < 0) TEST_ERROR + if(H5Sclose(mem_space) < 0) TEST_ERROR + if(H5Sclose(dspace) < 0) TEST_ERROR + if(H5Pclose(cparms) < 0) TEST_ERROR + if(H5Pclose(dxpl) < 0) TEST_ERROR + if(H5Fclose(filter_file) < 0) TEST_ERROR + + /* Remove the test file */ + remove(filename); + + PASSED(); + return SUCCEED; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Sclose(mem_space); + H5Sclose(dspace); + H5Pclose(cparms); + H5Pclose(dxpl); + H5Fclose(filter_file); + } H5E_END_TRY; + + H5_FAILED(); + return FAIL; +} /* test_flt_msk_with_skip_compress() */ + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Tests functions related to chunk information + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Binh-Minh Ribler + * November 5, 2018 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + hid_t fapl = H5I_INVALID_HID; /* File access property list */ + int nerrors = 0; /* Number of errors so far */ + + h5_reset(); + + /* Create a copy of file access property list */ + if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) TEST_ERROR + + /* Test basic operations on the chunk query functions */ + nerrors += test_basic_query(fapl) < 0 ? 1 : 0; + + /* Tests getting chunk information of version 1.8 and prior */ + nerrors += test_get_chunk_info_highest_v18(fapl) < 0 ? 1 : 0; + + /* Tests getting chunk information of version 1.10 */ + nerrors += test_get_chunk_info_v110(fapl) < 0 ? 1 : 0; + + /* Tests getting filter mask when compression filter is skipped */ + nerrors += test_flt_msk_with_skip_compress(fapl) < 0 ? 1 : 0; + + if(nerrors) + TEST_ERROR + + HDprintf("All chunk query tests passed.\n"); + + h5_cleanup(FILENAME, fapl); + + return SUCCEED; + +error: + nerrors = MAX(1, nerrors); + HDprintf("***** %d QUERY CHUNK INFO TEST%s FAILED! *****\n", + nerrors, 1 == nerrors ? "" : "S"); + return FAIL; +} + +/**************************************************************************** + Additional tests to be added: +- do the query when extending the dataset (shrink or expand) +- verify that invalid input parameters are handled properly + +****************************************************************************/ |