diff options
| -rw-r--r-- | c++/test/dsets.cpp | 33 | ||||
| -rw-r--r-- | c++/test/tfilter.cpp | 45 | ||||
| -rw-r--r-- | src/H5B2private.h | 1 | ||||
| -rw-r--r-- | src/H5Dint.c | 31 | ||||
| -rw-r--r-- | src/H5Dio.c | 109 | ||||
| -rw-r--r-- | src/H5Distore.c | 283 | ||||
| -rw-r--r-- | src/H5Dpkg.h | 6 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 2 | ||||
| -rw-r--r-- | src/H5HFcache.c | 4 | ||||
| -rw-r--r-- | src/H5HFhuge.c | 4 | ||||
| -rw-r--r-- | src/H5Odtype.c | 2 | ||||
| -rw-r--r-- | src/H5Opline.c | 78 | ||||
| -rw-r--r-- | src/H5Pdcpl.c | 2 | ||||
| -rw-r--r-- | src/H5T.c | 5 | ||||
| -rw-r--r-- | src/H5Tcommit.c | 32 | ||||
| -rw-r--r-- | src/H5Tprivate.h | 1 | ||||
| -rw-r--r-- | src/H5V.c | 8 | ||||
| -rw-r--r-- | src/H5Z.c | 289 | ||||
| -rw-r--r-- | src/H5Zdeflate.c | 49 | ||||
| -rw-r--r-- | src/H5Zdtype_modify.c | 1316 | ||||
| -rw-r--r-- | src/H5Zfletcher32.c | 41 | ||||
| -rw-r--r-- | src/H5Znbit.c | 40 | ||||
| -rw-r--r-- | src/H5Zprivate.h | 8 | ||||
| -rw-r--r-- | src/H5Zpublic.h | 55 | ||||
| -rw-r--r-- | src/H5Zscaleoffset.c | 33 | ||||
| -rw-r--r-- | src/H5Zshuffle.c | 47 | ||||
| -rw-r--r-- | src/H5Zszip.c | 47 | ||||
| -rw-r--r-- | src/H5vers.txt | 5 | ||||
| -rw-r--r-- | src/H5version.h | 64 | ||||
| -rw-r--r-- | test/change_dtypes.c | 91 | ||||
| -rw-r--r-- | test/cmpd_dset.c | 443 | ||||
| -rw-r--r-- | test/dsets.c | 153 | ||||
| -rw-r--r-- | tools/h5dump/h5dumpgentest.c | 83 |
33 files changed, 3064 insertions, 346 deletions
diff --git a/c++/test/dsets.cpp b/c++/test/dsets.cpp index 97ba6ca..c921123 100644 --- a/c++/test/dsets.cpp +++ b/c++/test/dsets.cpp @@ -59,8 +59,13 @@ const H5std_string DSET_BOGUS_NAME ("bogus"); const int H5Z_FILTER_BOGUS = 305; // Local prototypes +#ifndef H5_USE_16_API +static size_t filter_bogus(unsigned int flags, hsize_t chunk_offset, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +#else static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +#endif /*------------------------------------------------------------------------- @@ -352,6 +357,7 @@ test_tconv( H5File& file) } // test_tconv /* This message derives from H5Z */ +#ifndef H5_USE_16_API const H5Z_class_t H5Z_BOGUS[1] = {{ H5Z_CLASS_T_VERS, /* H5Z_class_t version number */ H5Z_FILTER_BOGUS, /* Filter id number */ @@ -359,9 +365,23 @@ const H5Z_class_t H5Z_BOGUS[1] = {{ "bogus", /* Filter name for debugging */ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ - NULL, /* The "reset local" callback */ + NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ + (H5Z_func_t)filter_bogus, /* The actual filter function */ +}}; +#else +const H5Z_class_t H5Z_BOGUS[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version number */ + H5Z_FILTER_BOGUS, /* Filter id number */ + "bogus", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ (H5Z_func_t)filter_bogus, /* The actual filter function */ }}; +#endif /*------------------------------------------------------------------------- * Function: bogus @@ -379,14 +399,19 @@ const H5Z_class_t H5Z_BOGUS[1] = {{ * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API static size_t /*bogus(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, const unsigned int UNUSED cd_values[], size_t nbytes, size_t UNUSED *buf_size, void UNUSED **buf) BMR: removed UNUSED for now until asking Q. or R. to pass compilation*/ -filter_bogus(unsigned int flags, size_t cd_nelmts, - const unsigned int cd_values[], size_t nbytes, - size_t *buf_size, void **buf) +filter_bogus(unsigned int flags, hsize_t chunk_offset, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#else +static size_t +filter_bogus(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], + size_t nbytes, size_t *buf_size, void **buf) +#endif { return nbytes; } diff --git a/c++/test/tfilter.cpp b/c++/test/tfilter.cpp index a667f2a..b88e122 100644 --- a/c++/test/tfilter.cpp +++ b/c++/test/tfilter.cpp @@ -57,7 +57,8 @@ static herr_t test_filter_internal(hid_t fid, const char *name, hid_t dcpl, /* Temporary filter IDs used for testing */ #define H5Z_FILTER_BOGUS 305 -static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, +#ifndef H5_USE_16_API +static size_t filter_bogus(unsigned int flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ const H5Z_class_t H5Z_BOGUS[1] = {{ @@ -68,6 +69,46 @@ const H5Z_class_t H5Z_BOGUS[1] = {{ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ + (H5Z_func_t)filter_bogus, /* The actual filter function */ +}}; + +/*------------------------------------------------------------------------- + * Function: filter_bogus + * + * Purpose: A bogus compression method that doesn't do anything. + * + * Return: Success: Data chunk size + * + * Failure: 0 + * + * Programmer: Robb Matzke + * Tuesday, April 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static size_t +filter_bogus(unsigned int UNUSED flags, hsize_t UNUSED chunk_offset, size_t UNUSED cd_nelmts, + const unsigned int UNUSED *cd_values, size_t nbytes, size_t UNUSED *buf_size, + void UNUSED **buf) +{ + return nbytes; +} +#else +static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +/* This message derives from H5Z */ +const H5Z_class_t H5Z_BOGUS[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_BOGUS, /* Filter id number */ + "bogus", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ (H5Z_func_t)filter_bogus, /* The actual filter function */ }}; @@ -95,6 +136,8 @@ filter_bogus(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, return nbytes; } +#endif + /*------------------------------------------------------------------------- * Function: test_null_filter * diff --git a/src/H5B2private.h b/src/H5B2private.h index 6d8172c..db2e785 100644 --- a/src/H5B2private.h +++ b/src/H5B2private.h @@ -54,6 +54,7 @@ typedef enum H5B2_subid_t { H5B2_SOHM_INDEX_ID, /* B-tree is an index for shared object header messages */ H5B2_ATTR_DENSE_NAME_ID, /* B-tree is for indexing 'name' field for "dense" attribute storage on objects */ H5B2_ATTR_DENSE_CORDER_ID, /* B-tree is for indexing 'creation order' field for "dense" attribute storage on objects */ + H5B2_MODIFY_TYPE_ID, /* B-tree is for indexing 'datatype' field for filter of modifying datatype */ H5B2_NUM_BTREE_ID /* Number of B-tree IDs (must be last) */ } H5B2_subid_t; diff --git a/src/H5Dint.c b/src/H5Dint.c index 272b00a..6a636db 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -1228,6 +1228,7 @@ H5D_open(const H5G_loc_t *loc, hid_t dxpl_id) HGOTO_ERROR(H5E_DATASET, H5E_CANTINC, NULL, "can't increment object count") } /* end else */ +#ifndef H5_USE_16_API /* Get a file ID for the file */ if((fid = H5F_get_id(dataset->oloc.file)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "unable to get file ID") @@ -1236,11 +1237,11 @@ H5D_open(const H5G_loc_t *loc, hid_t dxpl_id) * Make the "reset local" filter callbacks for this dataset. It's mainly for some filters * like datatype modification which need some info after the dataset is reopened. * If the library has been closed and reopened, the user-defined filters aren't - * registered in the library anymore. This function for those filters can't be + * registered in the library anymore. This function is for those filters can't be * found and may fail. Therefore ignore the error if it's this case. */ H5E_BEGIN_TRY { - H5Z_reset_local(dataset->shared->dcpl_id, dataset->shared->type_id, fid); + H5Z_reset_local(dataset->shared->dcpl_id, dataset->shared->type_id, fid, TRUE); } H5E_END_TRY; /* Release the file ID */ @@ -1248,6 +1249,7 @@ H5D_open(const H5G_loc_t *loc, hid_t dxpl_id) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEFILE, NULL, "can't close file") fid = -1; +#endif ret_value = dataset; @@ -1519,6 +1521,7 @@ done: herr_t H5D_close(H5D_t *dataset) { + hid_t fid; unsigned free_failed = FALSE; herr_t ret_value = SUCCEED; /* Return value */ @@ -1533,6 +1536,10 @@ H5D_close(H5D_t *dataset) H5D_istore_stats(dataset, FALSE); #endif /* H5F_ISTORE_DEBUG */ + /* Get a file ID for the file */ + if((fid = H5F_get_id(dataset->oloc.file)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "unable to get file ID") + dataset->shared->fo_count--; if(dataset->shared->fo_count == 0) { /* Flush the dataset's information */ @@ -1588,6 +1595,12 @@ H5D_close(H5D_t *dataset) #endif /* NDEBUG */ } /* end switch */ /*lint !e788 All appropriate cases are covered */ +#ifndef H5_USE_16_API + H5E_BEGIN_TRY { + H5Z_close_local(dataset->shared->dcpl_id, dataset->shared->type_id, fid); + } H5E_END_TRY; +#endif + /* * Release datatype, dataspace and creation property list -- there isn't * much we can do if one of these fails, so we just continue. @@ -1625,8 +1638,18 @@ H5D_close(H5D_t *dataset) if(H5FO_top_count(dataset->oloc.file, dataset->oloc.addr) == 0) if(H5O_close(&(dataset->oloc)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to close") + + H5E_BEGIN_TRY { + H5Z_close_local(dataset->shared->dcpl_id, dataset->shared->type_id, fid); + } H5E_END_TRY; } /* end else */ + /* Release the file ID */ + if(H5I_dec_ref(fid) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEFILE, NULL, "can't close file") + + fid = -1; + /* Release the dataset's path info */ if(H5G_name_free(&(dataset->path)) < 0) free_failed = TRUE; @@ -2277,13 +2300,13 @@ H5D_check_filters(H5D_t *dataset) if(H5Z_can_apply(dataset->shared->dcpl_id, dataset->shared->type_id, fid) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "can't apply filters") - dataset->shared->checked_filters = TRUE; - /* Release the file ID */ if(H5I_dec_ref(fid) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEFILE, NULL, "can't close file") fid = -1; + + dataset->shared->checked_filters = TRUE; } /* end if */ } /* end if */ } /* end if */ diff --git a/src/H5Dio.c b/src/H5Dio.c index e05836d..dcc15bf 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -1449,8 +1449,15 @@ H5D_chunk_read(H5D_io_info_t *io_info, hsize_t nelmts, store.chunk.offset = chunk_info->coords; store.chunk.index = chunk_info->index; - /* Load the chunk into cache and lock it. */ chunk_addr = H5D_istore_get_addr(io_info, &udata); + + /* Translate the offset of the chunck into single-dimensional offset for + * the filter of modifying dataset's datatype */ + if((io_info->store->chunk.linear_offset = H5D_istore_calc_offset( + dataset->shared->space, io_info->store->chunk.offset)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to calculate chunk offset") + + /* Load the chunk into cache and lock it. */ if(H5D_istore_if_load(io_info, chunk_addr)) { if(NULL == (chunk = H5D_istore_lock(io_info, &udata, FALSE, &idx_hint))) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to read raw data chunk") @@ -1573,6 +1580,12 @@ H5D_chunk_read(H5D_io_info_t *io_info, hsize_t nelmts, /* Load the chunk into cache and lock it. */ chunk_addr = H5D_istore_get_addr(io_info, &udata); + /* Translate the offset of the chunck into single-dimensional offset for + * the filter of modifying dataset's datatype */ + if((io_info->store->chunk.linear_offset = H5D_istore_calc_offset( + dataset->shared->space, io_info->store->chunk.offset)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to calculate chunk offset") + if(H5D_istore_if_load(io_info, chunk_addr)) { if(NULL == (chunk = H5D_istore_lock(io_info, &udata, FALSE, &idx_hint))) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to read raw data chunk") @@ -1782,6 +1795,8 @@ H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, H5D_istore_ud1_t udata; /*B-tree pass-through */ unsigned idx_hint = 0; /* Cache index hint */ hbool_t relax=TRUE; /* Whether whole chunk is selected */ + hbool_t has_mdtype_filter=FALSE;/* Whether filter for modifying dset's dtype + * is enabled */ herr_t ret_value = SUCCEED; /*return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_chunk_write) @@ -1810,6 +1825,23 @@ H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, dst_type_size = H5T_get_size(dataset->shared->type); max_type_size = MAX(src_type_size, dst_type_size); + /* If the filter for modifying dataset's datatype is enabled, don't relax + * because we need the offset of the chunk to pass through H5Z_pipeline. */ + if(dataset->shared->dcpl_id != H5P_DATASET_CREATE_DEFAULT) { + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret = SUCCEED; /* Return value */ + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(dataset->shared->dcpl_id, H5P_DATASET_CREATE))) + HGOTO_ERROR(H5E_DATASET, H5E_BADATOM, FAIL, "can't find object for ID") + + H5E_BEGIN_TRY { + ret = H5P_get_filter_by_id(plist, H5Z_FILTER_DTYPE_MODIFY, NULL, NULL, NULL, (size_t)0, NULL, NULL); + } H5E_END_TRY; + + if(ret >= 0) + has_mdtype_filter = TRUE; + } /* end if */ + /* * If there is no type conversion then write directly from the * application's buffer. This saves at least one mem-to-mem copy. @@ -1851,9 +1883,15 @@ H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, * simply allocate space instead of load the chunk. */ chunk_addr = H5D_istore_get_addr(io_info, &udata); + /* Translate the offset of the chunck into single-dimensional offset for + * the filter of modifying dataset's datatype */ + if((io_info->store->chunk.linear_offset = H5D_istore_calc_offset( + dataset->shared->space, io_info->store->chunk.offset)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to calculate chunk offset") + if(H5D_istore_if_load(io_info, chunk_addr)) { accessed_bytes = chunk_info->chunk_points * dst_type_size; - if(accessed_bytes != dataset->shared->layout.u.chunk.size) + if(has_mdtype_filter || accessed_bytes != dataset->shared->layout.u.chunk.size) relax = FALSE; if(NULL == (chunk = H5D_istore_lock(io_info, &udata, relax, &idx_hint))) @@ -1977,6 +2015,12 @@ H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, store.chunk.offset = chunk_info->coords; store.chunk.index = chunk_info->index; + /* Translate the offset of the chunck into single-dimensional offset for + * the filter of modifying dataset's datatype */ + if((io_info->store->chunk.linear_offset = H5D_istore_calc_offset( + dataset->shared->space, io_info->store->chunk.offset)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to calculate chunk offset") + /* Load the chunk into cache. But if the whole chunk is written, * simply allocate space instead of load the chunk. */ chunk_addr = H5D_istore_get_addr(io_info, &udata); @@ -1987,7 +2031,7 @@ H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, relax=FALSE; if(relax) { accessed_bytes = H5S_GET_SELECT_NPOINTS(chunk_info->mspace) * src_type_size; - if(accessed_bytes != dataset->shared->layout.u.chunk.size) + if(has_mdtype_filter || accessed_bytes != dataset->shared->layout.u.chunk.size) relax = FALSE; } @@ -2381,7 +2425,7 @@ H5Dmodify_dtype(hid_t dset_id, hid_t new_type_id) H5D_t *dset = NULL; H5T_t *new_type = NULL; - herr_t ret_value; + herr_t ret_value = SUCCEED; FUNC_ENTER_API(H5Dmodify_dtype, FAIL) H5TRACE2("e", "ii", dset_id, new_type_id); @@ -2465,13 +2509,31 @@ H5D_modify_dtype(H5F_t *file, H5D_t *dataset, hid_t type_id, H5T_t *type, hid_t HGOTO_ERROR(H5E_DATASET, H5E_BADATOM, FAIL, "can't find object for ID") H5E_BEGIN_TRY { - ret = H5P_get_filter_by_id(plist, H5Z_FILTER_DTYPE_MODIFY, NULL, NULL, NULL, (size_t)0, NULL, NULL); + ret = H5P_get_filter_by_id(plist, H5Z_FILTER_DTYPE_MODIFY, NULL, NULL, + NULL, (size_t)0, NULL, NULL); } H5E_END_TRY; if(ret >= 0) filter_enabled = TRUE; } /* end if */ + /* If the filter is present, generally only support committed or shared datatype. + * We also allow integer or floating number because of their limited encoding size. + */ + if(filter_enabled && !H5T_committed(type) && + !H5SM_can_share(file, dxpl_id, NULL, NULL, H5O_DTYPE_ID, type)) { + H5T_class_t tclass; + if((tclass = H5T_get_class(type, TRUE)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "can't find datatype class") + + if(tclass != H5T_INTEGER && tclass != H5T_FLOAT) + HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "violate datatype restriction") + } + + /* Check if the datatype should be (or are already) shared in the SOHM table */ + if(H5SM_try_share(file, dxpl_id, NULL, H5O_DTYPE_ID, dataset->shared->type, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADMESG, FAIL, "trying to share datatype failed") + /* If space hasn't been allocated and not using external storage, * set the flag HAS_DATA to FALSE. If the dataset is compact, * the flag is always TRUE. @@ -2526,30 +2588,29 @@ H5D_modify_dtype(H5F_t *file, H5D_t *dataset, hid_t type_id, H5T_t *type, hid_t default: HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "dataset should be contiguous or chunked") } + } else if(filter_enabled && has_data && H5D_CHUNKED == dataset->shared->layout.type) { + /* Convert the chunks already in the cache */ + if(H5D_istore_conv_dtype(dataset, type_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTCONVERT, FAIL, "can't convert data cache") } /* Get a file ID for the file */ if((fid = H5F_get_id(file)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get file ID") - /* Copy & initialize datatype for the dataset struct */ - if(H5D_init_type(file, dataset, type_id, type) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't copy datatype") - - /* Check if the datatype should be (or are already) shared in the SOHM table */ - if(H5SM_try_share(file, dxpl_id, NULL, H5O_DTYPE_ID, dataset->shared->type, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_BADMESG, FAIL, "trying to share datatype failed") - /* - * Check whether datatype is committed & increment ref count. (to maintain - * ref. count incr/decr similarity with "shared message" type of datatype sharing) + * Check whether the old datatype is committed & decrement ref count. */ if(H5T_committed(dataset->shared->type)) { - /* Increment the reference count on the shared datatype */ - if(H5T_link(dataset->shared->type, 1, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_LINKCOUNT, FAIL, "unable to adjust shared datatype link count") + /* Decrement the reference count on the committed datatype */ + if(H5T_link(dataset->shared->type, -1, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_LINKCOUNT, FAIL, "unable to adjust committed datatype link count") } /* end if */ + /* Copy & initialize datatype for the dataset struct */ + if(H5D_init_type(file, dataset, type_id, type) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't copy datatype") + /* Open the object header of the original dataset for updating datatype and layout */ if(H5O_open(&(dataset->oloc)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to open dataset object header") @@ -2561,8 +2622,8 @@ H5D_modify_dtype(H5F_t *file, H5D_t *dataset, hid_t type_id, H5T_t *type, hid_t /* Delete the old datatype message */ if(H5O_msg_remove(&(dataset->oloc), H5O_DTYPE_ID, 0, FALSE, dxpl_id) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTDELETE, FAIL, "unable to delete old dtype message") - - /* Add the new datatype message */ + + /* Add the new datatype message. Ref count for committed dtype is incremented here. */ if(H5O_msg_append_oh(file, dxpl_id, oh, H5O_DTYPE_ID, H5O_MSG_FLAG_CONSTANT, 0, dataset->shared->type) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update datatype header message") @@ -2606,7 +2667,9 @@ H5D_modify_dtype(H5F_t *file, H5D_t *dataset, hid_t type_id, H5T_t *type, hid_t * allocation time is early, since space may not be allocated) */ fill_prop = &(dataset->shared->dcpl_cache.fill); - if(H5O_msg_append_oh(file, dxpl_id, oh, H5O_LAYOUT_ID, ((fill_prop->alloc_time == H5D_ALLOC_TIME_EARLY && H5D_COMPACT != dataset->shared->layout.type) ? H5O_MSG_FLAG_CONSTANT : 0), 0, &(dataset->shared->layout)) < 0) + if(H5O_msg_append_oh(file, dxpl_id, oh, H5O_LAYOUT_ID, + ((fill_prop->alloc_time == H5D_ALLOC_TIME_EARLY && H5D_COMPACT != dataset->shared->layout.type) ? H5O_MSG_FLAG_CONSTANT : 0), + 0, &(dataset->shared->layout)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update layout") /* @@ -2666,7 +2729,7 @@ H5D_modify_dtype(H5F_t *file, H5D_t *dataset, hid_t type_id, H5T_t *type, hid_t HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to close dataset") } else if(filter_enabled) { /* Make the "reset local" filter callbacks for this dataset */ - if(H5Z_reset_local(dataset->shared->dcpl_id, dataset->shared->type_id, fid) < 0) + if(H5Z_reset_local(dataset->shared->dcpl_id, dataset->shared->type_id, fid, FALSE) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set local filter parameters") /* Release the file ID */ @@ -2675,7 +2738,7 @@ H5D_modify_dtype(H5F_t *file, H5D_t *dataset, hid_t type_id, H5T_t *type, hid_t fid = -1; } - + done: /* Release the file ID */ if(fid>=0 && H5I_dec_ref(fid) < 0) diff --git a/src/H5Distore.c b/src/H5Distore.c index 2d47a5a..bc265c6 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -123,6 +123,7 @@ typedef struct H5D_rdcc_ent_t { size_t alloc_size; /*amount allocated for the chunk */ uint8_t *chunk; /*the unfiltered chunk data */ unsigned idx; /*index in hash table */ + hssize_t linear_offset; /* Chunk's coordinates translated into one-dimensional offset */ struct H5D_rdcc_ent_t *next;/*next item in doubly-linked list */ struct H5D_rdcc_ent_t *prev;/*previous item in doubly-linked list */ } H5D_rdcc_ent_t; @@ -752,6 +753,8 @@ H5D_istore_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key, int cmp; unsigned u; H5B_ins_t ret_value; + + htri_t dis = FALSE; FUNC_ENTER_NOAPI_NOINIT(H5D_istore_insert) @@ -811,12 +814,15 @@ H5D_istore_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key, ret_value = H5B_INS_NOOP; } - } else if (H5V_hyper_disjointp(udata->common.mesg->u.chunk.ndims, + } else if ((dis = H5V_hyper_disjointp(udata->common.mesg->u.chunk.ndims, lt_key->offset, udata->common.mesg->u.chunk.dim, - udata->common.offset, udata->common.mesg->u.chunk.dim)) { - HDassert(H5V_hyper_disjointp(udata->common.mesg->u.chunk.ndims, + udata->common.offset, udata->common.mesg->u.chunk.dim)) == TRUE) { + + dis = H5V_hyper_disjointp(udata->common.mesg->u.chunk.ndims, rt_key->offset, udata->common.mesg->u.chunk.dim, - udata->common.offset, udata->common.mesg->u.chunk.dim)); + udata->common.offset, udata->common.mesg->u.chunk.dim); + HDassert(dis); + /* * Split this node, inserting the new new node to the right of the * current node. The MD_KEY is where the split occurs. @@ -993,6 +999,7 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, H5Z_EDC_t edc_read = H5Z_NO_EDC; size_t nbytes = lt_key->nbytes; H5Z_cb_t cb_struct; + hsize_t linear_offset = 0; /* No real use */ int ret_value = H5_ITER_CONT; /* Return value */ @@ -1042,7 +1049,8 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, if(is_compressed && (is_vlen || fix_ref)) { unsigned filter_mask = lt_key->filter_mask; - if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, edc_read, cb_struct, &nbytes, &buf_size, &buf) < 0) + if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, edc_read, linear_offset, + cb_struct, &nbytes, &buf_size, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "data pipeline read failed") } /* end if */ @@ -1103,7 +1111,7 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, /* Need to compress variable-length & reference data elements before writing to file */ if(is_compressed && (is_vlen || fix_ref) ) { - if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), edc_read, + if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), edc_read, linear_offset, cb_struct, &nbytes, &buf_size, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed") udata_dst.nbytes = nbytes; @@ -1147,6 +1155,7 @@ H5D_istore_iter_copy_conv(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, H5D_istore_it_ud4_t *udata = (H5D_istore_it_ud4_t *)_udata; const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; H5D_istore_ud1_t udata_dst; /* User data about new destination chunk */ + hsize_t linear_offset = 0; /* No real use */ /* General information about chunk copy */ void *bkg = udata->bkg; @@ -1206,7 +1215,7 @@ H5D_istore_iter_copy_conv(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, if(is_compressed) { unsigned filter_mask = lt_key->filter_mask; - if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, edc_read, cb_struct, &src_nbytes, &buf_size, &buf) < 0) + if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, edc_read, linear_offset, cb_struct, &src_nbytes, &buf_size, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "data pipeline read failed") } /* end if */ @@ -1230,7 +1239,7 @@ H5D_istore_iter_copy_conv(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, /* Need to compress variable-length & reference data elements before writing to file */ if(is_compressed) { - if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), edc_read, + if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), edc_read, linear_offset, cb_struct, &dst_nbytes, &buf_size, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed") udata->buf = buf; @@ -1251,7 +1260,7 @@ H5D_istore_iter_copy_conv(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_iter_copy() */ +} /* end H5D_istore_iter_copy_conv() */ /*------------------------------------------------------------------------- @@ -1452,7 +1461,6 @@ H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_ udata.nbytes = ent->chunk_size; udata.addr = HADDR_UNDEF; - /* Should the chunk be filtered before writing it to disk? */ if(io_info->dset->shared->dcpl_cache.pline.nused) { if(!reset) { /* @@ -1476,8 +1484,11 @@ H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_ point_of_no_return = TRUE; ent->chunk = NULL; } /* end else */ - if(H5Z_pipeline(&(io_info->dset->shared->dcpl_cache.pline), 0, &(udata.filter_mask), io_info->dxpl_cache->err_detect, - io_info->dxpl_cache->filter_cb, &(udata.nbytes), &alloc, &buf) < 0) + + if(H5Z_pipeline(&(io_info->dset->shared->dcpl_cache.pline), 0, + &(udata.filter_mask), io_info->dxpl_cache->err_detect, + (hsize_t)ent->linear_offset, io_info->dxpl_cache->filter_cb, + &(udata.nbytes), &alloc, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "output pipeline failed") } /* end if */ @@ -1546,6 +1557,7 @@ static herr_t H5D_istore_preempt(const H5D_io_info_t *io_info, H5D_rdcc_ent_t * ent, hbool_t flush) { H5D_rdcc_t *rdcc = &(io_info->dset->shared->cache.chunk); + hid_t dcpl_id = io_info->dset->shared->dcpl_id; herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_istore_preempt) @@ -1566,6 +1578,9 @@ H5D_istore_preempt(const H5D_io_info_t *io_info, H5D_rdcc_ent_t * ent, hbool_t f ent->chunk = (uint8_t *)H5D_istore_chunk_xfree(ent->chunk, &(io_info->dset->shared->dcpl_cache.pline)); } /* end else */ + if(H5Z_evict_local(dcpl_id, /*unused*/0, /*unused*/0, (hsize_t)ent->linear_offset) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "failed to evict local filter info") + /* Unlink from list */ if(ent->prev) ent->prev->next = ent->next; @@ -1644,6 +1659,174 @@ done: /*------------------------------------------------------------------------- + * Function: H5D_istore_conv_dtype + * + * Purpose: Convert all the chunks in the cache if the datatype for + * dataset has been changed (called by H5Dmodify_dtype). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 16 April 2008 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_istore_conv_dtype(H5D_t *dset, hid_t new_type_id) +{ + H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); + const H5O_pline_t *pline = &(dset->shared->dcpl_cache.pline); /* I/O pipeline info */ + H5D_rdcc_ent_t *ent, *next; + hid_t orig_type_id = dset->shared->type_id; + H5T_t *new_type, *orig_type; + H5T_t *new_type_copy; + hid_t new_type_copy_id; + size_t orig_type_size, new_type_size; + size_t nelmts; + unsigned char *new_chunk = NULL; + unsigned char *bkg=NULL; /* Pointer to background buffer */ + H5T_path_t *tpath=NULL; /*type conversion info */ + herr_t ret_value = SUCCEED; /* Return value */ + htri_t is_vlen = FALSE; + hbool_t vlen_conv = FALSE; + hid_t dxpl_id; + H5P_genplist_t *def_dx_plist = NULL, *dx_plist = NULL; + + FUNC_ENTER_NOAPI(H5D_istore_conv_dtype, FAIL) + + /* If there's no chunks in the cache, skip this function */ + if(!rdcc->head) + HGOTO_DONE(SUCCEED); + + /* The current datatype of the dataset */ + if(NULL==(orig_type=(H5T_t *)H5I_object_verify(orig_type_id, H5I_DATATYPE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype") + + /* The new datatype of the dataset */ + if(NULL==(new_type=(H5T_t *)H5I_object_verify(new_type_id, H5I_DATATYPE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype") + + /* Make a copy of the new datatype to avoid changing the original new type */ + if(NULL==(new_type_copy=H5T_copy(new_type, H5T_COPY_ALL))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy data type"); + + if((new_type_copy_id = H5I_register(H5I_DATATYPE, new_type_copy)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register datatype atom") + + /* Mark the new datatype as being on disk now because the data in the cache + * hasn't been converted to memory type yet. */ + if(H5T_set_loc(new_type_copy, dset->oloc.file, H5T_LOC_DISK) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't set datatype location") + + /* Get the object of default dataset creation property list */ + if(NULL == (def_dx_plist = (H5P_genplist_t *)H5I_object(H5P_DATASET_XFER_DEFAULT))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get dataset creation property list") + + /* Create a new default property list */ + if((dxpl_id = H5P_copy_plist(def_dx_plist)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "can't copy dataset creation property list") + + if(NULL == (dx_plist = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get dataset creation property list") + + /* If the datatype is or contains vlen, set the property to indicate no conversion + * is needed. */ + if((is_vlen = H5T_detect_class(orig_type, H5T_VLEN)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to detect dtatypes") + + if(is_vlen ) { + vlen_conv = FALSE; + + if(H5P_set(dx_plist, H5D_XFER_VLEN_CONV_NAME, &vlen_conv) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "Error setting vlen conv flag") + } + + if((orig_type_size = H5T_get_size(orig_type)) == 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "can't get size of original type") + + if((new_type_size = H5T_get_size(new_type_copy)) == 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "can't get size of new type") + + /* If the new type is the same as the current, simply finish the function */ + if(0 == H5T_cmp(orig_type, new_type_copy, FALSE)) + HGOTO_DONE(SUCCEED) + + /* Figure out the number of elements in a chunk */ + nelmts = dset->shared->layout.u.chunk.size/orig_type_size; + + /* Find the conversion function */ + if (NULL==(tpath=H5T_path_find(orig_type, new_type_copy, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst data types"); + + if (H5T_path_bkg(tpath) && (NULL==(bkg=(unsigned char*)H5MM_malloc(nelmts*(new_type_size > orig_type_size ? new_type_size : orig_type_size))))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate buffer") + + /* Loop over all entries in the chunk cache */ + for(ent = rdcc->head; ent; ent = next) { + next = ent->next; + ent->locked = TRUE; + + if(new_type_size > orig_type_size) { + if(NULL == (new_chunk = H5D_istore_chunk_alloc(nelmts*new_type_size, pline))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + + HDmemcpy(new_chunk, ent->chunk, ent->chunk_size); + + /* Convert the data */ + if (H5T_convert(tpath, orig_type_id, new_type_copy_id, nelmts, (size_t)0, + (size_t)0, new_chunk, bkg, dxpl_id)<0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "data type conversion failed"); + + H5MM_free(ent->chunk); + ent->chunk = (uint8_t*)new_chunk; + } else if (new_type_size < orig_type_size) { + if(NULL == (new_chunk = H5D_istore_chunk_alloc(nelmts*new_type_size, pline))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + + /* Convert the data */ + if (H5T_convert(tpath, orig_type_id, new_type_copy_id, nelmts, (size_t)0, + (size_t)0, ent->chunk, bkg, dxpl_id)<0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "data type conversion failed"); + + HDmemcpy(new_chunk, ent->chunk, nelmts*new_type_size); + + H5MM_free(ent->chunk); + ent->chunk = (uint8_t*)new_chunk; + } else { + /* Convert the data */ + if (H5T_convert(tpath, orig_type_id, new_type_copy_id, nelmts, (size_t)0, + (size_t)0, ent->chunk, bkg, dxpl_id)<0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "data type conversion failed"); + } + + ent->chunk_size = nelmts*new_type_size; + ent->alloc_size = ent->chunk_size; + ent->rd_count = ent->chunk_size; + ent->wr_count = ent->chunk_size; + ent->offset[dset->shared->layout.u.chunk.ndims-1] = new_type_size; + ent->dirty = TRUE; + ent->locked = FALSE; + + /* Update some local variables for the filter pipeline */ + if(H5Z_change_local(dset->shared->dcpl_id, new_type_copy_id, /*unused*/0, + (hsize_t)ent->linear_offset) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "failed to change local filter info") + } /* end for */ + + rdcc->nbytes = new_type_size*rdcc->nbytes/orig_type_size; + + H5MM_free(bkg); + + if (H5I_dec_ref(new_type_copy_id) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "problem freeing id"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_istore_conv_dtype() */ + + + +/*------------------------------------------------------------------------- * Function: H5D_istore_dest * * Purpose: Destroy the entire chunk cache by flushing dirty entries, @@ -1908,6 +2091,52 @@ done: /*------------------------------------------------------------------------- + * Function: H5D_istore_calc_offset + * + * Purpose: Private function to translate from the multi-dimensional + * offset of a chunk in a dataspace to linear offset. At this + * moment, it's used for the filter for modifying dataset's + * datatype. + * + * Return: Success: Linear offset of the chunk. + * + * Failure: Negative + * + * Programmer: Raymond Lu + * 12 March 2008 + * + * Modification: + *------------------------------------------------------------------------- + */ +hssize_t H5D_istore_calc_offset(H5S_t *space, hsize_t offset[]) +{ + hsize_t linear_offset = 0, tmp_offset = 0; + int space_ndims; /* Dataset's space rank */ + hsize_t space_dim[H5O_LAYOUT_NDIMS]; /* Dataset's dataspace dimensions */ + int u,v; /*counters */ + hssize_t ret_value; + + FUNC_ENTER_NOAPI(H5D_istore_calc_offset, FAIL) + + /* Retrieve the dataset dimensions */ + if((space_ndims = H5S_get_simple_extent_dims(space, space_dim, NULL)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get simple dataspace info") + + for(u=0; u<space_ndims; u++) { + tmp_offset = offset[u]; + for(v=u+1; v<space_ndims; v++) + tmp_offset *= space_dim[v]; + linear_offset += tmp_offset; + } + + ret_value = linear_offset; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} + + +/*------------------------------------------------------------------------- * Function: H5D_istore_lock * * Purpose: Return a pointer to a dataset chunk. The pointer points @@ -2038,8 +2267,10 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, HGOTO_ERROR(H5E_IO, H5E_READERROR, NULL, "unable to read raw data chunk") if(pline->nused) - if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &(udata->filter_mask), io_info->dxpl_cache->err_detect, - io_info->dxpl_cache->filter_cb, &(udata->nbytes), &chunk_alloc, &chunk) < 0) + if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &(udata->filter_mask), + io_info->dxpl_cache->err_detect, (hsize_t)io_info->store->chunk.linear_offset, + io_info->dxpl_cache->filter_cb, &(udata->nbytes), &chunk_alloc, + &chunk) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, NULL, "data pipeline read failed") #ifdef H5D_ISTORE_DEBUG rdcc->nmisses++; @@ -2126,6 +2357,7 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, assert(NULL==rdcc->slot[idx]); rdcc->slot[idx] = ent; ent->idx = idx; + ent->linear_offset = io_info->store->chunk.linear_offset; rdcc->nbytes += chunk_size; rdcc->nused++; @@ -2856,7 +3088,7 @@ H5D_istore_chunk_alloc(size_t size, const H5O_pline_t *pline) { void *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_chunk_alloc) + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_chunk_alloc) HDassert(size); HDassert(pline); @@ -2928,6 +3160,7 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) H5D_io_info_t io_info; /* Dataset I/O info */ H5D_storage_t store; /* Dataset storage information */ hsize_t chunk_offset[H5O_LAYOUT_NDIMS]; /* Offset of current chunk */ + hssize_t linear_offset = 0; size_t orig_chunk_size; /* Original size of chunk in bytes */ unsigned filter_mask = 0; /* Filter mask for chunks that have them */ const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ @@ -3012,6 +3245,7 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) || pline->nused > 0) should_fill = TRUE; + /* Check if fill values should be written to chunks */ if(should_fill) { /* Initialize the fill value buffer */ @@ -3033,7 +3267,9 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) size_t buf_size = orig_chunk_size; /* Push the chunk through the filters */ - if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &orig_chunk_size, &buf_size, &fb_info.fill_buf) < 0) + if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, + (hsize_t)linear_offset, dxpl_cache->filter_cb, &orig_chunk_size, + &buf_size, &fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed") } /* end if */ } /* end if */ @@ -3091,7 +3327,9 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) size_t nbytes = fb_info.fill_buf_size; /* Push the chunk through the filters */ - if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &nbytes, &buf_size, &fb_info.fill_buf) < 0) + if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, + (hsize_t)linear_offset, dxpl_cache->filter_cb, &nbytes, + &buf_size, &fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed") /* Keep the number of bytes the chunk turned in to */ @@ -3157,6 +3395,12 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) break; } /* end else */ } /* end for */ + + /* Translate the offset of the chunck into single-dimensional offset for + * the filter of modifying dataset's datatype */ + if((linear_offset = H5D_istore_calc_offset(dset->shared->space, chunk_offset)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to calculate chunk offset") + } /* end while */ #ifdef H5_HAVE_PARALLEL @@ -3785,7 +4029,7 @@ H5D_istore_delete(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout) /* Delete entire B-tree */ if(H5B_delete(f, dxpl_id, H5B_ISTORE, tmp_layout.u.chunk.addr, &udata)<0) - HGOTO_ERROR(H5E_IO, H5E_CANTDELETE, 0, "unable to delete chunk B-tree") + HGOTO_ERROR(H5E_IO, H5E_CANTDELETE, FAIL, "unable to delete chunk B-tree") /* Free the raw B-tree node buffer */ if(tmp_layout.u.chunk.btree_shared==NULL) @@ -4217,8 +4461,6 @@ H5D_istore_copy_conv(H5F_t *file, const H5D_t *dset_src, const H5D_t *dset_dst, if(H5P_set(plist, H5D_XFER_VLEN_CONV_NAME, &vlen_conv) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "Error setting vlen conv flag") - /*if(H5P_get(plist, H5D_XFER_VLEN_CONV_NAME, &vlen_conv) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "Error getting vlen conv flag")*/ } /* Set up the conversion functions */ @@ -4492,4 +4734,3 @@ H5D_istore_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int inden done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_debug() */ - diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 300fb18..09b595d 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -112,7 +112,7 @@ typedef struct H5D_io_info_t { hbool_t xfer_opt_mode_changed; hbool_t using_mpi_vfd; /* Whether the file is using an MPI-based VFD */ #endif /* H5_HAVE_PARALLEL */ - const H5D_storage_t *store; /* Dataset storage info */ + H5D_storage_t *store; /* Dataset storage info */ H5D_io_ops_t ops; /* I/O operation function pointers */ #ifdef H5S_DEBUG H5S_iostats_t *stats; /* I/O statistics */ @@ -454,8 +454,8 @@ H5_DLL haddr_t H5D_istore_get_addr(const H5D_io_info_t *io_info, H5_DLL herr_t H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, H5O_layout_t *layout_dst, H5T_t *src_dtype, H5O_copy_t *cpy_info, H5O_pline_t *pline, hid_t dxpl_id); -H5_DLL herr_t H5D_istore_copy_conv(H5F_t *file, const H5D_t *dset_src, const H5D_t *dset_dst, - H5O_copy_t UNUSED *cpy_info, hid_t dxpl_id); +H5_DLL herr_t H5D_istore_copy_conv(H5F_t *file, const H5D_t *dset_src, const H5D_t *dset_dst, H5O_copy_t UNUSED *cpy_info, hid_t dxpl_id); +H5_DLL herr_t H5D_istore_conv_dtype(H5D_t *dset, hid_t new_type_id); H5_DLL void * H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, hbool_t relax, unsigned *idx_hint/*in,out*/); H5_DLL herr_t H5D_istore_unlock(const H5D_io_info_t *io_info, diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index c5192b2..bab035c 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -112,6 +112,7 @@ typedef struct H5D_t H5D_t; typedef struct { hsize_t index; /* "Index" of chunk in dataset (must be first for TBBT routines) */ hsize_t *offset; /* Chunk's coordinates in elements */ + hssize_t linear_offset; /* Chunk's coordinates translated into one-dimensional offset */ } H5D_chunk_storage_t; typedef struct { @@ -185,6 +186,7 @@ H5_DLL herr_t H5D_istore_delete(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout); H5_DLL herr_t H5D_istore_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int indent, int fwidth, unsigned ndims); +H5_DLL hssize_t H5D_istore_calc_offset(H5S_t *space, hsize_t offset[]); #endif /* _H5Dprivate_H */ diff --git a/src/H5HFcache.c b/src/H5HFcache.c index 7454d76..9a83ec1 100644 --- a/src/H5HFcache.c +++ b/src/H5HFcache.c @@ -1290,7 +1290,7 @@ HDfprintf(stderr, "%s: read_size = %Zu, read_buf = %p\n", FUNC, read_size, read_ /* Push direct block data through I/O filter pipeline */ nbytes = read_size; - if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_ENABLE_EDC, + if(H5Z_pipeline(&(hdr->pline), (H5Z_FLAG_REVERSE | H5Z_FLAG_SKIP_CRECORD), &filter_mask, H5Z_ENABLE_EDC, 0, filter_cb, &nbytes, &read_size, &read_buf) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, NULL, "output pipeline failed") #ifdef QAK @@ -1465,7 +1465,7 @@ H5HF_cache_dblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, /* Push direct block data through I/O filter pipeline */ nbytes = write_size; - if(H5Z_pipeline(&(hdr->pline), 0, &filter_mask, H5Z_ENABLE_EDC, + if(H5Z_pipeline(&(hdr->pline), (0 | H5Z_FLAG_SKIP_CRECORD), &filter_mask, H5Z_ENABLE_EDC, 0, filter_cb, &nbytes, &write_size, &write_buf) < 0) HGOTO_ERROR(H5E_HEAP, H5E_WRITEERROR, FAIL, "output pipeline failed") #ifdef QAK diff --git a/src/H5HFhuge.c b/src/H5HFhuge.c index 198ef62..1634938 100644 --- a/src/H5HFhuge.c +++ b/src/H5HFhuge.c @@ -349,7 +349,7 @@ HDfprintf(stderr, "%s: obj_size = %Zu\n", FUNC, obj_size); /* Push direct block data through I/O filter pipeline */ nbytes = write_size; - if(H5Z_pipeline(&(hdr->pline), 0, &filter_mask, H5Z_NO_EDC, + if(H5Z_pipeline(&(hdr->pline), (0 | H5Z_FLAG_SKIP_CRECORD), &filter_mask, H5Z_NO_EDC, 0, filter_cb, &nbytes, &write_size, &write_buf) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, FAIL, "output pipeline failed") #ifdef QAK @@ -675,7 +675,7 @@ H5HF_huge_op_real(H5HF_hdr_t *hdr, hid_t dxpl_id, const uint8_t *id, /* De-filter the object */ read_size = nbytes = obj_size; - if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_NO_EDC, filter_cb, &nbytes, &read_size, &read_buf) < 0) + if(H5Z_pipeline(&(hdr->pline), (H5Z_FLAG_REVERSE | H5Z_FLAG_SKIP_CRECORD), &filter_mask, H5Z_NO_EDC, 0, filter_cb, &nbytes, &read_size, &read_buf) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, FAIL, "input filter failed") obj_size = nbytes; } /* end if */ diff --git a/src/H5Odtype.c b/src/H5Odtype.c index ac07edf..389d989 100644 --- a/src/H5Odtype.c +++ b/src/H5Odtype.c @@ -743,7 +743,7 @@ H5O_dtype_encode_helper(const H5F_t *f, uint8_t **pp, const H5T_t *dt) for(i = 0; i < dt->shared->u.compnd.nmembs; i++) { /* Sanity check */ /* (compound datatypes w/array members must be encoded w/version >= 2) */ - HDassert(dt->shared->u.compnd.memb[i].type->shared->type != H5T_ARRAY || dt->shared->version >= H5O_DTYPE_VERSION_2); + HDassert((dt->shared->u.compnd.memb[i].type)->shared->type != H5T_ARRAY || dt->shared->version >= H5O_DTYPE_VERSION_2); /* Name */ HDstrcpy((char*)(*pp), dt->shared->u.compnd.memb[i].name); diff --git a/src/H5Opline.c b/src/H5Opline.c index 9a926c1..20b060d 100644 --- a/src/H5Opline.c +++ b/src/H5Opline.c @@ -24,13 +24,13 @@ #define H5Z_PACKAGE /*suppress error about including H5Zpkg */ #include "H5private.h" /* Generic Functions */ +#include "H5Dprivate.h" /* Error handling */ #include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ #include "H5Zpkg.h" /* Data filters */ - /* PRIVATE PROTOTYPES */ static herr_t H5O_pline_encode(H5F_t *f, uint8_t *p, const void *mesg); static void *H5O_pline_decode(H5F_t *f, hid_t dxpl_id, unsigned mesg_flags, const uint8_t *p); @@ -38,6 +38,8 @@ static void *H5O_pline_copy(const void *_mesg, void *_dest); static size_t H5O_pline_size(const H5F_t *f, const void *_mesg); static herr_t H5O_pline_reset(void *_mesg); static herr_t H5O_pline_free(void *_mesg); +static herr_t H5O_pline_delete(H5F_t *f, hid_t dxpl_id, H5O_t UNUSED *open_oh, + void *_mesg); static herr_t H5O_pline_pre_copy_file(H5F_t *file_src, const void *mesg_src, hbool_t *deleted, const H5O_copy_t *cpy_info, void *_udata); static herr_t H5O_pline_debug(H5F_t *f, hid_t dxpl_id, const void *_mesg, @@ -52,7 +54,7 @@ static herr_t H5O_pline_debug(H5F_t *f, hid_t dxpl_id, const void *_mesg, #define H5O_SHARED_SIZE H5O_pline_shared_size #define H5O_SHARED_SIZE_REAL H5O_pline_size #define H5O_SHARED_DELETE H5O_pline_shared_delete -#undef H5O_SHARED_DELETE_REAL +#define H5O_SHARED_DELETE_REAL H5O_pline_delete #define H5O_SHARED_LINK H5O_pline_shared_link #undef H5O_SHARED_LINK_REAL #define H5O_SHARED_COPY_FILE H5O_pline_shared_copy_file @@ -551,6 +553,78 @@ H5O_pline_free(void *mesg) /*------------------------------------------------------------------------- + * Function: H5O_pline_delete + * + * Purpose: Free file space referenced by message + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 13 August 2008 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_pline_delete(H5F_t *f, hid_t dxpl_id, H5O_t UNUSED *open_oh, void *_mesg) +{ + H5O_pline_t *mesg = (H5O_pline_t *)_mesg; + H5O_pline_t *pline = NULL; + H5D_layout_t layout; /* Dataset's layout information */ + hid_t dcpl_id, file_id; + H5P_genplist_t *dc_plist = NULL, *def_dc_plist=NULL; /* New Property list */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5O_pline_delete) + + /* check args */ + HDassert(f); + HDassert(mesg); + + /* Get the object of default dataset creation property list */ + if(NULL == (def_dc_plist = (H5P_genplist_t *)H5I_object(H5P_DATASET_CREATE_DEFAULT))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get dataset creation property list") + + /* Create a new default property list */ + if((dcpl_id = H5P_copy_plist(def_dc_plist)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "can't copy dataset creation property list") + + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get dataset creation property list") + + /* Make a copy of filter pipeline and insert into the property list. Only the filter + * for modifying dataset's datatype has the callback function to delete the local + * values. This filter only needs the property of pipeline and chunking. */ + if((pline = H5O_pline_copy(mesg, NULL)) == NULL) + HGOTO_ERROR(H5E_PLINE, H5E_CANTCOPY, FAIL, "unable to copy pipeline") + + if(H5P_set(dc_plist, H5D_CRT_DATA_PIPELINE_NAME, pline) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to set pipeline") + + layout = H5D_CHUNKED; + if(H5P_set(dc_plist, H5D_CRT_LAYOUT_NAME, &layout) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve layout") + + if((file_id = H5F_get_id(f)) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get file ID") + + /* Call the function of the filter pipeline */ + if(H5Z_delete_local(dcpl_id, 0, file_id) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTDELETE, FAIL, "can't get dataset creation property list") + + if(H5I_dec_ref(dcpl_id) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "can't close property list ID"); + + if(H5I_dec_ref(file_id) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "can't close file ID"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_pline_delete() */ + + +/*------------------------------------------------------------------------- * Function: H5O_pline_pre_copy_file * * Purpose: Perform any necessary actions before copying message between diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index fa92315..747a5f0 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -2064,7 +2064,7 @@ H5Pset_dtype_modifiable(hid_t plist_id) H5O_pline_t pline; H5P_genplist_t *plist; /* Property list pointer */ size_t cd_nelmts=3; /* Number of filter parameters */ - unsigned cd_values[3]={0,0}; /* Filter parameters */ + unsigned cd_values[3]={0,0,0};/* Filter parameters */ herr_t ret_value=SUCCEED; /* return value */ FUNC_ENTER_API(H5Pset_dtype_modifiable, FAIL) @@ -5404,6 +5404,11 @@ H5T_dtype_is_valid(H5T_t *dtype, H5T_t *new_type) } } break; + case H5T_TIME: + if(new_class != H5T_TIME) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FALSE, "new type must be time type") + + break; case H5T_STRING: if(new_class != H5T_STRING) HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FALSE, "new type must be a string type") diff --git a/src/H5Tcommit.c b/src/H5Tcommit.c index bcb9962..8bcae49 100644 --- a/src/H5Tcommit.c +++ b/src/H5Tcommit.c @@ -434,6 +434,35 @@ H5T_committed(const H5T_t *type) /*------------------------------------------------------------------------- + * Function: H5T_immutable + * + * Purpose: Determines if a datatype is predefined or not. + * + * Return: Success: TRUE if predefined, FALSE otherwise. + * + * Programmer: Raymond Lu + * 4 August 2008 + * + *------------------------------------------------------------------------- + */ +htri_t +H5T_immutable(const H5T_t *type) +{ + /* Use no-init for efficiency */ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5T_immutable) + + HDassert(type); + +if(H5T_STATE_IMMUTABLE == type->shared->state) + fprintf(stderr, "H5T_immutable: state=%d\n", type->shared->state); +else + fprintf(stderr, "H5T_immutable: state isn't immutable\n"); + + FUNC_LEAVE_NOAPI(H5T_STATE_IMMUTABLE == type->shared->state) +} /* end H5T_immutable() */ + + +/*------------------------------------------------------------------------- * Function: H5T_link * * Purpose: Adjust the link count for an object header by adding @@ -456,7 +485,8 @@ H5T_link(const H5T_t *type, int adjust, hid_t dxpl_id) FUNC_ENTER_NOAPI(H5T_link, FAIL) HDassert(type); - HDassert(type->sh_loc.type == H5O_SHARE_TYPE_COMMITTED); + HDassert(type->sh_loc.type == H5O_SHARE_TYPE_COMMITTED || + type->sh_loc.type == H5O_SHARE_TYPE_SOHM); /* Adjust the link count on the named datatype */ if((ret_value = H5O_link(&type->oloc, adjust, dxpl_id)) < 0) diff --git a/src/H5Tprivate.h b/src/H5Tprivate.h index c102ead..8d5071a 100644 --- a/src/H5Tprivate.h +++ b/src/H5Tprivate.h @@ -130,6 +130,7 @@ H5_DLL H5R_type_t H5T_get_ref_type(const H5T_t *dt); /* Operations on named datatypes */ H5_DLL H5T_t *H5T_open(const H5G_loc_t *loc, hid_t dxpl_id); H5_DLL htri_t H5T_committed(const H5T_t *type); +H5_DLL htri_t H5T_immutable(const H5T_t *type); H5_DLL int H5T_link(const H5T_t *type, int adjust, hid_t dxpl_id); H5_DLL herr_t H5T_update_shared(H5T_t *type); @@ -416,10 +416,10 @@ H5V_hyper_disjointp(unsigned n, if (0==size1[u] || 0==size2[u]) HGOTO_DONE(TRUE) - if (((offset1?offset1[u]:0) < (offset2?offset2[u]:0) && - ((offset1?offset1[u]:0) + size1[u] <= (offset2?offset2[u]:0))) || - ((offset2?offset2[u]:0) < (offset1?offset1[u]:0) && - ((offset2?offset2[u]:0) + size2[u] <= (offset1?offset1[u]:0)))) + if ((((offset1?offset1[u]:0) < (offset2?offset2[u]:0)) && + ((offset1?offset1[u]:0) + (size1?size1[u]:0) <= (offset2?offset2[u]:0))) || + (((offset2?offset2[u]:0) < (offset1?offset1[u]:0)) && + ((offset2?offset2[u]:0) + (size2?size2[u]:0) <= (offset1?offset1[u]:0)))) HGOTO_DONE(TRUE) } @@ -48,7 +48,11 @@ typedef struct H5Z_stats_t { typedef enum { H5Z_PRELUDE_CAN_APPLY, /* Call "can apply" callback */ H5Z_PRELUDE_SET_LOCAL, /* Call "set local" callback */ - H5Z_PRELUDE_RESET_LOCAL /* Call "reset local" callback */ + H5Z_PRELUDE_RESET_LOCAL, /* Call "reset local" callback */ + H5Z_PRELUDE_CHANGE_LOCAL, /* Call "change local" callback*/ + H5Z_PRELUDE_EVICT_LOCAL, /* Call "evict local" callback */ + H5Z_PRELUDE_DELETE_LOCAL, /* Call "delete local" callback*/ + H5Z_PRELUDE_CLOSE_LOCAL /* Call "close local" callback */ } H5Z_prelude_type_t; /* Local variables */ @@ -472,14 +476,14 @@ done: */ static herr_t H5Z_prelude_callback(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_type, - hid_t file_id) + hid_t file_id, hsize_t chunk_offset, hbool_t from_reopen) { herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5Z_prelude_callback) assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); - assert (H5I_DATATYPE==H5I_get_type(type_id)); + /*assert (H5I_DATATYPE==H5I_get_type(type_id));*/ /* Check if the property list is non-default */ if(dcpl_id!=H5P_DATASET_CREATE_DEFAULT) { @@ -545,15 +549,20 @@ H5Z_prelude_callback(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_ty /* Make correct callback */ switch(prelude_type) { case H5Z_PRELUDE_CAN_APPLY: +#ifndef H5_USE_16_API /* Check if filter is configured to be able to encode */ if(! fclass->encoder_present) HGOTO_ERROR(H5E_PLINE, H5E_NOENCODER, FAIL, "Filter present but encoding is disabled."); - +#endif /* Check if there is a "can apply" callback */ if(fclass->can_apply) { /* Make callback to filter's "can apply" function */ +#ifdef H5_USE_16_API + herr_t status=(fclass->can_apply)(dcpl_id, type_id, space_id); +#else herr_t status=(fclass->can_apply)(dcpl_id, type_id, space_id, file_id); +#endif /* Check return value */ if(status<=0) { @@ -577,7 +586,11 @@ H5Z_prelude_callback(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_ty /* Check if there is a "set local" callback */ if(fclass->set_local) { /* Make callback to filter's "set local" function */ +#ifdef H5_USE_16_API + if((fclass->set_local)(dcpl_id, type_id, space_id)<0) { +#else if((fclass->set_local)(dcpl_id, type_id, space_id, file_id)<0) { +#endif /* We're leaving, so close dataspace */ if(H5I_dec_ref(space_id)<0) HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace") @@ -588,11 +601,72 @@ H5Z_prelude_callback(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_ty } /* end if */ break; +#ifndef H5_USE_16_API case H5Z_PRELUDE_RESET_LOCAL: /* Check if there is a "reset local" callback */ if(fclass->reset_local) { /* Make callback to filter's "set local" function */ - if((fclass->reset_local)(dcpl_id, type_id, space_id, file_id)<0) { + if((fclass->reset_local)(dcpl_id, type_id, space_id, file_id, from_reopen)<0) { + /* We're leaving, so close dataspace */ + if(H5I_dec_ref(space_id)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace") + + /* Indicate error during filter callback */ + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "error during user callback") + } /* end if */ + } /* end if */ + break; + + case H5Z_PRELUDE_DELETE_LOCAL: + /* Check if there is a "delete local" callback */ + if(fclass->delete_local) { + /* Make callback to filter's "delete local" function */ + if((fclass->delete_local)(dcpl_id)<0) { + /* We're leaving, so close dataspace */ + if(H5I_dec_ref(space_id)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace") + + /* Indicate error during filter callback */ + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "error during user callback") + } /* end if */ + } /* end if */ + break; + + case H5Z_PRELUDE_CLOSE_LOCAL: + /* Check if there is a "close local" callback */ + if(fclass->close_local) { + /* Make callback to filter's "close local" function */ + if((fclass->close_local)(dcpl_id)<0) { + /* We're leaving, so close dataspace */ + if(H5I_dec_ref(space_id)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace") + + /* Indicate error during filter callback */ + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "error during user callback") + } /* end if */ + } /* end if */ + break; + + case H5Z_PRELUDE_CHANGE_LOCAL: + /* Check if there is a "change local" callback */ + if(fclass->change_local) { + /* Make callback to filter's "change local" function */ + if((fclass->change_local)(dcpl_id, chunk_offset)<0) { + /* We're leaving, so close dataspace */ + if(H5I_dec_ref(space_id)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace") + + /* Indicate error during filter callback */ + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "error during user callback") + } /* end if */ + } /* end if */ + break; + + case H5Z_PRELUDE_EVICT_LOCAL: + /* Check if there is a "evict local" callback */ + if(fclass->evict_local) { + /* Make callback to filter's "evict local" function */ + if((fclass->evict_local)(dcpl_id, chunk_offset)<0) { /* We're leaving, so close dataspace */ if(H5I_dec_ref(space_id)<0) HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace") @@ -602,6 +676,7 @@ H5Z_prelude_callback(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_ty } /* end if */ } /* end if */ break; +#endif default: assert("invalid prelude type" && 0); @@ -653,7 +728,8 @@ H5Z_can_apply (hid_t dcpl_id, hid_t type_id, hid_t file_id) assert (H5I_DATATYPE==H5I_get_type(type_id)); /* Make "can apply" callbacks for filters in pipeline */ - if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_CAN_APPLY, file_id)<0) + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_CAN_APPLY, file_id, + /*UNUSED*/0, /*UNUSED*/0)<0) HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "unable to apply filter") done: @@ -693,7 +769,8 @@ H5Z_set_local (hid_t dcpl_id, hid_t type_id, hid_t file_id) assert (H5I_DATATYPE==H5I_get_type(type_id)); /* Make "set local" callbacks for filters in pipeline */ - if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_SET_LOCAL, file_id)<0) + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_SET_LOCAL, file_id, + /*UNUSED*/0, /*UNUSED*/0)<0) HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not set") done: @@ -723,7 +800,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Z_reset_local (hid_t dcpl_id, hid_t type_id, hid_t file_id) +H5Z_reset_local (hid_t dcpl_id, hid_t type_id, hid_t file_id, hbool_t from_reopen) { herr_t ret_value=SUCCEED; /* Return value */ @@ -733,7 +810,8 @@ H5Z_reset_local (hid_t dcpl_id, hid_t type_id, hid_t file_id) assert (H5I_DATATYPE==H5I_get_type(type_id)); /* Make "reset local" callbacks for filters in pipeline */ - if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_RESET_LOCAL, file_id)<0) + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_RESET_LOCAL, file_id, + /*UNUSED*/0, from_reopen)<0) HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not set") done: @@ -742,6 +820,134 @@ done: /*------------------------------------------------------------------------- + * Function: H5Z_change_local + * + * Purpose: Makes callback to change the local values for filters. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 12 March 2008 + * + * Modifications: + *------------------------------------------------------------------------- + */ +herr_t +H5Z_change_local (hid_t dcpl_id, hid_t type_id, hid_t file_id, hsize_t chunk_offset) +{ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_change_local,FAIL) + + assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Make "change local" callbacks for filters in pipeline */ + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_CHANGE_LOCAL, file_id, + chunk_offset, /*UNUSED*/0)<0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not changed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_change_local() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_evict_local + * + * Purpose: Makes callback to evict the local values for filters. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 12 March 2008 + * + * Modifications: + *------------------------------------------------------------------------- + */ +herr_t +H5Z_evict_local (hid_t dcpl_id, hid_t type_id, hid_t file_id, hsize_t chunk_offset) +{ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_evict_local,FAIL) + + assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Make "change local" callbacks for filters in pipeline */ + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_EVICT_LOCAL, file_id, + chunk_offset, /*UNUSED*/0)<0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not changed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_evict_local() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_delete_local + * + * Purpose: Makes callback to delete the local values for filters. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 11 August 2008 + * + * Modifications: + *------------------------------------------------------------------------- + */ +herr_t +H5Z_delete_local(hid_t dcpl_id, hid_t type_id, hid_t file_id) +{ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_delete_local,FAIL) + + assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Make "delete local" callbacks for filters in pipeline */ + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_DELETE_LOCAL, file_id, + /*UNUSED*/0, /*UNUSED*/0)<0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not deleted") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_delete_local() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_close_local + * + * Purpose: Makes callback to close the local values for filters. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 12 March 2008 + * + * Modifications: + *------------------------------------------------------------------------- + */ +herr_t +H5Z_close_local (hid_t dcpl_id, hid_t type_id, hid_t file_id) +{ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_close_local,FAIL) + + assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Make "close local" callbacks for filters in pipeline */ + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_CLOSE_LOCAL, file_id, + /*UNUSED*/0, /*UNUSED*/0)<0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not closed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_close_local() */ + + +/*------------------------------------------------------------------------- * Function: H5Z_modify * * Purpose: Modify filter parameters for specified pipeline. @@ -1010,14 +1216,17 @@ done: * Programmer: Robb Matzke * Tuesday, August 4, 1998 * - * Modifications: + * Modifications: Raymond Lu + * 12 March 2008 + * Added an additional parameter of CHUNK_OFFSET for the filter + * of modifying dataset's datatype. * *------------------------------------------------------------------------- */ herr_t H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, unsigned *filter_mask/*in,out*/, H5Z_EDC_t edc_read, - H5Z_cb_t cb_struct, size_t *nbytes/*in,out*/, + hsize_t chunk_offset, H5Z_cb_t cb_struct, size_t *nbytes/*in,out*/, size_t *buf_size/*in,out*/, void **buf/*in,out*/) { size_t i, idx, new_nbytes; @@ -1058,8 +1267,12 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, #endif tmp_flags=flags|(pline->filter[idx].flags); tmp_flags|=(edc_read== H5Z_DISABLE_EDC) ? H5Z_FLAG_SKIP_EDC : 0; - new_nbytes = (fclass->filter)(tmp_flags, pline->filter[idx].cd_nelmts, - pline->filter[idx].cd_values, *nbytes, buf_size, buf); + +#ifdef H5_USE_16_API + new_nbytes = (fclass->filter)(tmp_flags, pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, *nbytes, buf_size, buf); +#else + new_nbytes = (fclass->filter)(tmp_flags, chunk_offset, pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, *nbytes, buf_size, buf); +#endif #ifdef H5Z_DEBUG H5_timer_end(&(fstats->stats[1].timer), &timer); @@ -1099,8 +1312,13 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, fstats=&H5Z_stat_table_g[fclass_idx]; H5_timer_begin(&timer); #endif - new_nbytes = (fclass->filter)(flags|(pline->filter[idx].flags), pline->filter[idx].cd_nelmts, - pline->filter[idx].cd_values, *nbytes, buf_size, buf); + +#ifdef H5_USE_16_API + new_nbytes = (fclass->filter)(flags|(pline->filter[idx].flags), pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, *nbytes, buf_size, buf); +#else + new_nbytes = (fclass->filter)(flags|(pline->filter[idx].flags), chunk_offset, pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, *nbytes, buf_size, buf); +#endif + #ifdef H5Z_DEBUG H5_timer_end(&(fstats->stats[0].timer), &timer); fstats->stats[0].total += MAX(*nbytes, new_nbytes); @@ -1161,7 +1379,7 @@ H5Z_filter_info(const H5O_pline_t *pline, H5Z_filter_t filter) break; /* Check if the filter was not already in the pipeline */ - if(idx>pline->nused) + if(idx>=pline->nused) HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, NULL, "filter not in pipeline") /* Set return value */ @@ -1303,15 +1521,19 @@ done: /*------------------------------------------------------------------------- - * Function: H5Zget_filter_info + * Function: H5Zget_filter_info + * + * Purpose: Gets information about a pipeline data filter and stores it + * in filter_config_flags. * - * Purpose: Gets information about a pipeline data filter and stores it - * in filter_config_flags. + * Return: zero on success / negative on failure * - * Return: zero on success / negative on failure + * Programmer: James Laird and Nat Furrer + * Monday, June 7, 2004 * - * Programmer: James Laird and Nat Furrer - * Monday, June 7, 2004 + * Modification: Raymond Lu + * 12 March 2008 + * Added compatibility for v1.6. * *------------------------------------------------------------------------- */ @@ -1324,6 +1546,28 @@ H5Zget_filter_info(H5Z_filter_t filter, unsigned int *filter_config_flags) FUNC_ENTER_API(H5Zget_filter_info, FAIL) H5TRACE2("e", "Zf*Iu", filter, filter_config_flags); +#ifdef H5_USE_16_API + if (filter_config_flags != NULL) + { + if (filter == H5Z_FILTER_SZIP) + { + *filter_config_flags = 0; +#ifdef H5_HAVE_FILTER_SZIP + if(SZ_encoder_enabled()>0) + *filter_config_flags |= H5Z_FILTER_CONFIG_ENCODE_ENABLED; +#endif /* H5_HAVE_FILTER_SZIP */ + *filter_config_flags |= H5Z_FILTER_CONFIG_DECODE_ENABLED; + } + else + *filter_config_flags = H5Z_FILTER_CONFIG_DECODE_ENABLED | H5Z_FILTER_CONFIG_ENCODE_ENABLED; + + /* Make sure the filter exists */ + if (H5Z_find(filter) == NULL) { + *filter_config_flags = 0; + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Filter not defined") + } + } +#else /* Look up the filter class info */ if(NULL == (fclass = H5Z_find(filter))) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Filter not defined") @@ -1337,6 +1581,7 @@ H5Zget_filter_info(H5Z_filter_t filter, unsigned int *filter_config_flags) if(fclass->decoder_present) *filter_config_flags |= H5Z_FILTER_CONFIG_DECODE_ENABLED; } /* end if */ +#endif done: FUNC_LEAVE_API(ret_value) diff --git a/src/H5Zdeflate.c b/src/H5Zdeflate.c index a954c2a..b8fc124 100644 --- a/src/H5Zdeflate.c +++ b/src/H5Zdeflate.c @@ -32,22 +32,42 @@ # include "zlib.h" #endif +#ifndef H5_USE_16_API /* Local function prototypes */ -static size_t H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); +static size_t H5Z_filter_deflate (unsigned flags, hsize_t UNUSED chunk_offset, + size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ const H5Z_class_t H5Z_DEFLATE[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_DEFLATE, /* Filter id number */ - 1, /* encoder_present flag (set to true) */ - 1, /* decoder_present flag (set to true) */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ "deflate", /* Filter name for debugging */ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ H5Z_filter_deflate, /* The actual filter function */ }}; +#else +/* Local function prototypes */ +static size_t H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], + size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class_t H5Z_DEFLATE[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_DEFLATE, /* Filter id number */ + "deflate", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + H5Z_filter_deflate, /* The actual filter function */ +}}; +#endif #define H5Z_DEFLATE_SIZE_ADJUST(s) (HDceil(((double)(s))*1.001)+12) @@ -68,14 +88,19 @@ const H5Z_class_t H5Z_DEFLATE[1] = {{ * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API static size_t -H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, - size_t *buf_size, void **buf) +H5Z_filter_deflate (unsigned flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#else +static size_t +H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], + size_t nbytes, size_t *buf_size, void **buf) +#endif { void *outbuf = NULL; /* Pointer to new buffer */ int status; /* Status from zlib operation */ - size_t ret_value; /* Return value */ + size_t ret_value = nbytes; /* Return value */ FUNC_ENTER_NOAPI(H5Z_filter_deflate, 0) @@ -83,7 +108,7 @@ H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, if (cd_nelmts!=1 || cd_values[0]>9) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid deflate aggression level") - if (flags & H5Z_FLAG_REVERSE) { + if (flags & H5Z_FLAG_REVERSE) { /*read*/ /* Input; uncompress */ z_stream z_strm; /* zlib parameters */ size_t nalloc = *buf_size; /* Number of bytes for output (compressed) buffer */ @@ -148,8 +173,7 @@ H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, /* Finish uncompressing the stream */ (void)inflateEnd(&z_strm); - } - else { + } else if (!(flags & H5Z_FLAG_INVMASK) || (flags & H5Z_FLAG_SKIP_CRECORD)) { /*Write*/ /* * Output; compress but fail if the result would be larger than the * input. The library doesn't provide in-place compression, so we @@ -197,4 +221,3 @@ done: FUNC_LEAVE_NOAPI(ret_value) } #endif /* H5_HAVE_FILTER_DEFLATE */ - diff --git a/src/H5Zdtype_modify.c b/src/H5Zdtype_modify.c index b3f074f..9a99cd8 100644 --- a/src/H5Zdtype_modify.c +++ b/src/H5Zdtype_modify.c @@ -21,21 +21,53 @@ #define H5Z_PACKAGE /*suppress error about including H5Zpkg */ #include "H5private.h" /* Generic Functions */ +#include "H5B2private.h" /* v2 B-trees */ #include "H5Eprivate.h" /* Error handling */ #include "H5Dprivate.h" /* Dataset */ #include "H5Fprivate.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MMprivate.h" /* Memory management */ #include "H5SMprivate.h" /* Shared Object Header Messages */ +#include "H5Tprivate.h" /* Datatype */ #include "H5Zpkg.h" /* Data filters */ #ifdef H5_HAVE_FILTER_DTYPE_MODIFY +/* Current version of the filter*/ +#define H5Z_DTYPE_MODIFY_VERS (1) + +/* Number of client data elements */ +#define CD_NELMTS 16 + +#define H5Z_MDTYPE_DEC 0 +#define H5Z_MDTYPE_INC 1 + +/* v2 B-tree creation macros for 'datatype' field index */ +/* How to determine the node size? */ +#define H5Z_DTYPE_BT2_NODE_SIZE 128 +#define H5Z_DTYPE_BT2_MERGE_PERC 40 +#define H5Z_DTYPE_BT2_SPLIT_PERC 100 + /* Local function prototypes */ static herr_t H5Z_set_local_dtype_modify(hid_t dcpl_id, hid_t dtype_id, hid_t space_id, hid_t file_id); static herr_t H5Z_reset_local_dtype_modify(hid_t dcpl_id, hid_t dtype_id, hid_t space_id, - hid_t file_id); -static size_t H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, + hid_t file_id, hbool_t from_reopen); +static herr_t H5Z_change_local_dtype_modify(hid_t dcpl_id, hsize_t chunk_offset); +static herr_t H5Z_evict_local_dtype_modify(hid_t dcpl_id, hsize_t chunk_offset); +static herr_t H5Z_delete_local_dtype_modify(hid_t dcpl_id); +static herr_t H5Z_close_local_dtype_modify(hid_t dcpl_id); +static herr_t H5Z_free_slist_node(void *item, void UNUSED *key, void UNUSED *operator_data); +static herr_t H5Z_btree2_dtype_store(void *_nrecord, const void *_udata); +static herr_t H5Z_btree2_dtype_retrieve(void *udata, const void *nrecord); +static herr_t H5Z_btree2_dtype_encode(const H5F_t *f, uint8_t *raw, const void *_nrecord); +static herr_t H5Z_btree2_dtype_decode(const H5F_t *f, const uint8_t *raw, void *_nrecord); +static herr_t H5Z_btree2_dtype_compare(const void *_bt2_udata, const void *_bt2_rec); +static herr_t H5Z_btree2_dtype_find_cb(void *record, void *op_data); +static herr_t H5Z_btree2_dtype_remove_cb(void *record, void *op_data); +static herr_t H5Z_btree2_dtype_delete_cb(void *record, void *op_data); + +static size_t H5Z_filter_dtype_modify (unsigned flags, hsize_t chunk_offset, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ @@ -47,15 +79,53 @@ H5Z_class_t H5Z_DTYPE_MODIFY[1] = {{ "dtype_modify", /* Filter name for debugging */ NULL, /* The "can apply" callback */ H5Z_set_local_dtype_modify, /* The "set local" callback */ - H5Z_reset_local_dtype_modify, /* The "reset local" callback */ + H5Z_reset_local_dtype_modify, /* The "reset local" callback */ + H5Z_change_local_dtype_modify,/* The "change local" callback*/ + H5Z_evict_local_dtype_modify, /* The "evict local" callback */ + H5Z_delete_local_dtype_modify, /* The "delete local" callback */ + H5Z_close_local_dtype_modify, /* The "close local" callback */ H5Z_filter_dtype_modify, /* The actual filter function */ }}; +typedef struct H5Z_slist_node_t { + H5T_t *dtype; + hbool_t type_modified; + hsize_t key; +} H5Z_slist_node_t; + +/* Typedef for native 'name' field index records in the v2 B-tree */ +typedef struct H5Z_mdtype_bt2_rec_t { + uint8_t version; + uint8_t is_shared; + uint64_t numb_ref; + H5T_t *dtype; + hbool_t found; + uint8_t adjust; +} H5Z_mdtype_bt2_rec_t; + +/* The v2 B-tree class for indexing 'datatype' field */ +const H5B2_class_t H5Z_BT2_DTYPE[1]={{ /* B-tree class information */ + H5B2_MODIFY_TYPE_ID, /* Type of B-tree */ + sizeof(H5Z_mdtype_bt2_rec_t), /* Size of native record */ + H5Z_btree2_dtype_store, /* Record storage callback */ + H5Z_btree2_dtype_retrieve, /* Record retrieval callback */ + H5Z_btree2_dtype_compare, /* Record comparison callback */ + H5Z_btree2_dtype_encode, /* Record encoding callback */ + H5Z_btree2_dtype_decode, /* Record decoding callback */ + NULL /* Record debugging callback */ +}}; + + +/* Declare a free list to manage H5Z_slist_node_t objects */ +H5FL_DEFINE_STATIC(H5Z_slist_node_t); + /*------------------------------------------------------------------------- * Function: H5Z_set_local_dtype_modify * - * Purpose: Set the "local" dataset parameters for datatype modification + * Purpose: Set the "local" dataset parameters for datatype modification. + * They are the ID of the dataset's datatype and the file + * pointer. * * Return: Success: Non-negative * Failure: Negative @@ -71,36 +141,149 @@ static herr_t H5Z_set_local_dtype_modify(hid_t dcpl_id, hid_t dtype_id, hid_t UNUSED space_id, hid_t file_id) { + H5P_genplist_t *dc_plist = NULL; /* New Property list */ H5F_t *file=NULL; /* File object for file ID */ - size_t cd_nelmts = 1; /* Number of filter parameters */ - unsigned cd_values[8]={0,0,0,0,0,0,0,0}; /* Filter parameters */ + H5T_t *dt = NULL; /* Datatype object for datatype ID */ + H5SL_t *chunk_slist=NULL; /* Skip list for keeping record of chunks */ + size_t dt_size; /* the size of datatype */ + size_t bt2_rrec_size; /* v2 B-tree raw record size */ + haddr_t baddr; /* Address of B-tree to store dtype info */ + size_t cd_nelmts = 0; /* Number of filter parameters */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned baddr_count = 0; /* Keeps track of btree address size */ + unsigned cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ uint8_t *pp = NULL; herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5Z_set_local_dtype_modify, FAIL) + /* Get the dataset's property list object */ + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + if(NULL == (file = H5I_object_verify(file_id, H5I_FILE))) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID") - /* The first element of the client data is the ID of dataset's datatype */ - cd_values[0] = dtype_id; + /* Check args */ + if (NULL == (dt = H5I_object_verify(dtype_id,H5I_DATATYPE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype"); - pp = (uint8_t*)((unsigned *)cd_values + 1); + /* The first element of the client data is reserved for COUNT, which is + * copied toward the end of this function. COUNT has two digits and + * keeps track of the number of elements in CD_VALUES except itself (first + * digit) and the size of file pointer and skip list pointer (second digit). + */ + cd_nelmts = 1; + + /* The second element of the client data is the ID of dataset's datatype. */ + cd_values[1] = dtype_id; + cd_nelmts += 1; + count = 10; + + pp = (uint8_t*)((unsigned *)cd_values + 2); /* - * The second is the file pointer copied as unsigned integer. Depending - * on the size of pointer, it may take the space of 1 - 2 elements. + * The third is the file pointer copied as unsigned integer. The size of + * the pointer can be 4 or 8 bytes. Turn on the first digit of COUNT + * if the pointer size is 8 bytes. 8-byte pointer should take 2 CD_NELMTS. */ HDmemcpy(pp, &file, sizeof(H5F_t*)); - if(4 == sizeof(H5F_t*)) + count +=10; + if(4 == sizeof(H5F_t*)) { cd_nelmts += 1; - else if(8 == sizeof(H5F_t*)) + pp += 4; + } else if(8 == sizeof(H5F_t*)) { cd_nelmts += 2; - else + count +=1; + pp += 8; + } else HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "not implemented yet") + /* Create skip list to keep track of visited chunk. It's created here + * because the filter needs it when the dataset is just created and the + * data is being written to the file. But keep in mind that the client + * data is saved in the file. Everything is invalid when the file is + * reopened and the client data is retrieved. */ + if((chunk_slist = H5SL_create(H5SL_TYPE_HSIZE, 0.5, (size_t)16)) == NULL) + HGOTO_ERROR(H5E_PLINE, H5E_CANTCREATE, FAIL, "can't create skip list for chunks") + + /* The fourth element is the skip list for chunks. Again the second digit + * indicates whether the pointer is 8 bytes. */ + HDmemcpy(pp, &chunk_slist, sizeof(H5SL_t*)); + count += 10; + if(4 == sizeof(H5SL_t*)) { + cd_nelmts += 1; + pp += 4; + } else if(8 == sizeof(H5SL_t*)) { + count += 1; + cd_nelmts += 2; + pp += 8; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "not implemented yet") + + /* Set the COUNT */ + cd_values[0] = count; + + /* Generally only support committed or shared datatype. Check it + * here because this function can be called by H5D_create. We also allow + * integer or floating number because of their limited encoding size. + */ + if(!H5T_committed(dt) && + !H5SM_can_share(file, H5AC_dxpl_id, NULL, NULL, H5O_DTYPE_ID, dt)) { + H5T_class_t tclass; + if((tclass = H5T_get_class(dt, TRUE)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "can't find datatype class") + + if(tclass != H5T_INTEGER && tclass != H5T_FLOAT) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "violate datatype restriction") + } + + /* Check if the datatype should be (or are already) shared in the SOHM table */ + if(H5SM_try_share(file, H5AC_dxpl_id, NULL, H5O_DTYPE_ID, dt, NULL) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADMESG, FAIL, "trying to share datatype failed") + + /* Get the encoding size of the dataset's datatype */ + if((dt_size = H5O_msg_raw_size(file, H5O_DTYPE_ID, FALSE, dt))==0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADSIZE, 0, "can't find datatype size") + + /* If the datatype is committed or shared, its size should be the address plus + * index of heap, which is 12. If the datatype is predefined, the encoding + * size should be at most 20 bytes (for floating number. See H5O_dtype_size in + * H5Odtype.c). Because the datatype may be changed later, we make it big + * enough now. + */ + if(dt_size < 20) + dt_size = 20; + + /* Create a B-tree to store the info of all the dtypes used by the chunks. */ + bt2_rrec_size = 1 + /* version number */ + 1 + /* flag for shared datatype */ + dt_size + /* datatype info */ + 8; /* Number of reference */ + + if(H5B2_create(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, (size_t)H5Z_DTYPE_BT2_NODE_SIZE, + bt2_rrec_size, H5Z_DTYPE_BT2_SPLIT_PERC, H5Z_DTYPE_BT2_MERGE_PERC, &baddr/*out*/) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "can't create B-tree") + + /* the B-tree address can be 8 or 16 byte in size */ + if(8 == sizeof(haddr_t)) { + baddr_count = 2; + cd_nelmts += 2; + } else if(16 == sizeof(haddr_t)) { + baddr_count = 4; + cd_nelmts += 4; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "not implemented yet") + + HDmemcpy(pp, &baddr_count, sizeof(unsigned)); + pp += 4; + cd_nelmts += 1; + + HDmemcpy(pp, &baddr, sizeof(haddr_t)); + /* Modify the filter's parameters for this dataset */ - if(H5Pmodify_filter(dcpl_id, H5Z_FILTER_DTYPE_MODIFY, H5Z_FLAG_MANDATORY, (size_t)cd_nelmts, cd_values) < 0) + if(H5P_modify_filter(dc_plist, H5Z_FILTER_DTYPE_MODIFY, H5Z_FLAG_MANDATORY, + (size_t)cd_nelmts, cd_values) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't set local parameters") done: @@ -114,7 +297,8 @@ done: * Purpose: Reset the "local" dataset parameters for datatype * modification. This function is the same as * H5Z_set_local_dtype_modify. But it's called after the - * dataset has been created. + * dataset has been created to update some info like file_id + * and datatype. It's called by H5D_modify_dtype and H5D_open. * * Return: Success: Non-negative * Failure: Negative @@ -128,14 +312,150 @@ done: */ static herr_t H5Z_reset_local_dtype_modify(hid_t dcpl_id, hid_t dtype_id, hid_t UNUSED space_id, - hid_t file_id) + hid_t file_id, hbool_t from_reopen) { + H5P_genplist_t *dc_plist = NULL; /* New Property list */ + H5F_t *file=NULL; /* File object for file ID */ + H5SL_t *chunk_slist=NULL; /* Skip list for keeping record of chunks */ + size_t cd_nelmts = 0; /* Number of filter parameters */ + size_t orig_cd_nelmts = CD_NELMTS; /* Number of filter parameters */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned orig_count = 0; /* Keeps track of pointer size */ + unsigned baddr_count = 0; /* Keeps track of pointer size */ + unsigned orig_baddr_count = 0; /* Keeps track of pointer size */ + unsigned cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ + unsigned orig_cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ + uint8_t *pp = NULL; + uint8_t *orig_pp = NULL; herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5Z_reset_local_dtype_modify, FAIL) - if((ret_value = H5Z_set_local_dtype_modify(dcpl_id, dtype_id, space_id, file_id))<0) - HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't reset local parameters") + /* Get the dataset's property list object */ + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + + if(H5P_get_filter_by_id(dc_plist, H5Z_FILTER_DTYPE_MODIFY, NULL, + &orig_cd_nelmts, orig_cd_values, (size_t)0, NULL, NULL) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get filter info") + + orig_count = orig_cd_values[0]; + + if(NULL == (file = H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID") + + /* The first element of the client data is reserved for COUNT, which is + * copied toward the end of this function. COUNT has two digits and + * keeps track of the number of elements in CD_VALUES except itself (first + * digit) and the size of file pointer and skip list pointer (second digit). + */ + cd_nelmts = 1; + + /* The second element of the client data is the ID of dataset's datatype */ + cd_values[1] = dtype_id; + cd_nelmts += 1; + count = 10; + + pp = (uint8_t*)((unsigned *)cd_values + 2); + + /* + * The third is the file pointer copied as unsigned integer. Depending + * on the size of pointer, it may take the space of 1 - 2 elements. + */ + HDmemcpy(pp, &file, sizeof(H5F_t*)); + if(4 == sizeof(H5F_t*)) { + cd_nelmts += 1; + count += 10; + pp += 4; + } else if(8 == sizeof(H5F_t*)) { + cd_nelmts += 2; + count += 10; + count += 1; + pp += 8; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "not implemented yet") + + /* Check whether the skip list for visited chunks has been created. If the first + * digit of original COUNT is 2, there're only 2 elements, and the skip list hasn't + * been created. If this function is called by H5Dopen and FROM_REOPEN is set, the + * pointer value is invalid because the client data was saved in the file. Create + * a new skip list in this case. If it's 3 and FROM_REOPEN isn't set, the skip + * list has been created. Simply copy the pointer over. + */ + if(orig_count < 30 || from_reopen) { + /* Create skip list to keep track of visited chunk */ + if((chunk_slist = H5SL_create(H5SL_TYPE_HSIZE, 0.5, (size_t)16)) == NULL) + HGOTO_ERROR(H5E_PLINE, H5E_CANTCREATE, FAIL, "can't create skip list for chunks") + + /* The fourth element is the skip list for chunks. Again the second digit + * indicates whether the pointer is 8 bytes. */ + HDmemcpy(pp, &chunk_slist, sizeof(H5SL_t*)); + count += 10; + if(4 == sizeof(H5SL_t*)) { + cd_nelmts += 1; + pp += 4; + } else if(8 == sizeof(H5SL_t*)) { + count += 1; + cd_nelmts += 2; + pp += 8; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "not implemented yet") + + if(orig_count == 20) + orig_pp = (uint8_t*)((unsigned *)orig_cd_values + 3); + else if(orig_count > 20 && orig_count <= 30) + orig_pp = (uint8_t*)((unsigned *)orig_cd_values + 4); + else if(orig_count > 30) + orig_pp = (uint8_t*)((unsigned *)orig_cd_values + 6); + } else if (orig_count == 30 && !from_reopen) { + /* Copy the skip list. The size of pointer is 4 bytes. */ + orig_pp = (uint8_t*)((unsigned *)orig_cd_values + 3); + HDmemcpy(pp, orig_pp, 4); + count += 10; + cd_nelmts += 1; + pp += 4; + orig_pp += 4; + } else if (orig_count > 30 && !from_reopen) { + /* Copy the skip list. The size of pointer is 8 bytes. */ + orig_pp = (uint8_t*)((unsigned *)orig_cd_values + 4); + HDmemcpy(pp, orig_pp, 8); + count += 10; + count += 1; + cd_nelmts += 2; + pp += 8; + orig_pp += 8; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "not implemented yet") + + /* Set the COUNT */ + cd_values[0] = count; + + /* Copy B-tree address to the new client data */ + HDmemcpy(&orig_baddr_count, orig_pp, sizeof(unsigned)); + orig_pp += 4; + + /* the B-tree address can be 8 or 16 byte in size */ + if(orig_baddr_count == 2) { + baddr_count = 2; + HDmemcpy(pp, &baddr_count, sizeof(unsigned)); + pp += 4; + cd_nelmts += 1; + HDmemcpy(pp, orig_pp, 8); + cd_nelmts += 2; + } else if(orig_baddr_count == 4) { + baddr_count = 4; + HDmemcpy(pp, &baddr_count, sizeof(unsigned)); + pp += 4; + cd_nelmts += 1; + HDmemcpy(pp, orig_pp, 16); + cd_nelmts += 4; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTCOPY, FAIL, "not implemented yet") + + /* Modify the filter's parameters for this dataset */ + if(H5P_modify_filter(dc_plist, H5Z_FILTER_DTYPE_MODIFY, H5Z_FLAG_MANDATORY, + (size_t)cd_nelmts, cd_values) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't set local parameters") done: FUNC_LEAVE_NOAPI(ret_value) @@ -143,6 +463,631 @@ done: /*------------------------------------------------------------------------- + * Function: H5Z_change_local_dtype_modify + * + * Purpose: Search the node of the skip list for chunk info and update it. + * This function is called by H5D_istore_conv_dtype when the + * datatype is changed and all the chunks in the cache have + * to be converted. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Raymond Lu + * 16 April 2008 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_change_local_dtype_modify(hid_t dcpl_id, hsize_t chunk_offset) +{ + H5P_genplist_t *dc_plist = NULL; /* New Property list */ + H5SL_t *chunk_slist=NULL; /* Skip list for keeping record of chunks */ + H5Z_slist_node_t *node; + size_t cd_nelmts=CD_NELMTS; /* Number of filter parameters */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ + uint8_t *pp = NULL; + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_change_local_dtype_modify, FAIL) + + HDassert(H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Get the dataset's property list object */ + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + + if(H5P_get_filter_by_id(dc_plist, H5Z_FILTER_DTYPE_MODIFY, NULL, &cd_nelmts, cd_values, + (size_t)0, NULL, NULL) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get filter info") + + /* If the skip list exists, update the dtype modification info for the chunk */ + count = cd_values[0]; + + if(count >= 30) { + pp = (uint8_t*)((unsigned *)cd_values + 2); + + /* Depending on the size of pointer, retrieve the value of the pointer to + * the skip list. */ + if (count == 30) { + pp += 4; + HDmemcpy(&chunk_slist, pp, 4); + } else if (count > 30) { + pp += 8; + HDmemcpy(&chunk_slist, pp, 8); + } + + if(!chunk_slist) + HGOTO_ERROR(H5E_SLIST, H5E_CANTGET, FAIL, "can't find skip list") + + /* Search the skip list and update the node */ + if(NULL != (node = H5SL_search(chunk_slist, &chunk_offset))) + node->type_modified = TRUE; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_change_local_dtype_modify */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_evict_local_dtype_modify + * + * Purpose: Search the node of the skip list for chunk and remove the + * node. This function is called when the chunk is evicted + * from the cache. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Raymond Lu + * 16 April 2008 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_evict_local_dtype_modify(hid_t dcpl_id, hsize_t chunk_offset) +{ + H5P_genplist_t *dc_plist = NULL; /* New Property list */ + H5SL_t *chunk_slist=NULL; /* Skip list for keeping record of chunks */ + H5Z_slist_node_t *node; + size_t cd_nelmts=CD_NELMTS; /* Number of filter parameters */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ + uint8_t *pp = NULL; + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_evict_local_dtype_modify, FAIL) + + HDassert(H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Get the dataset's property list object */ + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + + if(H5P_get_filter_by_id(dc_plist, H5Z_FILTER_DTYPE_MODIFY, NULL, &cd_nelmts, cd_values, + (size_t)0, NULL, NULL) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get filter info") + + /* If the skip list exists, remove the node for the chunk */ + count = cd_values[0]; + + if(count >= 30 ) { + pp = (uint8_t*)((unsigned *)cd_values + 2); + + /* Depending on the size of pointer, retrieve the value of the pointer to + * the skip list. */ + if (count == 30) { + pp += 4; + HDmemcpy(&chunk_slist, pp, 4); + } else if (count > 30) { + pp += 8; + HDmemcpy(&chunk_slist, pp, 8); + } + + if(!chunk_slist) + HGOTO_ERROR(H5E_SLIST, H5E_CANTGET, FAIL, "can't find skip list") + + /* Search the skip list and update the node */ + if(NULL != (node = H5SL_remove(chunk_slist, &chunk_offset))) { + /* Check whether datatype is committed & decrement ref count + * (to maintain ref. count incr/decr similarity with "shared message" + * type of datatype sharing) + */ + if(H5T_committed(node->dtype)) { + /* Decrement the reference count on the committed datatype */ + if(H5T_link(node->dtype, -1, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, FAIL, "unable to adjust shared datatype link count") + } /* end if */ + + if(H5T_close(node->dtype) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "can't close data type") + + H5FL_FREE(H5Z_slist_node_t, node); + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_evict_local_dtype_modify */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_delete_local_dtype_modify + * + * Purpose: When the dataset is deleted, free the nodes of the skip list + * and delete it. Also decrement the reference counts of the + * datatypes in B-tree nodes, free the nodes of the B-tree and + * delete it. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Raymond Lu + * 11 August 2008 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_delete_local_dtype_modify(hid_t dcpl_id) +{ + H5P_genplist_t *dc_plist = NULL; /* New Property list */ + size_t cd_nelmts=CD_NELMTS; /* Number of filter parameters */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ + uint8_t *pp = NULL; + H5F_t *file = NULL; + unsigned baddr_count = 0; /* Keeps track of pointer size */ + haddr_t baddr; /* B-tree address */ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_delete_local_dtype_modify, FAIL) + + HDassert(H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Get the dataset's property list object */ + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + + if(H5P_get_filter_by_id(dc_plist, H5Z_FILTER_DTYPE_MODIFY, NULL, &cd_nelmts, cd_values, + (size_t)0, NULL, NULL) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get filter info") + + count = cd_values[0]; + pp = (uint8_t*)((unsigned *)cd_values + 2); + + /* Depending on number of elements and the size of pointer, retrieve the value + * of file pointer and skip list pointer. The first digit of CD_NELMTS indicates + * the number of elements in CD_VALUES. The second digit indicates whether the + * pointers are 8 bytes. 0 stands for 4 bytes. Greater than 0 means 8 bytes. + */ + if (count == 20) { + HDmemcpy(&file, pp, 4); + pp += 4; + } else if (20 < count && count < 30) { + HDmemcpy(&file, pp, 8); + pp += 8; + } else if (count == 30) { + HDmemcpy(&file, pp, 4); + pp += 4; + /* Advance over the skip list address */ + pp += 4; + } else if (count > 30) { + HDmemcpy(&file, pp, 8); + pp += 8; + /* Advance over the skip list address */ + pp += 8; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, 0, "not implemented yet") + + HDmemcpy(&baddr_count, pp, sizeof(unsigned)); + pp += 4; + + /* the B-tree address can be 8 or 16 byte in size */ + if(baddr_count == 2) { + HDmemcpy(&baddr, pp, 8); + } else if(baddr_count == 4) { + HDmemcpy(&baddr, pp, 16); + } + + /* Delete the entire B-tree and decrement reference count of datatype */ + if(H5B2_delete(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, H5Z_btree2_dtype_delete_cb, + NULL) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't delete B-tree") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_delete_local_dtype_modify() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_close_local_dtype_modify + * + * Purpose: When the dataset is closed, free the nodes of the skip list + * and delete it. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Raymond Lu + * 22 Feb 2008 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_close_local_dtype_modify(hid_t dcpl_id) +{ + H5P_genplist_t *dc_plist = NULL; /* New Property list */ + H5SL_t *chunk_slist=NULL; /* Skip list for keeping record of chunks */ + size_t cd_nelmts=CD_NELMTS; /* Number of filter parameters */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ + uint8_t *pp = NULL; + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_close_local_dtype_modify, FAIL) + + HDassert(H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Get the dataset's property list object */ + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + + if(H5P_get_filter_by_id(dc_plist, H5Z_FILTER_DTYPE_MODIFY, NULL, &cd_nelmts, cd_values, (size_t)0, NULL, NULL) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get filter info") + + count = cd_values[0]; + pp = (uint8_t*)((unsigned *)cd_values + 2); + + /* Depending on the size of pointer, retrieve the value of the pointer to + * the skip list. */ + if (count == 30) { + pp += 4; + HDmemcpy(&chunk_slist, pp, 4); + } else if (count > 30) { + pp += 8; + HDmemcpy(&chunk_slist, pp, 8); + } + + /* Free the skip list */ + if(chunk_slist && H5SL_destroy(chunk_slist, H5Z_free_slist_node, NULL) < 0) + HGOTO_ERROR(H5E_SLIST, H5E_CANTFREE, FAIL, "can't free skip list") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_close_local_dtype_modify() */ + +/*------------------------------------------------------------------------- + * Function: H5Z_free_slist_node + * + * Purpose: Private function for H5SL_destroy. Frees the nodes of + * skip list. + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Raymond Lu + * 22 February 2008 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_free_slist_node(void *item, void UNUSED *key, void UNUSED *operator_data/*in,out*/) +{ + H5T_t *dtype = ((H5Z_slist_node_t*)item)->dtype; + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_free_slist_node, FAIL) + + /* Check whether datatype is committed & decrement ref count + * (to maintain ref. count incr/decr similarity with "shared message" + * type of datatype sharing) + */ + if(H5T_committed(dtype)) { + /* Decrement the reference count on the committed datatype */ + if(H5T_link(dtype, -1, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, FAIL, "unable to adjust shared datatype link count") + } /* end if */ + + if(H5T_close(dtype) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "can't close data type") + + H5FL_FREE(H5Z_slist_node_t, item); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_free_slist_node */ + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_store + * + * Purpose: Store user information into native record for v2 B-tree + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_store(void *_nrecord, const void *_udata) +{ + const H5Z_mdtype_bt2_rec_t *udata = (const H5Z_mdtype_bt2_rec_t *)_udata; + H5Z_mdtype_bt2_rec_t *nrecord = (H5Z_mdtype_bt2_rec_t *)_nrecord; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(H5Z_btree2_dtype_store, FAIL) + + /* Copy user information info native record */ + nrecord->version = udata->version; + nrecord->is_shared = udata->is_shared; + nrecord->numb_ref = udata->numb_ref; + + /* Make a copy of the datatype because Btree resides in the cache separately. + * The dataset can be closed first. */ + if((nrecord->dtype = H5T_copy(udata->dtype, H5T_COPY_ALL)) == NULL) + HGOTO_ERROR(H5E_PLINE, H5E_CANTCOPY, FAIL, "can't copy datatype") + +done: + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5Z_btree2_dtype_store() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_retrieve + * + * Purpose: Retrieve native information from record for v2 B-tree + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_retrieve(void *udata, const void *nrecord) +{ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5Z_btree2_dtype_retrieve) + + *(H5Z_mdtype_bt2_rec_t *)udata = *(const H5Z_mdtype_bt2_rec_t *)nrecord; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5Z_btree2_dtype_retrieve() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_encode + * + * Purpose: Encode native information into raw form for storing on disk + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_encode(const H5F_t *f, uint8_t *raw, const void *_nrecord) +{ + const H5Z_mdtype_bt2_rec_t *nrecord = (const H5Z_mdtype_bt2_rec_t *)_nrecord; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(H5Z_btree2_dtype_encode, FAIL) + + /* Encode the record's fields */ + *raw++ = nrecord->version; + + /* Encode the flag indicating whether the dtype is shared in Btree node buffer */ + *raw++ = nrecord->is_shared; + + UINT64ENCODE(raw, nrecord->numb_ref); + + /* encode datatype info into Btree node buffer */ + if(H5O_msg_encode((H5F_t *)f, H5O_DTYPE_ID, FALSE, (unsigned char *)raw, + (const void *)nrecord->dtype)<0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "can't encode object") + + if(H5T_close(nrecord->dtype) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CLOSEERROR, FAIL, "can't close datatype") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5Z_btree2_dtype_encode() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_decode + * + * Purpose: Decode raw disk form of record into native form + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_decode(const H5F_t *f, const uint8_t *raw, void *_nrecord) +{ + H5Z_mdtype_bt2_rec_t *nrecord = (H5Z_mdtype_bt2_rec_t *)_nrecord; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(H5Z_btree2_dtype_decode, FAIL) + + /* Decode the record's fields */ + nrecord->version = *raw++; + nrecord->is_shared = *raw++; + UINT64DECODE(raw, nrecord->numb_ref); + + /* + * Decode the original datatype info from the chunk of data which might + * be shared or not shared. + */ + if(nrecord->is_shared && (nrecord->dtype=(H5T_t *)H5O_msg_decode((H5F_t *)f, + H5AC_dxpl_id, H5O_DTYPE_ID, (const unsigned char *)raw, H5O_MSG_FLAG_SHARED))==NULL) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "can't decode object") + else if(!nrecord->is_shared && (nrecord->dtype=(H5T_t *)H5O_msg_decode((H5F_t *)f, + H5AC_dxpl_id, H5O_DTYPE_ID, (const unsigned char *)raw, H5O_MSG_FLAG_WAS_UNKNOWN))==NULL) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "can't decode object") + +done: + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5Z_btree2_dtype_decode() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_compare + * + * Purpose: Compare two native information records, according to some key + * + * Return: <0 if rec1 < rec2 + * =0 if rec1 == rec2 + * >0 if rec1 > rec2 + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_compare(const void *_bt2_udata, const void *_bt2_rec) +{ + const H5Z_mdtype_bt2_rec_t *bt2_udata = (const H5Z_mdtype_bt2_rec_t *)_bt2_udata; + const H5Z_mdtype_bt2_rec_t *bt2_rec = (const H5Z_mdtype_bt2_rec_t *)_bt2_rec; + herr_t ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5Z_btree2_dtype_compare) + + /* Sanity check */ + HDassert(bt2_udata); + HDassert(bt2_rec); + + ret_value = H5T_cmp(bt2_udata->dtype, bt2_rec->dtype, FALSE); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5Z_btree2_dtype_compare() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_find_cb + * + * Purpose: Callback function for H5B2_find function. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_find_cb(void *record, void *op_data) +{ + H5Z_mdtype_bt2_rec_t *my_record = (H5Z_mdtype_bt2_rec_t *)record; + H5Z_mdtype_bt2_rec_t *udata = (H5Z_mdtype_bt2_rec_t *)op_data; + int compare; + herr_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5Z_btree2_dtype_find_cb) + + compare = H5T_cmp(my_record->dtype, udata->dtype, FALSE); + + if(compare == 0) { + if(udata->adjust == H5Z_MDTYPE_INC) + my_record->numb_ref++; + else if(udata->adjust == H5Z_MDTYPE_DEC) + my_record->numb_ref--; + + udata->numb_ref = my_record->numb_ref; + udata->found = TRUE; + ret_value = SUCCEED; + } + + FUNC_LEAVE_NOAPI(ret_value) +} + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_remove_cb + * + * Purpose: Callback function for H5B2_remove function. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_remove_cb(void *record, void *op_data) +{ + H5Z_mdtype_bt2_rec_t *my_record = (H5Z_mdtype_bt2_rec_t *)record; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_btree2_dtype_remove_cb, FAIL) + + /* Close the datatype saved in Btree node */ + if(H5T_close(my_record->dtype) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CLOSEERROR, FAIL, "can't close datatype") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_delete_cb + * + * Purpose: Callback function for H5B2_delete function. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_delete_cb(void *record, void *op_data) +{ + H5Z_mdtype_bt2_rec_t *my_record = (H5Z_mdtype_bt2_rec_t *)record; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_btree2_dtype_delete_cb, FAIL) + + /* Check whether datatype is committed & decrement ref count + * (to maintain ref. count incr/decr similarity with "shared message" + * type of datatype sharing) + */ + if(H5T_committed(my_record->dtype)) { + /* Decrement the reference count on the committed datatype */ + if(H5T_link(my_record->dtype, (int)-my_record->numb_ref, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, FAIL, "unable to adjust shared datatype link count") + } /* end if */ + + if(H5T_close(my_record->dtype) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CLOSEERROR, FAIL, "can't close datatype") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} + + +/*------------------------------------------------------------------------- * Function: H5Z_filter_dtype_modify * * Purpose: Implement the I/O filter of the datatype modification for @@ -163,8 +1108,8 @@ done: */ /* ARGSUSED */ static size_t -H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], - size_t nbytes, size_t *buf_size, void **buf) +H5Z_filter_dtype_modify (unsigned flags, hsize_t chunk_offset, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) { unsigned char *outbuf = NULL; /* Pointer to new buffer */ hid_t dtype_id; @@ -173,49 +1118,101 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val size_t type_msg_size; size_t nelmts, dbuf_size; H5F_t *file = NULL; + H5SL_t *chunk_slist=NULL; /* Skip list for keeping record of chunks */ + H5Z_slist_node_t *node; uint8_t *pp = NULL; unsigned char *dst=NULL; /* Temporary pointer to destination buffer */ - size_t ret_value; /* Return value */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned baddr_count = 0; /* Keeps track of pointer size */ + haddr_t baddr; /* B-tree address */ + int8_t is_shared = FALSE; /* Whether dtype is committed or shared */ + size_t ret_value=nbytes; /* Return value */ FUNC_ENTER_NOAPI(H5Z_filter_dtype_modify, 0) - assert(cd_nelmts==2 || cd_nelmts==3); + count = cd_values[0]; + + assert(cd_nelmts>=2 && cd_nelmts<=10); + assert(count>=20 && count<40); /* Get the dataset's datatype */ - dtype_id = cd_values[0]; - pp = (uint8_t*)((unsigned *)cd_values + 1); + dtype_id = cd_values[1]; + pp = (uint8_t*)((unsigned *)cd_values + 2); - /* Depending on the size of pointer, retrieve the value of file pointer */ - if(2 == cd_nelmts) { + /* Depending on number of elements and the size of pointer, retrieve the value + * of file pointer and skip list pointer. The first digit of CD_NELMTS indicates + * the number of elements in CD_VALUES. The second digit indicates whether the + * pointers are 8 bytes. 0 stands for 4 bytes. Greater than 0 means 8 bytes. + */ + if (count == 20) { HDmemcpy(&file, pp, 4); - } else if(3 == cd_nelmts) { + pp += 4; + } else if (20 < count && count < 30) { HDmemcpy(&file, pp, 8); + pp += 8; + } else if (count == 30) { + HDmemcpy(&file, pp, 4); + pp += 4; + HDmemcpy(&chunk_slist, pp, 4); + pp += 4; + } else if (count > 30) { + HDmemcpy(&file, pp, 8); + pp += 8; + HDmemcpy(&chunk_slist, pp, 8); + pp += 8; } else HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, 0, "not implemented yet") + HDmemcpy(&baddr_count, pp, sizeof(unsigned)); + pp += 4; + + /* the B-tree address can be 8 or 16 byte in size */ + if(baddr_count == 2) { + HDmemcpy(&baddr, pp, 8); + } else if(baddr_count == 4) { + HDmemcpy(&baddr, pp, 16); + } + /* The current datatype of the dataset */ if(NULL==(dtype=(H5T_t *)H5I_object_verify(dtype_id, H5I_DATATYPE))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, 0, "not a datatype") - if (flags & H5Z_FLAG_REVERSE) { /* Read */ + if (flags & H5Z_FLAG_REVERSE) { /*Read*/ + uint8_t version = 0; + hid_t orig_type_id; + size_t info_size = 0; /* the size of the encoded information except raw data */ + + /* Retrieve and verify version number */ + HDmemcpy(&version, *buf, sizeof(uint8_t)); + + if(version != H5Z_DTYPE_MODIFY_VERS) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, 0, "wrong version number") + + info_size = 1; + + /* Retrieve the flag indicating whether the dtype is committed or shared */ + outbuf = (unsigned char*)*buf + 1; + HDmemcpy(&is_shared, outbuf, sizeof(int8_t)); + info_size += 1; + outbuf += 1; + /* * Decode the original datatype info from the chunk of data which might - * be shared or not shared. Try to treat it as shared then non-shared. + * be shared or not shared. */ - H5E_BEGIN_TRY { - orig_type=(H5T_t *)H5O_msg_decode(file, H5AC_dxpl_id, H5O_DTYPE_ID, - (unsigned char *)*buf, H5O_MSG_FLAG_SHARED); - } H5E_END_TRY; - - if(!orig_type) { - if((orig_type=(H5T_t *)H5O_msg_decode(file, H5AC_dxpl_id, H5O_DTYPE_ID, (unsigned char *)*buf, H5O_MSG_FLAG_WAS_UNKNOWN))==NULL) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, 0, "can't decode object") - } + if(is_shared && (orig_type=(H5T_t *)H5O_msg_decode(file, H5AC_dxpl_id, H5O_DTYPE_ID, + outbuf, H5O_MSG_FLAG_SHARED))==NULL) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, 0, "can't decode object") + else if(!is_shared && (orig_type=(H5T_t *)H5O_msg_decode(file, H5AC_dxpl_id, + H5O_DTYPE_ID, outbuf, H5O_MSG_FLAG_WAS_UNKNOWN))==NULL) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, 0, "can't decode object") /* Get the encoding size of the original datatype */ if((type_msg_size = H5O_msg_raw_size(file, H5O_DTYPE_ID, FALSE, orig_type))==0) HGOTO_ERROR(H5E_DATATYPE, H5E_BADSIZE, 0, "can't find datatype size") + info_size += type_msg_size; + /* Calculate the size of buffer containing the new data */ if(0 != H5T_cmp(dtype, orig_type, FALSE)) { if((dtype_size = H5T_get_size(dtype))==0) @@ -225,19 +1222,19 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val nelmts = (nbytes-type_msg_size)/orig_size; if(dtype_size <= orig_size) - dbuf_size = nbytes-type_msg_size; + dbuf_size = nbytes-info_size; else dbuf_size = nelmts*dtype_size; } else - dbuf_size = nbytes-type_msg_size; + dbuf_size = nbytes-info_size; /* Allocate the buffer for the data */ if (NULL==(dst=(unsigned char*)H5MM_malloc(dbuf_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate buffer") /* Copy the raw data */ - outbuf = (unsigned char*)*buf + type_msg_size; - HDmemcpy((void*)dst, (void*)outbuf, nbytes-type_msg_size); + outbuf = (unsigned char*)*buf + info_size; + HDmemcpy((void*)dst, (void*)outbuf, nbytes-info_size); /* Free input buffer */ H5MM_xfree(*buf); @@ -248,35 +1245,31 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val */ if(0 != H5T_cmp(dtype, orig_type, FALSE)) { H5P_genplist_t *plist; /* Property list pointer */ - hbool_t is_vlen = FALSE; /* Flag to indicate VL type */ + htri_t is_vlen = FALSE; /* Flag to indicate VL type */ hbool_t vlen_conv = TRUE; /* Transfer property to indicate no conversion for vlen */ H5T_path_t *tpath=NULL; /*type conversion info */ unsigned char *bkg=NULL; /* Pointer to background buffer */ - hid_t orig_type_id; /* If the datatype is or contains vlen, set the property to indicate no conversion * is needed for vlen. The file to memory conversion will take place at a * higher level. */ - if((is_vlen = H5T_detect_class(orig_type, H5T_VLEN)) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to detect dtatypes") + if((is_vlen = H5T_detect_class(orig_type, H5T_VLEN)) == FAIL) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, 0, "unable to detect dtatypes") if(is_vlen ) { vlen_conv = FALSE; if(NULL == (plist = H5P_object_verify(H5AC_dxpl_id, H5P_DATASET_XFER))) - HGOTO_ERROR(H5E_PLIST, H5E_BADATOM, FAIL, "can't find object for ID") + HGOTO_ERROR(H5E_PLIST, H5E_BADATOM, 0, "can't find object for ID") if(H5P_set(plist, H5D_XFER_VLEN_CONV_NAME, &vlen_conv) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "Error setting vlen conv flag") + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, 0, "Error setting vlen conv flag") } /* Find the conversion function */ - if (NULL==(tpath=H5T_path_find(orig_type, dtype, NULL, NULL, /*H5P_DEFAULT*/H5AC_dxpl_id, FALSE))) + if (NULL==(tpath=H5T_path_find(orig_type, dtype, NULL, NULL, H5AC_dxpl_id, FALSE))) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, 0, "unable to convert between src and dst data types"); /* Register the original type and return the ID */ - /*if((orig_type_copy = H5T_copy(orig_type,H5T_COPY_ALL)) == NULL) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, 0, "unable to copy data type")*/ - if((orig_type_id = H5I_register(H5I_DATATYPE, orig_type)) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, 0, "unable to register data type") @@ -286,7 +1279,7 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val } /* Convert the data */ - if (H5T_convert(tpath, orig_type_id, dtype_id, nelmts, (size_t)0, (size_t)0, dst, bkg, /*H5P_DATASET_XFER_DEFAULT*/H5AC_dxpl_id)<0) + if (H5T_convert(tpath, orig_type_id, dtype_id, nelmts, (size_t)0, (size_t)0, dst, bkg, H5AC_dxpl_id)<0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, 0, "data type conversion failed"); /* Set the property of vlen conversion back to normal */ @@ -294,12 +1287,9 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val vlen_conv = TRUE; if(H5P_set(plist, H5D_XFER_VLEN_CONV_NAME, &vlen_conv) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "Error setting vlen conv flag") + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, 0, "Error setting vlen conv flag") } - if(H5Tclose(orig_type_id)<0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCLOSEOBJ, 0, "unable to close data type ID") - if(bkg) H5MM_xfree(bkg); } @@ -308,41 +1298,95 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val if(0 != H5T_cmp(dtype, orig_type, FALSE)) *buf_size = nelmts*dtype_size; else - *buf_size = nbytes - type_msg_size; + *buf_size = nbytes - info_size; *buf = (void*)dst; dst = NULL; outbuf = NULL; ret_value = *buf_size; - } else { /* Write */ - /* Mark the new datatype as being on disk now */ - if(H5T_set_loc(dtype, file, H5T_LOC_DISK) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, 0, "invalid datatype location") + /* If the caller isn't from fractal heap (the flag H5Z_FLAG_SKIP_CRECORD isn't + * set) and the current chunk isn't in the skip list, put the current datatype + * for this chunk in. When this datatype is being changed during the write + * call, the filter knows to decrement the reference count of the current type. + * The key of the node is the linear offset of the chunk in the dataspace. This + * linear offset is translated from the multi-dimensional offsets. + */ + if(!(flags & H5Z_FLAG_SKIP_CRECORD) && chunk_slist) { + /* Allocate space for the shared structure */ + if(NULL == (node = H5FL_MALLOC(H5Z_slist_node_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed") + node->key = chunk_offset; + + if((NULL == H5SL_search(chunk_slist, &(node->key))) && H5T_committed(orig_type)) { + if ((NULL==(node->dtype=H5T_copy(orig_type, H5T_COPY_ALL)))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, 0, "unable to copy data type"); + + /* Check whether datatype is committed & increment ref count + * (to maintain ref. count incr/decr similarity with "shared message" + * type of datatype sharing) + */ + /* Increment the reference count on the committed datatype */ + if(H5T_link(node->dtype, 1, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, 0, "unable to adjust shared datatype link count") + + /* set the flag for the later increment and decrement of reference count */ + if(0 != H5T_cmp(dtype, orig_type, FALSE)) + node->type_modified = TRUE; + else + node->type_modified = FALSE; + + if(H5SL_insert(chunk_slist, node, &(node->key)) < 0) + HGOTO_ERROR(H5E_SLIST, H5E_CANTINSERT, 0, "can't insert chunk node into skip list") + } + } + + /* Close the ID registered for doing data conversion */ + if(0 != H5T_cmp(dtype, orig_type, FALSE)) { + if(H5Tclose(orig_type_id)<0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCLOSEOBJ, 0, "unable to close data type ID") + } + } else if (!(flags & H5Z_FLAG_INVMASK) || (flags & H5Z_FLAG_SKIP_CRECORD)) { /*Write*/ + uint8_t version = H5Z_DTYPE_MODIFY_VERS; + size_t final_size = 0; + htri_t is_shared_int; + herr_t found = FAIL; + H5Z_mdtype_bt2_rec_t udata; + hsize_t bnode_numb = 0; + + /* Set the flag for committed or shared datatype */ + if(H5T_committed(dtype)) + is_shared = TRUE; /* Check if the datatype should be (or are already) shared in the SOHM table */ - if(H5SM_try_share(file, H5AC_dxpl_id, NULL, H5O_DTYPE_ID, dtype, NULL) < 0) + if((is_shared_int = H5SM_try_share(file, H5AC_dxpl_id, NULL, H5O_DTYPE_ID, dtype, NULL)) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_BADMESG, 0, "trying to share datatype failed") - - /* Check whether datatype is committed & increment ref count - * (to maintain ref. count incr/decr similarity with "shared message" - * type of datatype sharing) - */ - if(H5T_committed(dtype)) { - /* Increment the reference count on the shared datatype */ - if(H5T_link(dtype, 1, H5AC_dxpl_id) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, 0, "unable to adjust shared datatype link count") - } /* end if */ + else if(is_shared_int) + is_shared = TRUE; /* Get the encoding size of the dataset's datatype */ if((type_msg_size = H5O_msg_raw_size(file, H5O_DTYPE_ID, FALSE, dtype))==0) HGOTO_ERROR(H5E_DATATYPE, H5E_BADSIZE, 0, "can't find datatype size") + /* Size of final chunk */ + final_size = 1 /* Version number */ + + 1 /* Whether dtype is committed or shared */ + + type_msg_size /* Datatype message size */ + + nbytes; /* Raw data size */ + /* Allocate the buffer for the datatype info and the data */ - if (NULL==(dst=outbuf=(unsigned char*)H5MM_malloc(type_msg_size+nbytes))) + if (NULL==(dst=outbuf=(unsigned char*)H5MM_malloc(final_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate buffer") - /* prepend datatype info to raw data for storage */ + /* Encode version number */ + HDmemcpy(dst, &version, sizeof(uint8_t)); + dst += 1; + + /* Encode the flag indicating whether the dtype is shared in Btree node buffer */ + HDmemcpy(dst, &is_shared, sizeof(int8_t)); + dst += 1; + + /* prepend datatype info to raw data for storage as shared */ if(H5O_msg_encode(file, H5O_DTYPE_ID, FALSE, dst, dtype)<0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTENCODE, 0, "can't encode object") @@ -354,15 +1398,129 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val H5MM_xfree(*buf); /* Set return values */ - *buf_size = type_msg_size + nbytes; + *buf_size = final_size; *buf = (void*)outbuf; outbuf = NULL; ret_value = *buf_size; + + /* Adjust reference count for the old and new datatypes. */ + if(!(flags & H5Z_FLAG_SKIP_CRECORD) && chunk_slist) { + /* Decrement the reference count of the old datatype and increment + * the reference count of the new datatype if the chunk is in skip list. */ + if((NULL != (node = H5SL_search(chunk_slist, &chunk_offset))) + && H5T_committed(node->dtype) && node->type_modified) { + + udata.version = H5Z_DTYPE_MODIFY_VERS; + udata.is_shared = is_shared; + udata.dtype = node->dtype; + udata.numb_ref = 1; + udata.found = FALSE; + udata.adjust = H5Z_MDTYPE_DEC; + + /* Find out the number of nodes in the Btree */ + if(H5B2_get_nrec(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, &bnode_numb) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, 0, "can't get number of nodes in Btree") + + /* Find the old datatype and decrement the number of references to the type */ + if(bnode_numb) + H5B2_find(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, &udata, + H5Z_btree2_dtype_find_cb, &udata); + + /* If found the old type and no other chunk uses it, delete the node from the + * Btree */ + if(udata.found && !udata.numb_ref) { + /*delete the node*/ + if(H5B2_remove(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, &udata, + H5Z_btree2_dtype_remove_cb, NULL) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, 0, "can't remove node in Btree") + + bnode_numb--; + } + + /* Decrement the reference count of the old datatype in the skipped list. + * 1 for being prepended to the chunk. 1 is for the copy in the skip list.*/ + if(H5T_link(node->dtype, -2, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, 0, "unable to adjust shared datatype link count") + + if(H5T_close(node->dtype) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, 0, "can't close data type") + + /* Update the dtype in the skip list to the current dtype */ + if ((NULL==(node->dtype=H5T_copy(dtype, H5T_COPY_ALL)))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, 0, "unable to copy data type"); + + /* Increment the reference count on the new datatype. 1 is encoded with + * the chunk. 1 is the copy in the skip list */ + if(H5T_committed(node->dtype) && H5T_link(node->dtype, 2, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, 0, "unable to adjust shared datatype link count") + + node->type_modified = FALSE; + + /* Update the node in the skip list? */ + + /* If the dtype info isn't in Btree, insert it and its reference number + * into Btree */ + udata.dtype = dtype; + udata.numb_ref = 1; + udata.found = FALSE; + udata.adjust = H5Z_MDTYPE_INC; + + /* Find the new datatype in the Btree and increment the number of references + * to the type */ + if(bnode_numb) + H5B2_find(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, &udata, + H5Z_btree2_dtype_find_cb, &udata); + + if(udata.found == FALSE && H5B2_insert(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, + baddr, &udata) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINSERT, 0, "can't insert B-tree") + } else if(!node && H5T_committed(dtype)) { + /* Insert the chunk in the skip list. Increment type reference count. */ + /* Allocate space for the node for the skip list */ + if(NULL == (node = H5FL_MALLOC(H5Z_slist_node_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed") + node->key = chunk_offset; + node->type_modified = FALSE; + + if ((NULL==(node->dtype=H5T_copy(dtype, H5T_COPY_ALL)))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, 0, "unable to copy data type"); + + /* Increment the reference count on the new datatype. 1 is encoded with + * the chunk. 1 is the copy in the skip list */ + if(H5T_link(dtype, 2, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, 0, "unable to adjust shared datatype link count") + + /* Insert into the skip list */ + if(H5SL_insert(chunk_slist, node, &(node->key)) < 0) + HGOTO_ERROR(H5E_SLIST, H5E_CANTINSERT, 0, "can't insert chunk node into skip list") + + udata.version = H5Z_DTYPE_MODIFY_VERS; + udata.is_shared = is_shared; + udata.dtype = dtype; + udata.numb_ref = 1; + udata.found = FALSE; + udata.adjust = H5Z_MDTYPE_INC; + + /* Find out the number of nodes in the Btree */ + if(H5B2_get_nrec(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, &bnode_numb) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, 0, "can't get number of nodes in Btree") + + /* Find the datatype and increment the number of references to the type */ + if(bnode_numb) + H5B2_find(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, &udata, + H5Z_btree2_dtype_find_cb, &udata); + /*HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, 0, "can't locate dtype in Btree")*/ + + /* If the dtype info isn't in Btree, insert it and its reference number + * into Btree */ + if(udata.found == FALSE && H5B2_insert(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, + baddr, &udata) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINSERT, 0, "can't insert B-tree") + } + } } done: - /*if(outbuf) - H5MM_xfree(outbuf);*/ FUNC_LEAVE_NOAPI(ret_value) } diff --git a/src/H5Zfletcher32.c b/src/H5Zfletcher32.c index 2434537..a520cea 100644 --- a/src/H5Zfletcher32.c +++ b/src/H5Zfletcher32.c @@ -30,21 +30,40 @@ #ifdef H5_HAVE_FILTER_FLETCHER32 /* Local function prototypes */ -static size_t H5Z_filter_fletcher32 (unsigned flags, size_t cd_nelmts, +#ifndef H5_USE_16_API +static size_t H5Z_filter_fletcher32 (unsigned flags, hsize_t UNUSED chunk_offset, + size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class_t H5Z_FLETCHER32[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_FLETCHER32, /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "fletcher32", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ + H5Z_filter_fletcher32, /* The actual filter function */ +}}; +#else +static size_t H5Z_filter_fletcher32 (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ const H5Z_class_t H5Z_FLETCHER32[1] = {{ H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_FLETCHER32, /* Filter id number */ - 1, /* encoder_present flag (set to true) */ - 1, /* decoder_present flag (set to true) */ "fletcher32", /* Filter name for debugging */ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ - NULL, /* The "reset local" callback */ H5Z_filter_fletcher32, /* The actual filter function */ }}; +#endif #define FLETCHER_LEN 4 @@ -74,9 +93,15 @@ const H5Z_class_t H5Z_FLETCHER32[1] = {{ *------------------------------------------------------------------------- */ /* ARGSUSED */ +#ifndef H5_USE_16_API +static size_t +H5Z_filter_fletcher32 (unsigned flags, hsize_t UNUSED chunk_offset, size_t UNUSED cd_nelmts, + const unsigned UNUSED cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#else static size_t -H5Z_filter_fletcher32 (unsigned flags, size_t UNUSED cd_nelmts, const unsigned UNUSED cd_values[], - size_t nbytes, size_t *buf_size, void **buf) +H5Z_filter_fletcher32 (unsigned flags, size_t UNUSED cd_nelmts, + const unsigned UNUSED cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#endif { void *outbuf = NULL; /* Pointer to new buffer */ unsigned char *src = (unsigned char*)(*buf); @@ -84,7 +109,7 @@ H5Z_filter_fletcher32 (unsigned flags, size_t UNUSED cd_nelmts, const unsigned U uint32_t reversed_fletcher; /* Possible wrong checksum value */ uint8_t c[4]; uint8_t tmp; - size_t ret_value; /* Return value */ + size_t ret_value = nbytes; /* Return value */ FUNC_ENTER_NOAPI(H5Z_filter_fletcher32, 0) @@ -134,7 +159,7 @@ H5Z_filter_fletcher32 (unsigned flags, size_t UNUSED cd_nelmts, const unsigned U /* Set return values */ /* (Re-use the input buffer, just note that the size is smaller by the size of the checksum) */ ret_value = nbytes-FLETCHER_LEN; - } else { /* Write */ + } else if (!(flags & H5Z_FLAG_INVMASK)){ /* Write */ unsigned char *dst; /* Temporary pointer to destination buffer */ /* Compute checksum (can't fail) */ diff --git a/src/H5Znbit.c b/src/H5Znbit.c index 9827a9d..fe2054c 100644 --- a/src/H5Znbit.c +++ b/src/H5Znbit.c @@ -37,14 +37,6 @@ typedef struct { int offset; /* datatype offset */ } parms_atomic; -/* Local function prototypes */ -static herr_t H5Z_can_apply_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id, - hid_t UNUSED file_id); -static herr_t H5Z_set_local_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id, - hid_t UNUSED file_id); -static size_t H5Z_filter_nbit(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], - size_t nbytes, size_t *buf_size, void **buf); - static void H5Z_calc_parms_nooptype(void); static void H5Z_calc_parms_atomic(void); static herr_t H5Z_calc_parms_array(const H5T_t *type); @@ -81,16 +73,28 @@ static void H5Z_nbit_compress_one_compound(unsigned char *data, size_t data_offs static void H5Z_nbit_compress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer, size_t *buffer_size, const unsigned parms[]); +/* Local function prototypes */ +static herr_t H5Z_can_apply_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t UNUSED file_id); +static herr_t H5Z_set_local_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t UNUSED file_id); +static size_t H5Z_filter_nbit(unsigned flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + /* This message derives from H5Z */ H5Z_class_t H5Z_NBIT[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_NBIT, /* Filter id number */ - 1, /* Assume encoder present: check before registering */ - 1, /* decoder_present flag (set to true) */ - "nbit", /* Filter name for debugging */ + 1, /* Assume encoder present: check before registering */ + 1, /* decoder_present flag (set to true) */ + "nbit", /* Filter name for debugging */ H5Z_can_apply_nbit, /* The "can apply" callback */ H5Z_set_local_nbit, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ H5Z_filter_nbit, /* The actual filter function */ }}; @@ -871,10 +875,10 @@ done: *------------------------------------------------------------------------- */ static size_t -H5Z_filter_nbit(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], - size_t nbytes, size_t *buf_size, void **buf) +H5Z_filter_nbit(unsigned flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) { - size_t ret_value = 0; /* return value */ + size_t ret_value = nbytes; /* return value */ size_t size_out = 0; /* size of output buffer */ unsigned d_nelmts = 0; /* number of elements in the chunk */ unsigned char *outbuf = NULL; /* pointer to new output buffer */ @@ -899,7 +903,7 @@ H5Z_filter_nbit(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], d_nelmts = cd_values[2]; /* input; decompress */ - if (flags & H5Z_FLAG_REVERSE) { + if (flags & H5Z_FLAG_REVERSE) { /*Read*/ size_out = d_nelmts * cd_values[4]; /* cd_values[4] stores datatype size */ /* allocate memory space for decompressed buffer */ @@ -908,9 +912,9 @@ H5Z_filter_nbit(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], /* decompress the buffer */ H5Z_nbit_decompress(outbuf, d_nelmts, *buf, cd_values); - } + /* output; compress */ - else { + } else if (!(flags & H5Z_FLAG_INVMASK)) { /*Write*/ assert(nbytes == d_nelmts * cd_values[4]); size_out = nbytes; diff --git a/src/H5Zprivate.h b/src/H5Zprivate.h index 890125c..fd6327c 100644 --- a/src/H5Zprivate.h +++ b/src/H5Zprivate.h @@ -80,13 +80,17 @@ H5_DLL herr_t H5Z_modify(const struct H5O_pline_t *pline, H5Z_filter_t filter, unsigned flags, size_t cd_nelmts, const unsigned int cd_values[]); H5_DLL herr_t H5Z_pipeline(const struct H5O_pline_t *pline, unsigned flags, unsigned *filter_mask/*in,out*/, - H5Z_EDC_t edc_read, H5Z_cb_t cb_struct, + H5Z_EDC_t edc_read, hsize_t chunk_offset, H5Z_cb_t cb_struct, size_t *nbytes/*in,out*/, size_t *buf_size/*in,out*/, void **buf/*in,out*/); H5_DLL H5Z_class_t *H5Z_find(H5Z_filter_t id); H5_DLL herr_t H5Z_can_apply(hid_t dcpl_id, hid_t type_id, hid_t file_id); H5_DLL herr_t H5Z_set_local(hid_t dcpl_id, hid_t type_id, hid_t file_id); -H5_DLL herr_t H5Z_reset_local(hid_t dcpl_id, hid_t type_id, hid_t file_id); +H5_DLL herr_t H5Z_reset_local(hid_t dcpl_id, hid_t type_id, hid_t file_id, hbool_t from_reopen); +H5_DLL herr_t H5Z_change_local (hid_t dcpl_id, hid_t type_id, hid_t file_id, hsize_t chunk_offset); +H5_DLL herr_t H5Z_evict_local (hid_t dcpl_id, hid_t type_id, hid_t file_id, hsize_t chunk_offset); +H5_DLL herr_t H5Z_delete_local(hid_t dcpl_id, hid_t type_id, hid_t file_id); +H5_DLL herr_t H5Z_close_local(hid_t dcpl_id, hid_t type_id, hid_t file_id); H5_DLL H5Z_filter_info_t *H5Z_filter_info(const struct H5O_pline_t *pline, H5Z_filter_t filter); H5_DLL htri_t H5Z_all_filters_avail(const struct H5O_pline_t *pline); diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index 7ef9e98..c0ee263 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -64,6 +64,8 @@ typedef int H5Z_filter_t; #define H5Z_FLAG_INVMASK 0xff00 /*invocation flag mask */ #define H5Z_FLAG_REVERSE 0x0100 /*reverse direction; read */ #define H5Z_FLAG_SKIP_EDC 0x0200 /*skip EDC filters for read */ +#define H5Z_FLAG_SKIP_CRECORD 0x0400 /*skip record keeping for chunks. Only for the + *filter of modifying dset's dtype*/ /* Special parameters for szip compression */ /* [These are aliases for the similar definitions in szlib.h, which we can't @@ -84,7 +86,7 @@ typedef enum H5Z_SO_scale_type_t { } H5Z_SO_scale_type_t; /* Current version of the H5Z_class_t struct */ -#define H5Z_CLASS_T_VERS (1) +#define H5Z_CLASS_T_VERS (2) /* Values to decide if EDC is enabled for reading data */ typedef enum H5Z_EDC_t { @@ -140,7 +142,7 @@ extern "C" { * The "can_apply" callback returns positive a valid combination, zero for an * invalid combination and negative for an error. */ -typedef herr_t (*H5Z_can_apply_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id, +typedef herr_t (*H5Z_can_apply_func2_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t file_id); /* @@ -170,9 +172,15 @@ typedef herr_t (*H5Z_can_apply_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space * again. The function prototype of "reset_local" callback is identical to * the "set_local". */ -typedef herr_t (*H5Z_set_local_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id, +typedef herr_t (*H5Z_set_local_func2_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t file_id); +typedef herr_t (*H5Z_reset_local_func2_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t file_id, hbool_t from_reopen); + +typedef herr_t (*H5Z_change_local_func_t)(hid_t dcpl_id, hsize_t chunk_offset); +typedef herr_t (*H5Z_close_local_func_t)(hid_t dcpl_id); + /* * A filter gets definition flags and invocation flags (defined above), the * client data array and size defined when the filter was added to the @@ -188,7 +196,7 @@ typedef herr_t (*H5Z_set_local_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space * buffer. If an error occurs then the function should return zero and leave * all pointer arguments unchanged. */ -typedef size_t (*H5Z_func_t)(unsigned int flags, size_t cd_nelmts, +typedef size_t (*H5Z_func2_t)(unsigned int flags, hsize_t chunk_offset, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf); @@ -196,17 +204,45 @@ typedef size_t (*H5Z_func_t)(unsigned int flags, size_t cd_nelmts, * The filter table maps filter identification numbers to structs that * contain a pointers to the filter function and timing statistics. */ -typedef struct H5Z_class_t { +typedef struct H5Z_class2_t { int version; /* Version number of the H5Z_class_t struct */ H5Z_filter_t id; /* Filter ID number */ unsigned encoder_present; /* Does this filter have an encoder? */ unsigned decoder_present; /* Does this filter have a decoder? */ const char *name; /* Comment for debugging */ - H5Z_can_apply_func_t can_apply; /* The "can apply" callback for a filter */ - H5Z_set_local_func_t set_local; /* The "set local" callback for a filter */ - H5Z_set_local_func_t reset_local; /* The "reset local" callback for a filter */ + H5Z_can_apply_func2_t can_apply; /* The "can apply" callback for a filter */ + H5Z_set_local_func2_t set_local; /* The "set local" callback for a filter */ + H5Z_reset_local_func2_t reset_local; /* The "reset local" callback for a filter */ + H5Z_change_local_func_t change_local; /* The "change local" callback for a filter */ + H5Z_change_local_func_t evict_local; /* The "evict local" callback for a filter */ + H5Z_close_local_func_t delete_local; /* The "delete local" callback for a filter */ + H5Z_close_local_func_t close_local; /* The "close local" callback for a filter */ + H5Z_func2_t filter; /* The actual filter function */ +} H5Z_class2_t; + +/* Symbols defined for compatibility with previous versions of the HDF5 API. + * + * Use of these symbols is deprecated. + */ +#ifndef H5_NO_DEPRECATED_SYMBOLS + +/* Typedefs */ +typedef herr_t (*H5Z_can_apply_func1_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id); +typedef herr_t (*H5Z_set_local_func1_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id); +typedef size_t (*H5Z_func1_t)(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf); + +typedef struct H5Z_class1_t { + int version; /* Version number of the H5Z_class_t struct */ + H5Z_filter_t id; /* Filter ID number */ + const char *name; /* Comment for debugging */ + H5Z_can_apply_func1_t can_apply; /* The "can apply" callback for a filter */ + H5Z_set_local_func1_t set_local; /* The "set local" callback for a filter */ H5Z_func_t filter; /* The actual filter function */ -} H5Z_class_t; +} H5Z_class1_t; + +#endif /* H5_NO_DEPRECATED_SYMBOLS */ H5_DLL herr_t H5Zregister(const H5Z_class_t *cls); H5_DLL herr_t H5Zunregister(H5Z_filter_t id); @@ -217,4 +253,3 @@ H5_DLL herr_t H5Zget_filter_info(H5Z_filter_t filter, unsigned int *filter_confi } #endif #endif - diff --git a/src/H5Zscaleoffset.c b/src/H5Zscaleoffset.c index 12e9aee..b3d62f5 100644 --- a/src/H5Zscaleoffset.c +++ b/src/H5Zscaleoffset.c @@ -40,17 +40,11 @@ enum H5Z_scaleoffset_type {t_bad=0, t_uchar=1, t_ushort, t_uint, t_ulong, t_ulon /* Local function prototypes */ static double H5Z_scaleoffset_rnd(double val); -static herr_t H5Z_can_apply_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id, - hid_t UNUSED file_id); static enum H5Z_scaleoffset_type H5Z_scaleoffset_get_type(unsigned dtype_class, unsigned dtype_size, unsigned dtype_sign); static herr_t H5Z_scaleoffset_set_parms_fillval(H5P_genplist_t *dcpl_plist, const H5T_t *type, enum H5Z_scaleoffset_type scale_type, unsigned cd_values[], int need_convert, hid_t dxpl_id); -static herr_t H5Z_set_local_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id, - hid_t UNUSED file_id); -static size_t H5Z_filter_scaleoffset(unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); static void H5Z_scaleoffset_convert(void *buf, unsigned d_nelmts, size_t dtype_size); static unsigned H5Z_scaleoffset_log2(unsigned long_long num); static void H5Z_scaleoffset_precompress_i(void *data, unsigned d_nelmts, @@ -81,16 +75,27 @@ static void H5Z_scaleoffset_decompress(unsigned char *data, unsigned d_nelmts, static void H5Z_scaleoffset_compress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer, size_t buffer_size, parms_atomic p); +static herr_t H5Z_can_apply_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t UNUSED file_id); +static herr_t H5Z_set_local_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t UNUSED file_id); +static size_t H5Z_filter_scaleoffset(unsigned flags, hsize_t UNUSED chunk_offset, + size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + /* This message derives from H5Z */ H5Z_class_t H5Z_SCALEOFFSET[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ - H5Z_FILTER_SCALEOFFSET, /* Filter id number */ - 1, /* Assume encoder present: check before registering */ - 1, /* decoder_present flag (set to true) */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_SCALEOFFSET, /* Filter id number */ + 1, /* Assume encoder present: check before registering */ + 1, /* decoder_present flag (set to true) */ "scaleoffset", /* Filter name for debugging */ H5Z_can_apply_scaleoffset, /* The "can apply" callback */ H5Z_set_local_scaleoffset, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ H5Z_filter_scaleoffset, /* The actual filter function */ }}; @@ -911,8 +916,8 @@ done: *------------------------------------------------------------------------- */ static size_t -H5Z_filter_scaleoffset (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], - size_t nbytes, size_t *buf_size, void **buf) +H5Z_filter_scaleoffset (unsigned flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) { size_t ret_value = 0; /* return value */ size_t size_out = 0; /* size of output buffer */ @@ -1008,7 +1013,7 @@ H5Z_filter_scaleoffset (unsigned flags, size_t cd_nelmts, const unsigned cd_valu p.mem_order = H5T_native_order_g; /* input; decompress */ - if (flags & H5Z_FLAG_REVERSE) { + if (flags & H5Z_FLAG_REVERSE) { /*Read*/ /* retrieve values of minbits and minval from input compressed buffer * retrieve them corresponding to how they are stored during compression */ @@ -1090,7 +1095,7 @@ H5Z_filter_scaleoffset (unsigned flags, size_t cd_nelmts, const unsigned cd_valu H5Z_scaleoffset_convert(outbuf, d_nelmts, p.size); } /* output; compress */ - else { + else if (!(flags & H5Z_FLAG_INVMASK)) { /*Write*/ assert(nbytes == d_nelmts * p.size); /* before preprocess, convert to memory endianness order if needed */ diff --git a/src/H5Zshuffle.c b/src/H5Zshuffle.c index cc0bb5a..d808014 100644 --- a/src/H5Zshuffle.c +++ b/src/H5Zshuffle.c @@ -27,22 +27,44 @@ #ifdef H5_HAVE_FILTER_SHUFFLE /* Local function prototypes */ +#ifndef H5_USE_16_API +static herr_t H5Z_set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t UNUSED file_id); +static size_t H5Z_filter_shuffle(unsigned flags, hsize_t UNUSED chunk_offset, + size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, + void **buf); + +/* This message derives from H5Z */ +const H5Z_class_t H5Z_SHUFFLE[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_SHUFFLE, /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "shuffle", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + H5Z_set_local_shuffle, /* The "set local" callback */ + NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ + H5Z_filter_shuffle, /* The actual filter function */ +}}; +#else static herr_t H5Z_set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t space_id); -static size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, +static size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ const H5Z_class_t H5Z_SHUFFLE[1] = {{ H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_SHUFFLE, /* Filter id number */ - 1, /* encoder_present flag (set to true) */ - 1, /* decoder_present flag (set to true) */ "shuffle", /* Filter name for debugging */ NULL, /* The "can apply" callback */ H5Z_set_local_shuffle, /* The "set local" callback */ - NULL, /* The "reset local" callback */ H5Z_filter_shuffle, /* The actual filter function */ }}; +#endif /* Local macros */ #define H5Z_SHUFFLE_USER_NPARMS 0 /* Number of parameters that users can set */ @@ -67,8 +89,13 @@ const H5Z_class_t H5Z_SHUFFLE[1] = {{ *------------------------------------------------------------------------- */ /* ARGSUSED */ +#ifndef H5_USE_16_API +static herr_t H5Z_set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t UNUSED file_id) +#else static herr_t H5Z_set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t UNUSED space_id) +#endif { H5P_genplist_t *dcpl_plist; /* Property list pointer */ const H5T_t *type; /* Datatype */ @@ -126,9 +153,15 @@ done: * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API +static size_t H5Z_filter_shuffle(unsigned flags, hsize_t UNUSED chunk_offset, + size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, + void **buf) +#else static size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#endif { void *dest = NULL; /* Buffer to deposit [un]shuffled bytes into */ unsigned char *_src=NULL; /* Alias for source buffer */ @@ -140,7 +173,7 @@ H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t j; /* Local index variable */ #endif /* NO_DUFFS_DEVICE */ size_t leftover; /* Extra bytes at end of buffer */ - size_t ret_value; /* Return value */ + size_t ret_value=nbytes; /* Return value */ FUNC_ENTER_NOAPI(H5Z_filter_shuffle, 0) @@ -163,7 +196,7 @@ H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], if (NULL==(dest = H5MM_malloc(nbytes))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for shuffle buffer") - if(flags & H5Z_FLAG_REVERSE) { + if(flags & H5Z_FLAG_REVERSE) { /*Read*/ /* Get the pointer to the source buffer */ _src =(unsigned char *)(*buf); @@ -218,7 +251,7 @@ H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], HDmemcpy((void*)_dest, (void*)_src, leftover); } } /* end if */ - else { + else if (!(flags & H5Z_FLAG_INVMASK)) { /*Write*/ /* Get the pointer to the destination buffer */ _dest =(unsigned char *)dest; diff --git a/src/H5Zszip.c b/src/H5Zszip.c index 191d7e7..4263537 100644 --- a/src/H5Zszip.c +++ b/src/H5Zszip.c @@ -33,12 +33,13 @@ # include "szlib.h" #endif +#ifndef H5_USE_16_API /* Local function prototypes */ static herr_t H5Z_can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t UNUSED file_id); static herr_t H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t UNUSED file_id); -static size_t H5Z_filter_szip (unsigned flags, size_t cd_nelmts, +static size_t H5Z_filter_szip (unsigned flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ @@ -51,8 +52,29 @@ H5Z_class_t H5Z_SZIP[1] = {{ H5Z_can_apply_szip, /* The "can apply" callback */ H5Z_set_local_szip, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ H5Z_filter_szip, /* The actual filter function */ }}; +#else +/* Local function prototypes */ +static herr_t H5Z_can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static herr_t H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static size_t H5Z_filter_szip (unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +H5Z_class_t H5Z_SZIP[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_SZIP, /* Filter id number */ + "szip", /* Filter name for debugging */ + H5Z_can_apply_szip, /* The "can apply" callback */ + H5Z_set_local_szip, /* The "set local" callback */ + H5Z_filter_szip, /* The actual filter function */ +}}; +#endif /* Local macros */ #define H5Z_SZIP_USER_NPARMS 2 /* Number of parameters that users can set */ @@ -86,9 +108,14 @@ H5Z_class_t H5Z_SZIP[1] = {{ * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API static herr_t H5Z_can_apply_szip(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id, hid_t UNUSED file_id) +#else +static herr_t +H5Z_can_apply_szip(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id) +#endif { const H5T_t *type; /* Datatype */ unsigned dtype_size; /* Datatype's size (in bits) */ @@ -144,8 +171,13 @@ done: * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API static herr_t H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t UNUSED file_id) +#else +static herr_t +H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id) +#endif { H5P_genplist_t *dcpl_plist; /* Property list pointer */ const H5T_t *type; /* Datatype */ @@ -285,9 +317,15 @@ done: * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API +static size_t +H5Z_filter_szip (unsigned flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#else static size_t -H5Z_filter_szip (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], +H5Z_filter_szip (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#endif { size_t ret_value = 0; /* Return value */ size_t size_out = 0; /* Size of output buffer */ @@ -318,7 +356,7 @@ H5Z_filter_szip (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], H5_ASSIGN_OVERFLOW(sz_param.pixels_per_scanline,cd_values[H5Z_SZIP_PARM_PPS],unsigned,int); /* Input; uncompress */ - if (flags & H5Z_FLAG_REVERSE) { + if (flags & H5Z_FLAG_REVERSE) { /*Read*/ uint32_t stored_nalloc; /* Number of bytes the compressed block will expand into */ size_t nalloc; /* Number of bytes the compressed block will expand into */ @@ -347,7 +385,7 @@ H5Z_filter_szip (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], ret_value = nalloc; } /* Output; compress */ - else { + else if (!(flags & H5Z_FLAG_INVMASK)) { /*Write*/ unsigned char *dst = NULL; /* Temporary pointer to new output buffer */ /* Allocate space for the compressed buffer & header (assume data won't get bigger) */ @@ -381,4 +419,3 @@ done: } #endif /* H5_HAVE_FILTER_SZIP */ - diff --git a/src/H5vers.txt b/src/H5vers.txt index 00ab84b..8cd03cb 100644 --- a/src/H5vers.txt +++ b/src/H5vers.txt @@ -71,4 +71,7 @@ FUNCTION: H5Topen; ; v10, v18 # (although not required, it's easier to compare this file with the headers # generated if the list below is in alphanumeric sort order - QAK) TYPEDEF: H5E_auto; v10, v18 - +TYPEDEF: H5Z_func; v10, v18 +TYPEDEF: H5Z_can_apply_func; v10, v18 +TYPEDEF: H5Z_set_local_func; v10, v18 +TYPEDEF: H5Z_class; v10, v18 diff --git a/src/H5version.h b/src/H5version.h index eab972c..1c9d8ff 100644 --- a/src/H5version.h +++ b/src/H5version.h @@ -130,6 +130,22 @@ #define H5E_auto_t_vers 1 #endif /* !defined(H5E_auto_t_vers) */ +#if !defined(H5Z_can_apply_func_t_vers) +#define H5Z_can_apply_func_t_vers 1 +#endif /* !defined(H5Z_can_apply_func_t_vers) */ + +#if !defined(H5Z_class_t_vers) +#define H5Z_class_t_vers 1 +#endif /* !defined(H5Z_class_t_vers) */ + +#if !defined(H5Z_func_t_vers) +#define H5Z_func_t_vers 1 +#endif /* !defined(H5Z_func_t_vers) */ + +#if !defined(H5Z_set_local_func_t_vers) +#define H5Z_set_local_func_t_vers 1 +#endif /* !defined(H5Z_set_local_func_t_vers) */ + #endif /* H5_USE_16_API */ @@ -395,5 +411,53 @@ #error "H5E_auto_t_vers set to invalid value" #endif /* H5E_auto_t_vers */ + +#if !defined(H5Z_can_apply_func_t_vers) || H5Z_can_apply_func_t_vers == 2 +#ifndef H5Z_can_apply_func_t_vers +#define H5Z_can_apply_func_t_vers 2 +#endif /* H5Z_can_apply_func_t_vers */ +#define H5Z_can_apply_func_t H5Z_can_apply_func2_t +#elif H5Z_can_apply_func_t_vers == 1 +#define H5Z_can_apply_func_t H5Z_can_apply_func1_t +#else /* H5Z_can_apply_func_t_vers */ +#error "H5Z_can_apply_func_t_vers set to invalid value" +#endif /* H5Z_can_apply_func_t_vers */ + + +#if !defined(H5Z_class_t_vers) || H5Z_class_t_vers == 2 +#ifndef H5Z_class_t_vers +#define H5Z_class_t_vers 2 +#endif /* H5Z_class_t_vers */ +#define H5Z_class_t H5Z_class2_t +#elif H5Z_class_t_vers == 1 +#define H5Z_class_t H5Z_class1_t +#else /* H5Z_class_t_vers */ +#error "H5Z_class_t_vers set to invalid value" +#endif /* H5Z_class_t_vers */ + + +#if !defined(H5Z_func_t_vers) || H5Z_func_t_vers == 2 +#ifndef H5Z_func_t_vers +#define H5Z_func_t_vers 2 +#endif /* H5Z_func_t_vers */ +#define H5Z_func_t H5Z_func2_t +#elif H5Z_func_t_vers == 1 +#define H5Z_func_t H5Z_func1_t +#else /* H5Z_func_t_vers */ +#error "H5Z_func_t_vers set to invalid value" +#endif /* H5Z_func_t_vers */ + + +#if !defined(H5Z_set_local_func_t_vers) || H5Z_set_local_func_t_vers == 2 +#ifndef H5Z_set_local_func_t_vers +#define H5Z_set_local_func_t_vers 2 +#endif /* H5Z_set_local_func_t_vers */ +#define H5Z_set_local_func_t H5Z_set_local_func2_t +#elif H5Z_set_local_func_t_vers == 1 +#define H5Z_set_local_func_t H5Z_set_local_func1_t +#else /* H5Z_set_local_func_t_vers */ +#error "H5Z_set_local_func_t_vers set to invalid value" +#endif /* H5Z_set_local_func_t_vers */ + #endif /* H5version_H */ diff --git a/test/change_dtypes.c b/test/change_dtypes.c index 0897de9..67adb64 100644 --- a/test/change_dtypes.c +++ b/test/change_dtypes.c @@ -20,6 +20,14 @@ #include "h5test.h" +#ifdef H5_USE_16_API +int main(void) +{ + printf("Test skipped because backward compatbility with v1.6 is configured in\n"); + return 0; +} +#else /* H5_USE_16_API */ + #define STR_LEN 16 #define NX 10u #define NY 20u @@ -43,6 +51,7 @@ const char *DSET_NAME[] = { "enum", "array", "reference", + "btree_node", NULL }; @@ -128,6 +137,84 @@ error: /*------------------------------------------------------------------------- + * Function: test_btree_node + * + * Purpose: Creates a simple dataset of an integer type and tries to + * change the datatype to a bigger size. Test whether the + * B-tree node works properly in the filter for modifying + * dataset's datatype. The B-tree is for storing the number + * of datatype reference by the data chunks. + * + * Return: Success: 0 + * + * Failure: 1 + * + * Programmer: Raymond Lu + * 19 November 2007 + * + *------------------------------------------------------------------------- + */ +static int +test_btree_node(hid_t file) +{ + hid_t dset; + hid_t space; + hid_t dcpl; + hsize_t dims[2] = {NX, NY}; + hsize_t chunk_dims[2] = {NX/2, NY/2}; + int *wbuf = NULL, *ptr = NULL; + unsigned i; + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + TEST_ERROR; + + if((space=H5Screate_simple(2, dims, NULL)) < 0) + TEST_ERROR; + + if(H5Pset_dtype_modifiable(dcpl)<0) + TEST_ERROR; + + /* Set chunking */ + if(H5Pset_chunk(dcpl, 2, chunk_dims)<0) + TEST_ERROR; + + if((dset=H5Dcreate2(file, DSET_NAME[10], H5T_STD_U32BE, space, H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + TEST_ERROR; + + wbuf = (int *)malloc(sizeof(int)*NX*NY); + ptr = wbuf; + for(i = 0; i<NX*NY; i++) { + *ptr = i; + ptr++; + } + + if(H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) + TEST_ERROR; + + if(H5Dmodify_dtype(dset, H5T_STD_U64LE) < 0) + TEST_ERROR; + + if(H5Dclose(dset) < 0) + TEST_ERROR; + + free(wbuf); + + PASSED(); + return 0; + +error: + H5_FAILED(); + H5E_BEGIN_TRY { + H5Sclose(space); + H5Dclose(dset); + } H5E_END_TRY; + return 1; +} + + +/*------------------------------------------------------------------------- * Function: test_float * * Purpose: Creates a simple dataset of a float type and tries to @@ -805,6 +892,9 @@ int main(void) TESTING("dataset of reference type:"); nerrors += test_reference(fid); + TESTING("size of B-tree node"); + nerrors += test_btree_node(fid); + if(H5Fclose(fid) < 0) nerrors++; @@ -819,3 +909,4 @@ int main(void) return 0; } +#endif /* H5_USE_16_API */ diff --git a/test/cmpd_dset.c b/test/cmpd_dset.c index 7a7f8fa..759549a 100644 --- a/test/cmpd_dset.c +++ b/test/cmpd_dset.c @@ -124,9 +124,16 @@ typedef struct { hvl_t r; long_long s, t, u; } stype4; +typedef struct { + int a, b, c[8], d, e; + float f, g, h[16], i, j; + double k, l, m, n; + long o, p, q; + hvl_t r; + long_long s, t, u; + char v, w; +} stype5; -/*#define NX 10u -#define NY 20u*/ #define NX 100u #define NY 200u #define COMP_NX 8u @@ -140,15 +147,14 @@ const unsigned test_type_flags[H5O_SHMESG_MAX_NINDEXES] = H5O_MESG_SDSPACE_FLAG, H5O_MESG_PLINE_FLAG, 0, 0}; -#ifndef TMP + const unsigned test_minsizes[H5O_SHMESG_MAX_NINDEXES] = {0, 2, 40, 100, 3, 1000}; #define TEST_L2B 65 #define TEST_B2L 64 -#else -const unsigned test_minsizes[H5O_SHMESG_MAX_NINDEXES] = {250,250,250,250,250,250}; -#define TEST_L2B 50 -#define TEST_B2L 40 -#endif + +#define COMMITTED_DTYPE 0 +#define UNCOMMITTED_DTYPE 1 +#define MIXED_DTYPE 2 /*------------------------------------------------------------------------- @@ -237,7 +243,6 @@ test_compound (char *filename, hid_t fapl) /* Create xfer properties to preserve initialized data */ if ((PRESERVE = H5Pcreate (H5P_DATASET_XFER)) < 0) goto error; - if (H5Pset_preserve (PRESERVE, 1) < 0) goto error; /* *###################################################################### @@ -897,17 +902,17 @@ initialize_stype1(void *buf, const size_t num) s_ptr->d = i*8+6; s_ptr->e = i*8+7; - s_ptr->f = i*2/3; - s_ptr->g = i*2/3+1; + s_ptr->f = (float)i*2/3; + s_ptr->g = (float)i*2/3+1; for(j=0; j<16; j++) - s_ptr->h[j] = i*j/5+j; - s_ptr->i = i*2/3+2; - s_ptr->j = i*2/3+3; - - s_ptr->k = i/7+1; - s_ptr->l = i/7+2; - s_ptr->m = i/7+3; - s_ptr->n = i/7+4; + s_ptr->h[j] = (float)i*j/5+j; + s_ptr->i = (float)i*2/3+2; + s_ptr->j = (float)i*2/3+3; + + s_ptr->k = (double)i/7+1; + s_ptr->l = (double)i/7+2; + s_ptr->m = (double)i/7+3; + s_ptr->n = (double)i/7+4; } } @@ -940,17 +945,17 @@ initialize_stype2(void *buf, const size_t num) s_ptr->d = i*8+6; s_ptr->e = i*8+7; - s_ptr->f = i*2/3; - s_ptr->g = i*2/3+1; + s_ptr->f = (float)i*2/3; + s_ptr->g = (float)i*2/3+1; for(j=0; j<16; j++) - s_ptr->h[j] = i*j/5+j; - s_ptr->i = i*2/3+2; - s_ptr->j = i*2/3+3; + s_ptr->h[j] = (float)i*j/5+j; + s_ptr->i = (float)i*2/3+2; + s_ptr->j = (float)i*2/3+3; - s_ptr->k = i/7+1; - s_ptr->l = i/7+2; - s_ptr->m = i/7+3; - s_ptr->n = i/7+4; + s_ptr->k = (double)i/7+1; + s_ptr->l = (double)i/7+2; + s_ptr->m = (double)i/7+3; + s_ptr->n = (double)i/7+4; s_ptr->o = i*3+0; s_ptr->p = i*3+1; @@ -1009,7 +1014,7 @@ initialize_stype3(void *buf, const size_t num) *------------------------------------------------------------------------- */ static void -initialize_stype4(void *buf, const size_t num) +initialize_stype4(void *buf, const size_t num, int addition) { size_t i, j; stype4 *s_ptr; @@ -1023,17 +1028,17 @@ initialize_stype4(void *buf, const size_t num) s_ptr->d = i*8+6; s_ptr->e = i*8+7; - s_ptr->f = i*2/3; - s_ptr->g = i*2/3+1; + s_ptr->f = (float)i*2/3; + s_ptr->g = (float)i*2/3+1; for(j=0; j<16; j++) - s_ptr->h[j] = i*j/5+j; - s_ptr->i = i*2/3+2; - s_ptr->j = i*2/3+3; + s_ptr->h[j] = (float)i*j/5+j; + s_ptr->i = (float)i*2/3+2; + s_ptr->j = (float)i*2/3+3; - s_ptr->k = i/7+1; - s_ptr->l = i/7+2; - s_ptr->m = i/7+3; - s_ptr->n = i/7+4; + s_ptr->k = (double)i/7+1; + s_ptr->l = (double)i/7+2; + s_ptr->m = (double)i/7+3; + s_ptr->n = (double)i/7+4; s_ptr->o = i*3+0; s_ptr->p = i*3+1; @@ -1044,9 +1049,9 @@ initialize_stype4(void *buf, const size_t num) for(j=0; j<4; j++) ((unsigned int *)s_ptr->r.p)[j]=i*10+j; - s_ptr->s = i*5+1; - s_ptr->t = i*5+2; - s_ptr->u = i*5+3; + s_ptr->s = i*5+1+addition; + s_ptr->t = i*5+2+addition; + s_ptr->u = i*5+3+addition; } } @@ -1282,6 +1287,75 @@ error: /*------------------------------------------------------------------------- + * Function: create_stype5 + * + * Purpose: Create HDF5 compound datatype for stype5. + * + * Return: Success: datatype ID + * + * Failure: negative + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static hid_t +create_stype5(void) +{ + hid_t array_dt1, array_dt2, vl_tid, tid; + const hsize_t eight = 8, sixteen = 16; + + /* Build hdf5 datatypes */ + if((array_dt1 = H5Tarray_create2(H5T_NATIVE_INT,1, &eight)) < 0) + goto error; + if((array_dt2 = H5Tarray_create2(H5T_NATIVE_FLOAT,1, &sixteen)) < 0) + goto error; + if((vl_tid = H5Tvlen_create (H5T_NATIVE_UINT)) < 0) + goto error; + + if((tid = H5Tcreate(H5T_COMPOUND, sizeof(stype5))) < 0 || + H5Tinsert(tid, "a", HOFFSET(stype5, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(tid, "b", HOFFSET(stype5, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(tid, "c", HOFFSET(stype5, c), array_dt1) < 0 || + H5Tinsert(tid, "d", HOFFSET(stype5, d), H5T_NATIVE_INT) < 0 || + H5Tinsert(tid, "e", HOFFSET(stype5, e), H5T_NATIVE_INT) < 0 || + H5Tinsert(tid, "f", HOFFSET(stype5, f), H5T_NATIVE_FLOAT) < 0 || + H5Tinsert(tid, "g", HOFFSET(stype5, g), H5T_NATIVE_FLOAT) < 0 || + H5Tinsert(tid, "h", HOFFSET(stype5, h), array_dt2) < 0 || + H5Tinsert(tid, "i", HOFFSET(stype5, i), H5T_NATIVE_FLOAT) < 0 || + H5Tinsert(tid, "j", HOFFSET(stype5, j), H5T_NATIVE_FLOAT) < 0 || + H5Tinsert(tid, "k", HOFFSET(stype5, k), H5T_NATIVE_DOUBLE) < 0 || + H5Tinsert(tid, "l", HOFFSET(stype5, l), H5T_NATIVE_DOUBLE) < 0 || + H5Tinsert(tid, "m", HOFFSET(stype5, m), H5T_NATIVE_DOUBLE) < 0 || + H5Tinsert(tid, "n", HOFFSET(stype5, n), H5T_NATIVE_DOUBLE) < 0 || + H5Tinsert(tid, "o", HOFFSET(stype5, o), H5T_NATIVE_LONG) < 0 || + H5Tinsert(tid, "p", HOFFSET(stype5, p), H5T_NATIVE_LONG) < 0 || + H5Tinsert(tid, "q", HOFFSET(stype5, q), H5T_NATIVE_LONG) < 0 || + H5Tinsert(tid, "r", HOFFSET(stype5, r), vl_tid) < 0 || + H5Tinsert(tid, "s", HOFFSET(stype5, s), H5T_NATIVE_LLONG) < 0 || + H5Tinsert(tid, "t", HOFFSET(stype5, t), H5T_NATIVE_LLONG) < 0 || + H5Tinsert(tid, "u", HOFFSET(stype5, u), H5T_NATIVE_LLONG) < 0 || + H5Tinsert(tid, "v", HOFFSET(stype5, v), H5T_NATIVE_CHAR) < 0 || + H5Tinsert(tid, "w", HOFFSET(stype5, w), H5T_NATIVE_CHAR) < 0) + goto error; + + if(H5Tclose(array_dt1) < 0) + goto error; + if(H5Tclose(array_dt2) < 0) + goto error; + if(H5Tclose(vl_tid) < 0) + goto error; + + return tid; + +error: + return FAIL; +} + + +/*------------------------------------------------------------------------- * Function: compare_data * * Purpose: Compare data of stype1 and stype2. @@ -1332,6 +1406,7 @@ compare_data(void *src_data, void *dst_data, hbool_t src_subset) !DBL_ABS_EQUAL(s_ptr->n, d_ptr->n) ) { H5_FAILED(); + printf(" i=%d\n", i); printf(" src={a=%d, b=%d, c=[%d,%d,%d,%d,%d,%d,%d,%d], d=%d, e=%d, f=%f, g=%f, h=[%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f], i=%f, j=%f, k=%f, l=%f, m=%f, n=%f}\n", s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2], @@ -1348,6 +1423,7 @@ compare_data(void *src_data, void *dst_data, hbool_t src_subset) d_ptr->h[9],d_ptr->h[10],d_ptr->h[11],d_ptr->h[12],d_ptr->h[13], d_ptr->h[14], d_ptr->h[15], d_ptr->i,d_ptr->j,d_ptr->k,d_ptr->l, d_ptr->m,d_ptr->n); + goto error; } } @@ -1547,6 +1623,102 @@ error: /*------------------------------------------------------------------------- + * Function: compare_stype4_stype5 + * + * Purpose: Compare data between stype4 and stype5. + * + * Return: Success: 0 + * + * Failure: negative + * + * Programmer: Raymond Lu + * Friday, 14 September 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static int +compare_stype4_stype5(void *src_data, void *dst_data, size_t nelmts) +{ + stype4 *s_ptr; + stype5 *d_ptr; + size_t i, j; + + for(i = 0; i < nelmts; i++) { + s_ptr = ((stype4*)src_data) + i; + d_ptr = ((stype5*)dst_data) + i; + + if (s_ptr->a != d_ptr->a || + s_ptr->b != d_ptr->b || + s_ptr->c[0] != d_ptr->c[0] || + s_ptr->c[1] != d_ptr->c[1] || + s_ptr->c[2] != d_ptr->c[2] || + s_ptr->c[3] != d_ptr->c[3] || + s_ptr->d != d_ptr->d || + s_ptr->e != d_ptr->e || + !FLT_ABS_EQUAL(s_ptr->f, d_ptr->f) || + !FLT_ABS_EQUAL(s_ptr->g, d_ptr->g) || + !FLT_ABS_EQUAL(s_ptr->h[0], d_ptr->h[0]) || + !FLT_ABS_EQUAL(s_ptr->h[1], d_ptr->h[1]) || + !FLT_ABS_EQUAL(s_ptr->i, d_ptr->i) || + !FLT_ABS_EQUAL(s_ptr->j, d_ptr->j) || + !DBL_ABS_EQUAL(s_ptr->k, d_ptr->k) || + !DBL_ABS_EQUAL(s_ptr->l, d_ptr->l) || + !DBL_ABS_EQUAL(s_ptr->m, d_ptr->m) || + !DBL_ABS_EQUAL(s_ptr->n, d_ptr->n) || + s_ptr->o != d_ptr->o || + s_ptr->p != d_ptr->p || + s_ptr->q != d_ptr->q || + s_ptr->s != d_ptr->s || + s_ptr->t != d_ptr->t || + s_ptr->u != d_ptr->u ) { + + H5_FAILED(); + printf(" i=%d\n", i); + printf(" src={a=%d, b=%d, c=[%d,%d,%d,%d,%d,%d,%d,%d], d=%d, e=%d, f=%f, g=%f, h=[%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f], i=%f, j=%f, k=%f, l=%f, m=%f, n=%f, o=%d, p=%d, q=%d, s=%ld, t=%ld, u=%ld}\n", + s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2], + s_ptr->c[3], s_ptr->c[4], s_ptr->c[5], s_ptr->c[6], s_ptr->c[7], + s_ptr->d, s_ptr->e, s_ptr->f, s_ptr->g,s_ptr->h[0],s_ptr->h[1],s_ptr->h[2], + s_ptr->h[3],s_ptr->h[4],s_ptr->h[5],s_ptr->h[6],s_ptr->h[7],s_ptr->h[8], + s_ptr->h[9],s_ptr->h[10],s_ptr->h[11],s_ptr->h[12],s_ptr->h[13],s_ptr->h[14], + s_ptr->h[15], s_ptr->i,s_ptr->j,s_ptr->k,s_ptr->l,s_ptr->m,s_ptr->n, + s_ptr->o, s_ptr->p, s_ptr->q, s_ptr->s, s_ptr->t, s_ptr->u); + printf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d,%d,%d,%d,%d], d=%d, e=%d, f=%f, g=%f, h=[%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f], i=%f, j=%f, k=%f, l=%f, m=%f, n=%f, o=%d, p=%d, q=%d, s=%ld, t=%ld, u=%ld}\n", + d_ptr->a, d_ptr->b, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2], + d_ptr->c[3], d_ptr->c[4], d_ptr->c[5], d_ptr->c[6], d_ptr->c[7], + d_ptr->d, d_ptr->e, d_ptr->f, d_ptr->g,d_ptr->h[0],d_ptr->h[1],d_ptr->h[2], + d_ptr->h[3],d_ptr->h[4],d_ptr->h[5],d_ptr->h[6],d_ptr->h[7],d_ptr->h[8], + d_ptr->h[9],d_ptr->h[10],d_ptr->h[11],d_ptr->h[12],d_ptr->h[13], + d_ptr->h[14], d_ptr->h[15], d_ptr->i,d_ptr->j,d_ptr->k,d_ptr->l, + d_ptr->m,d_ptr->n, d_ptr->o, d_ptr->p, d_ptr->q, d_ptr->s, d_ptr->t, + d_ptr->u); + goto error; + } + + if(s_ptr->r.len!=d_ptr->r.len) { + H5_FAILED(); + printf(" i=%d\n", i); + printf("VL data lengths don't match!, src len=%d, dst len=%d\n",(int)s_ptr->r.len,(int)d_ptr->r.len); + goto error; + } /* end if */ + for(j=0; j<s_ptr->r.len; j++) { + if( ((unsigned int *)s_ptr->r.p)[j] != ((unsigned int *)d_ptr->r.p)[j] ) { + H5_FAILED(); + printf(" i=%d\n", i); + printf("VL data values don't match!, src r.p[%d]=%u, dst r.p[%d]=%u\n",(int)j, ((unsigned int *)s_ptr->r.p)[j], (int)j, ((unsigned int *)d_ptr->r.p)[j]); + goto error; + } + } + } + + return SUCCEED; + +error: + return FAIL; +} + + +/*------------------------------------------------------------------------- * Function: test_hdf5_src_subset * * Purpose: Test the optimization of compound data writing, rewriting, @@ -1659,9 +1831,6 @@ test_hdf5_src_subset(char *filename, hid_t fapl) if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) goto error; - if(H5Pset_preserve(dxpl, TRUE) < 0) - goto error; - /* Rewrite contiguous data set */ if((dataset = H5Dopen2(file, DSET_NAME[0], H5P_DEFAULT)) < 0) goto error; @@ -1813,7 +1982,7 @@ test_hdf5_dst_subset(char *filename, hid_t fapl) rbuf = (void*)malloc(NX * NY * sizeof(stype1)); rew_buf = (void*)malloc(NX * NY * sizeof(stype4)); - initialize_stype4(rew_buf, (size_t)NX*NY); + initialize_stype4(rew_buf, (size_t)NX*NY, 0); /* Create dataset creation property list */ if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) @@ -1863,9 +2032,6 @@ test_hdf5_dst_subset(char *filename, hid_t fapl) if((dxpl = H5Pcreate (H5P_DATASET_XFER)) < 0) goto error; - if(H5Pset_preserve(dxpl, TRUE) < 0) - goto error; - /* Rewrite contiguous data set */ if((dataset = H5Dopen2(file, DSET_NAME[2], H5P_DEFAULT)) < 0) goto error; @@ -1973,23 +2139,23 @@ error: */ #ifdef H5_HAVE_FILTER_DTYPE_MODIFY static int -test_hdf5_change_dtype_chunked(char *filename, hid_t fapl) +test_hdf5_change_dtype_chunked(char *filename, hid_t fapl, int dtype_flag) { hid_t file; - hid_t rew_tid, src_tid, dst_tid; + hid_t rew_tid, rew2_tid, src_tid, dst_tid; hid_t dataset; hid_t space, space2, mspace; hid_t fcpl, dcpl, dxpl; + H5O_info_t type_info; hsize_t dims[2] = {NX, NY}; hsize_t chunk_dims[2] = {NX/10, NY/10}; hsize_t mem_dims[2] = {NX/2, NY}; - void *orig=NULL, *rew_buf=NULL, *rbuf=NULL; + void *orig=NULL, *rew_buf=NULL, *rbuf=NULL, *rbuf2=NULL; void *rew_half_buf=NULL, *new_rbuf_half=NULL, *orig_rbuf_half=NULL; hsize_t hs_offset[2] = {0,0}; /* Hyperslab offset */ const hsize_t hs_size[2] = {NX/2, NY}; /* Hyperslab size */ const hsize_t hs_stride[2] = {1,1}; /* Hyperslab stride */ const hsize_t hs_count[2] = {1,1}; /* Hyperslab count */ - unsigned num_indexes; unsigned x; if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) @@ -2022,6 +2188,22 @@ test_hdf5_change_dtype_chunked(char *filename, hid_t fapl) if ((rew_tid=create_stype4())<0) goto error; + if ((rew2_tid=create_stype5())<0) + goto error; + + /* Determine whether to test committed or uncomitted data types */ + if((COMMITTED_DTYPE == dtype_flag || MIXED_DTYPE == dtype_flag) && H5Tcommit2(file, + "src type", src_tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) + goto error; + + if(COMMITTED_DTYPE == dtype_flag && H5Tcommit2(file, "rewrite type", rew_tid, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) + goto error; + + if((COMMITTED_DTYPE == dtype_flag || MIXED_DTYPE == dtype_flag) && H5Tcommit2(file, + "second rewrite type", rew2_tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) + goto error; + /* Create the data space */ if((space = H5Screate_simple(2, dims, NULL))<0) goto error; @@ -2034,15 +2216,16 @@ test_hdf5_change_dtype_chunked(char *filename, hid_t fapl) initialize_stype2(orig, (size_t)NX*NY); rbuf = (void*)malloc(NX * NY * sizeof(stype1)); + rbuf2 = (void*)malloc(NX * NY * sizeof(stype5)); orig_rbuf_half = (void*)malloc(NX/2 * NY * sizeof(stype2)); new_rbuf_half = (void*)malloc(NX/2 * NY * sizeof(stype4)); rew_half_buf = (void*)malloc((NX/2) * NY * sizeof(stype4)); - initialize_stype4(rew_half_buf, (size_t)(NX/2)*NY); + initialize_stype4(rew_half_buf, (size_t)(NX/2)*NY, 2); rew_buf = (void*)malloc(NX * NY * sizeof(stype4)); - initialize_stype4(rew_buf, (size_t)NX*NY); + initialize_stype4(rew_buf, (size_t)NX*NY, 1); /* Create dataset creation property list */ if((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) @@ -2158,13 +2341,9 @@ test_hdf5_change_dtype_chunked(char *filename, hid_t fapl) /* *###################################################################### - * STEP 4. Rewrite the whole data. + * STEP 4. Rewrite the whole data with the new datatype. */ - TESTING("rewriting data with a subset of original data type"); - - /* Create xfer properties to preserve initialized data */ - if ((dxpl = H5Pcreate (H5P_DATASET_XFER))<0) - goto error; + TESTING("rewriting data with the new data type"); /* Rewrite chunked data set */ if((dataset = H5Dopen2(file, DSET_NAME[3], H5P_DEFAULT))<0) @@ -2198,6 +2377,115 @@ test_hdf5_change_dtype_chunked(char *filename, hid_t fapl) if(H5Dclose(dataset) < 0) goto error; + PASSED(); + + /* + *###################################################################### + * STEP 6. Change datatype after rewrite the whole data. The data in + * the cache should be converted and flushed to the file when the + * dataset is closed. + */ + TESTING("change datatype after rewriting the data"); + + /* Rewrite chunked data set */ + if((dataset = H5Dopen2(file, DSET_NAME[3], H5P_DEFAULT))<0) + goto error; + + /* Modify the data buffer a little */ + initialize_stype4(rew_buf, (size_t)NX*NY, 3); + + /* Write the data to the dataset */ + if(H5Dwrite(dataset, rew_tid, H5S_ALL, H5S_ALL, dxpl, rew_buf)<0) + goto error; + + /* Modify the datatype of the chunked dataset now. It should only change + * the data of the chunks in the cache, not the data in the file. */ + if(H5Dmodify_dtype(dataset, rew2_tid)<0) + goto error; + +#ifdef TMP + /* There seems to be some problem with this operation - temporarily disable it */ + /* Read the data back with rew2_tid (stype5) and compare it with rew_buf + * of rew_tid (stype4). */ + if(H5Dread(dataset, rew2_tid, H5S_ALL, H5S_ALL, dxpl, rbuf2)<0) + goto error; + + if(compare_stype4_stype5(rew_buf, rbuf2, (size_t)NX*NY) < 0) + goto error; +#endif + + if(H5Dclose(dataset) < 0) + goto error; + + /* The reference count for the original dtype should be only 1, for + * it's a committed type and saved in the file, and no other reference. */ + if(H5Tcommitted(src_tid)) { + if(H5Oget_info(src_tid, &type_info) < 0) + goto error; + + if(type_info.rc != 1) + goto error; + } + + /* The reference count for the changed dtype should be 100, for it's + * a committed type, and there're 100 chunks with one of them being + * changed to the new dtype, and the dataset's dtype has been changed, + * too. */ + if(H5Tcommitted(rew_tid)) { + if(H5Oget_info(rew_tid, &type_info) < 0) + goto error; + + if(type_info.rc != 100) + goto error; + } + + /* The reference count for the second changed dtype should be 3, for + * it's a committed type, and one chunk has been converted in the cached + * and flushed to the file, and the dataset's dtype has been changed, + * too. */ + if(H5Tcommitted(rew2_tid)) { + if(H5Oget_info(rew2_tid, &type_info) < 0) + goto error; + + if(type_info.rc != 3) + goto error; + } + + PASSED(); + + /* + *###################################################################### + * STEP 7. Delete the dataset and verify the reference count of the + * data types. + */ + TESTING("delete the dataset"); + + /* delete the data set */ + if(H5Ldelete(file, DSET_NAME[3], H5P_DEFAULT)<0) + goto error; + + /* The reference count for the changed dtype should be 1, for it's + * a committed type, and the dataset refered to it has been deleted. + */ + if(H5Tcommitted(rew_tid)) { + if(H5Oget_info(rew_tid, &type_info) < 0) + goto error; + + if(type_info.rc != 1) + goto error; + } + + /* The reference count for the second changed dtype should be 1, for + * it's a committed type, and the dataset refered to it has been deleted. + */ + if(H5Tcommitted(rew2_tid)) { + if(H5Oget_info(rew2_tid, &type_info) < 0) + goto error; + + if(type_info.rc != 1) + goto error; + } + /* Finishing test and release resources */ if(H5Sclose(space) < 0) goto error; @@ -2217,6 +2505,9 @@ test_hdf5_change_dtype_chunked(char *filename, hid_t fapl) goto error; if(H5Tclose(rew_tid)<0) goto error; + if(H5Tclose(rew2_tid)<0) + goto error; + if(H5Fclose(file) < 0) goto error; @@ -2264,7 +2555,7 @@ static int test_hdf5_change_dtype_no_filter(char *filename, hid_t fapl, htri_t is_contig) { hid_t file; - hid_t rew_tid, src_tid, dst_tid; + hid_t rew_tid, src_tid; hid_t dataset; hid_t space; hid_t dcpl, dxpl; @@ -2272,8 +2563,6 @@ test_hdf5_change_dtype_no_filter(char *filename, hid_t fapl, htri_t is_contig) hsize_t chunk_dims[2] = {NX/10, NY/10}; /*hsize_t chunk_dims[2] = {NX, NY};*/ void *orig=NULL, *rew_buf=NULL, *rbuf1=NULL, *rbuf2=NULL; - unsigned num_indexes; - unsigned x; /* Create the file for this test */ if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) @@ -2297,7 +2586,7 @@ test_hdf5_change_dtype_no_filter(char *filename, hid_t fapl, htri_t is_contig) rbuf1 = (void*)malloc(NX * NY * sizeof(stype2)); rew_buf = (void*)malloc(NX * NY * sizeof(stype4)); - initialize_stype4(rew_buf, (size_t)NX*NY); + initialize_stype4(rew_buf, (size_t)NX*NY, 0); rbuf2 = (void*)malloc(NX * NY * sizeof(stype4)); @@ -2461,14 +2750,12 @@ static int test_hdf5_change_dtype_compact(char *filename, hid_t fapl) { hid_t file; - hid_t rew_tid, src_tid, dst_tid; + hid_t rew_tid, src_tid; hid_t dataset; hid_t space; hid_t dcpl, dxpl; hsize_t dims[2] = {COMP_NX, COMP_NY}; void *orig=NULL, *rew_buf=NULL, *rbuf1=NULL, *rbuf2=NULL; - unsigned num_indexes; - unsigned x; /* Create the file for this test */ if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) @@ -2492,7 +2779,7 @@ test_hdf5_change_dtype_compact(char *filename, hid_t fapl) rbuf1 = (void*)malloc(COMP_NX * COMP_NY * sizeof(stype2)); rew_buf = (void*)malloc(COMP_NX * COMP_NY * sizeof(stype4)); - initialize_stype4(rew_buf, (size_t)COMP_NX*COMP_NY); + initialize_stype4(rew_buf, (size_t)COMP_NX*COMP_NY, 0); rbuf2 = (void*)malloc(COMP_NX * COMP_NY * sizeof(stype4)); @@ -2694,21 +2981,31 @@ main (int argc, char *argv[]) h5_fixname(FILENAME[2], fapl_id, fname, sizeof(fname)); nerrors += test_hdf5_dst_subset(fname, fapl_id); - puts("Testing the filter for modifying datatype of chunked dataset:"); +#ifndef H5_USE_16_API + puts("Testing the filter for modifying datatype of chunked dataset with uncommitted dtype:"); h5_fixname(FILENAME[3], fapl_id, fname, sizeof(fname)); - nerrors += test_hdf5_change_dtype_chunked(fname, fapl_id); + nerrors += test_hdf5_change_dtype_chunked(fname, fapl_id, UNCOMMITTED_DTYPE); - puts("Testing modifying datatype of chunked dataset with no filter:"); + puts("Testing the filter for modifying datatype of chunked dataset with committed and uncommitted dtype:"); h5_fixname(FILENAME[4], fapl_id, fname, sizeof(fname)); + nerrors += test_hdf5_change_dtype_chunked(fname, fapl_id, MIXED_DTYPE); + + puts("Testing the filter for modifying datatype of chunked dataset with committed dtype:"); + h5_fixname(FILENAME[5], fapl_id, fname, sizeof(fname)); + nerrors += test_hdf5_change_dtype_chunked(fname, fapl_id, COMMITTED_DTYPE); + + puts("Testing modifying datatype of chunked dataset with no filter:"); + h5_fixname(FILENAME[6], fapl_id, fname, sizeof(fname)); nerrors += test_hdf5_change_dtype_no_filter(fname, fapl_id, FALSE); puts("Testing modifying datatype of contiguous dataset:"); - h5_fixname(FILENAME[5], fapl_id, fname, sizeof(fname)); + h5_fixname(FILENAME[7], fapl_id, fname, sizeof(fname)); nerrors += test_hdf5_change_dtype_no_filter(fname, fapl_id, TRUE); puts("Testing modifying datatype of compact dataset:"); - h5_fixname(FILENAME[6], fapl_id, fname, sizeof(fname)); + h5_fixname(FILENAME[8], fapl_id, fname, sizeof(fname)); nerrors += test_hdf5_change_dtype_compact(fname, fapl_id); +#endif if (nerrors) { printf("***** %u FAILURE%s! *****\n", diff --git a/test/dsets.c b/test/dsets.c index d5949eb..464cfad 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -160,17 +160,28 @@ const char *FILENAME[] = { int points[DSET_DIM1][DSET_DIM2], check[DSET_DIM1][DSET_DIM2]; double points_dbl[DSET_DIM1][DSET_DIM2], check_dbl[DSET_DIM1][DSET_DIM2]; +#ifndef H5_USE_16_API /* Local prototypes for filter functions */ -static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, +static size_t filter_bogus(unsigned int flags, hsize_t chunk_offset, size_t cd_nelmts, const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); static herr_t can_apply_bogus(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t file_id); static herr_t set_local_bogus2(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t file_id); +static size_t filter_bogus2(unsigned int flags, hsize_t chunk_offset, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +static size_t filter_corrupt(unsigned int flags, hsize_t chunk_offset, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +#else +static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +static herr_t can_apply_bogus(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static herr_t set_local_bogus2(hid_t dcpl_id, hid_t type_id, hid_t space_id); static size_t filter_bogus2(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); static size_t filter_corrupt(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +#endif /*------------------------------------------------------------------------- @@ -991,16 +1002,31 @@ test_tconv(hid_t file) } /* This message derives from H5Z */ +#ifndef H5_USE_16_API const H5Z_class_t H5Z_BOGUS[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_BOGUS, /* Filter id number */ - 1, 1, /* Encoding and decoding enabled */ + 1, 1, /* Encoding and decoding enabled*/ "bogus", /* Filter name for debugging */ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ + filter_bogus, /* The actual filter function */ +}}; +#else +const H5Z_class_t H5Z_BOGUS[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_BOGUS, /* Filter id number */ + "bogus", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ filter_bogus, /* The actual filter function */ }}; +#endif /*------------------------------------------------------------------------- @@ -1019,9 +1045,13 @@ const H5Z_class_t H5Z_BOGUS[1] = {{ * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API static herr_t can_apply_bogus(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id, hid_t UNUSED file_id) +#else +can_apply_bogus(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id) +#endif { if(H5Tequal(type_id,H5T_NATIVE_DOUBLE)) return 0; @@ -1046,10 +1076,17 @@ can_apply_bogus(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id, * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API +static size_t +filter_bogus(unsigned int UNUSED flags, hsize_t UNUSED chunk_offset, + size_t UNUSED cd_nelmts, const unsigned int UNUSED *cd_values, + size_t nbytes, size_t UNUSED *buf_size, void UNUSED **buf) +#else static size_t -filter_bogus(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, - const unsigned int UNUSED *cd_values, size_t nbytes, +filter_bogus(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, + const unsigned int UNUSED *cd_values, size_t nbytes, size_t UNUSED *buf_size, void UNUSED **buf) +#endif { return nbytes; } @@ -1072,9 +1109,14 @@ filter_bogus(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API static herr_t set_local_bogus2(hid_t dcpl_id, hid_t type_id, hid_t UNUSED space_id, hid_t UNUSED file_id) +#else +static herr_t +set_local_bogus2(hid_t dcpl_id, hid_t type_id, hid_t UNUSED space_id) +#endif { unsigned add_on=0; /* Value to add to data going through */ unsigned flags; /* Filter flags */ @@ -1126,10 +1168,15 @@ set_local_bogus2(hid_t dcpl_id, hid_t type_id, hid_t UNUSED space_id, * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API +static size_t +filter_bogus2(unsigned int flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf) +#else static size_t filter_bogus2(unsigned int flags, size_t cd_nelmts, - const unsigned int *cd_values, size_t nbytes, - size_t *buf_size, void **buf) + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf) +#endif { /* Check for the correct number of parameters */ if(cd_nelmts!=BOGUS2_ALL_NPARMS) @@ -1168,16 +1215,31 @@ filter_bogus2(unsigned int flags, size_t cd_nelmts, } /* This message derives from H5Z */ +#ifndef H5_USE_16_API const H5Z_class_t H5Z_CORRUPT[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_CORRUPT, /* Filter id number */ - 1, 1, /* Encoding and decoding enabled */ + 1, 1, /* Encoding and decoding enabled*/ "corrupt", /* Filter name for debugging */ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ + filter_corrupt, /* The actual filter function */ +}}; +#else +const H5Z_class_t H5Z_CORRUPT[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_CORRUPT, /* Filter id number */ + "corrupt", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ filter_corrupt, /* The actual filter function */ }}; +#endif /*------------------------------------------------------------------------- @@ -1198,10 +1260,15 @@ const H5Z_class_t H5Z_CORRUPT[1] = {{ * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API +static size_t +filter_corrupt(unsigned int flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf) +#else static size_t filter_corrupt(unsigned int flags, size_t cd_nelmts, - const unsigned int *cd_values, size_t nbytes, - size_t *buf_size, void **buf) + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf) +#endif { size_t ret_value = 0; unsigned char *dst = (unsigned char*)(*buf); @@ -1355,7 +1422,8 @@ test_filter_internal(hid_t fid, const char *name, hid_t dcpl, int if_fletcher32, /* Create the dataset */ if((dataset = H5Dcreate2(fid, name, H5T_NATIVE_INT, sid, H5P_DEFAULT, - dcpl, H5P_DEFAULT)) < 0) goto error; + dcpl, H5P_DEFAULT)) < 0) + goto error; PASSED(); @@ -2018,8 +2086,16 @@ UNUSED data_corrupt[1] = 33; data_corrupt[2] = 27; - /* Temporarily disable this test because the changes in chunk caching conflicts with - * the way this test is conducted. -slu 2007/7/20 */ + /* This is a shaky test because any changes in chunk caching may conflict with + * the way this test is conducted. In Step 4 of test_filter_internal, the + * program tries to modify some data. When it does H5Dwrite, the library will + * overwrite the entire chunks if all data in the chunks are being modified. + * However, if the data being modified doesn't cover the whole chunks or the + * filter for modifying dataset's datatype is enabled, the chunks will be read + * first which will cause the Fletcher32 checksum to fail. (See how the RELAX + * variable is defined in H5D_chunk_write of H5Dio.c and H5D_istore_lock of + * H5Distore.c.) - SLU 2008/3/5 + */ if(H5Zregister (H5Z_CORRUPT) < 0) goto error; if(H5Pset_filter(dc, H5Z_FILTER_CORRUPT, 0, (size_t)3, data_corrupt) < 0) goto error; @@ -2124,6 +2200,7 @@ UNUSED * STEP 5: Test datatype modification by itself. *---------------------------------------------------------- */ +#ifndef H5_USE_16_API #ifdef H5_HAVE_FILTER_DTYPE_MODIFY puts("Testing datatype modification filter"); if((dc = H5Pcreate(H5P_DATASET_CREATE))<0) goto error; @@ -2139,6 +2216,7 @@ UNUSED SKIPPED(); puts(" Datatype modification filter not enabled"); #endif /* H5_HAVE_FILTER_DTYPE_MODIFY */ +#endif /* H5_USE_16_API */ /*---------------------------------------------------------- * STEP 6: Test shuffle + deflate + dtype modification. @@ -2148,11 +2226,17 @@ UNUSED puts("Testing dtype modify+shuffle+deflate filters(dtype modify first)"); if((dc = H5Pcreate(H5P_DATASET_CREATE))<0) goto error; if (H5Pset_chunk (dc, 2, chunk_size)<0) goto error; +#ifndef H5_USE_16_API if (H5Pset_dtype_modifiable (dc)<0) goto error; +#endif /* H5_USE_16_API */ if (H5Pset_shuffle (dc)<0) goto error; if (H5Pset_deflate (dc, 6)<0) goto error; +#ifndef H5_USE_16_API if(test_filter_internal(file,DSET_SHUF_DEF_DTMOD_NAME,dc,ENABLE_FLETCHER32,DATA_NOT_CORRUPTED,CHANGE_DTYPE,&combo_size)<0) goto error; +#else + if(test_filter_internal(file,DSET_SHUF_DEF_DTMOD_NAME,dc,ENABLE_FLETCHER32,DATA_NOT_CORRUPTED,DONT_CHANGE_DTYPE,&combo_size)<0) goto error; +#endif /* H5_USE_16_API */ /* Clean up objects used for this test */ if (H5Pclose (dc)<0) goto error; @@ -4879,16 +4963,31 @@ test_types(hid_t file) } /* This message derives from H5Z */ +#ifndef H5_USE_16_API const H5Z_class_t H5Z_CAN_APPLY_TEST[1] = {{ - H5Z_CLASS_T_VERS, + H5Z_CLASS_T_VERS, H5Z_FILTER_BOGUS3, /* Filter id number */ - 1, 1, + 1, 1, "bogus", /* Filter name for debugging */ can_apply_bogus, /* The "can apply" callback */ NULL, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ filter_bogus, /* The actual filter function */ }}; +#else +const H5Z_class_t H5Z_CAN_APPLY_TEST[1] = {{ + H5Z_CLASS_T_VERS, + H5Z_FILTER_BOGUS3, /* Filter id number */ + "bogus", /* Filter name for debugging */ + can_apply_bogus, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + filter_bogus, /* The actual filter function */ +}}; +#endif /*------------------------------------------------------------------------- @@ -5240,16 +5339,31 @@ error: /* This message derives from H5Z */ +#ifndef H5_USE_16_API const H5Z_class_t H5Z_SET_LOCAL_TEST[1] = {{ - H5Z_CLASS_T_VERS, + H5Z_CLASS_T_VERS, H5Z_FILTER_BOGUS2, /* Filter id number */ - 1, 1, + 1, 1, "bogus2", /* Filter name for debugging */ NULL, /* The "can apply" callback */ set_local_bogus2, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ filter_bogus2, /* The actual filter function */ }}; +#else +const H5Z_class_t H5Z_SET_LOCAL_TEST[1] = {{ + H5Z_CLASS_T_VERS, + H5Z_FILTER_BOGUS2, /* Filter id number */ + "bogus2", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + set_local_bogus2, /* The "set local" callback */ + filter_bogus2, /* The actual filter function */ +}}; +#endif /*------------------------------------------------------------------------- @@ -6580,6 +6694,7 @@ main(void) nerrors += (test_tconv(file) < 0 ? 1 : 0); nerrors += (test_filters(file, my_fapl) < 0 ? 1 : 0); nerrors += (test_onebyte_shuffle(file) < 0 ? 1 : 0); +#ifndef H5_USE_16_API nerrors += (test_nbit_int(file) < 0 ? 1 : 0); nerrors += (test_nbit_float(file) < 0 ? 1 : 0); nerrors += (test_nbit_double(file) < 0 ? 1 : 0); @@ -6593,6 +6708,7 @@ main(void) nerrors += (test_scaleoffset_float_2(file) < 0 ? 1 : 0); nerrors += (test_scaleoffset_double(file) < 0 ? 1 : 0); nerrors += (test_scaleoffset_double_2(file) < 0 ? 1 : 0); +#endif nerrors += (test_multiopen (file) < 0 ? 1 : 0); nerrors += (test_types(file) < 0 ? 1 : 0); nerrors += (test_userblock_offset(my_fapl) < 0 ? 1 : 0); @@ -6635,4 +6751,3 @@ error: nerrors, 1 == nerrors ? "" : "S"); return 1; } - diff --git a/tools/h5dump/h5dumpgentest.c b/tools/h5dump/h5dumpgentest.c index f72fe8f..a994d42 100644 --- a/tools/h5dump/h5dumpgentest.c +++ b/tools/h5dump/h5dumpgentest.c @@ -102,18 +102,18 @@ static int write_dset( hid_t loc_id, int rank, hsize_t *dims, const char *dset_name, hid_t tid, void *buf ); +#define MYFILTER_ID 405 +#ifndef H5_USE_16_API /* a filter operation callback function */ static size_t -myfilter(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, - const unsigned int UNUSED *cd_values, size_t nbytes, - size_t UNUSED *buf_size, void UNUSED **buf); +myfilter(unsigned int UNUSED flags, hsize_t UNUSED chunk_offset, size_t UNUSED cd_nelmts, + const unsigned int UNUSED *cd_values, size_t nbytes, size_t UNUSED *buf_size, + void UNUSED **buf); /* a "set local" callback */ static herr_t set_local_myfilter(hid_t dcpl_id, hid_t tid, hid_t UNUSED sid, hid_t UNUSED file_id); -#define MYFILTER_ID 405 - /* This message derives from H5Z */ const H5Z_class_t H5Z_MYFILTER[1] = {{ H5Z_CLASS_T_VERS, @@ -123,9 +123,33 @@ const H5Z_class_t H5Z_MYFILTER[1] = {{ NULL, /* The "can apply" callback */ set_local_myfilter, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ myfilter, /* The actual filter function */ }}; +#else +/* a filter operation callback function */ +static size_t +myfilter(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, + const unsigned int UNUSED *cd_values, size_t nbytes, size_t UNUSED *buf_size, + void UNUSED **buf); +/* a "set local" callback */ +static herr_t +set_local_myfilter(hid_t dcpl_id, hid_t tid, hid_t UNUSED sid); + +/* This message derives from H5Z */ +const H5Z_class_t H5Z_MYFILTER[1] = {{ + H5Z_CLASS_T_VERS, + MYFILTER_ID, /* Filter id number */ + "myfilter", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + set_local_myfilter, /* The "set local" callback */ + myfilter, /* The actual filter function */ +}}; +#endif /* A UD link traversal function. Shouldn't actually be called. */ static hid_t UD_traverse(UNUSED const char * link_name, UNUSED hid_t cur_group, @@ -4925,6 +4949,51 @@ static void gent_filters(void) } +#ifndef H5_USE_16_API +/*------------------------------------------------------------------------- + * Function: myfilter + * + * Purpose: filter operation callback function; the filter does nothing + * + *------------------------------------------------------------------------- + */ +static size_t +myfilter(unsigned int UNUSED flags, hsize_t chunk_offset, size_t UNUSED cd_nelmts, + const unsigned int UNUSED *cd_values, size_t nbytes, size_t UNUSED *buf_size, + void UNUSED **buf) +{ + return nbytes; +} + + +/*------------------------------------------------------------------------- + * Function: set_local_myfilter + * + * Purpose: filter operation "set local" callback + * + *------------------------------------------------------------------------- + */ + +static herr_t +set_local_myfilter(hid_t dcpl_id, hid_t UNUSED tid, hid_t UNUSED sid, hid_t UNUSED file_id) +{ + unsigned flags; /* Filter flags */ + size_t cd_nelmts = 0; /* Number of filter parameters */ + unsigned cd_values[2] = {5, 6}; /* Filter parameters */ + + /* Get the filter's current parameters */ + if(H5Pget_filter_by_id2(dcpl_id, MYFILTER_ID, &flags, &cd_nelmts, cd_values, 0, NULL, NULL) < 0) + return(FAIL); + + cd_nelmts = 2; + + /* Modify the filter's parameters for this dataset */ + if(H5Pmodify_filter(dcpl_id, MYFILTER_ID, flags, cd_nelmts, cd_values) < 0) + return(FAIL); + + return(SUCCEED); +} +#else /*------------------------------------------------------------------------- * Function: myfilter * @@ -4950,7 +5019,7 @@ myfilter(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, */ static herr_t -set_local_myfilter(hid_t dcpl_id, hid_t UNUSED tid, hid_t UNUSED sid, hid_t UNUSED file_id) +set_local_myfilter(hid_t dcpl_id, hid_t UNUSED tid, hid_t UNUSED sid) { unsigned flags; /* Filter flags */ size_t cd_nelmts = 0; /* Number of filter parameters */ @@ -4969,6 +5038,8 @@ set_local_myfilter(hid_t dcpl_id, hid_t UNUSED tid, hid_t UNUSED sid, hid_t UNUS return(SUCCEED); } +#endif + /*------------------------------------------------------------------------- * Function: gent_fcontents * |