diff options
| author | Raymond Lu <songyulu@hdfgroup.org> | 2008-10-03 14:55:22 (GMT) |
|---|---|---|
| committer | Raymond Lu <songyulu@hdfgroup.org> | 2008-10-03 14:55:22 (GMT) |
| commit | 3e1fd51a8a3dceb8229a0a34c5562f121e3f684d (patch) | |
| tree | 5c0ff6033e9651e9dd6b8b1bc32cb80fe580dc28 | |
| parent | e1632d2284b4ad4da38be88900d197091b81cc89 (diff) | |
| download | hdf5-3e1fd51a8a3dceb8229a0a34c5562f121e3f684d.zip hdf5-3e1fd51a8a3dceb8229a0a34c5562f121e3f684d.tar.gz hdf5-3e1fd51a8a3dceb8229a0a34c5562f121e3f684d.tar.bz2 | |
[svn-r15765] This is the second checkin for the filter of modifying dataset's datatype. There're a lot
of changes since the last checkin. Basically, I put a skipped list in the filter to keep
track of the old (current) datatype for the chunk when the data chunk is read into the
memory. When the datatype is changed, the filter will decrement the reference count for
the old datatype. Another significant change is putting a B-tree in the filter to record
the reference count of some datatype from the data chunks. When the dataset is deleted,
the filter will decrement this many reference count for the datatype all at once to save
time.
This project is wrapped up at this point temporarily because of the flaws in the design.
Quincey will talk to the people in Chicago about redesigning it.
I've tested it on smirom, kagiso, and linew.
| -rw-r--r-- | c++/test/dsets.cpp | 33 | ||||
| -rw-r--r-- | c++/test/tfilter.cpp | 45 | ||||
| -rw-r--r-- | src/H5B2private.h | 1 | ||||
| -rw-r--r-- | src/H5Dint.c | 31 | ||||
| -rw-r--r-- | src/H5Dio.c | 109 | ||||
| -rw-r--r-- | src/H5Distore.c | 283 | ||||
| -rw-r--r-- | src/H5Dpkg.h | 6 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 2 | ||||
| -rw-r--r-- | src/H5HFcache.c | 4 | ||||
| -rw-r--r-- | src/H5HFhuge.c | 4 | ||||
| -rw-r--r-- | src/H5Odtype.c | 2 | ||||
| -rw-r--r-- | src/H5Opline.c | 78 | ||||
| -rw-r--r-- | src/H5Pdcpl.c | 2 | ||||
| -rw-r--r-- | src/H5T.c | 5 | ||||
| -rw-r--r-- | src/H5Tcommit.c | 32 | ||||
| -rw-r--r-- | src/H5Tprivate.h | 1 | ||||
| -rw-r--r-- | src/H5V.c | 8 | ||||
| -rw-r--r-- | src/H5Z.c | 289 | ||||
| -rw-r--r-- | src/H5Zdeflate.c | 49 | ||||
| -rw-r--r-- | src/H5Zdtype_modify.c | 1316 | ||||
| -rw-r--r-- | src/H5Zfletcher32.c | 41 | ||||
| -rw-r--r-- | src/H5Znbit.c | 40 | ||||
| -rw-r--r-- | src/H5Zprivate.h | 8 | ||||
| -rw-r--r-- | src/H5Zpublic.h | 55 | ||||
| -rw-r--r-- | src/H5Zscaleoffset.c | 33 | ||||
| -rw-r--r-- | src/H5Zshuffle.c | 47 | ||||
| -rw-r--r-- | src/H5Zszip.c | 47 | ||||
| -rw-r--r-- | src/H5vers.txt | 5 | ||||
| -rw-r--r-- | src/H5version.h | 64 | ||||
| -rw-r--r-- | test/change_dtypes.c | 91 | ||||
| -rw-r--r-- | test/cmpd_dset.c | 443 | ||||
| -rw-r--r-- | test/dsets.c | 153 | ||||
| -rw-r--r-- | tools/h5dump/h5dumpgentest.c | 83 |
33 files changed, 3064 insertions, 346 deletions
diff --git a/c++/test/dsets.cpp b/c++/test/dsets.cpp index 97ba6ca..c921123 100644 --- a/c++/test/dsets.cpp +++ b/c++/test/dsets.cpp @@ -59,8 +59,13 @@ const H5std_string DSET_BOGUS_NAME ("bogus"); const int H5Z_FILTER_BOGUS = 305; // Local prototypes +#ifndef H5_USE_16_API +static size_t filter_bogus(unsigned int flags, hsize_t chunk_offset, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +#else static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +#endif /*------------------------------------------------------------------------- @@ -352,6 +357,7 @@ test_tconv( H5File& file) } // test_tconv /* This message derives from H5Z */ +#ifndef H5_USE_16_API const H5Z_class_t H5Z_BOGUS[1] = {{ H5Z_CLASS_T_VERS, /* H5Z_class_t version number */ H5Z_FILTER_BOGUS, /* Filter id number */ @@ -359,9 +365,23 @@ const H5Z_class_t H5Z_BOGUS[1] = {{ "bogus", /* Filter name for debugging */ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ - NULL, /* The "reset local" callback */ + NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ + (H5Z_func_t)filter_bogus, /* The actual filter function */ +}}; +#else +const H5Z_class_t H5Z_BOGUS[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version number */ + H5Z_FILTER_BOGUS, /* Filter id number */ + "bogus", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ (H5Z_func_t)filter_bogus, /* The actual filter function */ }}; +#endif /*------------------------------------------------------------------------- * Function: bogus @@ -379,14 +399,19 @@ const H5Z_class_t H5Z_BOGUS[1] = {{ * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API static size_t /*bogus(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, const unsigned int UNUSED cd_values[], size_t nbytes, size_t UNUSED *buf_size, void UNUSED **buf) BMR: removed UNUSED for now until asking Q. or R. to pass compilation*/ -filter_bogus(unsigned int flags, size_t cd_nelmts, - const unsigned int cd_values[], size_t nbytes, - size_t *buf_size, void **buf) +filter_bogus(unsigned int flags, hsize_t chunk_offset, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#else +static size_t +filter_bogus(unsigned int flags, size_t cd_nelmts, const unsigned int cd_values[], + size_t nbytes, size_t *buf_size, void **buf) +#endif { return nbytes; } diff --git a/c++/test/tfilter.cpp b/c++/test/tfilter.cpp index a667f2a..b88e122 100644 --- a/c++/test/tfilter.cpp +++ b/c++/test/tfilter.cpp @@ -57,7 +57,8 @@ static herr_t test_filter_internal(hid_t fid, const char *name, hid_t dcpl, /* Temporary filter IDs used for testing */ #define H5Z_FILTER_BOGUS 305 -static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, +#ifndef H5_USE_16_API +static size_t filter_bogus(unsigned int flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ const H5Z_class_t H5Z_BOGUS[1] = {{ @@ -68,6 +69,46 @@ const H5Z_class_t H5Z_BOGUS[1] = {{ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ + (H5Z_func_t)filter_bogus, /* The actual filter function */ +}}; + +/*------------------------------------------------------------------------- + * Function: filter_bogus + * + * Purpose: A bogus compression method that doesn't do anything. + * + * Return: Success: Data chunk size + * + * Failure: 0 + * + * Programmer: Robb Matzke + * Tuesday, April 21, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static size_t +filter_bogus(unsigned int UNUSED flags, hsize_t UNUSED chunk_offset, size_t UNUSED cd_nelmts, + const unsigned int UNUSED *cd_values, size_t nbytes, size_t UNUSED *buf_size, + void UNUSED **buf) +{ + return nbytes; +} +#else +static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +/* This message derives from H5Z */ +const H5Z_class_t H5Z_BOGUS[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_BOGUS, /* Filter id number */ + "bogus", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ (H5Z_func_t)filter_bogus, /* The actual filter function */ }}; @@ -95,6 +136,8 @@ filter_bogus(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, return nbytes; } +#endif + /*------------------------------------------------------------------------- * Function: test_null_filter * diff --git a/src/H5B2private.h b/src/H5B2private.h index 6d8172c..db2e785 100644 --- a/src/H5B2private.h +++ b/src/H5B2private.h @@ -54,6 +54,7 @@ typedef enum H5B2_subid_t { H5B2_SOHM_INDEX_ID, /* B-tree is an index for shared object header messages */ H5B2_ATTR_DENSE_NAME_ID, /* B-tree is for indexing 'name' field for "dense" attribute storage on objects */ H5B2_ATTR_DENSE_CORDER_ID, /* B-tree is for indexing 'creation order' field for "dense" attribute storage on objects */ + H5B2_MODIFY_TYPE_ID, /* B-tree is for indexing 'datatype' field for filter of modifying datatype */ H5B2_NUM_BTREE_ID /* Number of B-tree IDs (must be last) */ } H5B2_subid_t; diff --git a/src/H5Dint.c b/src/H5Dint.c index 272b00a..6a636db 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -1228,6 +1228,7 @@ H5D_open(const H5G_loc_t *loc, hid_t dxpl_id) HGOTO_ERROR(H5E_DATASET, H5E_CANTINC, NULL, "can't increment object count") } /* end else */ +#ifndef H5_USE_16_API /* Get a file ID for the file */ if((fid = H5F_get_id(dataset->oloc.file)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "unable to get file ID") @@ -1236,11 +1237,11 @@ H5D_open(const H5G_loc_t *loc, hid_t dxpl_id) * Make the "reset local" filter callbacks for this dataset. It's mainly for some filters * like datatype modification which need some info after the dataset is reopened. * If the library has been closed and reopened, the user-defined filters aren't - * registered in the library anymore. This function for those filters can't be + * registered in the library anymore. This function is for those filters can't be * found and may fail. Therefore ignore the error if it's this case. */ H5E_BEGIN_TRY { - H5Z_reset_local(dataset->shared->dcpl_id, dataset->shared->type_id, fid); + H5Z_reset_local(dataset->shared->dcpl_id, dataset->shared->type_id, fid, TRUE); } H5E_END_TRY; /* Release the file ID */ @@ -1248,6 +1249,7 @@ H5D_open(const H5G_loc_t *loc, hid_t dxpl_id) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEFILE, NULL, "can't close file") fid = -1; +#endif ret_value = dataset; @@ -1519,6 +1521,7 @@ done: herr_t H5D_close(H5D_t *dataset) { + hid_t fid; unsigned free_failed = FALSE; herr_t ret_value = SUCCEED; /* Return value */ @@ -1533,6 +1536,10 @@ H5D_close(H5D_t *dataset) H5D_istore_stats(dataset, FALSE); #endif /* H5F_ISTORE_DEBUG */ + /* Get a file ID for the file */ + if((fid = H5F_get_id(dataset->oloc.file)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, NULL, "unable to get file ID") + dataset->shared->fo_count--; if(dataset->shared->fo_count == 0) { /* Flush the dataset's information */ @@ -1588,6 +1595,12 @@ H5D_close(H5D_t *dataset) #endif /* NDEBUG */ } /* end switch */ /*lint !e788 All appropriate cases are covered */ +#ifndef H5_USE_16_API + H5E_BEGIN_TRY { + H5Z_close_local(dataset->shared->dcpl_id, dataset->shared->type_id, fid); + } H5E_END_TRY; +#endif + /* * Release datatype, dataspace and creation property list -- there isn't * much we can do if one of these fails, so we just continue. @@ -1625,8 +1638,18 @@ H5D_close(H5D_t *dataset) if(H5FO_top_count(dataset->oloc.file, dataset->oloc.addr) == 0) if(H5O_close(&(dataset->oloc)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to close") + + H5E_BEGIN_TRY { + H5Z_close_local(dataset->shared->dcpl_id, dataset->shared->type_id, fid); + } H5E_END_TRY; } /* end else */ + /* Release the file ID */ + if(H5I_dec_ref(fid) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEFILE, NULL, "can't close file") + + fid = -1; + /* Release the dataset's path info */ if(H5G_name_free(&(dataset->path)) < 0) free_failed = TRUE; @@ -2277,13 +2300,13 @@ H5D_check_filters(H5D_t *dataset) if(H5Z_can_apply(dataset->shared->dcpl_id, dataset->shared->type_id, fid) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "can't apply filters") - dataset->shared->checked_filters = TRUE; - /* Release the file ID */ if(H5I_dec_ref(fid) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEFILE, NULL, "can't close file") fid = -1; + + dataset->shared->checked_filters = TRUE; } /* end if */ } /* end if */ } /* end if */ diff --git a/src/H5Dio.c b/src/H5Dio.c index e05836d..dcc15bf 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -1449,8 +1449,15 @@ H5D_chunk_read(H5D_io_info_t *io_info, hsize_t nelmts, store.chunk.offset = chunk_info->coords; store.chunk.index = chunk_info->index; - /* Load the chunk into cache and lock it. */ chunk_addr = H5D_istore_get_addr(io_info, &udata); + + /* Translate the offset of the chunck into single-dimensional offset for + * the filter of modifying dataset's datatype */ + if((io_info->store->chunk.linear_offset = H5D_istore_calc_offset( + dataset->shared->space, io_info->store->chunk.offset)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to calculate chunk offset") + + /* Load the chunk into cache and lock it. */ if(H5D_istore_if_load(io_info, chunk_addr)) { if(NULL == (chunk = H5D_istore_lock(io_info, &udata, FALSE, &idx_hint))) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to read raw data chunk") @@ -1573,6 +1580,12 @@ H5D_chunk_read(H5D_io_info_t *io_info, hsize_t nelmts, /* Load the chunk into cache and lock it. */ chunk_addr = H5D_istore_get_addr(io_info, &udata); + /* Translate the offset of the chunck into single-dimensional offset for + * the filter of modifying dataset's datatype */ + if((io_info->store->chunk.linear_offset = H5D_istore_calc_offset( + dataset->shared->space, io_info->store->chunk.offset)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to calculate chunk offset") + if(H5D_istore_if_load(io_info, chunk_addr)) { if(NULL == (chunk = H5D_istore_lock(io_info, &udata, FALSE, &idx_hint))) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "unable to read raw data chunk") @@ -1782,6 +1795,8 @@ H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, H5D_istore_ud1_t udata; /*B-tree pass-through */ unsigned idx_hint = 0; /* Cache index hint */ hbool_t relax=TRUE; /* Whether whole chunk is selected */ + hbool_t has_mdtype_filter=FALSE;/* Whether filter for modifying dset's dtype + * is enabled */ herr_t ret_value = SUCCEED; /*return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_chunk_write) @@ -1810,6 +1825,23 @@ H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, dst_type_size = H5T_get_size(dataset->shared->type); max_type_size = MAX(src_type_size, dst_type_size); + /* If the filter for modifying dataset's datatype is enabled, don't relax + * because we need the offset of the chunk to pass through H5Z_pipeline. */ + if(dataset->shared->dcpl_id != H5P_DATASET_CREATE_DEFAULT) { + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret = SUCCEED; /* Return value */ + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(dataset->shared->dcpl_id, H5P_DATASET_CREATE))) + HGOTO_ERROR(H5E_DATASET, H5E_BADATOM, FAIL, "can't find object for ID") + + H5E_BEGIN_TRY { + ret = H5P_get_filter_by_id(plist, H5Z_FILTER_DTYPE_MODIFY, NULL, NULL, NULL, (size_t)0, NULL, NULL); + } H5E_END_TRY; + + if(ret >= 0) + has_mdtype_filter = TRUE; + } /* end if */ + /* * If there is no type conversion then write directly from the * application's buffer. This saves at least one mem-to-mem copy. @@ -1851,9 +1883,15 @@ H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, * simply allocate space instead of load the chunk. */ chunk_addr = H5D_istore_get_addr(io_info, &udata); + /* Translate the offset of the chunck into single-dimensional offset for + * the filter of modifying dataset's datatype */ + if((io_info->store->chunk.linear_offset = H5D_istore_calc_offset( + dataset->shared->space, io_info->store->chunk.offset)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to calculate chunk offset") + if(H5D_istore_if_load(io_info, chunk_addr)) { accessed_bytes = chunk_info->chunk_points * dst_type_size; - if(accessed_bytes != dataset->shared->layout.u.chunk.size) + if(has_mdtype_filter || accessed_bytes != dataset->shared->layout.u.chunk.size) relax = FALSE; if(NULL == (chunk = H5D_istore_lock(io_info, &udata, relax, &idx_hint))) @@ -1977,6 +2015,12 @@ H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, store.chunk.offset = chunk_info->coords; store.chunk.index = chunk_info->index; + /* Translate the offset of the chunck into single-dimensional offset for + * the filter of modifying dataset's datatype */ + if((io_info->store->chunk.linear_offset = H5D_istore_calc_offset( + dataset->shared->space, io_info->store->chunk.offset)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to calculate chunk offset") + /* Load the chunk into cache. But if the whole chunk is written, * simply allocate space instead of load the chunk. */ chunk_addr = H5D_istore_get_addr(io_info, &udata); @@ -1987,7 +2031,7 @@ H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, relax=FALSE; if(relax) { accessed_bytes = H5S_GET_SELECT_NPOINTS(chunk_info->mspace) * src_type_size; - if(accessed_bytes != dataset->shared->layout.u.chunk.size) + if(has_mdtype_filter || accessed_bytes != dataset->shared->layout.u.chunk.size) relax = FALSE; } @@ -2381,7 +2425,7 @@ H5Dmodify_dtype(hid_t dset_id, hid_t new_type_id) H5D_t *dset = NULL; H5T_t *new_type = NULL; - herr_t ret_value; + herr_t ret_value = SUCCEED; FUNC_ENTER_API(H5Dmodify_dtype, FAIL) H5TRACE2("e", "ii", dset_id, new_type_id); @@ -2465,13 +2509,31 @@ H5D_modify_dtype(H5F_t *file, H5D_t *dataset, hid_t type_id, H5T_t *type, hid_t HGOTO_ERROR(H5E_DATASET, H5E_BADATOM, FAIL, "can't find object for ID") H5E_BEGIN_TRY { - ret = H5P_get_filter_by_id(plist, H5Z_FILTER_DTYPE_MODIFY, NULL, NULL, NULL, (size_t)0, NULL, NULL); + ret = H5P_get_filter_by_id(plist, H5Z_FILTER_DTYPE_MODIFY, NULL, NULL, + NULL, (size_t)0, NULL, NULL); } H5E_END_TRY; if(ret >= 0) filter_enabled = TRUE; } /* end if */ + /* If the filter is present, generally only support committed or shared datatype. + * We also allow integer or floating number because of their limited encoding size. + */ + if(filter_enabled && !H5T_committed(type) && + !H5SM_can_share(file, dxpl_id, NULL, NULL, H5O_DTYPE_ID, type)) { + H5T_class_t tclass; + if((tclass = H5T_get_class(type, TRUE)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "can't find datatype class") + + if(tclass != H5T_INTEGER && tclass != H5T_FLOAT) + HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "violate datatype restriction") + } + + /* Check if the datatype should be (or are already) shared in the SOHM table */ + if(H5SM_try_share(file, dxpl_id, NULL, H5O_DTYPE_ID, dataset->shared->type, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADMESG, FAIL, "trying to share datatype failed") + /* If space hasn't been allocated and not using external storage, * set the flag HAS_DATA to FALSE. If the dataset is compact, * the flag is always TRUE. @@ -2526,30 +2588,29 @@ H5D_modify_dtype(H5F_t *file, H5D_t *dataset, hid_t type_id, H5T_t *type, hid_t default: HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "dataset should be contiguous or chunked") } + } else if(filter_enabled && has_data && H5D_CHUNKED == dataset->shared->layout.type) { + /* Convert the chunks already in the cache */ + if(H5D_istore_conv_dtype(dataset, type_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTCONVERT, FAIL, "can't convert data cache") } /* Get a file ID for the file */ if((fid = H5F_get_id(file)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get file ID") - /* Copy & initialize datatype for the dataset struct */ - if(H5D_init_type(file, dataset, type_id, type) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't copy datatype") - - /* Check if the datatype should be (or are already) shared in the SOHM table */ - if(H5SM_try_share(file, dxpl_id, NULL, H5O_DTYPE_ID, dataset->shared->type, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_BADMESG, FAIL, "trying to share datatype failed") - /* - * Check whether datatype is committed & increment ref count. (to maintain - * ref. count incr/decr similarity with "shared message" type of datatype sharing) + * Check whether the old datatype is committed & decrement ref count. */ if(H5T_committed(dataset->shared->type)) { - /* Increment the reference count on the shared datatype */ - if(H5T_link(dataset->shared->type, 1, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_LINKCOUNT, FAIL, "unable to adjust shared datatype link count") + /* Decrement the reference count on the committed datatype */ + if(H5T_link(dataset->shared->type, -1, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_LINKCOUNT, FAIL, "unable to adjust committed datatype link count") } /* end if */ + /* Copy & initialize datatype for the dataset struct */ + if(H5D_init_type(file, dataset, type_id, type) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't copy datatype") + /* Open the object header of the original dataset for updating datatype and layout */ if(H5O_open(&(dataset->oloc)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to open dataset object header") @@ -2561,8 +2622,8 @@ H5D_modify_dtype(H5F_t *file, H5D_t *dataset, hid_t type_id, H5T_t *type, hid_t /* Delete the old datatype message */ if(H5O_msg_remove(&(dataset->oloc), H5O_DTYPE_ID, 0, FALSE, dxpl_id) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTDELETE, FAIL, "unable to delete old dtype message") - - /* Add the new datatype message */ + + /* Add the new datatype message. Ref count for committed dtype is incremented here. */ if(H5O_msg_append_oh(file, dxpl_id, oh, H5O_DTYPE_ID, H5O_MSG_FLAG_CONSTANT, 0, dataset->shared->type) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update datatype header message") @@ -2606,7 +2667,9 @@ H5D_modify_dtype(H5F_t *file, H5D_t *dataset, hid_t type_id, H5T_t *type, hid_t * allocation time is early, since space may not be allocated) */ fill_prop = &(dataset->shared->dcpl_cache.fill); - if(H5O_msg_append_oh(file, dxpl_id, oh, H5O_LAYOUT_ID, ((fill_prop->alloc_time == H5D_ALLOC_TIME_EARLY && H5D_COMPACT != dataset->shared->layout.type) ? H5O_MSG_FLAG_CONSTANT : 0), 0, &(dataset->shared->layout)) < 0) + if(H5O_msg_append_oh(file, dxpl_id, oh, H5O_LAYOUT_ID, + ((fill_prop->alloc_time == H5D_ALLOC_TIME_EARLY && H5D_COMPACT != dataset->shared->layout.type) ? H5O_MSG_FLAG_CONSTANT : 0), + 0, &(dataset->shared->layout)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update layout") /* @@ -2666,7 +2729,7 @@ H5D_modify_dtype(H5F_t *file, H5D_t *dataset, hid_t type_id, H5T_t *type, hid_t HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to close dataset") } else if(filter_enabled) { /* Make the "reset local" filter callbacks for this dataset */ - if(H5Z_reset_local(dataset->shared->dcpl_id, dataset->shared->type_id, fid) < 0) + if(H5Z_reset_local(dataset->shared->dcpl_id, dataset->shared->type_id, fid, FALSE) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to set local filter parameters") /* Release the file ID */ @@ -2675,7 +2738,7 @@ H5D_modify_dtype(H5F_t *file, H5D_t *dataset, hid_t type_id, H5T_t *type, hid_t fid = -1; } - + done: /* Release the file ID */ if(fid>=0 && H5I_dec_ref(fid) < 0) diff --git a/src/H5Distore.c b/src/H5Distore.c index 2d47a5a..bc265c6 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -123,6 +123,7 @@ typedef struct H5D_rdcc_ent_t { size_t alloc_size; /*amount allocated for the chunk */ uint8_t *chunk; /*the unfiltered chunk data */ unsigned idx; /*index in hash table */ + hssize_t linear_offset; /* Chunk's coordinates translated into one-dimensional offset */ struct H5D_rdcc_ent_t *next;/*next item in doubly-linked list */ struct H5D_rdcc_ent_t *prev;/*previous item in doubly-linked list */ } H5D_rdcc_ent_t; @@ -752,6 +753,8 @@ H5D_istore_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key, int cmp; unsigned u; H5B_ins_t ret_value; + + htri_t dis = FALSE; FUNC_ENTER_NOAPI_NOINIT(H5D_istore_insert) @@ -811,12 +814,15 @@ H5D_istore_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_lt_key, ret_value = H5B_INS_NOOP; } - } else if (H5V_hyper_disjointp(udata->common.mesg->u.chunk.ndims, + } else if ((dis = H5V_hyper_disjointp(udata->common.mesg->u.chunk.ndims, lt_key->offset, udata->common.mesg->u.chunk.dim, - udata->common.offset, udata->common.mesg->u.chunk.dim)) { - HDassert(H5V_hyper_disjointp(udata->common.mesg->u.chunk.ndims, + udata->common.offset, udata->common.mesg->u.chunk.dim)) == TRUE) { + + dis = H5V_hyper_disjointp(udata->common.mesg->u.chunk.ndims, rt_key->offset, udata->common.mesg->u.chunk.dim, - udata->common.offset, udata->common.mesg->u.chunk.dim)); + udata->common.offset, udata->common.mesg->u.chunk.dim); + HDassert(dis); + /* * Split this node, inserting the new new node to the right of the * current node. The MD_KEY is where the split occurs. @@ -993,6 +999,7 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, H5Z_EDC_t edc_read = H5Z_NO_EDC; size_t nbytes = lt_key->nbytes; H5Z_cb_t cb_struct; + hsize_t linear_offset = 0; /* No real use */ int ret_value = H5_ITER_CONT; /* Return value */ @@ -1042,7 +1049,8 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, if(is_compressed && (is_vlen || fix_ref)) { unsigned filter_mask = lt_key->filter_mask; - if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, edc_read, cb_struct, &nbytes, &buf_size, &buf) < 0) + if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, edc_read, linear_offset, + cb_struct, &nbytes, &buf_size, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "data pipeline read failed") } /* end if */ @@ -1103,7 +1111,7 @@ H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, /* Need to compress variable-length & reference data elements before writing to file */ if(is_compressed && (is_vlen || fix_ref) ) { - if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), edc_read, + if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), edc_read, linear_offset, cb_struct, &nbytes, &buf_size, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed") udata_dst.nbytes = nbytes; @@ -1147,6 +1155,7 @@ H5D_istore_iter_copy_conv(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, H5D_istore_it_ud4_t *udata = (H5D_istore_it_ud4_t *)_udata; const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; H5D_istore_ud1_t udata_dst; /* User data about new destination chunk */ + hsize_t linear_offset = 0; /* No real use */ /* General information about chunk copy */ void *bkg = udata->bkg; @@ -1206,7 +1215,7 @@ H5D_istore_iter_copy_conv(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, if(is_compressed) { unsigned filter_mask = lt_key->filter_mask; - if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, edc_read, cb_struct, &src_nbytes, &buf_size, &buf) < 0) + if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, edc_read, linear_offset, cb_struct, &src_nbytes, &buf_size, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "data pipeline read failed") } /* end if */ @@ -1230,7 +1239,7 @@ H5D_istore_iter_copy_conv(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, /* Need to compress variable-length & reference data elements before writing to file */ if(is_compressed) { - if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), edc_read, + if(H5Z_pipeline(pline, 0, &(udata_dst.filter_mask), edc_read, linear_offset, cb_struct, &dst_nbytes, &buf_size, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, H5_ITER_ERROR, "output pipeline failed") udata->buf = buf; @@ -1251,7 +1260,7 @@ H5D_istore_iter_copy_conv(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_istore_iter_copy() */ +} /* end H5D_istore_iter_copy_conv() */ /*------------------------------------------------------------------------- @@ -1452,7 +1461,6 @@ H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_ udata.nbytes = ent->chunk_size; udata.addr = HADDR_UNDEF; - /* Should the chunk be filtered before writing it to disk? */ if(io_info->dset->shared->dcpl_cache.pline.nused) { if(!reset) { /* @@ -1476,8 +1484,11 @@ H5D_istore_flush_entry(const H5D_io_info_t *io_info, H5D_rdcc_ent_t *ent, hbool_ point_of_no_return = TRUE; ent->chunk = NULL; } /* end else */ - if(H5Z_pipeline(&(io_info->dset->shared->dcpl_cache.pline), 0, &(udata.filter_mask), io_info->dxpl_cache->err_detect, - io_info->dxpl_cache->filter_cb, &(udata.nbytes), &alloc, &buf) < 0) + + if(H5Z_pipeline(&(io_info->dset->shared->dcpl_cache.pline), 0, + &(udata.filter_mask), io_info->dxpl_cache->err_detect, + (hsize_t)ent->linear_offset, io_info->dxpl_cache->filter_cb, + &(udata.nbytes), &alloc, &buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, FAIL, "output pipeline failed") } /* end if */ @@ -1546,6 +1557,7 @@ static herr_t H5D_istore_preempt(const H5D_io_info_t *io_info, H5D_rdcc_ent_t * ent, hbool_t flush) { H5D_rdcc_t *rdcc = &(io_info->dset->shared->cache.chunk); + hid_t dcpl_id = io_info->dset->shared->dcpl_id; herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_istore_preempt) @@ -1566,6 +1578,9 @@ H5D_istore_preempt(const H5D_io_info_t *io_info, H5D_rdcc_ent_t * ent, hbool_t f ent->chunk = (uint8_t *)H5D_istore_chunk_xfree(ent->chunk, &(io_info->dset->shared->dcpl_cache.pline)); } /* end else */ + if(H5Z_evict_local(dcpl_id, /*unused*/0, /*unused*/0, (hsize_t)ent->linear_offset) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "failed to evict local filter info") + /* Unlink from list */ if(ent->prev) ent->prev->next = ent->next; @@ -1644,6 +1659,174 @@ done: /*------------------------------------------------------------------------- + * Function: H5D_istore_conv_dtype + * + * Purpose: Convert all the chunks in the cache if the datatype for + * dataset has been changed (called by H5Dmodify_dtype). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 16 April 2008 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_istore_conv_dtype(H5D_t *dset, hid_t new_type_id) +{ + H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); + const H5O_pline_t *pline = &(dset->shared->dcpl_cache.pline); /* I/O pipeline info */ + H5D_rdcc_ent_t *ent, *next; + hid_t orig_type_id = dset->shared->type_id; + H5T_t *new_type, *orig_type; + H5T_t *new_type_copy; + hid_t new_type_copy_id; + size_t orig_type_size, new_type_size; + size_t nelmts; + unsigned char *new_chunk = NULL; + unsigned char *bkg=NULL; /* Pointer to background buffer */ + H5T_path_t *tpath=NULL; /*type conversion info */ + herr_t ret_value = SUCCEED; /* Return value */ + htri_t is_vlen = FALSE; + hbool_t vlen_conv = FALSE; + hid_t dxpl_id; + H5P_genplist_t *def_dx_plist = NULL, *dx_plist = NULL; + + FUNC_ENTER_NOAPI(H5D_istore_conv_dtype, FAIL) + + /* If there's no chunks in the cache, skip this function */ + if(!rdcc->head) + HGOTO_DONE(SUCCEED); + + /* The current datatype of the dataset */ + if(NULL==(orig_type=(H5T_t *)H5I_object_verify(orig_type_id, H5I_DATATYPE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype") + + /* The new datatype of the dataset */ + if(NULL==(new_type=(H5T_t *)H5I_object_verify(new_type_id, H5I_DATATYPE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype") + + /* Make a copy of the new datatype to avoid changing the original new type */ + if(NULL==(new_type_copy=H5T_copy(new_type, H5T_COPY_ALL))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy data type"); + + if((new_type_copy_id = H5I_register(H5I_DATATYPE, new_type_copy)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register datatype atom") + + /* Mark the new datatype as being on disk now because the data in the cache + * hasn't been converted to memory type yet. */ + if(H5T_set_loc(new_type_copy, dset->oloc.file, H5T_LOC_DISK) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't set datatype location") + + /* Get the object of default dataset creation property list */ + if(NULL == (def_dx_plist = (H5P_genplist_t *)H5I_object(H5P_DATASET_XFER_DEFAULT))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get dataset creation property list") + + /* Create a new default property list */ + if((dxpl_id = H5P_copy_plist(def_dx_plist)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "can't copy dataset creation property list") + + if(NULL == (dx_plist = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get dataset creation property list") + + /* If the datatype is or contains vlen, set the property to indicate no conversion + * is needed. */ + if((is_vlen = H5T_detect_class(orig_type, H5T_VLEN)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to detect dtatypes") + + if(is_vlen ) { + vlen_conv = FALSE; + + if(H5P_set(dx_plist, H5D_XFER_VLEN_CONV_NAME, &vlen_conv) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "Error setting vlen conv flag") + } + + if((orig_type_size = H5T_get_size(orig_type)) == 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "can't get size of original type") + + if((new_type_size = H5T_get_size(new_type_copy)) == 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "can't get size of new type") + + /* If the new type is the same as the current, simply finish the function */ + if(0 == H5T_cmp(orig_type, new_type_copy, FALSE)) + HGOTO_DONE(SUCCEED) + + /* Figure out the number of elements in a chunk */ + nelmts = dset->shared->layout.u.chunk.size/orig_type_size; + + /* Find the conversion function */ + if (NULL==(tpath=H5T_path_find(orig_type, new_type_copy, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst data types"); + + if (H5T_path_bkg(tpath) && (NULL==(bkg=(unsigned char*)H5MM_malloc(nelmts*(new_type_size > orig_type_size ? new_type_size : orig_type_size))))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate buffer") + + /* Loop over all entries in the chunk cache */ + for(ent = rdcc->head; ent; ent = next) { + next = ent->next; + ent->locked = TRUE; + + if(new_type_size > orig_type_size) { + if(NULL == (new_chunk = H5D_istore_chunk_alloc(nelmts*new_type_size, pline))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + + HDmemcpy(new_chunk, ent->chunk, ent->chunk_size); + + /* Convert the data */ + if (H5T_convert(tpath, orig_type_id, new_type_copy_id, nelmts, (size_t)0, + (size_t)0, new_chunk, bkg, dxpl_id)<0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "data type conversion failed"); + + H5MM_free(ent->chunk); + ent->chunk = (uint8_t*)new_chunk; + } else if (new_type_size < orig_type_size) { + if(NULL == (new_chunk = H5D_istore_chunk_alloc(nelmts*new_type_size, pline))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + + /* Convert the data */ + if (H5T_convert(tpath, orig_type_id, new_type_copy_id, nelmts, (size_t)0, + (size_t)0, ent->chunk, bkg, dxpl_id)<0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "data type conversion failed"); + + HDmemcpy(new_chunk, ent->chunk, nelmts*new_type_size); + + H5MM_free(ent->chunk); + ent->chunk = (uint8_t*)new_chunk; + } else { + /* Convert the data */ + if (H5T_convert(tpath, orig_type_id, new_type_copy_id, nelmts, (size_t)0, + (size_t)0, ent->chunk, bkg, dxpl_id)<0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "data type conversion failed"); + } + + ent->chunk_size = nelmts*new_type_size; + ent->alloc_size = ent->chunk_size; + ent->rd_count = ent->chunk_size; + ent->wr_count = ent->chunk_size; + ent->offset[dset->shared->layout.u.chunk.ndims-1] = new_type_size; + ent->dirty = TRUE; + ent->locked = FALSE; + + /* Update some local variables for the filter pipeline */ + if(H5Z_change_local(dset->shared->dcpl_id, new_type_copy_id, /*unused*/0, + (hsize_t)ent->linear_offset) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "failed to change local filter info") + } /* end for */ + + rdcc->nbytes = new_type_size*rdcc->nbytes/orig_type_size; + + H5MM_free(bkg); + + if (H5I_dec_ref(new_type_copy_id) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "problem freeing id"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_istore_conv_dtype() */ + + + +/*------------------------------------------------------------------------- * Function: H5D_istore_dest * * Purpose: Destroy the entire chunk cache by flushing dirty entries, @@ -1908,6 +2091,52 @@ done: /*------------------------------------------------------------------------- + * Function: H5D_istore_calc_offset + * + * Purpose: Private function to translate from the multi-dimensional + * offset of a chunk in a dataspace to linear offset. At this + * moment, it's used for the filter for modifying dataset's + * datatype. + * + * Return: Success: Linear offset of the chunk. + * + * Failure: Negative + * + * Programmer: Raymond Lu + * 12 March 2008 + * + * Modification: + *------------------------------------------------------------------------- + */ +hssize_t H5D_istore_calc_offset(H5S_t *space, hsize_t offset[]) +{ + hsize_t linear_offset = 0, tmp_offset = 0; + int space_ndims; /* Dataset's space rank */ + hsize_t space_dim[H5O_LAYOUT_NDIMS]; /* Dataset's dataspace dimensions */ + int u,v; /*counters */ + hssize_t ret_value; + + FUNC_ENTER_NOAPI(H5D_istore_calc_offset, FAIL) + + /* Retrieve the dataset dimensions */ + if((space_ndims = H5S_get_simple_extent_dims(space, space_dim, NULL)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get simple dataspace info") + + for(u=0; u<space_ndims; u++) { + tmp_offset = offset[u]; + for(v=u+1; v<space_ndims; v++) + tmp_offset *= space_dim[v]; + linear_offset += tmp_offset; + } + + ret_value = linear_offset; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} + + +/*------------------------------------------------------------------------- * Function: H5D_istore_lock * * Purpose: Return a pointer to a dataset chunk. The pointer points @@ -2038,8 +2267,10 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, HGOTO_ERROR(H5E_IO, H5E_READERROR, NULL, "unable to read raw data chunk") if(pline->nused) - if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &(udata->filter_mask), io_info->dxpl_cache->err_detect, - io_info->dxpl_cache->filter_cb, &(udata->nbytes), &chunk_alloc, &chunk) < 0) + if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &(udata->filter_mask), + io_info->dxpl_cache->err_detect, (hsize_t)io_info->store->chunk.linear_offset, + io_info->dxpl_cache->filter_cb, &(udata->nbytes), &chunk_alloc, + &chunk) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, NULL, "data pipeline read failed") #ifdef H5D_ISTORE_DEBUG rdcc->nmisses++; @@ -2126,6 +2357,7 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, assert(NULL==rdcc->slot[idx]); rdcc->slot[idx] = ent; ent->idx = idx; + ent->linear_offset = io_info->store->chunk.linear_offset; rdcc->nbytes += chunk_size; rdcc->nused++; @@ -2856,7 +3088,7 @@ H5D_istore_chunk_alloc(size_t size, const H5O_pline_t *pline) { void *ret_value = NULL; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5D_istore_chunk_alloc) + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_chunk_alloc) HDassert(size); HDassert(pline); @@ -2928,6 +3160,7 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) H5D_io_info_t io_info; /* Dataset I/O info */ H5D_storage_t store; /* Dataset storage information */ hsize_t chunk_offset[H5O_LAYOUT_NDIMS]; /* Offset of current chunk */ + hssize_t linear_offset = 0; size_t orig_chunk_size; /* Original size of chunk in bytes */ unsigned filter_mask = 0; /* Filter mask for chunks that have them */ const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ @@ -3012,6 +3245,7 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) || pline->nused > 0) should_fill = TRUE; + /* Check if fill values should be written to chunks */ if(should_fill) { /* Initialize the fill value buffer */ @@ -3033,7 +3267,9 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) size_t buf_size = orig_chunk_size; /* Push the chunk through the filters */ - if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &orig_chunk_size, &buf_size, &fb_info.fill_buf) < 0) + if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, + (hsize_t)linear_offset, dxpl_cache->filter_cb, &orig_chunk_size, + &buf_size, &fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed") } /* end if */ } /* end if */ @@ -3091,7 +3327,9 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) size_t nbytes = fb_info.fill_buf_size; /* Push the chunk through the filters */ - if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &nbytes, &buf_size, &fb_info.fill_buf) < 0) + if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, + (hsize_t)linear_offset, dxpl_cache->filter_cb, &nbytes, + &buf_size, &fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed") /* Keep the number of bytes the chunk turned in to */ @@ -3157,6 +3395,12 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) break; } /* end else */ } /* end for */ + + /* Translate the offset of the chunck into single-dimensional offset for + * the filter of modifying dataset's datatype */ + if((linear_offset = H5D_istore_calc_offset(dset->shared->space, chunk_offset)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to calculate chunk offset") + } /* end while */ #ifdef H5_HAVE_PARALLEL @@ -3785,7 +4029,7 @@ H5D_istore_delete(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout) /* Delete entire B-tree */ if(H5B_delete(f, dxpl_id, H5B_ISTORE, tmp_layout.u.chunk.addr, &udata)<0) - HGOTO_ERROR(H5E_IO, H5E_CANTDELETE, 0, "unable to delete chunk B-tree") + HGOTO_ERROR(H5E_IO, H5E_CANTDELETE, FAIL, "unable to delete chunk B-tree") /* Free the raw B-tree node buffer */ if(tmp_layout.u.chunk.btree_shared==NULL) @@ -4217,8 +4461,6 @@ H5D_istore_copy_conv(H5F_t *file, const H5D_t *dset_src, const H5D_t *dset_dst, if(H5P_set(plist, H5D_XFER_VLEN_CONV_NAME, &vlen_conv) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "Error setting vlen conv flag") - /*if(H5P_get(plist, H5D_XFER_VLEN_CONV_NAME, &vlen_conv) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "Error getting vlen conv flag")*/ } /* Set up the conversion functions */ @@ -4492,4 +4734,3 @@ H5D_istore_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int inden done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_debug() */ - diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 300fb18..09b595d 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -112,7 +112,7 @@ typedef struct H5D_io_info_t { hbool_t xfer_opt_mode_changed; hbool_t using_mpi_vfd; /* Whether the file is using an MPI-based VFD */ #endif /* H5_HAVE_PARALLEL */ - const H5D_storage_t *store; /* Dataset storage info */ + H5D_storage_t *store; /* Dataset storage info */ H5D_io_ops_t ops; /* I/O operation function pointers */ #ifdef H5S_DEBUG H5S_iostats_t *stats; /* I/O statistics */ @@ -454,8 +454,8 @@ H5_DLL haddr_t H5D_istore_get_addr(const H5D_io_info_t *io_info, H5_DLL herr_t H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, H5O_layout_t *layout_dst, H5T_t *src_dtype, H5O_copy_t *cpy_info, H5O_pline_t *pline, hid_t dxpl_id); -H5_DLL herr_t H5D_istore_copy_conv(H5F_t *file, const H5D_t *dset_src, const H5D_t *dset_dst, - H5O_copy_t UNUSED *cpy_info, hid_t dxpl_id); +H5_DLL herr_t H5D_istore_copy_conv(H5F_t *file, const H5D_t *dset_src, const H5D_t *dset_dst, H5O_copy_t UNUSED *cpy_info, hid_t dxpl_id); +H5_DLL herr_t H5D_istore_conv_dtype(H5D_t *dset, hid_t new_type_id); H5_DLL void * H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, hbool_t relax, unsigned *idx_hint/*in,out*/); H5_DLL herr_t H5D_istore_unlock(const H5D_io_info_t *io_info, diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index c5192b2..bab035c 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -112,6 +112,7 @@ typedef struct H5D_t H5D_t; typedef struct { hsize_t index; /* "Index" of chunk in dataset (must be first for TBBT routines) */ hsize_t *offset; /* Chunk's coordinates in elements */ + hssize_t linear_offset; /* Chunk's coordinates translated into one-dimensional offset */ } H5D_chunk_storage_t; typedef struct { @@ -185,6 +186,7 @@ H5_DLL herr_t H5D_istore_delete(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout); H5_DLL herr_t H5D_istore_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int indent, int fwidth, unsigned ndims); +H5_DLL hssize_t H5D_istore_calc_offset(H5S_t *space, hsize_t offset[]); #endif /* _H5Dprivate_H */ diff --git a/src/H5HFcache.c b/src/H5HFcache.c index 7454d76..9a83ec1 100644 --- a/src/H5HFcache.c +++ b/src/H5HFcache.c @@ -1290,7 +1290,7 @@ HDfprintf(stderr, "%s: read_size = %Zu, read_buf = %p\n", FUNC, read_size, read_ /* Push direct block data through I/O filter pipeline */ nbytes = read_size; - if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_ENABLE_EDC, + if(H5Z_pipeline(&(hdr->pline), (H5Z_FLAG_REVERSE | H5Z_FLAG_SKIP_CRECORD), &filter_mask, H5Z_ENABLE_EDC, 0, filter_cb, &nbytes, &read_size, &read_buf) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, NULL, "output pipeline failed") #ifdef QAK @@ -1465,7 +1465,7 @@ H5HF_cache_dblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, /* Push direct block data through I/O filter pipeline */ nbytes = write_size; - if(H5Z_pipeline(&(hdr->pline), 0, &filter_mask, H5Z_ENABLE_EDC, + if(H5Z_pipeline(&(hdr->pline), (0 | H5Z_FLAG_SKIP_CRECORD), &filter_mask, H5Z_ENABLE_EDC, 0, filter_cb, &nbytes, &write_size, &write_buf) < 0) HGOTO_ERROR(H5E_HEAP, H5E_WRITEERROR, FAIL, "output pipeline failed") #ifdef QAK diff --git a/src/H5HFhuge.c b/src/H5HFhuge.c index 198ef62..1634938 100644 --- a/src/H5HFhuge.c +++ b/src/H5HFhuge.c @@ -349,7 +349,7 @@ HDfprintf(stderr, "%s: obj_size = %Zu\n", FUNC, obj_size); /* Push direct block data through I/O filter pipeline */ nbytes = write_size; - if(H5Z_pipeline(&(hdr->pline), 0, &filter_mask, H5Z_NO_EDC, + if(H5Z_pipeline(&(hdr->pline), (0 | H5Z_FLAG_SKIP_CRECORD), &filter_mask, H5Z_NO_EDC, 0, filter_cb, &nbytes, &write_size, &write_buf) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, FAIL, "output pipeline failed") #ifdef QAK @@ -675,7 +675,7 @@ H5HF_huge_op_real(H5HF_hdr_t *hdr, hid_t dxpl_id, const uint8_t *id, /* De-filter the object */ read_size = nbytes = obj_size; - if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_NO_EDC, filter_cb, &nbytes, &read_size, &read_buf) < 0) + if(H5Z_pipeline(&(hdr->pline), (H5Z_FLAG_REVERSE | H5Z_FLAG_SKIP_CRECORD), &filter_mask, H5Z_NO_EDC, 0, filter_cb, &nbytes, &read_size, &read_buf) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, FAIL, "input filter failed") obj_size = nbytes; } /* end if */ diff --git a/src/H5Odtype.c b/src/H5Odtype.c index ac07edf..389d989 100644 --- a/src/H5Odtype.c +++ b/src/H5Odtype.c @@ -743,7 +743,7 @@ H5O_dtype_encode_helper(const H5F_t *f, uint8_t **pp, const H5T_t *dt) for(i = 0; i < dt->shared->u.compnd.nmembs; i++) { /* Sanity check */ /* (compound datatypes w/array members must be encoded w/version >= 2) */ - HDassert(dt->shared->u.compnd.memb[i].type->shared->type != H5T_ARRAY || dt->shared->version >= H5O_DTYPE_VERSION_2); + HDassert((dt->shared->u.compnd.memb[i].type)->shared->type != H5T_ARRAY || dt->shared->version >= H5O_DTYPE_VERSION_2); /* Name */ HDstrcpy((char*)(*pp), dt->shared->u.compnd.memb[i].name); diff --git a/src/H5Opline.c b/src/H5Opline.c index 9a926c1..20b060d 100644 --- a/src/H5Opline.c +++ b/src/H5Opline.c @@ -24,13 +24,13 @@ #define H5Z_PACKAGE /*suppress error about including H5Zpkg */ #include "H5private.h" /* Generic Functions */ +#include "H5Dprivate.h" /* Error handling */ #include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ #include "H5Zpkg.h" /* Data filters */ - /* PRIVATE PROTOTYPES */ static herr_t H5O_pline_encode(H5F_t *f, uint8_t *p, const void *mesg); static void *H5O_pline_decode(H5F_t *f, hid_t dxpl_id, unsigned mesg_flags, const uint8_t *p); @@ -38,6 +38,8 @@ static void *H5O_pline_copy(const void *_mesg, void *_dest); static size_t H5O_pline_size(const H5F_t *f, const void *_mesg); static herr_t H5O_pline_reset(void *_mesg); static herr_t H5O_pline_free(void *_mesg); +static herr_t H5O_pline_delete(H5F_t *f, hid_t dxpl_id, H5O_t UNUSED *open_oh, + void *_mesg); static herr_t H5O_pline_pre_copy_file(H5F_t *file_src, const void *mesg_src, hbool_t *deleted, const H5O_copy_t *cpy_info, void *_udata); static herr_t H5O_pline_debug(H5F_t *f, hid_t dxpl_id, const void *_mesg, @@ -52,7 +54,7 @@ static herr_t H5O_pline_debug(H5F_t *f, hid_t dxpl_id, const void *_mesg, #define H5O_SHARED_SIZE H5O_pline_shared_size #define H5O_SHARED_SIZE_REAL H5O_pline_size #define H5O_SHARED_DELETE H5O_pline_shared_delete -#undef H5O_SHARED_DELETE_REAL +#define H5O_SHARED_DELETE_REAL H5O_pline_delete #define H5O_SHARED_LINK H5O_pline_shared_link #undef H5O_SHARED_LINK_REAL #define H5O_SHARED_COPY_FILE H5O_pline_shared_copy_file @@ -551,6 +553,78 @@ H5O_pline_free(void *mesg) /*------------------------------------------------------------------------- + * Function: H5O_pline_delete + * + * Purpose: Free file space referenced by message + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 13 August 2008 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_pline_delete(H5F_t *f, hid_t dxpl_id, H5O_t UNUSED *open_oh, void *_mesg) +{ + H5O_pline_t *mesg = (H5O_pline_t *)_mesg; + H5O_pline_t *pline = NULL; + H5D_layout_t layout; /* Dataset's layout information */ + hid_t dcpl_id, file_id; + H5P_genplist_t *dc_plist = NULL, *def_dc_plist=NULL; /* New Property list */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5O_pline_delete) + + /* check args */ + HDassert(f); + HDassert(mesg); + + /* Get the object of default dataset creation property list */ + if(NULL == (def_dc_plist = (H5P_genplist_t *)H5I_object(H5P_DATASET_CREATE_DEFAULT))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get dataset creation property list") + + /* Create a new default property list */ + if((dcpl_id = H5P_copy_plist(def_dc_plist)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "can't copy dataset creation property list") + + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get dataset creation property list") + + /* Make a copy of filter pipeline and insert into the property list. Only the filter + * for modifying dataset's datatype has the callback function to delete the local + * values. This filter only needs the property of pipeline and chunking. */ + if((pline = H5O_pline_copy(mesg, NULL)) == NULL) + HGOTO_ERROR(H5E_PLINE, H5E_CANTCOPY, FAIL, "unable to copy pipeline") + + if(H5P_set(dc_plist, H5D_CRT_DATA_PIPELINE_NAME, pline) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to set pipeline") + + layout = H5D_CHUNKED; + if(H5P_set(dc_plist, H5D_CRT_LAYOUT_NAME, &layout) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve layout") + + if((file_id = H5F_get_id(f)) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't get file ID") + + /* Call the function of the filter pipeline */ + if(H5Z_delete_local(dcpl_id, 0, file_id) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTDELETE, FAIL, "can't get dataset creation property list") + + if(H5I_dec_ref(dcpl_id) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "can't close property list ID"); + + if(H5I_dec_ref(file_id) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTFREE, FAIL, "can't close file ID"); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_pline_delete() */ + + +/*------------------------------------------------------------------------- * Function: H5O_pline_pre_copy_file * * Purpose: Perform any necessary actions before copying message between diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index fa92315..747a5f0 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -2064,7 +2064,7 @@ H5Pset_dtype_modifiable(hid_t plist_id) H5O_pline_t pline; H5P_genplist_t *plist; /* Property list pointer */ size_t cd_nelmts=3; /* Number of filter parameters */ - unsigned cd_values[3]={0,0}; /* Filter parameters */ + unsigned cd_values[3]={0,0,0};/* Filter parameters */ herr_t ret_value=SUCCEED; /* return value */ FUNC_ENTER_API(H5Pset_dtype_modifiable, FAIL) @@ -5404,6 +5404,11 @@ H5T_dtype_is_valid(H5T_t *dtype, H5T_t *new_type) } } break; + case H5T_TIME: + if(new_class != H5T_TIME) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FALSE, "new type must be time type") + + break; case H5T_STRING: if(new_class != H5T_STRING) HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FALSE, "new type must be a string type") diff --git a/src/H5Tcommit.c b/src/H5Tcommit.c index bcb9962..8bcae49 100644 --- a/src/H5Tcommit.c +++ b/src/H5Tcommit.c @@ -434,6 +434,35 @@ H5T_committed(const H5T_t *type) /*------------------------------------------------------------------------- + * Function: H5T_immutable + * + * Purpose: Determines if a datatype is predefined or not. + * + * Return: Success: TRUE if predefined, FALSE otherwise. + * + * Programmer: Raymond Lu + * 4 August 2008 + * + *------------------------------------------------------------------------- + */ +htri_t +H5T_immutable(const H5T_t *type) +{ + /* Use no-init for efficiency */ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5T_immutable) + + HDassert(type); + +if(H5T_STATE_IMMUTABLE == type->shared->state) + fprintf(stderr, "H5T_immutable: state=%d\n", type->shared->state); +else + fprintf(stderr, "H5T_immutable: state isn't immutable\n"); + + FUNC_LEAVE_NOAPI(H5T_STATE_IMMUTABLE == type->shared->state) +} /* end H5T_immutable() */ + + +/*------------------------------------------------------------------------- * Function: H5T_link * * Purpose: Adjust the link count for an object header by adding @@ -456,7 +485,8 @@ H5T_link(const H5T_t *type, int adjust, hid_t dxpl_id) FUNC_ENTER_NOAPI(H5T_link, FAIL) HDassert(type); - HDassert(type->sh_loc.type == H5O_SHARE_TYPE_COMMITTED); + HDassert(type->sh_loc.type == H5O_SHARE_TYPE_COMMITTED || + type->sh_loc.type == H5O_SHARE_TYPE_SOHM); /* Adjust the link count on the named datatype */ if((ret_value = H5O_link(&type->oloc, adjust, dxpl_id)) < 0) diff --git a/src/H5Tprivate.h b/src/H5Tprivate.h index c102ead..8d5071a 100644 --- a/src/H5Tprivate.h +++ b/src/H5Tprivate.h @@ -130,6 +130,7 @@ H5_DLL H5R_type_t H5T_get_ref_type(const H5T_t *dt); /* Operations on named datatypes */ H5_DLL H5T_t *H5T_open(const H5G_loc_t *loc, hid_t dxpl_id); H5_DLL htri_t H5T_committed(const H5T_t *type); +H5_DLL htri_t H5T_immutable(const H5T_t *type); H5_DLL int H5T_link(const H5T_t *type, int adjust, hid_t dxpl_id); H5_DLL herr_t H5T_update_shared(H5T_t *type); @@ -416,10 +416,10 @@ H5V_hyper_disjointp(unsigned n, if (0==size1[u] || 0==size2[u]) HGOTO_DONE(TRUE) - if (((offset1?offset1[u]:0) < (offset2?offset2[u]:0) && - ((offset1?offset1[u]:0) + size1[u] <= (offset2?offset2[u]:0))) || - ((offset2?offset2[u]:0) < (offset1?offset1[u]:0) && - ((offset2?offset2[u]:0) + size2[u] <= (offset1?offset1[u]:0)))) + if ((((offset1?offset1[u]:0) < (offset2?offset2[u]:0)) && + ((offset1?offset1[u]:0) + (size1?size1[u]:0) <= (offset2?offset2[u]:0))) || + (((offset2?offset2[u]:0) < (offset1?offset1[u]:0)) && + ((offset2?offset2[u]:0) + (size2?size2[u]:0) <= (offset1?offset1[u]:0)))) HGOTO_DONE(TRUE) } @@ -48,7 +48,11 @@ typedef struct H5Z_stats_t { typedef enum { H5Z_PRELUDE_CAN_APPLY, /* Call "can apply" callback */ H5Z_PRELUDE_SET_LOCAL, /* Call "set local" callback */ - H5Z_PRELUDE_RESET_LOCAL /* Call "reset local" callback */ + H5Z_PRELUDE_RESET_LOCAL, /* Call "reset local" callback */ + H5Z_PRELUDE_CHANGE_LOCAL, /* Call "change local" callback*/ + H5Z_PRELUDE_EVICT_LOCAL, /* Call "evict local" callback */ + H5Z_PRELUDE_DELETE_LOCAL, /* Call "delete local" callback*/ + H5Z_PRELUDE_CLOSE_LOCAL /* Call "close local" callback */ } H5Z_prelude_type_t; /* Local variables */ @@ -472,14 +476,14 @@ done: */ static herr_t H5Z_prelude_callback(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_type, - hid_t file_id) + hid_t file_id, hsize_t chunk_offset, hbool_t from_reopen) { herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5Z_prelude_callback) assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); - assert (H5I_DATATYPE==H5I_get_type(type_id)); + /*assert (H5I_DATATYPE==H5I_get_type(type_id));*/ /* Check if the property list is non-default */ if(dcpl_id!=H5P_DATASET_CREATE_DEFAULT) { @@ -545,15 +549,20 @@ H5Z_prelude_callback(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_ty /* Make correct callback */ switch(prelude_type) { case H5Z_PRELUDE_CAN_APPLY: +#ifndef H5_USE_16_API /* Check if filter is configured to be able to encode */ if(! fclass->encoder_present) HGOTO_ERROR(H5E_PLINE, H5E_NOENCODER, FAIL, "Filter present but encoding is disabled."); - +#endif /* Check if there is a "can apply" callback */ if(fclass->can_apply) { /* Make callback to filter's "can apply" function */ +#ifdef H5_USE_16_API + herr_t status=(fclass->can_apply)(dcpl_id, type_id, space_id); +#else herr_t status=(fclass->can_apply)(dcpl_id, type_id, space_id, file_id); +#endif /* Check return value */ if(status<=0) { @@ -577,7 +586,11 @@ H5Z_prelude_callback(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_ty /* Check if there is a "set local" callback */ if(fclass->set_local) { /* Make callback to filter's "set local" function */ +#ifdef H5_USE_16_API + if((fclass->set_local)(dcpl_id, type_id, space_id)<0) { +#else if((fclass->set_local)(dcpl_id, type_id, space_id, file_id)<0) { +#endif /* We're leaving, so close dataspace */ if(H5I_dec_ref(space_id)<0) HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace") @@ -588,11 +601,72 @@ H5Z_prelude_callback(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_ty } /* end if */ break; +#ifndef H5_USE_16_API case H5Z_PRELUDE_RESET_LOCAL: /* Check if there is a "reset local" callback */ if(fclass->reset_local) { /* Make callback to filter's "set local" function */ - if((fclass->reset_local)(dcpl_id, type_id, space_id, file_id)<0) { + if((fclass->reset_local)(dcpl_id, type_id, space_id, file_id, from_reopen)<0) { + /* We're leaving, so close dataspace */ + if(H5I_dec_ref(space_id)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace") + + /* Indicate error during filter callback */ + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "error during user callback") + } /* end if */ + } /* end if */ + break; + + case H5Z_PRELUDE_DELETE_LOCAL: + /* Check if there is a "delete local" callback */ + if(fclass->delete_local) { + /* Make callback to filter's "delete local" function */ + if((fclass->delete_local)(dcpl_id)<0) { + /* We're leaving, so close dataspace */ + if(H5I_dec_ref(space_id)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace") + + /* Indicate error during filter callback */ + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "error during user callback") + } /* end if */ + } /* end if */ + break; + + case H5Z_PRELUDE_CLOSE_LOCAL: + /* Check if there is a "close local" callback */ + if(fclass->close_local) { + /* Make callback to filter's "close local" function */ + if((fclass->close_local)(dcpl_id)<0) { + /* We're leaving, so close dataspace */ + if(H5I_dec_ref(space_id)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace") + + /* Indicate error during filter callback */ + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "error during user callback") + } /* end if */ + } /* end if */ + break; + + case H5Z_PRELUDE_CHANGE_LOCAL: + /* Check if there is a "change local" callback */ + if(fclass->change_local) { + /* Make callback to filter's "change local" function */ + if((fclass->change_local)(dcpl_id, chunk_offset)<0) { + /* We're leaving, so close dataspace */ + if(H5I_dec_ref(space_id)<0) + HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace") + + /* Indicate error during filter callback */ + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "error during user callback") + } /* end if */ + } /* end if */ + break; + + case H5Z_PRELUDE_EVICT_LOCAL: + /* Check if there is a "evict local" callback */ + if(fclass->evict_local) { + /* Make callback to filter's "evict local" function */ + if((fclass->evict_local)(dcpl_id, chunk_offset)<0) { /* We're leaving, so close dataspace */ if(H5I_dec_ref(space_id)<0) HGOTO_ERROR (H5E_PLINE, H5E_CANTRELEASE, FAIL, "unable to close dataspace") @@ -602,6 +676,7 @@ H5Z_prelude_callback(hid_t dcpl_id, hid_t type_id, H5Z_prelude_type_t prelude_ty } /* end if */ } /* end if */ break; +#endif default: assert("invalid prelude type" && 0); @@ -653,7 +728,8 @@ H5Z_can_apply (hid_t dcpl_id, hid_t type_id, hid_t file_id) assert (H5I_DATATYPE==H5I_get_type(type_id)); /* Make "can apply" callbacks for filters in pipeline */ - if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_CAN_APPLY, file_id)<0) + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_CAN_APPLY, file_id, + /*UNUSED*/0, /*UNUSED*/0)<0) HGOTO_ERROR(H5E_PLINE, H5E_CANAPPLY, FAIL, "unable to apply filter") done: @@ -693,7 +769,8 @@ H5Z_set_local (hid_t dcpl_id, hid_t type_id, hid_t file_id) assert (H5I_DATATYPE==H5I_get_type(type_id)); /* Make "set local" callbacks for filters in pipeline */ - if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_SET_LOCAL, file_id)<0) + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_SET_LOCAL, file_id, + /*UNUSED*/0, /*UNUSED*/0)<0) HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not set") done: @@ -723,7 +800,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Z_reset_local (hid_t dcpl_id, hid_t type_id, hid_t file_id) +H5Z_reset_local (hid_t dcpl_id, hid_t type_id, hid_t file_id, hbool_t from_reopen) { herr_t ret_value=SUCCEED; /* Return value */ @@ -733,7 +810,8 @@ H5Z_reset_local (hid_t dcpl_id, hid_t type_id, hid_t file_id) assert (H5I_DATATYPE==H5I_get_type(type_id)); /* Make "reset local" callbacks for filters in pipeline */ - if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_RESET_LOCAL, file_id)<0) + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_RESET_LOCAL, file_id, + /*UNUSED*/0, from_reopen)<0) HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not set") done: @@ -742,6 +820,134 @@ done: /*------------------------------------------------------------------------- + * Function: H5Z_change_local + * + * Purpose: Makes callback to change the local values for filters. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 12 March 2008 + * + * Modifications: + *------------------------------------------------------------------------- + */ +herr_t +H5Z_change_local (hid_t dcpl_id, hid_t type_id, hid_t file_id, hsize_t chunk_offset) +{ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_change_local,FAIL) + + assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Make "change local" callbacks for filters in pipeline */ + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_CHANGE_LOCAL, file_id, + chunk_offset, /*UNUSED*/0)<0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not changed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_change_local() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_evict_local + * + * Purpose: Makes callback to evict the local values for filters. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 12 March 2008 + * + * Modifications: + *------------------------------------------------------------------------- + */ +herr_t +H5Z_evict_local (hid_t dcpl_id, hid_t type_id, hid_t file_id, hsize_t chunk_offset) +{ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_evict_local,FAIL) + + assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Make "change local" callbacks for filters in pipeline */ + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_EVICT_LOCAL, file_id, + chunk_offset, /*UNUSED*/0)<0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not changed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_evict_local() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_delete_local + * + * Purpose: Makes callback to delete the local values for filters. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 11 August 2008 + * + * Modifications: + *------------------------------------------------------------------------- + */ +herr_t +H5Z_delete_local(hid_t dcpl_id, hid_t type_id, hid_t file_id) +{ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_delete_local,FAIL) + + assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Make "delete local" callbacks for filters in pipeline */ + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_DELETE_LOCAL, file_id, + /*UNUSED*/0, /*UNUSED*/0)<0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not deleted") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_delete_local() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_close_local + * + * Purpose: Makes callback to close the local values for filters. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 12 March 2008 + * + * Modifications: + *------------------------------------------------------------------------- + */ +herr_t +H5Z_close_local (hid_t dcpl_id, hid_t type_id, hid_t file_id) +{ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_close_local,FAIL) + + assert (H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Make "close local" callbacks for filters in pipeline */ + if(H5Z_prelude_callback(dcpl_id, type_id, H5Z_PRELUDE_CLOSE_LOCAL, file_id, + /*UNUSED*/0, /*UNUSED*/0)<0) + HGOTO_ERROR(H5E_PLINE, H5E_SETLOCAL, FAIL, "local filter parameters not closed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_close_local() */ + + +/*------------------------------------------------------------------------- * Function: H5Z_modify * * Purpose: Modify filter parameters for specified pipeline. @@ -1010,14 +1216,17 @@ done: * Programmer: Robb Matzke * Tuesday, August 4, 1998 * - * Modifications: + * Modifications: Raymond Lu + * 12 March 2008 + * Added an additional parameter of CHUNK_OFFSET for the filter + * of modifying dataset's datatype. * *------------------------------------------------------------------------- */ herr_t H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, unsigned *filter_mask/*in,out*/, H5Z_EDC_t edc_read, - H5Z_cb_t cb_struct, size_t *nbytes/*in,out*/, + hsize_t chunk_offset, H5Z_cb_t cb_struct, size_t *nbytes/*in,out*/, size_t *buf_size/*in,out*/, void **buf/*in,out*/) { size_t i, idx, new_nbytes; @@ -1058,8 +1267,12 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, #endif tmp_flags=flags|(pline->filter[idx].flags); tmp_flags|=(edc_read== H5Z_DISABLE_EDC) ? H5Z_FLAG_SKIP_EDC : 0; - new_nbytes = (fclass->filter)(tmp_flags, pline->filter[idx].cd_nelmts, - pline->filter[idx].cd_values, *nbytes, buf_size, buf); + +#ifdef H5_USE_16_API + new_nbytes = (fclass->filter)(tmp_flags, pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, *nbytes, buf_size, buf); +#else + new_nbytes = (fclass->filter)(tmp_flags, chunk_offset, pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, *nbytes, buf_size, buf); +#endif #ifdef H5Z_DEBUG H5_timer_end(&(fstats->stats[1].timer), &timer); @@ -1099,8 +1312,13 @@ H5Z_pipeline(const H5O_pline_t *pline, unsigned flags, fstats=&H5Z_stat_table_g[fclass_idx]; H5_timer_begin(&timer); #endif - new_nbytes = (fclass->filter)(flags|(pline->filter[idx].flags), pline->filter[idx].cd_nelmts, - pline->filter[idx].cd_values, *nbytes, buf_size, buf); + +#ifdef H5_USE_16_API + new_nbytes = (fclass->filter)(flags|(pline->filter[idx].flags), pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, *nbytes, buf_size, buf); +#else + new_nbytes = (fclass->filter)(flags|(pline->filter[idx].flags), chunk_offset, pline->filter[idx].cd_nelmts, pline->filter[idx].cd_values, *nbytes, buf_size, buf); +#endif + #ifdef H5Z_DEBUG H5_timer_end(&(fstats->stats[0].timer), &timer); fstats->stats[0].total += MAX(*nbytes, new_nbytes); @@ -1161,7 +1379,7 @@ H5Z_filter_info(const H5O_pline_t *pline, H5Z_filter_t filter) break; /* Check if the filter was not already in the pipeline */ - if(idx>pline->nused) + if(idx>=pline->nused) HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, NULL, "filter not in pipeline") /* Set return value */ @@ -1303,15 +1521,19 @@ done: /*------------------------------------------------------------------------- - * Function: H5Zget_filter_info + * Function: H5Zget_filter_info + * + * Purpose: Gets information about a pipeline data filter and stores it + * in filter_config_flags. * - * Purpose: Gets information about a pipeline data filter and stores it - * in filter_config_flags. + * Return: zero on success / negative on failure * - * Return: zero on success / negative on failure + * Programmer: James Laird and Nat Furrer + * Monday, June 7, 2004 * - * Programmer: James Laird and Nat Furrer - * Monday, June 7, 2004 + * Modification: Raymond Lu + * 12 March 2008 + * Added compatibility for v1.6. * *------------------------------------------------------------------------- */ @@ -1324,6 +1546,28 @@ H5Zget_filter_info(H5Z_filter_t filter, unsigned int *filter_config_flags) FUNC_ENTER_API(H5Zget_filter_info, FAIL) H5TRACE2("e", "Zf*Iu", filter, filter_config_flags); +#ifdef H5_USE_16_API + if (filter_config_flags != NULL) + { + if (filter == H5Z_FILTER_SZIP) + { + *filter_config_flags = 0; +#ifdef H5_HAVE_FILTER_SZIP + if(SZ_encoder_enabled()>0) + *filter_config_flags |= H5Z_FILTER_CONFIG_ENCODE_ENABLED; +#endif /* H5_HAVE_FILTER_SZIP */ + *filter_config_flags |= H5Z_FILTER_CONFIG_DECODE_ENABLED; + } + else + *filter_config_flags = H5Z_FILTER_CONFIG_DECODE_ENABLED | H5Z_FILTER_CONFIG_ENCODE_ENABLED; + + /* Make sure the filter exists */ + if (H5Z_find(filter) == NULL) { + *filter_config_flags = 0; + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Filter not defined") + } + } +#else /* Look up the filter class info */ if(NULL == (fclass = H5Z_find(filter))) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Filter not defined") @@ -1337,6 +1581,7 @@ H5Zget_filter_info(H5Z_filter_t filter, unsigned int *filter_config_flags) if(fclass->decoder_present) *filter_config_flags |= H5Z_FILTER_CONFIG_DECODE_ENABLED; } /* end if */ +#endif done: FUNC_LEAVE_API(ret_value) diff --git a/src/H5Zdeflate.c b/src/H5Zdeflate.c index a954c2a..b8fc124 100644 --- a/src/H5Zdeflate.c +++ b/src/H5Zdeflate.c @@ -32,22 +32,42 @@ # include "zlib.h" #endif +#ifndef H5_USE_16_API /* Local function prototypes */ -static size_t H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); +static size_t H5Z_filter_deflate (unsigned flags, hsize_t UNUSED chunk_offset, + size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ const H5Z_class_t H5Z_DEFLATE[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_DEFLATE, /* Filter id number */ - 1, /* encoder_present flag (set to true) */ - 1, /* decoder_present flag (set to true) */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ "deflate", /* Filter name for debugging */ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ H5Z_filter_deflate, /* The actual filter function */ }}; +#else +/* Local function prototypes */ +static size_t H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], + size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class_t H5Z_DEFLATE[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_DEFLATE, /* Filter id number */ + "deflate", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + H5Z_filter_deflate, /* The actual filter function */ +}}; +#endif #define H5Z_DEFLATE_SIZE_ADJUST(s) (HDceil(((double)(s))*1.001)+12) @@ -68,14 +88,19 @@ const H5Z_class_t H5Z_DEFLATE[1] = {{ * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API static size_t -H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, - size_t *buf_size, void **buf) +H5Z_filter_deflate (unsigned flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#else +static size_t +H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], + size_t nbytes, size_t *buf_size, void **buf) +#endif { void *outbuf = NULL; /* Pointer to new buffer */ int status; /* Status from zlib operation */ - size_t ret_value; /* Return value */ + size_t ret_value = nbytes; /* Return value */ FUNC_ENTER_NOAPI(H5Z_filter_deflate, 0) @@ -83,7 +108,7 @@ H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, if (cd_nelmts!=1 || cd_values[0]>9) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, 0, "invalid deflate aggression level") - if (flags & H5Z_FLAG_REVERSE) { + if (flags & H5Z_FLAG_REVERSE) { /*read*/ /* Input; uncompress */ z_stream z_strm; /* zlib parameters */ size_t nalloc = *buf_size; /* Number of bytes for output (compressed) buffer */ @@ -148,8 +173,7 @@ H5Z_filter_deflate (unsigned flags, size_t cd_nelmts, /* Finish uncompressing the stream */ (void)inflateEnd(&z_strm); - } - else { + } else if (!(flags & H5Z_FLAG_INVMASK) || (flags & H5Z_FLAG_SKIP_CRECORD)) { /*Write*/ /* * Output; compress but fail if the result would be larger than the * input. The library doesn't provide in-place compression, so we @@ -197,4 +221,3 @@ done: FUNC_LEAVE_NOAPI(ret_value) } #endif /* H5_HAVE_FILTER_DEFLATE */ - diff --git a/src/H5Zdtype_modify.c b/src/H5Zdtype_modify.c index b3f074f..9a99cd8 100644 --- a/src/H5Zdtype_modify.c +++ b/src/H5Zdtype_modify.c @@ -21,21 +21,53 @@ #define H5Z_PACKAGE /*suppress error about including H5Zpkg */ #include "H5private.h" /* Generic Functions */ +#include "H5B2private.h" /* v2 B-trees */ #include "H5Eprivate.h" /* Error handling */ #include "H5Dprivate.h" /* Dataset */ #include "H5Fprivate.h" /* File access */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5MMprivate.h" /* Memory management */ #include "H5SMprivate.h" /* Shared Object Header Messages */ +#include "H5Tprivate.h" /* Datatype */ #include "H5Zpkg.h" /* Data filters */ #ifdef H5_HAVE_FILTER_DTYPE_MODIFY +/* Current version of the filter*/ +#define H5Z_DTYPE_MODIFY_VERS (1) + +/* Number of client data elements */ +#define CD_NELMTS 16 + +#define H5Z_MDTYPE_DEC 0 +#define H5Z_MDTYPE_INC 1 + +/* v2 B-tree creation macros for 'datatype' field index */ +/* How to determine the node size? */ +#define H5Z_DTYPE_BT2_NODE_SIZE 128 +#define H5Z_DTYPE_BT2_MERGE_PERC 40 +#define H5Z_DTYPE_BT2_SPLIT_PERC 100 + /* Local function prototypes */ static herr_t H5Z_set_local_dtype_modify(hid_t dcpl_id, hid_t dtype_id, hid_t space_id, hid_t file_id); static herr_t H5Z_reset_local_dtype_modify(hid_t dcpl_id, hid_t dtype_id, hid_t space_id, - hid_t file_id); -static size_t H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, + hid_t file_id, hbool_t from_reopen); +static herr_t H5Z_change_local_dtype_modify(hid_t dcpl_id, hsize_t chunk_offset); +static herr_t H5Z_evict_local_dtype_modify(hid_t dcpl_id, hsize_t chunk_offset); +static herr_t H5Z_delete_local_dtype_modify(hid_t dcpl_id); +static herr_t H5Z_close_local_dtype_modify(hid_t dcpl_id); +static herr_t H5Z_free_slist_node(void *item, void UNUSED *key, void UNUSED *operator_data); +static herr_t H5Z_btree2_dtype_store(void *_nrecord, const void *_udata); +static herr_t H5Z_btree2_dtype_retrieve(void *udata, const void *nrecord); +static herr_t H5Z_btree2_dtype_encode(const H5F_t *f, uint8_t *raw, const void *_nrecord); +static herr_t H5Z_btree2_dtype_decode(const H5F_t *f, const uint8_t *raw, void *_nrecord); +static herr_t H5Z_btree2_dtype_compare(const void *_bt2_udata, const void *_bt2_rec); +static herr_t H5Z_btree2_dtype_find_cb(void *record, void *op_data); +static herr_t H5Z_btree2_dtype_remove_cb(void *record, void *op_data); +static herr_t H5Z_btree2_dtype_delete_cb(void *record, void *op_data); + +static size_t H5Z_filter_dtype_modify (unsigned flags, hsize_t chunk_offset, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ @@ -47,15 +79,53 @@ H5Z_class_t H5Z_DTYPE_MODIFY[1] = {{ "dtype_modify", /* Filter name for debugging */ NULL, /* The "can apply" callback */ H5Z_set_local_dtype_modify, /* The "set local" callback */ - H5Z_reset_local_dtype_modify, /* The "reset local" callback */ + H5Z_reset_local_dtype_modify, /* The "reset local" callback */ + H5Z_change_local_dtype_modify,/* The "change local" callback*/ + H5Z_evict_local_dtype_modify, /* The "evict local" callback */ + H5Z_delete_local_dtype_modify, /* The "delete local" callback */ + H5Z_close_local_dtype_modify, /* The "close local" callback */ H5Z_filter_dtype_modify, /* The actual filter function */ }}; +typedef struct H5Z_slist_node_t { + H5T_t *dtype; + hbool_t type_modified; + hsize_t key; +} H5Z_slist_node_t; + +/* Typedef for native 'name' field index records in the v2 B-tree */ +typedef struct H5Z_mdtype_bt2_rec_t { + uint8_t version; + uint8_t is_shared; + uint64_t numb_ref; + H5T_t *dtype; + hbool_t found; + uint8_t adjust; +} H5Z_mdtype_bt2_rec_t; + +/* The v2 B-tree class for indexing 'datatype' field */ +const H5B2_class_t H5Z_BT2_DTYPE[1]={{ /* B-tree class information */ + H5B2_MODIFY_TYPE_ID, /* Type of B-tree */ + sizeof(H5Z_mdtype_bt2_rec_t), /* Size of native record */ + H5Z_btree2_dtype_store, /* Record storage callback */ + H5Z_btree2_dtype_retrieve, /* Record retrieval callback */ + H5Z_btree2_dtype_compare, /* Record comparison callback */ + H5Z_btree2_dtype_encode, /* Record encoding callback */ + H5Z_btree2_dtype_decode, /* Record decoding callback */ + NULL /* Record debugging callback */ +}}; + + +/* Declare a free list to manage H5Z_slist_node_t objects */ +H5FL_DEFINE_STATIC(H5Z_slist_node_t); + /*------------------------------------------------------------------------- * Function: H5Z_set_local_dtype_modify * - * Purpose: Set the "local" dataset parameters for datatype modification + * Purpose: Set the "local" dataset parameters for datatype modification. + * They are the ID of the dataset's datatype and the file + * pointer. * * Return: Success: Non-negative * Failure: Negative @@ -71,36 +141,149 @@ static herr_t H5Z_set_local_dtype_modify(hid_t dcpl_id, hid_t dtype_id, hid_t UNUSED space_id, hid_t file_id) { + H5P_genplist_t *dc_plist = NULL; /* New Property list */ H5F_t *file=NULL; /* File object for file ID */ - size_t cd_nelmts = 1; /* Number of filter parameters */ - unsigned cd_values[8]={0,0,0,0,0,0,0,0}; /* Filter parameters */ + H5T_t *dt = NULL; /* Datatype object for datatype ID */ + H5SL_t *chunk_slist=NULL; /* Skip list for keeping record of chunks */ + size_t dt_size; /* the size of datatype */ + size_t bt2_rrec_size; /* v2 B-tree raw record size */ + haddr_t baddr; /* Address of B-tree to store dtype info */ + size_t cd_nelmts = 0; /* Number of filter parameters */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned baddr_count = 0; /* Keeps track of btree address size */ + unsigned cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ uint8_t *pp = NULL; herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5Z_set_local_dtype_modify, FAIL) + /* Get the dataset's property list object */ + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + if(NULL == (file = H5I_object_verify(file_id, H5I_FILE))) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID") - /* The first element of the client data is the ID of dataset's datatype */ - cd_values[0] = dtype_id; + /* Check args */ + if (NULL == (dt = H5I_object_verify(dtype_id,H5I_DATATYPE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a datatype"); - pp = (uint8_t*)((unsigned *)cd_values + 1); + /* The first element of the client data is reserved for COUNT, which is + * copied toward the end of this function. COUNT has two digits and + * keeps track of the number of elements in CD_VALUES except itself (first + * digit) and the size of file pointer and skip list pointer (second digit). + */ + cd_nelmts = 1; + + /* The second element of the client data is the ID of dataset's datatype. */ + cd_values[1] = dtype_id; + cd_nelmts += 1; + count = 10; + + pp = (uint8_t*)((unsigned *)cd_values + 2); /* - * The second is the file pointer copied as unsigned integer. Depending - * on the size of pointer, it may take the space of 1 - 2 elements. + * The third is the file pointer copied as unsigned integer. The size of + * the pointer can be 4 or 8 bytes. Turn on the first digit of COUNT + * if the pointer size is 8 bytes. 8-byte pointer should take 2 CD_NELMTS. */ HDmemcpy(pp, &file, sizeof(H5F_t*)); - if(4 == sizeof(H5F_t*)) + count +=10; + if(4 == sizeof(H5F_t*)) { cd_nelmts += 1; - else if(8 == sizeof(H5F_t*)) + pp += 4; + } else if(8 == sizeof(H5F_t*)) { cd_nelmts += 2; - else + count +=1; + pp += 8; + } else HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "not implemented yet") + /* Create skip list to keep track of visited chunk. It's created here + * because the filter needs it when the dataset is just created and the + * data is being written to the file. But keep in mind that the client + * data is saved in the file. Everything is invalid when the file is + * reopened and the client data is retrieved. */ + if((chunk_slist = H5SL_create(H5SL_TYPE_HSIZE, 0.5, (size_t)16)) == NULL) + HGOTO_ERROR(H5E_PLINE, H5E_CANTCREATE, FAIL, "can't create skip list for chunks") + + /* The fourth element is the skip list for chunks. Again the second digit + * indicates whether the pointer is 8 bytes. */ + HDmemcpy(pp, &chunk_slist, sizeof(H5SL_t*)); + count += 10; + if(4 == sizeof(H5SL_t*)) { + cd_nelmts += 1; + pp += 4; + } else if(8 == sizeof(H5SL_t*)) { + count += 1; + cd_nelmts += 2; + pp += 8; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "not implemented yet") + + /* Set the COUNT */ + cd_values[0] = count; + + /* Generally only support committed or shared datatype. Check it + * here because this function can be called by H5D_create. We also allow + * integer or floating number because of their limited encoding size. + */ + if(!H5T_committed(dt) && + !H5SM_can_share(file, H5AC_dxpl_id, NULL, NULL, H5O_DTYPE_ID, dt)) { + H5T_class_t tclass; + if((tclass = H5T_get_class(dt, TRUE)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "can't find datatype class") + + if(tclass != H5T_INTEGER && tclass != H5T_FLOAT) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADTYPE, FAIL, "violate datatype restriction") + } + + /* Check if the datatype should be (or are already) shared in the SOHM table */ + if(H5SM_try_share(file, H5AC_dxpl_id, NULL, H5O_DTYPE_ID, dt, NULL) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADMESG, FAIL, "trying to share datatype failed") + + /* Get the encoding size of the dataset's datatype */ + if((dt_size = H5O_msg_raw_size(file, H5O_DTYPE_ID, FALSE, dt))==0) + HGOTO_ERROR(H5E_DATATYPE, H5E_BADSIZE, 0, "can't find datatype size") + + /* If the datatype is committed or shared, its size should be the address plus + * index of heap, which is 12. If the datatype is predefined, the encoding + * size should be at most 20 bytes (for floating number. See H5O_dtype_size in + * H5Odtype.c). Because the datatype may be changed later, we make it big + * enough now. + */ + if(dt_size < 20) + dt_size = 20; + + /* Create a B-tree to store the info of all the dtypes used by the chunks. */ + bt2_rrec_size = 1 + /* version number */ + 1 + /* flag for shared datatype */ + dt_size + /* datatype info */ + 8; /* Number of reference */ + + if(H5B2_create(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, (size_t)H5Z_DTYPE_BT2_NODE_SIZE, + bt2_rrec_size, H5Z_DTYPE_BT2_SPLIT_PERC, H5Z_DTYPE_BT2_MERGE_PERC, &baddr/*out*/) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "can't create B-tree") + + /* the B-tree address can be 8 or 16 byte in size */ + if(8 == sizeof(haddr_t)) { + baddr_count = 2; + cd_nelmts += 2; + } else if(16 == sizeof(haddr_t)) { + baddr_count = 4; + cd_nelmts += 4; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "not implemented yet") + + HDmemcpy(pp, &baddr_count, sizeof(unsigned)); + pp += 4; + cd_nelmts += 1; + + HDmemcpy(pp, &baddr, sizeof(haddr_t)); + /* Modify the filter's parameters for this dataset */ - if(H5Pmodify_filter(dcpl_id, H5Z_FILTER_DTYPE_MODIFY, H5Z_FLAG_MANDATORY, (size_t)cd_nelmts, cd_values) < 0) + if(H5P_modify_filter(dc_plist, H5Z_FILTER_DTYPE_MODIFY, H5Z_FLAG_MANDATORY, + (size_t)cd_nelmts, cd_values) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't set local parameters") done: @@ -114,7 +297,8 @@ done: * Purpose: Reset the "local" dataset parameters for datatype * modification. This function is the same as * H5Z_set_local_dtype_modify. But it's called after the - * dataset has been created. + * dataset has been created to update some info like file_id + * and datatype. It's called by H5D_modify_dtype and H5D_open. * * Return: Success: Non-negative * Failure: Negative @@ -128,14 +312,150 @@ done: */ static herr_t H5Z_reset_local_dtype_modify(hid_t dcpl_id, hid_t dtype_id, hid_t UNUSED space_id, - hid_t file_id) + hid_t file_id, hbool_t from_reopen) { + H5P_genplist_t *dc_plist = NULL; /* New Property list */ + H5F_t *file=NULL; /* File object for file ID */ + H5SL_t *chunk_slist=NULL; /* Skip list for keeping record of chunks */ + size_t cd_nelmts = 0; /* Number of filter parameters */ + size_t orig_cd_nelmts = CD_NELMTS; /* Number of filter parameters */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned orig_count = 0; /* Keeps track of pointer size */ + unsigned baddr_count = 0; /* Keeps track of pointer size */ + unsigned orig_baddr_count = 0; /* Keeps track of pointer size */ + unsigned cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ + unsigned orig_cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ + uint8_t *pp = NULL; + uint8_t *orig_pp = NULL; herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5Z_reset_local_dtype_modify, FAIL) - if((ret_value = H5Z_set_local_dtype_modify(dcpl_id, dtype_id, space_id, file_id))<0) - HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't reset local parameters") + /* Get the dataset's property list object */ + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + + if(H5P_get_filter_by_id(dc_plist, H5Z_FILTER_DTYPE_MODIFY, NULL, + &orig_cd_nelmts, orig_cd_values, (size_t)0, NULL, NULL) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get filter info") + + orig_count = orig_cd_values[0]; + + if(NULL == (file = H5I_object_verify(file_id, H5I_FILE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID") + + /* The first element of the client data is reserved for COUNT, which is + * copied toward the end of this function. COUNT has two digits and + * keeps track of the number of elements in CD_VALUES except itself (first + * digit) and the size of file pointer and skip list pointer (second digit). + */ + cd_nelmts = 1; + + /* The second element of the client data is the ID of dataset's datatype */ + cd_values[1] = dtype_id; + cd_nelmts += 1; + count = 10; + + pp = (uint8_t*)((unsigned *)cd_values + 2); + + /* + * The third is the file pointer copied as unsigned integer. Depending + * on the size of pointer, it may take the space of 1 - 2 elements. + */ + HDmemcpy(pp, &file, sizeof(H5F_t*)); + if(4 == sizeof(H5F_t*)) { + cd_nelmts += 1; + count += 10; + pp += 4; + } else if(8 == sizeof(H5F_t*)) { + cd_nelmts += 2; + count += 10; + count += 1; + pp += 8; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "not implemented yet") + + /* Check whether the skip list for visited chunks has been created. If the first + * digit of original COUNT is 2, there're only 2 elements, and the skip list hasn't + * been created. If this function is called by H5Dopen and FROM_REOPEN is set, the + * pointer value is invalid because the client data was saved in the file. Create + * a new skip list in this case. If it's 3 and FROM_REOPEN isn't set, the skip + * list has been created. Simply copy the pointer over. + */ + if(orig_count < 30 || from_reopen) { + /* Create skip list to keep track of visited chunk */ + if((chunk_slist = H5SL_create(H5SL_TYPE_HSIZE, 0.5, (size_t)16)) == NULL) + HGOTO_ERROR(H5E_PLINE, H5E_CANTCREATE, FAIL, "can't create skip list for chunks") + + /* The fourth element is the skip list for chunks. Again the second digit + * indicates whether the pointer is 8 bytes. */ + HDmemcpy(pp, &chunk_slist, sizeof(H5SL_t*)); + count += 10; + if(4 == sizeof(H5SL_t*)) { + cd_nelmts += 1; + pp += 4; + } else if(8 == sizeof(H5SL_t*)) { + count += 1; + cd_nelmts += 2; + pp += 8; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "not implemented yet") + + if(orig_count == 20) + orig_pp = (uint8_t*)((unsigned *)orig_cd_values + 3); + else if(orig_count > 20 && orig_count <= 30) + orig_pp = (uint8_t*)((unsigned *)orig_cd_values + 4); + else if(orig_count > 30) + orig_pp = (uint8_t*)((unsigned *)orig_cd_values + 6); + } else if (orig_count == 30 && !from_reopen) { + /* Copy the skip list. The size of pointer is 4 bytes. */ + orig_pp = (uint8_t*)((unsigned *)orig_cd_values + 3); + HDmemcpy(pp, orig_pp, 4); + count += 10; + cd_nelmts += 1; + pp += 4; + orig_pp += 4; + } else if (orig_count > 30 && !from_reopen) { + /* Copy the skip list. The size of pointer is 8 bytes. */ + orig_pp = (uint8_t*)((unsigned *)orig_cd_values + 4); + HDmemcpy(pp, orig_pp, 8); + count += 10; + count += 1; + cd_nelmts += 2; + pp += 8; + orig_pp += 8; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "not implemented yet") + + /* Set the COUNT */ + cd_values[0] = count; + + /* Copy B-tree address to the new client data */ + HDmemcpy(&orig_baddr_count, orig_pp, sizeof(unsigned)); + orig_pp += 4; + + /* the B-tree address can be 8 or 16 byte in size */ + if(orig_baddr_count == 2) { + baddr_count = 2; + HDmemcpy(pp, &baddr_count, sizeof(unsigned)); + pp += 4; + cd_nelmts += 1; + HDmemcpy(pp, orig_pp, 8); + cd_nelmts += 2; + } else if(orig_baddr_count == 4) { + baddr_count = 4; + HDmemcpy(pp, &baddr_count, sizeof(unsigned)); + pp += 4; + cd_nelmts += 1; + HDmemcpy(pp, orig_pp, 16); + cd_nelmts += 4; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTCOPY, FAIL, "not implemented yet") + + /* Modify the filter's parameters for this dataset */ + if(H5P_modify_filter(dc_plist, H5Z_FILTER_DTYPE_MODIFY, H5Z_FLAG_MANDATORY, + (size_t)cd_nelmts, cd_values) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTSET, FAIL, "can't set local parameters") done: FUNC_LEAVE_NOAPI(ret_value) @@ -143,6 +463,631 @@ done: /*------------------------------------------------------------------------- + * Function: H5Z_change_local_dtype_modify + * + * Purpose: Search the node of the skip list for chunk info and update it. + * This function is called by H5D_istore_conv_dtype when the + * datatype is changed and all the chunks in the cache have + * to be converted. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Raymond Lu + * 16 April 2008 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_change_local_dtype_modify(hid_t dcpl_id, hsize_t chunk_offset) +{ + H5P_genplist_t *dc_plist = NULL; /* New Property list */ + H5SL_t *chunk_slist=NULL; /* Skip list for keeping record of chunks */ + H5Z_slist_node_t *node; + size_t cd_nelmts=CD_NELMTS; /* Number of filter parameters */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ + uint8_t *pp = NULL; + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_change_local_dtype_modify, FAIL) + + HDassert(H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Get the dataset's property list object */ + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + + if(H5P_get_filter_by_id(dc_plist, H5Z_FILTER_DTYPE_MODIFY, NULL, &cd_nelmts, cd_values, + (size_t)0, NULL, NULL) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get filter info") + + /* If the skip list exists, update the dtype modification info for the chunk */ + count = cd_values[0]; + + if(count >= 30) { + pp = (uint8_t*)((unsigned *)cd_values + 2); + + /* Depending on the size of pointer, retrieve the value of the pointer to + * the skip list. */ + if (count == 30) { + pp += 4; + HDmemcpy(&chunk_slist, pp, 4); + } else if (count > 30) { + pp += 8; + HDmemcpy(&chunk_slist, pp, 8); + } + + if(!chunk_slist) + HGOTO_ERROR(H5E_SLIST, H5E_CANTGET, FAIL, "can't find skip list") + + /* Search the skip list and update the node */ + if(NULL != (node = H5SL_search(chunk_slist, &chunk_offset))) + node->type_modified = TRUE; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_change_local_dtype_modify */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_evict_local_dtype_modify + * + * Purpose: Search the node of the skip list for chunk and remove the + * node. This function is called when the chunk is evicted + * from the cache. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Raymond Lu + * 16 April 2008 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_evict_local_dtype_modify(hid_t dcpl_id, hsize_t chunk_offset) +{ + H5P_genplist_t *dc_plist = NULL; /* New Property list */ + H5SL_t *chunk_slist=NULL; /* Skip list for keeping record of chunks */ + H5Z_slist_node_t *node; + size_t cd_nelmts=CD_NELMTS; /* Number of filter parameters */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ + uint8_t *pp = NULL; + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_evict_local_dtype_modify, FAIL) + + HDassert(H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Get the dataset's property list object */ + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + + if(H5P_get_filter_by_id(dc_plist, H5Z_FILTER_DTYPE_MODIFY, NULL, &cd_nelmts, cd_values, + (size_t)0, NULL, NULL) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get filter info") + + /* If the skip list exists, remove the node for the chunk */ + count = cd_values[0]; + + if(count >= 30 ) { + pp = (uint8_t*)((unsigned *)cd_values + 2); + + /* Depending on the size of pointer, retrieve the value of the pointer to + * the skip list. */ + if (count == 30) { + pp += 4; + HDmemcpy(&chunk_slist, pp, 4); + } else if (count > 30) { + pp += 8; + HDmemcpy(&chunk_slist, pp, 8); + } + + if(!chunk_slist) + HGOTO_ERROR(H5E_SLIST, H5E_CANTGET, FAIL, "can't find skip list") + + /* Search the skip list and update the node */ + if(NULL != (node = H5SL_remove(chunk_slist, &chunk_offset))) { + /* Check whether datatype is committed & decrement ref count + * (to maintain ref. count incr/decr similarity with "shared message" + * type of datatype sharing) + */ + if(H5T_committed(node->dtype)) { + /* Decrement the reference count on the committed datatype */ + if(H5T_link(node->dtype, -1, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, FAIL, "unable to adjust shared datatype link count") + } /* end if */ + + if(H5T_close(node->dtype) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "can't close data type") + + H5FL_FREE(H5Z_slist_node_t, node); + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_evict_local_dtype_modify */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_delete_local_dtype_modify + * + * Purpose: When the dataset is deleted, free the nodes of the skip list + * and delete it. Also decrement the reference counts of the + * datatypes in B-tree nodes, free the nodes of the B-tree and + * delete it. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Raymond Lu + * 11 August 2008 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_delete_local_dtype_modify(hid_t dcpl_id) +{ + H5P_genplist_t *dc_plist = NULL; /* New Property list */ + size_t cd_nelmts=CD_NELMTS; /* Number of filter parameters */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ + uint8_t *pp = NULL; + H5F_t *file = NULL; + unsigned baddr_count = 0; /* Keeps track of pointer size */ + haddr_t baddr; /* B-tree address */ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_delete_local_dtype_modify, FAIL) + + HDassert(H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Get the dataset's property list object */ + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + + if(H5P_get_filter_by_id(dc_plist, H5Z_FILTER_DTYPE_MODIFY, NULL, &cd_nelmts, cd_values, + (size_t)0, NULL, NULL) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get filter info") + + count = cd_values[0]; + pp = (uint8_t*)((unsigned *)cd_values + 2); + + /* Depending on number of elements and the size of pointer, retrieve the value + * of file pointer and skip list pointer. The first digit of CD_NELMTS indicates + * the number of elements in CD_VALUES. The second digit indicates whether the + * pointers are 8 bytes. 0 stands for 4 bytes. Greater than 0 means 8 bytes. + */ + if (count == 20) { + HDmemcpy(&file, pp, 4); + pp += 4; + } else if (20 < count && count < 30) { + HDmemcpy(&file, pp, 8); + pp += 8; + } else if (count == 30) { + HDmemcpy(&file, pp, 4); + pp += 4; + /* Advance over the skip list address */ + pp += 4; + } else if (count > 30) { + HDmemcpy(&file, pp, 8); + pp += 8; + /* Advance over the skip list address */ + pp += 8; + } else + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, 0, "not implemented yet") + + HDmemcpy(&baddr_count, pp, sizeof(unsigned)); + pp += 4; + + /* the B-tree address can be 8 or 16 byte in size */ + if(baddr_count == 2) { + HDmemcpy(&baddr, pp, 8); + } else if(baddr_count == 4) { + HDmemcpy(&baddr, pp, 16); + } + + /* Delete the entire B-tree and decrement reference count of datatype */ + if(H5B2_delete(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, H5Z_btree2_dtype_delete_cb, + NULL) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, FAIL, "can't delete B-tree") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_delete_local_dtype_modify() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_close_local_dtype_modify + * + * Purpose: When the dataset is closed, free the nodes of the skip list + * and delete it. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Raymond Lu + * 22 Feb 2008 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_close_local_dtype_modify(hid_t dcpl_id) +{ + H5P_genplist_t *dc_plist = NULL; /* New Property list */ + H5SL_t *chunk_slist=NULL; /* Skip list for keeping record of chunks */ + size_t cd_nelmts=CD_NELMTS; /* Number of filter parameters */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned cd_values[CD_NELMTS]={0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}; /* Filter parameters */ + uint8_t *pp = NULL; + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_close_local_dtype_modify, FAIL) + + HDassert(H5I_GENPROP_LST==H5I_get_type(dcpl_id)); + + /* Get the dataset's property list object */ + if(NULL == (dc_plist = (H5P_genplist_t *)H5I_object(dcpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get dataset creation property list") + + if(H5P_get_filter_by_id(dc_plist, H5Z_FILTER_DTYPE_MODIFY, NULL, &cd_nelmts, cd_values, (size_t)0, NULL, NULL) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "can't get filter info") + + count = cd_values[0]; + pp = (uint8_t*)((unsigned *)cd_values + 2); + + /* Depending on the size of pointer, retrieve the value of the pointer to + * the skip list. */ + if (count == 30) { + pp += 4; + HDmemcpy(&chunk_slist, pp, 4); + } else if (count > 30) { + pp += 8; + HDmemcpy(&chunk_slist, pp, 8); + } + + /* Free the skip list */ + if(chunk_slist && H5SL_destroy(chunk_slist, H5Z_free_slist_node, NULL) < 0) + HGOTO_ERROR(H5E_SLIST, H5E_CANTFREE, FAIL, "can't free skip list") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_close_local_dtype_modify() */ + +/*------------------------------------------------------------------------- + * Function: H5Z_free_slist_node + * + * Purpose: Private function for H5SL_destroy. Frees the nodes of + * skip list. + * + * Return: Non-negative on success, negative on failure + * + * Programmer: Raymond Lu + * 22 February 2008 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_free_slist_node(void *item, void UNUSED *key, void UNUSED *operator_data/*in,out*/) +{ + H5T_t *dtype = ((H5Z_slist_node_t*)item)->dtype; + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_free_slist_node, FAIL) + + /* Check whether datatype is committed & decrement ref count + * (to maintain ref. count incr/decr similarity with "shared message" + * type of datatype sharing) + */ + if(H5T_committed(dtype)) { + /* Decrement the reference count on the committed datatype */ + if(H5T_link(dtype, -1, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, FAIL, "unable to adjust shared datatype link count") + } /* end if */ + + if(H5T_close(dtype) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "can't close data type") + + H5FL_FREE(H5Z_slist_node_t, item); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5Z_free_slist_node */ + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_store + * + * Purpose: Store user information into native record for v2 B-tree + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_store(void *_nrecord, const void *_udata) +{ + const H5Z_mdtype_bt2_rec_t *udata = (const H5Z_mdtype_bt2_rec_t *)_udata; + H5Z_mdtype_bt2_rec_t *nrecord = (H5Z_mdtype_bt2_rec_t *)_nrecord; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(H5Z_btree2_dtype_store, FAIL) + + /* Copy user information info native record */ + nrecord->version = udata->version; + nrecord->is_shared = udata->is_shared; + nrecord->numb_ref = udata->numb_ref; + + /* Make a copy of the datatype because Btree resides in the cache separately. + * The dataset can be closed first. */ + if((nrecord->dtype = H5T_copy(udata->dtype, H5T_COPY_ALL)) == NULL) + HGOTO_ERROR(H5E_PLINE, H5E_CANTCOPY, FAIL, "can't copy datatype") + +done: + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5Z_btree2_dtype_store() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_retrieve + * + * Purpose: Retrieve native information from record for v2 B-tree + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_retrieve(void *udata, const void *nrecord) +{ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5Z_btree2_dtype_retrieve) + + *(H5Z_mdtype_bt2_rec_t *)udata = *(const H5Z_mdtype_bt2_rec_t *)nrecord; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5Z_btree2_dtype_retrieve() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_encode + * + * Purpose: Encode native information into raw form for storing on disk + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_encode(const H5F_t *f, uint8_t *raw, const void *_nrecord) +{ + const H5Z_mdtype_bt2_rec_t *nrecord = (const H5Z_mdtype_bt2_rec_t *)_nrecord; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(H5Z_btree2_dtype_encode, FAIL) + + /* Encode the record's fields */ + *raw++ = nrecord->version; + + /* Encode the flag indicating whether the dtype is shared in Btree node buffer */ + *raw++ = nrecord->is_shared; + + UINT64ENCODE(raw, nrecord->numb_ref); + + /* encode datatype info into Btree node buffer */ + if(H5O_msg_encode((H5F_t *)f, H5O_DTYPE_ID, FALSE, (unsigned char *)raw, + (const void *)nrecord->dtype)<0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, FAIL, "can't encode object") + + if(H5T_close(nrecord->dtype) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CLOSEERROR, FAIL, "can't close datatype") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5Z_btree2_dtype_encode() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_decode + * + * Purpose: Decode raw disk form of record into native form + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_decode(const H5F_t *f, const uint8_t *raw, void *_nrecord) +{ + H5Z_mdtype_bt2_rec_t *nrecord = (H5Z_mdtype_bt2_rec_t *)_nrecord; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(H5Z_btree2_dtype_decode, FAIL) + + /* Decode the record's fields */ + nrecord->version = *raw++; + nrecord->is_shared = *raw++; + UINT64DECODE(raw, nrecord->numb_ref); + + /* + * Decode the original datatype info from the chunk of data which might + * be shared or not shared. + */ + if(nrecord->is_shared && (nrecord->dtype=(H5T_t *)H5O_msg_decode((H5F_t *)f, + H5AC_dxpl_id, H5O_DTYPE_ID, (const unsigned char *)raw, H5O_MSG_FLAG_SHARED))==NULL) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "can't decode object") + else if(!nrecord->is_shared && (nrecord->dtype=(H5T_t *)H5O_msg_decode((H5F_t *)f, + H5AC_dxpl_id, H5O_DTYPE_ID, (const unsigned char *)raw, H5O_MSG_FLAG_WAS_UNKNOWN))==NULL) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, FAIL, "can't decode object") + +done: + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5Z_btree2_dtype_decode() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_compare + * + * Purpose: Compare two native information records, according to some key + * + * Return: <0 if rec1 < rec2 + * =0 if rec1 == rec2 + * >0 if rec1 > rec2 + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_compare(const void *_bt2_udata, const void *_bt2_rec) +{ + const H5Z_mdtype_bt2_rec_t *bt2_udata = (const H5Z_mdtype_bt2_rec_t *)_bt2_udata; + const H5Z_mdtype_bt2_rec_t *bt2_rec = (const H5Z_mdtype_bt2_rec_t *)_bt2_rec; + herr_t ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5Z_btree2_dtype_compare) + + /* Sanity check */ + HDassert(bt2_udata); + HDassert(bt2_rec); + + ret_value = H5T_cmp(bt2_udata->dtype, bt2_rec->dtype, FALSE); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5Z_btree2_dtype_compare() */ + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_find_cb + * + * Purpose: Callback function for H5B2_find function. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_find_cb(void *record, void *op_data) +{ + H5Z_mdtype_bt2_rec_t *my_record = (H5Z_mdtype_bt2_rec_t *)record; + H5Z_mdtype_bt2_rec_t *udata = (H5Z_mdtype_bt2_rec_t *)op_data; + int compare; + herr_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5Z_btree2_dtype_find_cb) + + compare = H5T_cmp(my_record->dtype, udata->dtype, FALSE); + + if(compare == 0) { + if(udata->adjust == H5Z_MDTYPE_INC) + my_record->numb_ref++; + else if(udata->adjust == H5Z_MDTYPE_DEC) + my_record->numb_ref--; + + udata->numb_ref = my_record->numb_ref; + udata->found = TRUE; + ret_value = SUCCEED; + } + + FUNC_LEAVE_NOAPI(ret_value) +} + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_remove_cb + * + * Purpose: Callback function for H5B2_remove function. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_remove_cb(void *record, void *op_data) +{ + H5Z_mdtype_bt2_rec_t *my_record = (H5Z_mdtype_bt2_rec_t *)record; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_btree2_dtype_remove_cb, FAIL) + + /* Close the datatype saved in Btree node */ + if(H5T_close(my_record->dtype) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CLOSEERROR, FAIL, "can't close datatype") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} + + +/*------------------------------------------------------------------------- + * Function: H5Z_btree2_dtype_delete_cb + * + * Purpose: Callback function for H5B2_delete function. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Raymond Lu + * 5 August 2008 + *------------------------------------------------------------------------- + */ +static herr_t +H5Z_btree2_dtype_delete_cb(void *record, void *op_data) +{ + H5Z_mdtype_bt2_rec_t *my_record = (H5Z_mdtype_bt2_rec_t *)record; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5Z_btree2_dtype_delete_cb, FAIL) + + /* Check whether datatype is committed & decrement ref count + * (to maintain ref. count incr/decr similarity with "shared message" + * type of datatype sharing) + */ + if(H5T_committed(my_record->dtype)) { + /* Decrement the reference count on the committed datatype */ + if(H5T_link(my_record->dtype, (int)-my_record->numb_ref, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, FAIL, "unable to adjust shared datatype link count") + } /* end if */ + + if(H5T_close(my_record->dtype) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CLOSEERROR, FAIL, "can't close datatype") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} + + +/*------------------------------------------------------------------------- * Function: H5Z_filter_dtype_modify * * Purpose: Implement the I/O filter of the datatype modification for @@ -163,8 +1108,8 @@ done: */ /* ARGSUSED */ static size_t -H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], - size_t nbytes, size_t *buf_size, void **buf) +H5Z_filter_dtype_modify (unsigned flags, hsize_t chunk_offset, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) { unsigned char *outbuf = NULL; /* Pointer to new buffer */ hid_t dtype_id; @@ -173,49 +1118,101 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val size_t type_msg_size; size_t nelmts, dbuf_size; H5F_t *file = NULL; + H5SL_t *chunk_slist=NULL; /* Skip list for keeping record of chunks */ + H5Z_slist_node_t *node; uint8_t *pp = NULL; unsigned char *dst=NULL; /* Temporary pointer to destination buffer */ - size_t ret_value; /* Return value */ + unsigned count = 0; /* Keeps track of pointer size */ + unsigned baddr_count = 0; /* Keeps track of pointer size */ + haddr_t baddr; /* B-tree address */ + int8_t is_shared = FALSE; /* Whether dtype is committed or shared */ + size_t ret_value=nbytes; /* Return value */ FUNC_ENTER_NOAPI(H5Z_filter_dtype_modify, 0) - assert(cd_nelmts==2 || cd_nelmts==3); + count = cd_values[0]; + + assert(cd_nelmts>=2 && cd_nelmts<=10); + assert(count>=20 && count<40); /* Get the dataset's datatype */ - dtype_id = cd_values[0]; - pp = (uint8_t*)((unsigned *)cd_values + 1); + dtype_id = cd_values[1]; + pp = (uint8_t*)((unsigned *)cd_values + 2); - /* Depending on the size of pointer, retrieve the value of file pointer */ - if(2 == cd_nelmts) { + /* Depending on number of elements and the size of pointer, retrieve the value + * of file pointer and skip list pointer. The first digit of CD_NELMTS indicates + * the number of elements in CD_VALUES. The second digit indicates whether the + * pointers are 8 bytes. 0 stands for 4 bytes. Greater than 0 means 8 bytes. + */ + if (count == 20) { HDmemcpy(&file, pp, 4); - } else if(3 == cd_nelmts) { + pp += 4; + } else if (20 < count && count < 30) { HDmemcpy(&file, pp, 8); + pp += 8; + } else if (count == 30) { + HDmemcpy(&file, pp, 4); + pp += 4; + HDmemcpy(&chunk_slist, pp, 4); + pp += 4; + } else if (count > 30) { + HDmemcpy(&file, pp, 8); + pp += 8; + HDmemcpy(&chunk_slist, pp, 8); + pp += 8; } else HGOTO_ERROR(H5E_PLINE, H5E_CANTENCODE, 0, "not implemented yet") + HDmemcpy(&baddr_count, pp, sizeof(unsigned)); + pp += 4; + + /* the B-tree address can be 8 or 16 byte in size */ + if(baddr_count == 2) { + HDmemcpy(&baddr, pp, 8); + } else if(baddr_count == 4) { + HDmemcpy(&baddr, pp, 16); + } + /* The current datatype of the dataset */ if(NULL==(dtype=(H5T_t *)H5I_object_verify(dtype_id, H5I_DATATYPE))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, 0, "not a datatype") - if (flags & H5Z_FLAG_REVERSE) { /* Read */ + if (flags & H5Z_FLAG_REVERSE) { /*Read*/ + uint8_t version = 0; + hid_t orig_type_id; + size_t info_size = 0; /* the size of the encoded information except raw data */ + + /* Retrieve and verify version number */ + HDmemcpy(&version, *buf, sizeof(uint8_t)); + + if(version != H5Z_DTYPE_MODIFY_VERS) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, 0, "wrong version number") + + info_size = 1; + + /* Retrieve the flag indicating whether the dtype is committed or shared */ + outbuf = (unsigned char*)*buf + 1; + HDmemcpy(&is_shared, outbuf, sizeof(int8_t)); + info_size += 1; + outbuf += 1; + /* * Decode the original datatype info from the chunk of data which might - * be shared or not shared. Try to treat it as shared then non-shared. + * be shared or not shared. */ - H5E_BEGIN_TRY { - orig_type=(H5T_t *)H5O_msg_decode(file, H5AC_dxpl_id, H5O_DTYPE_ID, - (unsigned char *)*buf, H5O_MSG_FLAG_SHARED); - } H5E_END_TRY; - - if(!orig_type) { - if((orig_type=(H5T_t *)H5O_msg_decode(file, H5AC_dxpl_id, H5O_DTYPE_ID, (unsigned char *)*buf, H5O_MSG_FLAG_WAS_UNKNOWN))==NULL) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, 0, "can't decode object") - } + if(is_shared && (orig_type=(H5T_t *)H5O_msg_decode(file, H5AC_dxpl_id, H5O_DTYPE_ID, + outbuf, H5O_MSG_FLAG_SHARED))==NULL) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, 0, "can't decode object") + else if(!is_shared && (orig_type=(H5T_t *)H5O_msg_decode(file, H5AC_dxpl_id, + H5O_DTYPE_ID, outbuf, H5O_MSG_FLAG_WAS_UNKNOWN))==NULL) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDECODE, 0, "can't decode object") /* Get the encoding size of the original datatype */ if((type_msg_size = H5O_msg_raw_size(file, H5O_DTYPE_ID, FALSE, orig_type))==0) HGOTO_ERROR(H5E_DATATYPE, H5E_BADSIZE, 0, "can't find datatype size") + info_size += type_msg_size; + /* Calculate the size of buffer containing the new data */ if(0 != H5T_cmp(dtype, orig_type, FALSE)) { if((dtype_size = H5T_get_size(dtype))==0) @@ -225,19 +1222,19 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val nelmts = (nbytes-type_msg_size)/orig_size; if(dtype_size <= orig_size) - dbuf_size = nbytes-type_msg_size; + dbuf_size = nbytes-info_size; else dbuf_size = nelmts*dtype_size; } else - dbuf_size = nbytes-type_msg_size; + dbuf_size = nbytes-info_size; /* Allocate the buffer for the data */ if (NULL==(dst=(unsigned char*)H5MM_malloc(dbuf_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate buffer") /* Copy the raw data */ - outbuf = (unsigned char*)*buf + type_msg_size; - HDmemcpy((void*)dst, (void*)outbuf, nbytes-type_msg_size); + outbuf = (unsigned char*)*buf + info_size; + HDmemcpy((void*)dst, (void*)outbuf, nbytes-info_size); /* Free input buffer */ H5MM_xfree(*buf); @@ -248,35 +1245,31 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val */ if(0 != H5T_cmp(dtype, orig_type, FALSE)) { H5P_genplist_t *plist; /* Property list pointer */ - hbool_t is_vlen = FALSE; /* Flag to indicate VL type */ + htri_t is_vlen = FALSE; /* Flag to indicate VL type */ hbool_t vlen_conv = TRUE; /* Transfer property to indicate no conversion for vlen */ H5T_path_t *tpath=NULL; /*type conversion info */ unsigned char *bkg=NULL; /* Pointer to background buffer */ - hid_t orig_type_id; /* If the datatype is or contains vlen, set the property to indicate no conversion * is needed for vlen. The file to memory conversion will take place at a * higher level. */ - if((is_vlen = H5T_detect_class(orig_type, H5T_VLEN)) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to detect dtatypes") + if((is_vlen = H5T_detect_class(orig_type, H5T_VLEN)) == FAIL) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, 0, "unable to detect dtatypes") if(is_vlen ) { vlen_conv = FALSE; if(NULL == (plist = H5P_object_verify(H5AC_dxpl_id, H5P_DATASET_XFER))) - HGOTO_ERROR(H5E_PLIST, H5E_BADATOM, FAIL, "can't find object for ID") + HGOTO_ERROR(H5E_PLIST, H5E_BADATOM, 0, "can't find object for ID") if(H5P_set(plist, H5D_XFER_VLEN_CONV_NAME, &vlen_conv) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "Error setting vlen conv flag") + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, 0, "Error setting vlen conv flag") } /* Find the conversion function */ - if (NULL==(tpath=H5T_path_find(orig_type, dtype, NULL, NULL, /*H5P_DEFAULT*/H5AC_dxpl_id, FALSE))) + if (NULL==(tpath=H5T_path_find(orig_type, dtype, NULL, NULL, H5AC_dxpl_id, FALSE))) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, 0, "unable to convert between src and dst data types"); /* Register the original type and return the ID */ - /*if((orig_type_copy = H5T_copy(orig_type,H5T_COPY_ALL)) == NULL) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, 0, "unable to copy data type")*/ - if((orig_type_id = H5I_register(H5I_DATATYPE, orig_type)) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, 0, "unable to register data type") @@ -286,7 +1279,7 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val } /* Convert the data */ - if (H5T_convert(tpath, orig_type_id, dtype_id, nelmts, (size_t)0, (size_t)0, dst, bkg, /*H5P_DATASET_XFER_DEFAULT*/H5AC_dxpl_id)<0) + if (H5T_convert(tpath, orig_type_id, dtype_id, nelmts, (size_t)0, (size_t)0, dst, bkg, H5AC_dxpl_id)<0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, 0, "data type conversion failed"); /* Set the property of vlen conversion back to normal */ @@ -294,12 +1287,9 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val vlen_conv = TRUE; if(H5P_set(plist, H5D_XFER_VLEN_CONV_NAME, &vlen_conv) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "Error setting vlen conv flag") + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, 0, "Error setting vlen conv flag") } - if(H5Tclose(orig_type_id)<0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCLOSEOBJ, 0, "unable to close data type ID") - if(bkg) H5MM_xfree(bkg); } @@ -308,41 +1298,95 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val if(0 != H5T_cmp(dtype, orig_type, FALSE)) *buf_size = nelmts*dtype_size; else - *buf_size = nbytes - type_msg_size; + *buf_size = nbytes - info_size; *buf = (void*)dst; dst = NULL; outbuf = NULL; ret_value = *buf_size; - } else { /* Write */ - /* Mark the new datatype as being on disk now */ - if(H5T_set_loc(dtype, file, H5T_LOC_DISK) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, 0, "invalid datatype location") + /* If the caller isn't from fractal heap (the flag H5Z_FLAG_SKIP_CRECORD isn't + * set) and the current chunk isn't in the skip list, put the current datatype + * for this chunk in. When this datatype is being changed during the write + * call, the filter knows to decrement the reference count of the current type. + * The key of the node is the linear offset of the chunk in the dataspace. This + * linear offset is translated from the multi-dimensional offsets. + */ + if(!(flags & H5Z_FLAG_SKIP_CRECORD) && chunk_slist) { + /* Allocate space for the shared structure */ + if(NULL == (node = H5FL_MALLOC(H5Z_slist_node_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed") + node->key = chunk_offset; + + if((NULL == H5SL_search(chunk_slist, &(node->key))) && H5T_committed(orig_type)) { + if ((NULL==(node->dtype=H5T_copy(orig_type, H5T_COPY_ALL)))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, 0, "unable to copy data type"); + + /* Check whether datatype is committed & increment ref count + * (to maintain ref. count incr/decr similarity with "shared message" + * type of datatype sharing) + */ + /* Increment the reference count on the committed datatype */ + if(H5T_link(node->dtype, 1, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, 0, "unable to adjust shared datatype link count") + + /* set the flag for the later increment and decrement of reference count */ + if(0 != H5T_cmp(dtype, orig_type, FALSE)) + node->type_modified = TRUE; + else + node->type_modified = FALSE; + + if(H5SL_insert(chunk_slist, node, &(node->key)) < 0) + HGOTO_ERROR(H5E_SLIST, H5E_CANTINSERT, 0, "can't insert chunk node into skip list") + } + } + + /* Close the ID registered for doing data conversion */ + if(0 != H5T_cmp(dtype, orig_type, FALSE)) { + if(H5Tclose(orig_type_id)<0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCLOSEOBJ, 0, "unable to close data type ID") + } + } else if (!(flags & H5Z_FLAG_INVMASK) || (flags & H5Z_FLAG_SKIP_CRECORD)) { /*Write*/ + uint8_t version = H5Z_DTYPE_MODIFY_VERS; + size_t final_size = 0; + htri_t is_shared_int; + herr_t found = FAIL; + H5Z_mdtype_bt2_rec_t udata; + hsize_t bnode_numb = 0; + + /* Set the flag for committed or shared datatype */ + if(H5T_committed(dtype)) + is_shared = TRUE; /* Check if the datatype should be (or are already) shared in the SOHM table */ - if(H5SM_try_share(file, H5AC_dxpl_id, NULL, H5O_DTYPE_ID, dtype, NULL) < 0) + if((is_shared_int = H5SM_try_share(file, H5AC_dxpl_id, NULL, H5O_DTYPE_ID, dtype, NULL)) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_BADMESG, 0, "trying to share datatype failed") - - /* Check whether datatype is committed & increment ref count - * (to maintain ref. count incr/decr similarity with "shared message" - * type of datatype sharing) - */ - if(H5T_committed(dtype)) { - /* Increment the reference count on the shared datatype */ - if(H5T_link(dtype, 1, H5AC_dxpl_id) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, 0, "unable to adjust shared datatype link count") - } /* end if */ + else if(is_shared_int) + is_shared = TRUE; /* Get the encoding size of the dataset's datatype */ if((type_msg_size = H5O_msg_raw_size(file, H5O_DTYPE_ID, FALSE, dtype))==0) HGOTO_ERROR(H5E_DATATYPE, H5E_BADSIZE, 0, "can't find datatype size") + /* Size of final chunk */ + final_size = 1 /* Version number */ + + 1 /* Whether dtype is committed or shared */ + + type_msg_size /* Datatype message size */ + + nbytes; /* Raw data size */ + /* Allocate the buffer for the datatype info and the data */ - if (NULL==(dst=outbuf=(unsigned char*)H5MM_malloc(type_msg_size+nbytes))) + if (NULL==(dst=outbuf=(unsigned char*)H5MM_malloc(final_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "unable to allocate buffer") - /* prepend datatype info to raw data for storage */ + /* Encode version number */ + HDmemcpy(dst, &version, sizeof(uint8_t)); + dst += 1; + + /* Encode the flag indicating whether the dtype is shared in Btree node buffer */ + HDmemcpy(dst, &is_shared, sizeof(int8_t)); + dst += 1; + + /* prepend datatype info to raw data for storage as shared */ if(H5O_msg_encode(file, H5O_DTYPE_ID, FALSE, dst, dtype)<0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTENCODE, 0, "can't encode object") @@ -354,15 +1398,129 @@ H5Z_filter_dtype_modify (unsigned flags, size_t cd_nelmts, const unsigned cd_val H5MM_xfree(*buf); /* Set return values */ - *buf_size = type_msg_size + nbytes; + *buf_size = final_size; *buf = (void*)outbuf; outbuf = NULL; ret_value = *buf_size; + + /* Adjust reference count for the old and new datatypes. */ + if(!(flags & H5Z_FLAG_SKIP_CRECORD) && chunk_slist) { + /* Decrement the reference count of the old datatype and increment + * the reference count of the new datatype if the chunk is in skip list. */ + if((NULL != (node = H5SL_search(chunk_slist, &chunk_offset))) + && H5T_committed(node->dtype) && node->type_modified) { + + udata.version = H5Z_DTYPE_MODIFY_VERS; + udata.is_shared = is_shared; + udata.dtype = node->dtype; + udata.numb_ref = 1; + udata.found = FALSE; + udata.adjust = H5Z_MDTYPE_DEC; + + /* Find out the number of nodes in the Btree */ + if(H5B2_get_nrec(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, &bnode_numb) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, 0, "can't get number of nodes in Btree") + + /* Find the old datatype and decrement the number of references to the type */ + if(bnode_numb) + H5B2_find(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, &udata, + H5Z_btree2_dtype_find_cb, &udata); + + /* If found the old type and no other chunk uses it, delete the node from the + * Btree */ + if(udata.found && !udata.numb_ref) { + /*delete the node*/ + if(H5B2_remove(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, &udata, + H5Z_btree2_dtype_remove_cb, NULL) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, 0, "can't remove node in Btree") + + bnode_numb--; + } + + /* Decrement the reference count of the old datatype in the skipped list. + * 1 for being prepended to the chunk. 1 is for the copy in the skip list.*/ + if(H5T_link(node->dtype, -2, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, 0, "unable to adjust shared datatype link count") + + if(H5T_close(node->dtype) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, 0, "can't close data type") + + /* Update the dtype in the skip list to the current dtype */ + if ((NULL==(node->dtype=H5T_copy(dtype, H5T_COPY_ALL)))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, 0, "unable to copy data type"); + + /* Increment the reference count on the new datatype. 1 is encoded with + * the chunk. 1 is the copy in the skip list */ + if(H5T_committed(node->dtype) && H5T_link(node->dtype, 2, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, 0, "unable to adjust shared datatype link count") + + node->type_modified = FALSE; + + /* Update the node in the skip list? */ + + /* If the dtype info isn't in Btree, insert it and its reference number + * into Btree */ + udata.dtype = dtype; + udata.numb_ref = 1; + udata.found = FALSE; + udata.adjust = H5Z_MDTYPE_INC; + + /* Find the new datatype in the Btree and increment the number of references + * to the type */ + if(bnode_numb) + H5B2_find(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, &udata, + H5Z_btree2_dtype_find_cb, &udata); + + if(udata.found == FALSE && H5B2_insert(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, + baddr, &udata) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINSERT, 0, "can't insert B-tree") + } else if(!node && H5T_committed(dtype)) { + /* Insert the chunk in the skip list. Increment type reference count. */ + /* Allocate space for the node for the skip list */ + if(NULL == (node = H5FL_MALLOC(H5Z_slist_node_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed") + node->key = chunk_offset; + node->type_modified = FALSE; + + if ((NULL==(node->dtype=H5T_copy(dtype, H5T_COPY_ALL)))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, 0, "unable to copy data type"); + + /* Increment the reference count on the new datatype. 1 is encoded with + * the chunk. 1 is the copy in the skip list */ + if(H5T_link(dtype, 2, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_LINKCOUNT, 0, "unable to adjust shared datatype link count") + + /* Insert into the skip list */ + if(H5SL_insert(chunk_slist, node, &(node->key)) < 0) + HGOTO_ERROR(H5E_SLIST, H5E_CANTINSERT, 0, "can't insert chunk node into skip list") + + udata.version = H5Z_DTYPE_MODIFY_VERS; + udata.is_shared = is_shared; + udata.dtype = dtype; + udata.numb_ref = 1; + udata.found = FALSE; + udata.adjust = H5Z_MDTYPE_INC; + + /* Find out the number of nodes in the Btree */ + if(H5B2_get_nrec(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, &bnode_numb) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTGET, 0, "can't get number of nodes in Btree") + + /* Find the datatype and increment the number of references to the type */ + if(bnode_numb) + H5B2_find(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, baddr, &udata, + H5Z_btree2_dtype_find_cb, &udata); + /*HGOTO_ERROR(H5E_PLINE, H5E_NOTFOUND, 0, "can't locate dtype in Btree")*/ + + /* If the dtype info isn't in Btree, insert it and its reference number + * into Btree */ + if(udata.found == FALSE && H5B2_insert(file, H5AC_dxpl_id, H5Z_BT2_DTYPE, + baddr, &udata) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINSERT, 0, "can't insert B-tree") + } + } } done: - /*if(outbuf) - H5MM_xfree(outbuf);*/ FUNC_LEAVE_NOAPI(ret_value) } diff --git a/src/H5Zfletcher32.c b/src/H5Zfletcher32.c index 2434537..a520cea 100644 --- a/src/H5Zfletcher32.c +++ b/src/H5Zfletcher32.c @@ -30,21 +30,40 @@ #ifdef H5_HAVE_FILTER_FLETCHER32 /* Local function prototypes */ -static size_t H5Z_filter_fletcher32 (unsigned flags, size_t cd_nelmts, +#ifndef H5_USE_16_API +static size_t H5Z_filter_fletcher32 (unsigned flags, hsize_t UNUSED chunk_offset, + size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +const H5Z_class_t H5Z_FLETCHER32[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_FLETCHER32, /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "fletcher32", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ + H5Z_filter_fletcher32, /* The actual filter function */ +}}; +#else +static size_t H5Z_filter_fletcher32 (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ const H5Z_class_t H5Z_FLETCHER32[1] = {{ H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_FLETCHER32, /* Filter id number */ - 1, /* encoder_present flag (set to true) */ - 1, /* decoder_present flag (set to true) */ "fletcher32", /* Filter name for debugging */ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ - NULL, /* The "reset local" callback */ H5Z_filter_fletcher32, /* The actual filter function */ }}; +#endif #define FLETCHER_LEN 4 @@ -74,9 +93,15 @@ const H5Z_class_t H5Z_FLETCHER32[1] = {{ *------------------------------------------------------------------------- */ /* ARGSUSED */ +#ifndef H5_USE_16_API +static size_t +H5Z_filter_fletcher32 (unsigned flags, hsize_t UNUSED chunk_offset, size_t UNUSED cd_nelmts, + const unsigned UNUSED cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#else static size_t -H5Z_filter_fletcher32 (unsigned flags, size_t UNUSED cd_nelmts, const unsigned UNUSED cd_values[], - size_t nbytes, size_t *buf_size, void **buf) +H5Z_filter_fletcher32 (unsigned flags, size_t UNUSED cd_nelmts, + const unsigned UNUSED cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#endif { void *outbuf = NULL; /* Pointer to new buffer */ unsigned char *src = (unsigned char*)(*buf); @@ -84,7 +109,7 @@ H5Z_filter_fletcher32 (unsigned flags, size_t UNUSED cd_nelmts, const unsigned U uint32_t reversed_fletcher; /* Possible wrong checksum value */ uint8_t c[4]; uint8_t tmp; - size_t ret_value; /* Return value */ + size_t ret_value = nbytes; /* Return value */ FUNC_ENTER_NOAPI(H5Z_filter_fletcher32, 0) @@ -134,7 +159,7 @@ H5Z_filter_fletcher32 (unsigned flags, size_t UNUSED cd_nelmts, const unsigned U /* Set return values */ /* (Re-use the input buffer, just note that the size is smaller by the size of the checksum) */ ret_value = nbytes-FLETCHER_LEN; - } else { /* Write */ + } else if (!(flags & H5Z_FLAG_INVMASK)){ /* Write */ unsigned char *dst; /* Temporary pointer to destination buffer */ /* Compute checksum (can't fail) */ diff --git a/src/H5Znbit.c b/src/H5Znbit.c index 9827a9d..fe2054c 100644 --- a/src/H5Znbit.c +++ b/src/H5Znbit.c @@ -37,14 +37,6 @@ typedef struct { int offset; /* datatype offset */ } parms_atomic; -/* Local function prototypes */ -static herr_t H5Z_can_apply_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id, - hid_t UNUSED file_id); -static herr_t H5Z_set_local_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id, - hid_t UNUSED file_id); -static size_t H5Z_filter_nbit(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], - size_t nbytes, size_t *buf_size, void **buf); - static void H5Z_calc_parms_nooptype(void); static void H5Z_calc_parms_atomic(void); static herr_t H5Z_calc_parms_array(const H5T_t *type); @@ -81,16 +73,28 @@ static void H5Z_nbit_compress_one_compound(unsigned char *data, size_t data_offs static void H5Z_nbit_compress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer, size_t *buffer_size, const unsigned parms[]); +/* Local function prototypes */ +static herr_t H5Z_can_apply_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t UNUSED file_id); +static herr_t H5Z_set_local_nbit(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t UNUSED file_id); +static size_t H5Z_filter_nbit(unsigned flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + /* This message derives from H5Z */ H5Z_class_t H5Z_NBIT[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_NBIT, /* Filter id number */ - 1, /* Assume encoder present: check before registering */ - 1, /* decoder_present flag (set to true) */ - "nbit", /* Filter name for debugging */ + 1, /* Assume encoder present: check before registering */ + 1, /* decoder_present flag (set to true) */ + "nbit", /* Filter name for debugging */ H5Z_can_apply_nbit, /* The "can apply" callback */ H5Z_set_local_nbit, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ H5Z_filter_nbit, /* The actual filter function */ }}; @@ -871,10 +875,10 @@ done: *------------------------------------------------------------------------- */ static size_t -H5Z_filter_nbit(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], - size_t nbytes, size_t *buf_size, void **buf) +H5Z_filter_nbit(unsigned flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) { - size_t ret_value = 0; /* return value */ + size_t ret_value = nbytes; /* return value */ size_t size_out = 0; /* size of output buffer */ unsigned d_nelmts = 0; /* number of elements in the chunk */ unsigned char *outbuf = NULL; /* pointer to new output buffer */ @@ -899,7 +903,7 @@ H5Z_filter_nbit(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], d_nelmts = cd_values[2]; /* input; decompress */ - if (flags & H5Z_FLAG_REVERSE) { + if (flags & H5Z_FLAG_REVERSE) { /*Read*/ size_out = d_nelmts * cd_values[4]; /* cd_values[4] stores datatype size */ /* allocate memory space for decompressed buffer */ @@ -908,9 +912,9 @@ H5Z_filter_nbit(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], /* decompress the buffer */ H5Z_nbit_decompress(outbuf, d_nelmts, *buf, cd_values); - } + /* output; compress */ - else { + } else if (!(flags & H5Z_FLAG_INVMASK)) { /*Write*/ assert(nbytes == d_nelmts * cd_values[4]); size_out = nbytes; diff --git a/src/H5Zprivate.h b/src/H5Zprivate.h index 890125c..fd6327c 100644 --- a/src/H5Zprivate.h +++ b/src/H5Zprivate.h @@ -80,13 +80,17 @@ H5_DLL herr_t H5Z_modify(const struct H5O_pline_t *pline, H5Z_filter_t filter, unsigned flags, size_t cd_nelmts, const unsigned int cd_values[]); H5_DLL herr_t H5Z_pipeline(const struct H5O_pline_t *pline, unsigned flags, unsigned *filter_mask/*in,out*/, - H5Z_EDC_t edc_read, H5Z_cb_t cb_struct, + H5Z_EDC_t edc_read, hsize_t chunk_offset, H5Z_cb_t cb_struct, size_t *nbytes/*in,out*/, size_t *buf_size/*in,out*/, void **buf/*in,out*/); H5_DLL H5Z_class_t *H5Z_find(H5Z_filter_t id); H5_DLL herr_t H5Z_can_apply(hid_t dcpl_id, hid_t type_id, hid_t file_id); H5_DLL herr_t H5Z_set_local(hid_t dcpl_id, hid_t type_id, hid_t file_id); -H5_DLL herr_t H5Z_reset_local(hid_t dcpl_id, hid_t type_id, hid_t file_id); +H5_DLL herr_t H5Z_reset_local(hid_t dcpl_id, hid_t type_id, hid_t file_id, hbool_t from_reopen); +H5_DLL herr_t H5Z_change_local (hid_t dcpl_id, hid_t type_id, hid_t file_id, hsize_t chunk_offset); +H5_DLL herr_t H5Z_evict_local (hid_t dcpl_id, hid_t type_id, hid_t file_id, hsize_t chunk_offset); +H5_DLL herr_t H5Z_delete_local(hid_t dcpl_id, hid_t type_id, hid_t file_id); +H5_DLL herr_t H5Z_close_local(hid_t dcpl_id, hid_t type_id, hid_t file_id); H5_DLL H5Z_filter_info_t *H5Z_filter_info(const struct H5O_pline_t *pline, H5Z_filter_t filter); H5_DLL htri_t H5Z_all_filters_avail(const struct H5O_pline_t *pline); diff --git a/src/H5Zpublic.h b/src/H5Zpublic.h index 7ef9e98..c0ee263 100644 --- a/src/H5Zpublic.h +++ b/src/H5Zpublic.h @@ -64,6 +64,8 @@ typedef int H5Z_filter_t; #define H5Z_FLAG_INVMASK 0xff00 /*invocation flag mask */ #define H5Z_FLAG_REVERSE 0x0100 /*reverse direction; read */ #define H5Z_FLAG_SKIP_EDC 0x0200 /*skip EDC filters for read */ +#define H5Z_FLAG_SKIP_CRECORD 0x0400 /*skip record keeping for chunks. Only for the + *filter of modifying dset's dtype*/ /* Special parameters for szip compression */ /* [These are aliases for the similar definitions in szlib.h, which we can't @@ -84,7 +86,7 @@ typedef enum H5Z_SO_scale_type_t { } H5Z_SO_scale_type_t; /* Current version of the H5Z_class_t struct */ -#define H5Z_CLASS_T_VERS (1) +#define H5Z_CLASS_T_VERS (2) /* Values to decide if EDC is enabled for reading data */ typedef enum H5Z_EDC_t { @@ -140,7 +142,7 @@ extern "C" { * The "can_apply" callback returns positive a valid combination, zero for an * invalid combination and negative for an error. */ -typedef herr_t (*H5Z_can_apply_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id, +typedef herr_t (*H5Z_can_apply_func2_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t file_id); /* @@ -170,9 +172,15 @@ typedef herr_t (*H5Z_can_apply_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space * again. The function prototype of "reset_local" callback is identical to * the "set_local". */ -typedef herr_t (*H5Z_set_local_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id, +typedef herr_t (*H5Z_set_local_func2_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t file_id); +typedef herr_t (*H5Z_reset_local_func2_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t file_id, hbool_t from_reopen); + +typedef herr_t (*H5Z_change_local_func_t)(hid_t dcpl_id, hsize_t chunk_offset); +typedef herr_t (*H5Z_close_local_func_t)(hid_t dcpl_id); + /* * A filter gets definition flags and invocation flags (defined above), the * client data array and size defined when the filter was added to the @@ -188,7 +196,7 @@ typedef herr_t (*H5Z_set_local_func_t)(hid_t dcpl_id, hid_t type_id, hid_t space * buffer. If an error occurs then the function should return zero and leave * all pointer arguments unchanged. */ -typedef size_t (*H5Z_func_t)(unsigned int flags, size_t cd_nelmts, +typedef size_t (*H5Z_func2_t)(unsigned int flags, hsize_t chunk_offset, size_t cd_nelmts, const unsigned int cd_values[], size_t nbytes, size_t *buf_size, void **buf); @@ -196,17 +204,45 @@ typedef size_t (*H5Z_func_t)(unsigned int flags, size_t cd_nelmts, * The filter table maps filter identification numbers to structs that * contain a pointers to the filter function and timing statistics. */ -typedef struct H5Z_class_t { +typedef struct H5Z_class2_t { int version; /* Version number of the H5Z_class_t struct */ H5Z_filter_t id; /* Filter ID number */ unsigned encoder_present; /* Does this filter have an encoder? */ unsigned decoder_present; /* Does this filter have a decoder? */ const char *name; /* Comment for debugging */ - H5Z_can_apply_func_t can_apply; /* The "can apply" callback for a filter */ - H5Z_set_local_func_t set_local; /* The "set local" callback for a filter */ - H5Z_set_local_func_t reset_local; /* The "reset local" callback for a filter */ + H5Z_can_apply_func2_t can_apply; /* The "can apply" callback for a filter */ + H5Z_set_local_func2_t set_local; /* The "set local" callback for a filter */ + H5Z_reset_local_func2_t reset_local; /* The "reset local" callback for a filter */ + H5Z_change_local_func_t change_local; /* The "change local" callback for a filter */ + H5Z_change_local_func_t evict_local; /* The "evict local" callback for a filter */ + H5Z_close_local_func_t delete_local; /* The "delete local" callback for a filter */ + H5Z_close_local_func_t close_local; /* The "close local" callback for a filter */ + H5Z_func2_t filter; /* The actual filter function */ +} H5Z_class2_t; + +/* Symbols defined for compatibility with previous versions of the HDF5 API. + * + * Use of these symbols is deprecated. + */ +#ifndef H5_NO_DEPRECATED_SYMBOLS + +/* Typedefs */ +typedef herr_t (*H5Z_can_apply_func1_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id); +typedef herr_t (*H5Z_set_local_func1_t)(hid_t dcpl_id, hid_t type_id, hid_t space_id); +typedef size_t (*H5Z_func1_t)(unsigned int flags, size_t cd_nelmts, + const unsigned int cd_values[], size_t nbytes, + size_t *buf_size, void **buf); + +typedef struct H5Z_class1_t { + int version; /* Version number of the H5Z_class_t struct */ + H5Z_filter_t id; /* Filter ID number */ + const char *name; /* Comment for debugging */ + H5Z_can_apply_func1_t can_apply; /* The "can apply" callback for a filter */ + H5Z_set_local_func1_t set_local; /* The "set local" callback for a filter */ H5Z_func_t filter; /* The actual filter function */ -} H5Z_class_t; +} H5Z_class1_t; + +#endif /* H5_NO_DEPRECATED_SYMBOLS */ H5_DLL herr_t H5Zregister(const H5Z_class_t *cls); H5_DLL herr_t H5Zunregister(H5Z_filter_t id); @@ -217,4 +253,3 @@ H5_DLL herr_t H5Zget_filter_info(H5Z_filter_t filter, unsigned int *filter_confi } #endif #endif - diff --git a/src/H5Zscaleoffset.c b/src/H5Zscaleoffset.c index 12e9aee..b3d62f5 100644 --- a/src/H5Zscaleoffset.c +++ b/src/H5Zscaleoffset.c @@ -40,17 +40,11 @@ enum H5Z_scaleoffset_type {t_bad=0, t_uchar=1, t_ushort, t_uint, t_ulong, t_ulon /* Local function prototypes */ static double H5Z_scaleoffset_rnd(double val); -static herr_t H5Z_can_apply_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id, - hid_t UNUSED file_id); static enum H5Z_scaleoffset_type H5Z_scaleoffset_get_type(unsigned dtype_class, unsigned dtype_size, unsigned dtype_sign); static herr_t H5Z_scaleoffset_set_parms_fillval(H5P_genplist_t *dcpl_plist, const H5T_t *type, enum H5Z_scaleoffset_type scale_type, unsigned cd_values[], int need_convert, hid_t dxpl_id); -static herr_t H5Z_set_local_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id, - hid_t UNUSED file_id); -static size_t H5Z_filter_scaleoffset(unsigned flags, size_t cd_nelmts, - const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); static void H5Z_scaleoffset_convert(void *buf, unsigned d_nelmts, size_t dtype_size); static unsigned H5Z_scaleoffset_log2(unsigned long_long num); static void H5Z_scaleoffset_precompress_i(void *data, unsigned d_nelmts, @@ -81,16 +75,27 @@ static void H5Z_scaleoffset_decompress(unsigned char *data, unsigned d_nelmts, static void H5Z_scaleoffset_compress(unsigned char *data, unsigned d_nelmts, unsigned char *buffer, size_t buffer_size, parms_atomic p); +static herr_t H5Z_can_apply_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t UNUSED file_id); +static herr_t H5Z_set_local_scaleoffset(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t UNUSED file_id); +static size_t H5Z_filter_scaleoffset(unsigned flags, hsize_t UNUSED chunk_offset, + size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + /* This message derives from H5Z */ H5Z_class_t H5Z_SCALEOFFSET[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ - H5Z_FILTER_SCALEOFFSET, /* Filter id number */ - 1, /* Assume encoder present: check before registering */ - 1, /* decoder_present flag (set to true) */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_SCALEOFFSET, /* Filter id number */ + 1, /* Assume encoder present: check before registering */ + 1, /* decoder_present flag (set to true) */ "scaleoffset", /* Filter name for debugging */ H5Z_can_apply_scaleoffset, /* The "can apply" callback */ H5Z_set_local_scaleoffset, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ H5Z_filter_scaleoffset, /* The actual filter function */ }}; @@ -911,8 +916,8 @@ done: *------------------------------------------------------------------------- */ static size_t -H5Z_filter_scaleoffset (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], - size_t nbytes, size_t *buf_size, void **buf) +H5Z_filter_scaleoffset (unsigned flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) { size_t ret_value = 0; /* return value */ size_t size_out = 0; /* size of output buffer */ @@ -1008,7 +1013,7 @@ H5Z_filter_scaleoffset (unsigned flags, size_t cd_nelmts, const unsigned cd_valu p.mem_order = H5T_native_order_g; /* input; decompress */ - if (flags & H5Z_FLAG_REVERSE) { + if (flags & H5Z_FLAG_REVERSE) { /*Read*/ /* retrieve values of minbits and minval from input compressed buffer * retrieve them corresponding to how they are stored during compression */ @@ -1090,7 +1095,7 @@ H5Z_filter_scaleoffset (unsigned flags, size_t cd_nelmts, const unsigned cd_valu H5Z_scaleoffset_convert(outbuf, d_nelmts, p.size); } /* output; compress */ - else { + else if (!(flags & H5Z_FLAG_INVMASK)) { /*Write*/ assert(nbytes == d_nelmts * p.size); /* before preprocess, convert to memory endianness order if needed */ diff --git a/src/H5Zshuffle.c b/src/H5Zshuffle.c index cc0bb5a..d808014 100644 --- a/src/H5Zshuffle.c +++ b/src/H5Zshuffle.c @@ -27,22 +27,44 @@ #ifdef H5_HAVE_FILTER_SHUFFLE /* Local function prototypes */ +#ifndef H5_USE_16_API +static herr_t H5Z_set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t UNUSED file_id); +static size_t H5Z_filter_shuffle(unsigned flags, hsize_t UNUSED chunk_offset, + size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, + void **buf); + +/* This message derives from H5Z */ +const H5Z_class_t H5Z_SHUFFLE[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_SHUFFLE, /* Filter id number */ + 1, /* encoder_present flag (set to true) */ + 1, /* decoder_present flag (set to true) */ + "shuffle", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + H5Z_set_local_shuffle, /* The "set local" callback */ + NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ + H5Z_filter_shuffle, /* The actual filter function */ +}}; +#else static herr_t H5Z_set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t space_id); -static size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, +static size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ const H5Z_class_t H5Z_SHUFFLE[1] = {{ H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_SHUFFLE, /* Filter id number */ - 1, /* encoder_present flag (set to true) */ - 1, /* decoder_present flag (set to true) */ "shuffle", /* Filter name for debugging */ NULL, /* The "can apply" callback */ H5Z_set_local_shuffle, /* The "set local" callback */ - NULL, /* The "reset local" callback */ H5Z_filter_shuffle, /* The actual filter function */ }}; +#endif /* Local macros */ #define H5Z_SHUFFLE_USER_NPARMS 0 /* Number of parameters that users can set */ @@ -67,8 +89,13 @@ const H5Z_class_t H5Z_SHUFFLE[1] = {{ *------------------------------------------------------------------------- */ /* ARGSUSED */ +#ifndef H5_USE_16_API +static herr_t H5Z_set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t space_id, + hid_t UNUSED file_id) +#else static herr_t H5Z_set_local_shuffle(hid_t dcpl_id, hid_t type_id, hid_t UNUSED space_id) +#endif { H5P_genplist_t *dcpl_plist; /* Property list pointer */ const H5T_t *type; /* Datatype */ @@ -126,9 +153,15 @@ done: * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API +static size_t H5Z_filter_shuffle(unsigned flags, hsize_t UNUSED chunk_offset, + size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, + void **buf) +#else static size_t H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#endif { void *dest = NULL; /* Buffer to deposit [un]shuffled bytes into */ unsigned char *_src=NULL; /* Alias for source buffer */ @@ -140,7 +173,7 @@ H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t j; /* Local index variable */ #endif /* NO_DUFFS_DEVICE */ size_t leftover; /* Extra bytes at end of buffer */ - size_t ret_value; /* Return value */ + size_t ret_value=nbytes; /* Return value */ FUNC_ENTER_NOAPI(H5Z_filter_shuffle, 0) @@ -163,7 +196,7 @@ H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], if (NULL==(dest = H5MM_malloc(nbytes))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, 0, "memory allocation failed for shuffle buffer") - if(flags & H5Z_FLAG_REVERSE) { + if(flags & H5Z_FLAG_REVERSE) { /*Read*/ /* Get the pointer to the source buffer */ _src =(unsigned char *)(*buf); @@ -218,7 +251,7 @@ H5Z_filter_shuffle(unsigned flags, size_t cd_nelmts, const unsigned cd_values[], HDmemcpy((void*)_dest, (void*)_src, leftover); } } /* end if */ - else { + else if (!(flags & H5Z_FLAG_INVMASK)) { /*Write*/ /* Get the pointer to the destination buffer */ _dest =(unsigned char *)dest; diff --git a/src/H5Zszip.c b/src/H5Zszip.c index 191d7e7..4263537 100644 --- a/src/H5Zszip.c +++ b/src/H5Zszip.c @@ -33,12 +33,13 @@ # include "szlib.h" #endif +#ifndef H5_USE_16_API /* Local function prototypes */ static herr_t H5Z_can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t UNUSED file_id); static herr_t H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t UNUSED file_id); -static size_t H5Z_filter_szip (unsigned flags, size_t cd_nelmts, +static size_t H5Z_filter_szip (unsigned flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); /* This message derives from H5Z */ @@ -51,8 +52,29 @@ H5Z_class_t H5Z_SZIP[1] = {{ H5Z_can_apply_szip, /* The "can apply" callback */ H5Z_set_local_szip, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ H5Z_filter_szip, /* The actual filter function */ }}; +#else +/* Local function prototypes */ +static herr_t H5Z_can_apply_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static herr_t H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static size_t H5Z_filter_szip (unsigned flags, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf); + +/* This message derives from H5Z */ +H5Z_class_t H5Z_SZIP[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_SZIP, /* Filter id number */ + "szip", /* Filter name for debugging */ + H5Z_can_apply_szip, /* The "can apply" callback */ + H5Z_set_local_szip, /* The "set local" callback */ + H5Z_filter_szip, /* The actual filter function */ +}}; +#endif /* Local macros */ #define H5Z_SZIP_USER_NPARMS 2 /* Number of parameters that users can set */ @@ -86,9 +108,14 @@ H5Z_class_t H5Z_SZIP[1] = {{ * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API static herr_t H5Z_can_apply_szip(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id, hid_t UNUSED file_id) +#else +static herr_t +H5Z_can_apply_szip(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id) +#endif { const H5T_t *type; /* Datatype */ unsigned dtype_size; /* Datatype's size (in bits) */ @@ -144,8 +171,13 @@ done: * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API static herr_t H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t UNUSED file_id) +#else +static herr_t +H5Z_set_local_szip(hid_t dcpl_id, hid_t type_id, hid_t space_id) +#endif { H5P_genplist_t *dcpl_plist; /* Property list pointer */ const H5T_t *type; /* Datatype */ @@ -285,9 +317,15 @@ done: * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API +static size_t +H5Z_filter_szip (unsigned flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#else static size_t -H5Z_filter_szip (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], +H5Z_filter_szip (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], size_t nbytes, size_t *buf_size, void **buf) +#endif { size_t ret_value = 0; /* Return value */ size_t size_out = 0; /* Size of output buffer */ @@ -318,7 +356,7 @@ H5Z_filter_szip (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], H5_ASSIGN_OVERFLOW(sz_param.pixels_per_scanline,cd_values[H5Z_SZIP_PARM_PPS],unsigned,int); /* Input; uncompress */ - if (flags & H5Z_FLAG_REVERSE) { + if (flags & H5Z_FLAG_REVERSE) { /*Read*/ uint32_t stored_nalloc; /* Number of bytes the compressed block will expand into */ size_t nalloc; /* Number of bytes the compressed block will expand into */ @@ -347,7 +385,7 @@ H5Z_filter_szip (unsigned flags, size_t cd_nelmts, const unsigned cd_values[], ret_value = nalloc; } /* Output; compress */ - else { + else if (!(flags & H5Z_FLAG_INVMASK)) { /*Write*/ unsigned char *dst = NULL; /* Temporary pointer to new output buffer */ /* Allocate space for the compressed buffer & header (assume data won't get bigger) */ @@ -381,4 +419,3 @@ done: } #endif /* H5_HAVE_FILTER_SZIP */ - diff --git a/src/H5vers.txt b/src/H5vers.txt index 00ab84b..8cd03cb 100644 --- a/src/H5vers.txt +++ b/src/H5vers.txt @@ -71,4 +71,7 @@ FUNCTION: H5Topen; ; v10, v18 # (although not required, it's easier to compare this file with the headers # generated if the list below is in alphanumeric sort order - QAK) TYPEDEF: H5E_auto; v10, v18 - +TYPEDEF: H5Z_func; v10, v18 +TYPEDEF: H5Z_can_apply_func; v10, v18 +TYPEDEF: H5Z_set_local_func; v10, v18 +TYPEDEF: H5Z_class; v10, v18 diff --git a/src/H5version.h b/src/H5version.h index eab972c..1c9d8ff 100644 --- a/src/H5version.h +++ b/src/H5version.h @@ -130,6 +130,22 @@ #define H5E_auto_t_vers 1 #endif /* !defined(H5E_auto_t_vers) */ +#if !defined(H5Z_can_apply_func_t_vers) +#define H5Z_can_apply_func_t_vers 1 +#endif /* !defined(H5Z_can_apply_func_t_vers) */ + +#if !defined(H5Z_class_t_vers) +#define H5Z_class_t_vers 1 +#endif /* !defined(H5Z_class_t_vers) */ + +#if !defined(H5Z_func_t_vers) +#define H5Z_func_t_vers 1 +#endif /* !defined(H5Z_func_t_vers) */ + +#if !defined(H5Z_set_local_func_t_vers) +#define H5Z_set_local_func_t_vers 1 +#endif /* !defined(H5Z_set_local_func_t_vers) */ + #endif /* H5_USE_16_API */ @@ -395,5 +411,53 @@ #error "H5E_auto_t_vers set to invalid value" #endif /* H5E_auto_t_vers */ + +#if !defined(H5Z_can_apply_func_t_vers) || H5Z_can_apply_func_t_vers == 2 +#ifndef H5Z_can_apply_func_t_vers +#define H5Z_can_apply_func_t_vers 2 +#endif /* H5Z_can_apply_func_t_vers */ +#define H5Z_can_apply_func_t H5Z_can_apply_func2_t +#elif H5Z_can_apply_func_t_vers == 1 +#define H5Z_can_apply_func_t H5Z_can_apply_func1_t +#else /* H5Z_can_apply_func_t_vers */ +#error "H5Z_can_apply_func_t_vers set to invalid value" +#endif /* H5Z_can_apply_func_t_vers */ + + +#if !defined(H5Z_class_t_vers) || H5Z_class_t_vers == 2 +#ifndef H5Z_class_t_vers +#define H5Z_class_t_vers 2 +#endif /* H5Z_class_t_vers */ +#define H5Z_class_t H5Z_class2_t +#elif H5Z_class_t_vers == 1 +#define H5Z_class_t H5Z_class1_t +#else /* H5Z_class_t_vers */ +#error "H5Z_class_t_vers set to invalid value" +#endif /* H5Z_class_t_vers */ + + +#if !defined(H5Z_func_t_vers) || H5Z_func_t_vers == 2 +#ifndef H5Z_func_t_vers +#define H5Z_func_t_vers 2 +#endif /* H5Z_func_t_vers */ +#define H5Z_func_t H5Z_func2_t +#elif H5Z_func_t_vers == 1 +#define H5Z_func_t H5Z_func1_t +#else /* H5Z_func_t_vers */ +#error "H5Z_func_t_vers set to invalid value" +#endif /* H5Z_func_t_vers */ + + +#if !defined(H5Z_set_local_func_t_vers) || H5Z_set_local_func_t_vers == 2 +#ifndef H5Z_set_local_func_t_vers +#define H5Z_set_local_func_t_vers 2 +#endif /* H5Z_set_local_func_t_vers */ +#define H5Z_set_local_func_t H5Z_set_local_func2_t +#elif H5Z_set_local_func_t_vers == 1 +#define H5Z_set_local_func_t H5Z_set_local_func1_t +#else /* H5Z_set_local_func_t_vers */ +#error "H5Z_set_local_func_t_vers set to invalid value" +#endif /* H5Z_set_local_func_t_vers */ + #endif /* H5version_H */ diff --git a/test/change_dtypes.c b/test/change_dtypes.c index 0897de9..67adb64 100644 --- a/test/change_dtypes.c +++ b/test/change_dtypes.c @@ -20,6 +20,14 @@ #include "h5test.h" +#ifdef H5_USE_16_API +int main(void) +{ + printf("Test skipped because backward compatbility with v1.6 is configured in\n"); + return 0; +} +#else /* H5_USE_16_API */ + #define STR_LEN 16 #define NX 10u #define NY 20u @@ -43,6 +51,7 @@ const char *DSET_NAME[] = { "enum", "array", "reference", + "btree_node", NULL }; @@ -128,6 +137,84 @@ error: /*------------------------------------------------------------------------- + * Function: test_btree_node + * + * Purpose: Creates a simple dataset of an integer type and tries to + * change the datatype to a bigger size. Test whether the + * B-tree node works properly in the filter for modifying + * dataset's datatype. The B-tree is for storing the number + * of datatype reference by the data chunks. + * + * Return: Success: 0 + * + * Failure: 1 + * + * Programmer: Raymond Lu + * 19 November 2007 + * + *------------------------------------------------------------------------- + */ +static int +test_btree_node(hid_t file) +{ + hid_t dset; + hid_t space; + hid_t dcpl; + hsize_t dims[2] = {NX, NY}; + hsize_t chunk_dims[2] = {NX/2, NY/2}; + int *wbuf = NULL, *ptr = NULL; + unsigned i; + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + TEST_ERROR; + + if((space=H5Screate_simple(2, dims, NULL)) < 0) + TEST_ERROR; + + if(H5Pset_dtype_modifiable(dcpl)<0) + TEST_ERROR; + + /* Set chunking */ + if(H5Pset_chunk(dcpl, 2, chunk_dims)<0) + TEST_ERROR; + + if((dset=H5Dcreate2(file, DSET_NAME[10], H5T_STD_U32BE, space, H5P_DEFAULT, dcpl, + H5P_DEFAULT)) < 0) + TEST_ERROR; + + wbuf = (int *)malloc(sizeof(int)*NX*NY); + ptr = wbuf; + for(i = 0; i<NX*NY; i++) { + *ptr = i; + ptr++; + } + + if(H5Dwrite(dset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, wbuf) < 0) + TEST_ERROR; + + if(H5Dmodify_dtype(dset, H5T_STD_U64LE) < 0) + TEST_ERROR; + + if(H5Dclose(dset) < 0) + TEST_ERROR; + + free(wbuf); + + PASSED(); + return 0; + +error: + H5_FAILED(); + H5E_BEGIN_TRY { + H5Sclose(space); + H5Dclose(dset); + } H5E_END_TRY; + return 1; +} + + +/*------------------------------------------------------------------------- * Function: test_float * * Purpose: Creates a simple dataset of a float type and tries to @@ -805,6 +892,9 @@ int main(void) TESTING("dataset of reference type:"); nerrors += test_reference(fid); + TESTING("size of B-tree node"); + nerrors += test_btree_node(fid); + if(H5Fclose(fid) < 0) nerrors++; @@ -819,3 +909,4 @@ int main(void) return 0; } +#endif /* H5_USE_16_API */ diff --git a/test/cmpd_dset.c b/test/cmpd_dset.c index 7a7f8fa..759549a 100644 --- a/test/cmpd_dset.c +++ b/test/cmpd_dset.c @@ -124,9 +124,16 @@ typedef struct { hvl_t r; long_long s, t, u; } stype4; +typedef struct { + int a, b, c[8], d, e; + float f, g, h[16], i, j; + double k, l, m, n; + long o, p, q; + hvl_t r; + long_long s, t, u; + char v, w; +} stype5; -/*#define NX 10u -#define NY 20u*/ #define NX 100u #define NY 200u #define COMP_NX 8u @@ -140,15 +147,14 @@ const unsigned test_type_flags[H5O_SHMESG_MAX_NINDEXES] = H5O_MESG_SDSPACE_FLAG, H5O_MESG_PLINE_FLAG, 0, 0}; -#ifndef TMP + const unsigned test_minsizes[H5O_SHMESG_MAX_NINDEXES] = {0, 2, 40, 100, 3, 1000}; #define TEST_L2B 65 #define TEST_B2L 64 -#else -const unsigned test_minsizes[H5O_SHMESG_MAX_NINDEXES] = {250,250,250,250,250,250}; -#define TEST_L2B 50 -#define TEST_B2L 40 -#endif + +#define COMMITTED_DTYPE 0 +#define UNCOMMITTED_DTYPE 1 +#define MIXED_DTYPE 2 /*------------------------------------------------------------------------- @@ -237,7 +243,6 @@ test_compound (char *filename, hid_t fapl) /* Create xfer properties to preserve initialized data */ if ((PRESERVE = H5Pcreate (H5P_DATASET_XFER)) < 0) goto error; - if (H5Pset_preserve (PRESERVE, 1) < 0) goto error; /* *###################################################################### @@ -897,17 +902,17 @@ initialize_stype1(void *buf, const size_t num) s_ptr->d = i*8+6; s_ptr->e = i*8+7; - s_ptr->f = i*2/3; - s_ptr->g = i*2/3+1; + s_ptr->f = (float)i*2/3; + s_ptr->g = (float)i*2/3+1; for(j=0; j<16; j++) - s_ptr->h[j] = i*j/5+j; - s_ptr->i = i*2/3+2; - s_ptr->j = i*2/3+3; - - s_ptr->k = i/7+1; - s_ptr->l = i/7+2; - s_ptr->m = i/7+3; - s_ptr->n = i/7+4; + s_ptr->h[j] = (float)i*j/5+j; + s_ptr->i = (float)i*2/3+2; + s_ptr->j = (float)i*2/3+3; + + s_ptr->k = (double)i/7+1; + s_ptr->l = (double)i/7+2; + s_ptr->m = (double)i/7+3; + s_ptr->n = (double)i/7+4; } } @@ -940,17 +945,17 @@ initialize_stype2(void *buf, const size_t num) s_ptr->d = i*8+6; s_ptr->e = i*8+7; - s_ptr->f = i*2/3; - s_ptr->g = i*2/3+1; + s_ptr->f = (float)i*2/3; + s_ptr->g = (float)i*2/3+1; for(j=0; j<16; j++) - s_ptr->h[j] = i*j/5+j; - s_ptr->i = i*2/3+2; - s_ptr->j = i*2/3+3; + s_ptr->h[j] = (float)i*j/5+j; + s_ptr->i = (float)i*2/3+2; + s_ptr->j = (float)i*2/3+3; - s_ptr->k = i/7+1; - s_ptr->l = i/7+2; - s_ptr->m = i/7+3; - s_ptr->n = i/7+4; + s_ptr->k = (double)i/7+1; + s_ptr->l = (double)i/7+2; + s_ptr->m = (double)i/7+3; + s_ptr->n = (double)i/7+4; s_ptr->o = i*3+0; s_ptr->p = i*3+1; @@ -1009,7 +1014,7 @@ initialize_stype3(void *buf, const size_t num) *------------------------------------------------------------------------- */ static void -initialize_stype4(void *buf, const size_t num) +initialize_stype4(void *buf, const size_t num, int addition) { size_t i, j; stype4 *s_ptr; @@ -1023,17 +1028,17 @@ initialize_stype4(void *buf, const size_t num) s_ptr->d = i*8+6; s_ptr->e = i*8+7; - s_ptr->f = i*2/3; - s_ptr->g = i*2/3+1; + s_ptr->f = (float)i*2/3; + s_ptr->g = (float)i*2/3+1; for(j=0; j<16; j++) - s_ptr->h[j] = i*j/5+j; - s_ptr->i = i*2/3+2; - s_ptr->j = i*2/3+3; + s_ptr->h[j] = (float)i*j/5+j; + s_ptr->i = (float)i*2/3+2; + s_ptr->j = (float)i*2/3+3; - s_ptr->k = i/7+1; - s_ptr->l = i/7+2; - s_ptr->m = i/7+3; - s_ptr->n = i/7+4; + s_ptr->k = (double)i/7+1; + s_ptr->l = (double)i/7+2; + s_ptr->m = (double)i/7+3; + s_ptr->n = (double)i/7+4; s_ptr->o = i*3+0; s_ptr->p = i*3+1; @@ -1044,9 +1049,9 @@ initialize_stype4(void *buf, const size_t num) for(j=0; j<4; j++) ((unsigned int *)s_ptr->r.p)[j]=i*10+j; - s_ptr->s = i*5+1; - s_ptr->t = i*5+2; - s_ptr->u = i*5+3; + s_ptr->s = i*5+1+addition; + s_ptr->t = i*5+2+addition; + s_ptr->u = i*5+3+addition; } } @@ -1282,6 +1287,75 @@ error: /*------------------------------------------------------------------------- + * Function: create_stype5 + * + * Purpose: Create HDF5 compound datatype for stype5. + * + * Return: Success: datatype ID + * + * Failure: negative + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static hid_t +create_stype5(void) +{ + hid_t array_dt1, array_dt2, vl_tid, tid; + const hsize_t eight = 8, sixteen = 16; + + /* Build hdf5 datatypes */ + if((array_dt1 = H5Tarray_create2(H5T_NATIVE_INT,1, &eight)) < 0) + goto error; + if((array_dt2 = H5Tarray_create2(H5T_NATIVE_FLOAT,1, &sixteen)) < 0) + goto error; + if((vl_tid = H5Tvlen_create (H5T_NATIVE_UINT)) < 0) + goto error; + + if((tid = H5Tcreate(H5T_COMPOUND, sizeof(stype5))) < 0 || + H5Tinsert(tid, "a", HOFFSET(stype5, a), H5T_NATIVE_INT) < 0 || + H5Tinsert(tid, "b", HOFFSET(stype5, b), H5T_NATIVE_INT) < 0 || + H5Tinsert(tid, "c", HOFFSET(stype5, c), array_dt1) < 0 || + H5Tinsert(tid, "d", HOFFSET(stype5, d), H5T_NATIVE_INT) < 0 || + H5Tinsert(tid, "e", HOFFSET(stype5, e), H5T_NATIVE_INT) < 0 || + H5Tinsert(tid, "f", HOFFSET(stype5, f), H5T_NATIVE_FLOAT) < 0 || + H5Tinsert(tid, "g", HOFFSET(stype5, g), H5T_NATIVE_FLOAT) < 0 || + H5Tinsert(tid, "h", HOFFSET(stype5, h), array_dt2) < 0 || + H5Tinsert(tid, "i", HOFFSET(stype5, i), H5T_NATIVE_FLOAT) < 0 || + H5Tinsert(tid, "j", HOFFSET(stype5, j), H5T_NATIVE_FLOAT) < 0 || + H5Tinsert(tid, "k", HOFFSET(stype5, k), H5T_NATIVE_DOUBLE) < 0 || + H5Tinsert(tid, "l", HOFFSET(stype5, l), H5T_NATIVE_DOUBLE) < 0 || + H5Tinsert(tid, "m", HOFFSET(stype5, m), H5T_NATIVE_DOUBLE) < 0 || + H5Tinsert(tid, "n", HOFFSET(stype5, n), H5T_NATIVE_DOUBLE) < 0 || + H5Tinsert(tid, "o", HOFFSET(stype5, o), H5T_NATIVE_LONG) < 0 || + H5Tinsert(tid, "p", HOFFSET(stype5, p), H5T_NATIVE_LONG) < 0 || + H5Tinsert(tid, "q", HOFFSET(stype5, q), H5T_NATIVE_LONG) < 0 || + H5Tinsert(tid, "r", HOFFSET(stype5, r), vl_tid) < 0 || + H5Tinsert(tid, "s", HOFFSET(stype5, s), H5T_NATIVE_LLONG) < 0 || + H5Tinsert(tid, "t", HOFFSET(stype5, t), H5T_NATIVE_LLONG) < 0 || + H5Tinsert(tid, "u", HOFFSET(stype5, u), H5T_NATIVE_LLONG) < 0 || + H5Tinsert(tid, "v", HOFFSET(stype5, v), H5T_NATIVE_CHAR) < 0 || + H5Tinsert(tid, "w", HOFFSET(stype5, w), H5T_NATIVE_CHAR) < 0) + goto error; + + if(H5Tclose(array_dt1) < 0) + goto error; + if(H5Tclose(array_dt2) < 0) + goto error; + if(H5Tclose(vl_tid) < 0) + goto error; + + return tid; + +error: + return FAIL; +} + + +/*------------------------------------------------------------------------- * Function: compare_data * * Purpose: Compare data of stype1 and stype2. @@ -1332,6 +1406,7 @@ compare_data(void *src_data, void *dst_data, hbool_t src_subset) !DBL_ABS_EQUAL(s_ptr->n, d_ptr->n) ) { H5_FAILED(); + printf(" i=%d\n", i); printf(" src={a=%d, b=%d, c=[%d,%d,%d,%d,%d,%d,%d,%d], d=%d, e=%d, f=%f, g=%f, h=[%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f], i=%f, j=%f, k=%f, l=%f, m=%f, n=%f}\n", s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2], @@ -1348,6 +1423,7 @@ compare_data(void *src_data, void *dst_data, hbool_t src_subset) d_ptr->h[9],d_ptr->h[10],d_ptr->h[11],d_ptr->h[12],d_ptr->h[13], d_ptr->h[14], d_ptr->h[15], d_ptr->i,d_ptr->j,d_ptr->k,d_ptr->l, d_ptr->m,d_ptr->n); + goto error; } } @@ -1547,6 +1623,102 @@ error: /*------------------------------------------------------------------------- + * Function: compare_stype4_stype5 + * + * Purpose: Compare data between stype4 and stype5. + * + * Return: Success: 0 + * + * Failure: negative + * + * Programmer: Raymond Lu + * Friday, 14 September 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static int +compare_stype4_stype5(void *src_data, void *dst_data, size_t nelmts) +{ + stype4 *s_ptr; + stype5 *d_ptr; + size_t i, j; + + for(i = 0; i < nelmts; i++) { + s_ptr = ((stype4*)src_data) + i; + d_ptr = ((stype5*)dst_data) + i; + + if (s_ptr->a != d_ptr->a || + s_ptr->b != d_ptr->b || + s_ptr->c[0] != d_ptr->c[0] || + s_ptr->c[1] != d_ptr->c[1] || + s_ptr->c[2] != d_ptr->c[2] || + s_ptr->c[3] != d_ptr->c[3] || + s_ptr->d != d_ptr->d || + s_ptr->e != d_ptr->e || + !FLT_ABS_EQUAL(s_ptr->f, d_ptr->f) || + !FLT_ABS_EQUAL(s_ptr->g, d_ptr->g) || + !FLT_ABS_EQUAL(s_ptr->h[0], d_ptr->h[0]) || + !FLT_ABS_EQUAL(s_ptr->h[1], d_ptr->h[1]) || + !FLT_ABS_EQUAL(s_ptr->i, d_ptr->i) || + !FLT_ABS_EQUAL(s_ptr->j, d_ptr->j) || + !DBL_ABS_EQUAL(s_ptr->k, d_ptr->k) || + !DBL_ABS_EQUAL(s_ptr->l, d_ptr->l) || + !DBL_ABS_EQUAL(s_ptr->m, d_ptr->m) || + !DBL_ABS_EQUAL(s_ptr->n, d_ptr->n) || + s_ptr->o != d_ptr->o || + s_ptr->p != d_ptr->p || + s_ptr->q != d_ptr->q || + s_ptr->s != d_ptr->s || + s_ptr->t != d_ptr->t || + s_ptr->u != d_ptr->u ) { + + H5_FAILED(); + printf(" i=%d\n", i); + printf(" src={a=%d, b=%d, c=[%d,%d,%d,%d,%d,%d,%d,%d], d=%d, e=%d, f=%f, g=%f, h=[%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f], i=%f, j=%f, k=%f, l=%f, m=%f, n=%f, o=%d, p=%d, q=%d, s=%ld, t=%ld, u=%ld}\n", + s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2], + s_ptr->c[3], s_ptr->c[4], s_ptr->c[5], s_ptr->c[6], s_ptr->c[7], + s_ptr->d, s_ptr->e, s_ptr->f, s_ptr->g,s_ptr->h[0],s_ptr->h[1],s_ptr->h[2], + s_ptr->h[3],s_ptr->h[4],s_ptr->h[5],s_ptr->h[6],s_ptr->h[7],s_ptr->h[8], + s_ptr->h[9],s_ptr->h[10],s_ptr->h[11],s_ptr->h[12],s_ptr->h[13],s_ptr->h[14], + s_ptr->h[15], s_ptr->i,s_ptr->j,s_ptr->k,s_ptr->l,s_ptr->m,s_ptr->n, + s_ptr->o, s_ptr->p, s_ptr->q, s_ptr->s, s_ptr->t, s_ptr->u); + printf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d,%d,%d,%d,%d], d=%d, e=%d, f=%f, g=%f, h=[%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f], i=%f, j=%f, k=%f, l=%f, m=%f, n=%f, o=%d, p=%d, q=%d, s=%ld, t=%ld, u=%ld}\n", + d_ptr->a, d_ptr->b, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2], + d_ptr->c[3], d_ptr->c[4], d_ptr->c[5], d_ptr->c[6], d_ptr->c[7], + d_ptr->d, d_ptr->e, d_ptr->f, d_ptr->g,d_ptr->h[0],d_ptr->h[1],d_ptr->h[2], + d_ptr->h[3],d_ptr->h[4],d_ptr->h[5],d_ptr->h[6],d_ptr->h[7],d_ptr->h[8], + d_ptr->h[9],d_ptr->h[10],d_ptr->h[11],d_ptr->h[12],d_ptr->h[13], + d_ptr->h[14], d_ptr->h[15], d_ptr->i,d_ptr->j,d_ptr->k,d_ptr->l, + d_ptr->m,d_ptr->n, d_ptr->o, d_ptr->p, d_ptr->q, d_ptr->s, d_ptr->t, + d_ptr->u); + goto error; + } + + if(s_ptr->r.len!=d_ptr->r.len) { + H5_FAILED(); + printf(" i=%d\n", i); + printf("VL data lengths don't match!, src len=%d, dst len=%d\n",(int)s_ptr->r.len,(int)d_ptr->r.len); + goto error; + } /* end if */ + for(j=0; j<s_ptr->r.len; j++) { + if( ((unsigned int *)s_ptr->r.p)[j] != ((unsigned int *)d_ptr->r.p)[j] ) { + H5_FAILED(); + printf(" i=%d\n", i); + printf("VL data values don't match!, src r.p[%d]=%u, dst r.p[%d]=%u\n",(int)j, ((unsigned int *)s_ptr->r.p)[j], (int)j, ((unsigned int *)d_ptr->r.p)[j]); + goto error; + } + } + } + + return SUCCEED; + +error: + return FAIL; +} + + +/*------------------------------------------------------------------------- * Function: test_hdf5_src_subset * * Purpose: Test the optimization of compound data writing, rewriting, @@ -1659,9 +1831,6 @@ test_hdf5_src_subset(char *filename, hid_t fapl) if((dxpl = H5Pcreate(H5P_DATASET_XFER)) < 0) goto error; - if(H5Pset_preserve(dxpl, TRUE) < 0) - goto error; - /* Rewrite contiguous data set */ if((dataset = H5Dopen2(file, DSET_NAME[0], H5P_DEFAULT)) < 0) goto error; @@ -1813,7 +1982,7 @@ test_hdf5_dst_subset(char *filename, hid_t fapl) rbuf = (void*)malloc(NX * NY * sizeof(stype1)); rew_buf = (void*)malloc(NX * NY * sizeof(stype4)); - initialize_stype4(rew_buf, (size_t)NX*NY); + initialize_stype4(rew_buf, (size_t)NX*NY, 0); /* Create dataset creation property list */ if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) @@ -1863,9 +2032,6 @@ test_hdf5_dst_subset(char *filename, hid_t fapl) if((dxpl = H5Pcreate (H5P_DATASET_XFER)) < 0) goto error; - if(H5Pset_preserve(dxpl, TRUE) < 0) - goto error; - /* Rewrite contiguous data set */ if((dataset = H5Dopen2(file, DSET_NAME[2], H5P_DEFAULT)) < 0) goto error; @@ -1973,23 +2139,23 @@ error: */ #ifdef H5_HAVE_FILTER_DTYPE_MODIFY static int -test_hdf5_change_dtype_chunked(char *filename, hid_t fapl) +test_hdf5_change_dtype_chunked(char *filename, hid_t fapl, int dtype_flag) { hid_t file; - hid_t rew_tid, src_tid, dst_tid; + hid_t rew_tid, rew2_tid, src_tid, dst_tid; hid_t dataset; hid_t space, space2, mspace; hid_t fcpl, dcpl, dxpl; + H5O_info_t type_info; hsize_t dims[2] = {NX, NY}; hsize_t chunk_dims[2] = {NX/10, NY/10}; hsize_t mem_dims[2] = {NX/2, NY}; - void *orig=NULL, *rew_buf=NULL, *rbuf=NULL; + void *orig=NULL, *rew_buf=NULL, *rbuf=NULL, *rbuf2=NULL; void *rew_half_buf=NULL, *new_rbuf_half=NULL, *orig_rbuf_half=NULL; hsize_t hs_offset[2] = {0,0}; /* Hyperslab offset */ const hsize_t hs_size[2] = {NX/2, NY}; /* Hyperslab size */ const hsize_t hs_stride[2] = {1,1}; /* Hyperslab stride */ const hsize_t hs_count[2] = {1,1}; /* Hyperslab count */ - unsigned num_indexes; unsigned x; if((fcpl = H5Pcreate(H5P_FILE_CREATE)) < 0) @@ -2022,6 +2188,22 @@ test_hdf5_change_dtype_chunked(char *filename, hid_t fapl) if ((rew_tid=create_stype4())<0) goto error; + if ((rew2_tid=create_stype5())<0) + goto error; + + /* Determine whether to test committed or uncomitted data types */ + if((COMMITTED_DTYPE == dtype_flag || MIXED_DTYPE == dtype_flag) && H5Tcommit2(file, + "src type", src_tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) + goto error; + + if(COMMITTED_DTYPE == dtype_flag && H5Tcommit2(file, "rewrite type", rew_tid, + H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) + goto error; + + if((COMMITTED_DTYPE == dtype_flag || MIXED_DTYPE == dtype_flag) && H5Tcommit2(file, + "second rewrite type", rew2_tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT) < 0) + goto error; + /* Create the data space */ if((space = H5Screate_simple(2, dims, NULL))<0) goto error; @@ -2034,15 +2216,16 @@ test_hdf5_change_dtype_chunked(char *filename, hid_t fapl) initialize_stype2(orig, (size_t)NX*NY); rbuf = (void*)malloc(NX * NY * sizeof(stype1)); + rbuf2 = (void*)malloc(NX * NY * sizeof(stype5)); orig_rbuf_half = (void*)malloc(NX/2 * NY * sizeof(stype2)); new_rbuf_half = (void*)malloc(NX/2 * NY * sizeof(stype4)); rew_half_buf = (void*)malloc((NX/2) * NY * sizeof(stype4)); - initialize_stype4(rew_half_buf, (size_t)(NX/2)*NY); + initialize_stype4(rew_half_buf, (size_t)(NX/2)*NY, 2); rew_buf = (void*)malloc(NX * NY * sizeof(stype4)); - initialize_stype4(rew_buf, (size_t)NX*NY); + initialize_stype4(rew_buf, (size_t)NX*NY, 1); /* Create dataset creation property list */ if((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) @@ -2158,13 +2341,9 @@ test_hdf5_change_dtype_chunked(char *filename, hid_t fapl) /* *###################################################################### - * STEP 4. Rewrite the whole data. + * STEP 4. Rewrite the whole data with the new datatype. */ - TESTING("rewriting data with a subset of original data type"); - - /* Create xfer properties to preserve initialized data */ - if ((dxpl = H5Pcreate (H5P_DATASET_XFER))<0) - goto error; + TESTING("rewriting data with the new data type"); /* Rewrite chunked data set */ if((dataset = H5Dopen2(file, DSET_NAME[3], H5P_DEFAULT))<0) @@ -2198,6 +2377,115 @@ test_hdf5_change_dtype_chunked(char *filename, hid_t fapl) if(H5Dclose(dataset) < 0) goto error; + PASSED(); + + /* + *###################################################################### + * STEP 6. Change datatype after rewrite the whole data. The data in + * the cache should be converted and flushed to the file when the + * dataset is closed. + */ + TESTING("change datatype after rewriting the data"); + + /* Rewrite chunked data set */ + if((dataset = H5Dopen2(file, DSET_NAME[3], H5P_DEFAULT))<0) + goto error; + + /* Modify the data buffer a little */ + initialize_stype4(rew_buf, (size_t)NX*NY, 3); + + /* Write the data to the dataset */ + if(H5Dwrite(dataset, rew_tid, H5S_ALL, H5S_ALL, dxpl, rew_buf)<0) + goto error; + + /* Modify the datatype of the chunked dataset now. It should only change + * the data of the chunks in the cache, not the data in the file. */ + if(H5Dmodify_dtype(dataset, rew2_tid)<0) + goto error; + +#ifdef TMP + /* There seems to be some problem with this operation - temporarily disable it */ + /* Read the data back with rew2_tid (stype5) and compare it with rew_buf + * of rew_tid (stype4). */ + if(H5Dread(dataset, rew2_tid, H5S_ALL, H5S_ALL, dxpl, rbuf2)<0) + goto error; + + if(compare_stype4_stype5(rew_buf, rbuf2, (size_t)NX*NY) < 0) + goto error; +#endif + + if(H5Dclose(dataset) < 0) + goto error; + + /* The reference count for the original dtype should be only 1, for + * it's a committed type and saved in the file, and no other reference. */ + if(H5Tcommitted(src_tid)) { + if(H5Oget_info(src_tid, &type_info) < 0) + goto error; + + if(type_info.rc != 1) + goto error; + } + + /* The reference count for the changed dtype should be 100, for it's + * a committed type, and there're 100 chunks with one of them being + * changed to the new dtype, and the dataset's dtype has been changed, + * too. */ + if(H5Tcommitted(rew_tid)) { + if(H5Oget_info(rew_tid, &type_info) < 0) + goto error; + + if(type_info.rc != 100) + goto error; + } + + /* The reference count for the second changed dtype should be 3, for + * it's a committed type, and one chunk has been converted in the cached + * and flushed to the file, and the dataset's dtype has been changed, + * too. */ + if(H5Tcommitted(rew2_tid)) { + if(H5Oget_info(rew2_tid, &type_info) < 0) + goto error; + + if(type_info.rc != 3) + goto error; + } + + PASSED(); + + /* + *###################################################################### + * STEP 7. Delete the dataset and verify the reference count of the + * data types. + */ + TESTING("delete the dataset"); + + /* delete the data set */ + if(H5Ldelete(file, DSET_NAME[3], H5P_DEFAULT)<0) + goto error; + + /* The reference count for the changed dtype should be 1, for it's + * a committed type, and the dataset refered to it has been deleted. + */ + if(H5Tcommitted(rew_tid)) { + if(H5Oget_info(rew_tid, &type_info) < 0) + goto error; + + if(type_info.rc != 1) + goto error; + } + + /* The reference count for the second changed dtype should be 1, for + * it's a committed type, and the dataset refered to it has been deleted. + */ + if(H5Tcommitted(rew2_tid)) { + if(H5Oget_info(rew2_tid, &type_info) < 0) + goto error; + + if(type_info.rc != 1) + goto error; + } + /* Finishing test and release resources */ if(H5Sclose(space) < 0) goto error; @@ -2217,6 +2505,9 @@ test_hdf5_change_dtype_chunked(char *filename, hid_t fapl) goto error; if(H5Tclose(rew_tid)<0) goto error; + if(H5Tclose(rew2_tid)<0) + goto error; + if(H5Fclose(file) < 0) goto error; @@ -2264,7 +2555,7 @@ static int test_hdf5_change_dtype_no_filter(char *filename, hid_t fapl, htri_t is_contig) { hid_t file; - hid_t rew_tid, src_tid, dst_tid; + hid_t rew_tid, src_tid; hid_t dataset; hid_t space; hid_t dcpl, dxpl; @@ -2272,8 +2563,6 @@ test_hdf5_change_dtype_no_filter(char *filename, hid_t fapl, htri_t is_contig) hsize_t chunk_dims[2] = {NX/10, NY/10}; /*hsize_t chunk_dims[2] = {NX, NY};*/ void *orig=NULL, *rew_buf=NULL, *rbuf1=NULL, *rbuf2=NULL; - unsigned num_indexes; - unsigned x; /* Create the file for this test */ if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) @@ -2297,7 +2586,7 @@ test_hdf5_change_dtype_no_filter(char *filename, hid_t fapl, htri_t is_contig) rbuf1 = (void*)malloc(NX * NY * sizeof(stype2)); rew_buf = (void*)malloc(NX * NY * sizeof(stype4)); - initialize_stype4(rew_buf, (size_t)NX*NY); + initialize_stype4(rew_buf, (size_t)NX*NY, 0); rbuf2 = (void*)malloc(NX * NY * sizeof(stype4)); @@ -2461,14 +2750,12 @@ static int test_hdf5_change_dtype_compact(char *filename, hid_t fapl) { hid_t file; - hid_t rew_tid, src_tid, dst_tid; + hid_t rew_tid, src_tid; hid_t dataset; hid_t space; hid_t dcpl, dxpl; hsize_t dims[2] = {COMP_NX, COMP_NY}; void *orig=NULL, *rew_buf=NULL, *rbuf1=NULL, *rbuf2=NULL; - unsigned num_indexes; - unsigned x; /* Create the file for this test */ if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) @@ -2492,7 +2779,7 @@ test_hdf5_change_dtype_compact(char *filename, hid_t fapl) rbuf1 = (void*)malloc(COMP_NX * COMP_NY * sizeof(stype2)); rew_buf = (void*)malloc(COMP_NX * COMP_NY * sizeof(stype4)); - initialize_stype4(rew_buf, (size_t)COMP_NX*COMP_NY); + initialize_stype4(rew_buf, (size_t)COMP_NX*COMP_NY, 0); rbuf2 = (void*)malloc(COMP_NX * COMP_NY * sizeof(stype4)); @@ -2694,21 +2981,31 @@ main (int argc, char *argv[]) h5_fixname(FILENAME[2], fapl_id, fname, sizeof(fname)); nerrors += test_hdf5_dst_subset(fname, fapl_id); - puts("Testing the filter for modifying datatype of chunked dataset:"); +#ifndef H5_USE_16_API + puts("Testing the filter for modifying datatype of chunked dataset with uncommitted dtype:"); h5_fixname(FILENAME[3], fapl_id, fname, sizeof(fname)); - nerrors += test_hdf5_change_dtype_chunked(fname, fapl_id); + nerrors += test_hdf5_change_dtype_chunked(fname, fapl_id, UNCOMMITTED_DTYPE); - puts("Testing modifying datatype of chunked dataset with no filter:"); + puts("Testing the filter for modifying datatype of chunked dataset with committed and uncommitted dtype:"); h5_fixname(FILENAME[4], fapl_id, fname, sizeof(fname)); + nerrors += test_hdf5_change_dtype_chunked(fname, fapl_id, MIXED_DTYPE); + + puts("Testing the filter for modifying datatype of chunked dataset with committed dtype:"); + h5_fixname(FILENAME[5], fapl_id, fname, sizeof(fname)); + nerrors += test_hdf5_change_dtype_chunked(fname, fapl_id, COMMITTED_DTYPE); + + puts("Testing modifying datatype of chunked dataset with no filter:"); + h5_fixname(FILENAME[6], fapl_id, fname, sizeof(fname)); nerrors += test_hdf5_change_dtype_no_filter(fname, fapl_id, FALSE); puts("Testing modifying datatype of contiguous dataset:"); - h5_fixname(FILENAME[5], fapl_id, fname, sizeof(fname)); + h5_fixname(FILENAME[7], fapl_id, fname, sizeof(fname)); nerrors += test_hdf5_change_dtype_no_filter(fname, fapl_id, TRUE); puts("Testing modifying datatype of compact dataset:"); - h5_fixname(FILENAME[6], fapl_id, fname, sizeof(fname)); + h5_fixname(FILENAME[8], fapl_id, fname, sizeof(fname)); nerrors += test_hdf5_change_dtype_compact(fname, fapl_id); +#endif if (nerrors) { printf("***** %u FAILURE%s! *****\n", diff --git a/test/dsets.c b/test/dsets.c index d5949eb..464cfad 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -160,17 +160,28 @@ const char *FILENAME[] = { int points[DSET_DIM1][DSET_DIM2], check[DSET_DIM1][DSET_DIM2]; double points_dbl[DSET_DIM1][DSET_DIM2], check_dbl[DSET_DIM1][DSET_DIM2]; +#ifndef H5_USE_16_API /* Local prototypes for filter functions */ -static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, +static size_t filter_bogus(unsigned int flags, hsize_t chunk_offset, size_t cd_nelmts, const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); static herr_t can_apply_bogus(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t file_id); static herr_t set_local_bogus2(hid_t dcpl_id, hid_t type_id, hid_t space_id, hid_t file_id); +static size_t filter_bogus2(unsigned int flags, hsize_t chunk_offset, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +static size_t filter_corrupt(unsigned int flags, hsize_t chunk_offset, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +#else +static size_t filter_bogus(unsigned int flags, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +static herr_t can_apply_bogus(hid_t dcpl_id, hid_t type_id, hid_t space_id); +static herr_t set_local_bogus2(hid_t dcpl_id, hid_t type_id, hid_t space_id); static size_t filter_bogus2(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); static size_t filter_corrupt(unsigned int flags, size_t cd_nelmts, const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf); +#endif /*------------------------------------------------------------------------- @@ -991,16 +1002,31 @@ test_tconv(hid_t file) } /* This message derives from H5Z */ +#ifndef H5_USE_16_API const H5Z_class_t H5Z_BOGUS[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_BOGUS, /* Filter id number */ - 1, 1, /* Encoding and decoding enabled */ + 1, 1, /* Encoding and decoding enabled*/ "bogus", /* Filter name for debugging */ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ + filter_bogus, /* The actual filter function */ +}}; +#else +const H5Z_class_t H5Z_BOGUS[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_BOGUS, /* Filter id number */ + "bogus", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ filter_bogus, /* The actual filter function */ }}; +#endif /*------------------------------------------------------------------------- @@ -1019,9 +1045,13 @@ const H5Z_class_t H5Z_BOGUS[1] = {{ * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API static herr_t can_apply_bogus(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id, hid_t UNUSED file_id) +#else +can_apply_bogus(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id) +#endif { if(H5Tequal(type_id,H5T_NATIVE_DOUBLE)) return 0; @@ -1046,10 +1076,17 @@ can_apply_bogus(hid_t UNUSED dcpl_id, hid_t type_id, hid_t UNUSED space_id, * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API +static size_t +filter_bogus(unsigned int UNUSED flags, hsize_t UNUSED chunk_offset, + size_t UNUSED cd_nelmts, const unsigned int UNUSED *cd_values, + size_t nbytes, size_t UNUSED *buf_size, void UNUSED **buf) +#else static size_t -filter_bogus(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, - const unsigned int UNUSED *cd_values, size_t nbytes, +filter_bogus(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, + const unsigned int UNUSED *cd_values, size_t nbytes, size_t UNUSED *buf_size, void UNUSED **buf) +#endif { return nbytes; } @@ -1072,9 +1109,14 @@ filter_bogus(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API static herr_t set_local_bogus2(hid_t dcpl_id, hid_t type_id, hid_t UNUSED space_id, hid_t UNUSED file_id) +#else +static herr_t +set_local_bogus2(hid_t dcpl_id, hid_t type_id, hid_t UNUSED space_id) +#endif { unsigned add_on=0; /* Value to add to data going through */ unsigned flags; /* Filter flags */ @@ -1126,10 +1168,15 @@ set_local_bogus2(hid_t dcpl_id, hid_t type_id, hid_t UNUSED space_id, * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API +static size_t +filter_bogus2(unsigned int flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf) +#else static size_t filter_bogus2(unsigned int flags, size_t cd_nelmts, - const unsigned int *cd_values, size_t nbytes, - size_t *buf_size, void **buf) + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf) +#endif { /* Check for the correct number of parameters */ if(cd_nelmts!=BOGUS2_ALL_NPARMS) @@ -1168,16 +1215,31 @@ filter_bogus2(unsigned int flags, size_t cd_nelmts, } /* This message derives from H5Z */ +#ifndef H5_USE_16_API const H5Z_class_t H5Z_CORRUPT[1] = {{ - H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ H5Z_FILTER_CORRUPT, /* Filter id number */ - 1, 1, /* Encoding and decoding enabled */ + 1, 1, /* Encoding and decoding enabled*/ "corrupt", /* Filter name for debugging */ NULL, /* The "can apply" callback */ NULL, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ + filter_corrupt, /* The actual filter function */ +}}; +#else +const H5Z_class_t H5Z_CORRUPT[1] = {{ + H5Z_CLASS_T_VERS, /* H5Z_class_t version */ + H5Z_FILTER_CORRUPT, /* Filter id number */ + "corrupt", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + NULL, /* The "set local" callback */ filter_corrupt, /* The actual filter function */ }}; +#endif /*------------------------------------------------------------------------- @@ -1198,10 +1260,15 @@ const H5Z_class_t H5Z_CORRUPT[1] = {{ * *------------------------------------------------------------------------- */ +#ifndef H5_USE_16_API +static size_t +filter_corrupt(unsigned int flags, hsize_t UNUSED chunk_offset, size_t cd_nelmts, + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf) +#else static size_t filter_corrupt(unsigned int flags, size_t cd_nelmts, - const unsigned int *cd_values, size_t nbytes, - size_t *buf_size, void **buf) + const unsigned int *cd_values, size_t nbytes, size_t *buf_size, void **buf) +#endif { size_t ret_value = 0; unsigned char *dst = (unsigned char*)(*buf); @@ -1355,7 +1422,8 @@ test_filter_internal(hid_t fid, const char *name, hid_t dcpl, int if_fletcher32, /* Create the dataset */ if((dataset = H5Dcreate2(fid, name, H5T_NATIVE_INT, sid, H5P_DEFAULT, - dcpl, H5P_DEFAULT)) < 0) goto error; + dcpl, H5P_DEFAULT)) < 0) + goto error; PASSED(); @@ -2018,8 +2086,16 @@ UNUSED data_corrupt[1] = 33; data_corrupt[2] = 27; - /* Temporarily disable this test because the changes in chunk caching conflicts with - * the way this test is conducted. -slu 2007/7/20 */ + /* This is a shaky test because any changes in chunk caching may conflict with + * the way this test is conducted. In Step 4 of test_filter_internal, the + * program tries to modify some data. When it does H5Dwrite, the library will + * overwrite the entire chunks if all data in the chunks are being modified. + * However, if the data being modified doesn't cover the whole chunks or the + * filter for modifying dataset's datatype is enabled, the chunks will be read + * first which will cause the Fletcher32 checksum to fail. (See how the RELAX + * variable is defined in H5D_chunk_write of H5Dio.c and H5D_istore_lock of + * H5Distore.c.) - SLU 2008/3/5 + */ if(H5Zregister (H5Z_CORRUPT) < 0) goto error; if(H5Pset_filter(dc, H5Z_FILTER_CORRUPT, 0, (size_t)3, data_corrupt) < 0) goto error; @@ -2124,6 +2200,7 @@ UNUSED * STEP 5: Test datatype modification by itself. *---------------------------------------------------------- */ +#ifndef H5_USE_16_API #ifdef H5_HAVE_FILTER_DTYPE_MODIFY puts("Testing datatype modification filter"); if((dc = H5Pcreate(H5P_DATASET_CREATE))<0) goto error; @@ -2139,6 +2216,7 @@ UNUSED SKIPPED(); puts(" Datatype modification filter not enabled"); #endif /* H5_HAVE_FILTER_DTYPE_MODIFY */ +#endif /* H5_USE_16_API */ /*---------------------------------------------------------- * STEP 6: Test shuffle + deflate + dtype modification. @@ -2148,11 +2226,17 @@ UNUSED puts("Testing dtype modify+shuffle+deflate filters(dtype modify first)"); if((dc = H5Pcreate(H5P_DATASET_CREATE))<0) goto error; if (H5Pset_chunk (dc, 2, chunk_size)<0) goto error; +#ifndef H5_USE_16_API if (H5Pset_dtype_modifiable (dc)<0) goto error; +#endif /* H5_USE_16_API */ if (H5Pset_shuffle (dc)<0) goto error; if (H5Pset_deflate (dc, 6)<0) goto error; +#ifndef H5_USE_16_API if(test_filter_internal(file,DSET_SHUF_DEF_DTMOD_NAME,dc,ENABLE_FLETCHER32,DATA_NOT_CORRUPTED,CHANGE_DTYPE,&combo_size)<0) goto error; +#else + if(test_filter_internal(file,DSET_SHUF_DEF_DTMOD_NAME,dc,ENABLE_FLETCHER32,DATA_NOT_CORRUPTED,DONT_CHANGE_DTYPE,&combo_size)<0) goto error; +#endif /* H5_USE_16_API */ /* Clean up objects used for this test */ if (H5Pclose (dc)<0) goto error; @@ -4879,16 +4963,31 @@ test_types(hid_t file) } /* This message derives from H5Z */ +#ifndef H5_USE_16_API const H5Z_class_t H5Z_CAN_APPLY_TEST[1] = {{ - H5Z_CLASS_T_VERS, + H5Z_CLASS_T_VERS, H5Z_FILTER_BOGUS3, /* Filter id number */ - 1, 1, + 1, 1, "bogus", /* Filter name for debugging */ can_apply_bogus, /* The "can apply" callback */ NULL, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ filter_bogus, /* The actual filter function */ }}; +#else +const H5Z_class_t H5Z_CAN_APPLY_TEST[1] = {{ + H5Z_CLASS_T_VERS, + H5Z_FILTER_BOGUS3, /* Filter id number */ + "bogus", /* Filter name for debugging */ + can_apply_bogus, /* The "can apply" callback */ + NULL, /* The "set local" callback */ + filter_bogus, /* The actual filter function */ +}}; +#endif /*------------------------------------------------------------------------- @@ -5240,16 +5339,31 @@ error: /* This message derives from H5Z */ +#ifndef H5_USE_16_API const H5Z_class_t H5Z_SET_LOCAL_TEST[1] = {{ - H5Z_CLASS_T_VERS, + H5Z_CLASS_T_VERS, H5Z_FILTER_BOGUS2, /* Filter id number */ - 1, 1, + 1, 1, "bogus2", /* Filter name for debugging */ NULL, /* The "can apply" callback */ set_local_bogus2, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ filter_bogus2, /* The actual filter function */ }}; +#else +const H5Z_class_t H5Z_SET_LOCAL_TEST[1] = {{ + H5Z_CLASS_T_VERS, + H5Z_FILTER_BOGUS2, /* Filter id number */ + "bogus2", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + set_local_bogus2, /* The "set local" callback */ + filter_bogus2, /* The actual filter function */ +}}; +#endif /*------------------------------------------------------------------------- @@ -6580,6 +6694,7 @@ main(void) nerrors += (test_tconv(file) < 0 ? 1 : 0); nerrors += (test_filters(file, my_fapl) < 0 ? 1 : 0); nerrors += (test_onebyte_shuffle(file) < 0 ? 1 : 0); +#ifndef H5_USE_16_API nerrors += (test_nbit_int(file) < 0 ? 1 : 0); nerrors += (test_nbit_float(file) < 0 ? 1 : 0); nerrors += (test_nbit_double(file) < 0 ? 1 : 0); @@ -6593,6 +6708,7 @@ main(void) nerrors += (test_scaleoffset_float_2(file) < 0 ? 1 : 0); nerrors += (test_scaleoffset_double(file) < 0 ? 1 : 0); nerrors += (test_scaleoffset_double_2(file) < 0 ? 1 : 0); +#endif nerrors += (test_multiopen (file) < 0 ? 1 : 0); nerrors += (test_types(file) < 0 ? 1 : 0); nerrors += (test_userblock_offset(my_fapl) < 0 ? 1 : 0); @@ -6635,4 +6751,3 @@ error: nerrors, 1 == nerrors ? "" : "S"); return 1; } - diff --git a/tools/h5dump/h5dumpgentest.c b/tools/h5dump/h5dumpgentest.c index f72fe8f..a994d42 100644 --- a/tools/h5dump/h5dumpgentest.c +++ b/tools/h5dump/h5dumpgentest.c @@ -102,18 +102,18 @@ static int write_dset( hid_t loc_id, int rank, hsize_t *dims, const char *dset_name, hid_t tid, void *buf ); +#define MYFILTER_ID 405 +#ifndef H5_USE_16_API /* a filter operation callback function */ static size_t -myfilter(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, - const unsigned int UNUSED *cd_values, size_t nbytes, - size_t UNUSED *buf_size, void UNUSED **buf); +myfilter(unsigned int UNUSED flags, hsize_t UNUSED chunk_offset, size_t UNUSED cd_nelmts, + const unsigned int UNUSED *cd_values, size_t nbytes, size_t UNUSED *buf_size, + void UNUSED **buf); /* a "set local" callback */ static herr_t set_local_myfilter(hid_t dcpl_id, hid_t tid, hid_t UNUSED sid, hid_t UNUSED file_id); -#define MYFILTER_ID 405 - /* This message derives from H5Z */ const H5Z_class_t H5Z_MYFILTER[1] = {{ H5Z_CLASS_T_VERS, @@ -123,9 +123,33 @@ const H5Z_class_t H5Z_MYFILTER[1] = {{ NULL, /* The "can apply" callback */ set_local_myfilter, /* The "set local" callback */ NULL, /* The "reset local" callback */ + NULL, /* The "change local" callback */ + NULL, /* The "evict local" callback */ + NULL, /* The "delete local" callback */ + NULL, /* The "close local" callback */ myfilter, /* The actual filter function */ }}; +#else +/* a filter operation callback function */ +static size_t +myfilter(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, + const unsigned int UNUSED *cd_values, size_t nbytes, size_t UNUSED *buf_size, + void UNUSED **buf); +/* a "set local" callback */ +static herr_t +set_local_myfilter(hid_t dcpl_id, hid_t tid, hid_t UNUSED sid); + +/* This message derives from H5Z */ +const H5Z_class_t H5Z_MYFILTER[1] = {{ + H5Z_CLASS_T_VERS, + MYFILTER_ID, /* Filter id number */ + "myfilter", /* Filter name for debugging */ + NULL, /* The "can apply" callback */ + set_local_myfilter, /* The "set local" callback */ + myfilter, /* The actual filter function */ +}}; +#endif /* A UD link traversal function. Shouldn't actually be called. */ static hid_t UD_traverse(UNUSED const char * link_name, UNUSED hid_t cur_group, @@ -4925,6 +4949,51 @@ static void gent_filters(void) } +#ifndef H5_USE_16_API +/*------------------------------------------------------------------------- + * Function: myfilter + * + * Purpose: filter operation callback function; the filter does nothing + * + *------------------------------------------------------------------------- + */ +static size_t +myfilter(unsigned int UNUSED flags, hsize_t chunk_offset, size_t UNUSED cd_nelmts, + const unsigned int UNUSED *cd_values, size_t nbytes, size_t UNUSED *buf_size, + void UNUSED **buf) +{ + return nbytes; +} + + +/*------------------------------------------------------------------------- + * Function: set_local_myfilter + * + * Purpose: filter operation "set local" callback + * + *------------------------------------------------------------------------- + */ + +static herr_t +set_local_myfilter(hid_t dcpl_id, hid_t UNUSED tid, hid_t UNUSED sid, hid_t UNUSED file_id) +{ + unsigned flags; /* Filter flags */ + size_t cd_nelmts = 0; /* Number of filter parameters */ + unsigned cd_values[2] = {5, 6}; /* Filter parameters */ + + /* Get the filter's current parameters */ + if(H5Pget_filter_by_id2(dcpl_id, MYFILTER_ID, &flags, &cd_nelmts, cd_values, 0, NULL, NULL) < 0) + return(FAIL); + + cd_nelmts = 2; + + /* Modify the filter's parameters for this dataset */ + if(H5Pmodify_filter(dcpl_id, MYFILTER_ID, flags, cd_nelmts, cd_values) < 0) + return(FAIL); + + return(SUCCEED); +} +#else /*------------------------------------------------------------------------- * Function: myfilter * @@ -4950,7 +5019,7 @@ myfilter(unsigned int UNUSED flags, size_t UNUSED cd_nelmts, */ static herr_t -set_local_myfilter(hid_t dcpl_id, hid_t UNUSED tid, hid_t UNUSED sid, hid_t UNUSED file_id) +set_local_myfilter(hid_t dcpl_id, hid_t UNUSED tid, hid_t UNUSED sid) { unsigned flags; /* Filter flags */ size_t cd_nelmts = 0; /* Number of filter parameters */ @@ -4969,6 +5038,8 @@ set_local_myfilter(hid_t dcpl_id, hid_t UNUSED tid, hid_t UNUSED sid, hid_t UNUS return(SUCCEED); } +#endif + /*------------------------------------------------------------------------- * Function: gent_fcontents * |