diff options
author | David Young <dyoung@hdfgroup.org> | 2019-12-09 16:30:58 (GMT) |
---|---|---|
committer | David Young <dyoung@hdfgroup.org> | 2019-12-09 16:30:58 (GMT) |
commit | c8f533cfc33ac743227cbed8eba361c715a2976f (patch) | |
tree | bcae5320f80bac774647cacbbd8493604f9384d2 /src/H5Dint.c | |
parent | adcf8a315e82c0848d126e7e46b662930c081896 (diff) | |
download | hdf5-c8f533cfc33ac743227cbed8eba361c715a2976f.zip hdf5-c8f533cfc33ac743227cbed8eba361c715a2976f.tar.gz hdf5-c8f533cfc33ac743227cbed8eba361c715a2976f.tar.bz2 |
Merge all of my changes from merge-back-to-feature-vfd_swmr-attempt-1,
including the merge of `hdffv/hdf5/develop`, back to the branch that Vailin and
I share.
Now I need to put this branch on a fork with a less confusing name than
vchoi_fork!
Diffstat (limited to 'src/H5Dint.c')
-rw-r--r-- | src/H5Dint.c | 494 |
1 files changed, 338 insertions, 156 deletions
diff --git a/src/H5Dint.c b/src/H5Dint.c index 9d15ca6..97ce042 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -21,15 +21,17 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5CXprivate.h" /* API Contexts */ -#include "H5Dpkg.h" /* Datasets */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5FLprivate.h" /* Free Lists */ -#include "H5FOprivate.h" /* File objects */ -#include "H5Iprivate.h" /* IDs */ -#include "H5Lprivate.h" /* Links */ -#include "H5MMprivate.h" /* Memory management */ +#include "H5private.h" /* Generic Functions */ +#include "H5CXprivate.h" /* API Contexts */ +#include "H5Dpkg.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* Files */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5FOprivate.h" /* File objects */ +#include "H5Iprivate.h" /* IDs */ +#include "H5Lprivate.h" /* Links */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5VLprivate.h" /* Virtual Object Layer */ /****************/ @@ -52,12 +54,15 @@ static herr_t H5D__init_type(H5F_t *file, const H5D_t *dset, hid_t type_id, cons static herr_t H5D__cache_dataspace_info(const H5D_t *dset); static herr_t H5D__init_space(H5F_t *file, const H5D_t *dset, const H5S_t *space); static herr_t H5D__update_oh_info(H5F_t *file, H5D_t *dset, hid_t dapl_id); -static herr_t H5D__build_file_prefix(const H5D_t *dset, hid_t dapl_id, const char *prefix_type, char **file_prefix); +static herr_t H5D__build_file_prefix(const H5D_t *dset, H5F_prefix_open_t prefix_type, char **file_prefix); static herr_t H5D__open_oid(H5D_t *dataset, hid_t dapl_id); static herr_t H5D__init_storage(const H5D_io_info_t *io_info, hbool_t full_overwrite, hsize_t old_dim[]); static herr_t H5D__append_flush_setup(H5D_t *dset, hid_t dapl_id); -static herr_t H5D__close_cb(H5D_t *dset); +static herr_t H5D__close_cb(H5VL_object_t *dset_vol_obj); +static herr_t H5D__use_minimized_dset_headers(H5F_t *file, H5D_t *dset, hbool_t *minimize); +static herr_t H5D__prepare_minimized_oh(H5F_t *file, H5D_t *dset, H5O_loc_t *oloc); +static size_t H5D__calculate_minimum_header_size(H5F_t *file, H5D_t *dset, H5O_t *ohdr); /*********************/ @@ -107,6 +112,10 @@ static const H5I_class_t H5I_DATASET_CLS[1] = {{ /* Flag indicating "top" of interface has been initialized */ static hbool_t H5D_top_package_initialize_s = FALSE; +/* Prefixes of VDS and external file from the environment variables + * HDF5_EXTFILE_PREFIX and HDF5_VDS_PREFIX */ +static const char *H5D_prefix_ext_env = NULL; +static const char *H5D_prefix_vds_env = NULL; /*------------------------------------------------------------------------- @@ -183,6 +192,10 @@ H5D__init_package(void) /* Mark "top" of interface as initialized, too */ H5D_top_package_initialize_s = TRUE; + /* Retrieve the prefixes of VDS and external file from the environment variable */ + H5D_prefix_vds_env = HDgetenv("HDF5_VDS_PREFIX"); + H5D_prefix_ext_env = HDgetenv("HDF5_EXTFILE_PREFIX"); + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__init_package() */ @@ -289,23 +302,29 @@ H5D_term_package(void) *------------------------------------------------------------------------- */ static herr_t -H5D__close_cb(H5D_t *dset) +H5D__close_cb(H5VL_object_t *dset_vol_obj) { herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Sanity check */ - HDassert(dset); - HDassert(dset->oloc.file); - HDassert(dset->shared); - HDassert(dset->shared->fo_count > 0); + HDassert(dset_vol_obj); /* Close the dataset */ - if(H5D_close(dset) < 0) + if(H5VL_dataset_close(dset_vol_obj, H5P_DATASET_XFER_DEFAULT, H5_REQUEST_NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to close dataset"); done: + /* XXX: (MSC) Weird thing for datasets and filters: + * Always decrement the ref count on the VOL for datasets, since + * the ID is removed even if the close fails. + */ + + /* Free the VOL object */ + if(H5VL_free_object(dset_vol_obj) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "unable to free VOL object"); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__close_cb() */ @@ -459,7 +478,7 @@ H5D__new(hid_t dcpl_id, hbool_t creating, hbool_t vl_type) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* Copy the default dataset information */ - HDmemcpy(new_dset, &H5D_def_dset, sizeof(H5D_shared_t)); + H5MM_memcpy(new_dset, &H5D_def_dset, sizeof(H5D_shared_t)); /* If we are using the default dataset creation property list, during creation * don't bother to copy it, just increment the reference count @@ -528,7 +547,7 @@ H5D__init_type(H5F_t *file, const H5D_t *dset, hid_t type_id, const H5T_t *type) /* To use at least v18 format versions or not */ use_at_least_v18 = (H5F_LOW_BOUND(file) >= H5F_LIBVER_V18); - /* Copy the datatype if it's a custom datatype or if it'll change when it's location is changed */ + /* Copy the datatype if it's a custom datatype or if it'll change when its location is changed */ if(!immutable || relocatable || use_at_least_v18) { /* Copy datatype for dataset */ if((dset->shared->type = H5T_copy(type, H5T_COPY_ALL)) == NULL) @@ -551,7 +570,7 @@ H5D__init_type(H5F_t *file, const H5D_t *dset, hid_t type_id, const H5T_t *type) /* Get a datatype ID for the dataset's datatype */ if((dset->shared->type_id = H5I_register(H5I_DATATYPE, dset->shared->type, FALSE)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, "unable to register type") - } + } /* end if */ /* Not a custom datatype, just use it directly */ else { if(H5I_inc_ref(type_id, FALSE) < 0) @@ -560,7 +579,7 @@ H5D__init_type(H5F_t *file, const H5D_t *dset, hid_t type_id, const H5T_t *type) /* Use existing datatype */ dset->shared->type_id = type_id; dset->shared->type = (H5T_t *)type; /* (Cast away const OK - QAK) */ - } + } /* end else */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -651,6 +670,208 @@ done: /*------------------------------------------------------------------------- + * Function: H5D__use_minimized_dset_headers + * + * Purpose: Compartmentalize check for file- or dcpl-set values indicating + * to create a "minimized" dataset object header. + * Upon success, write resulting value to out pointer `minimize`. + * + * Return: Success: SUCCEED (0) (non-negative value) + * Failure: FAIL (-1) (negative value) + * + * Programmer: Jacob Smith + * 16 August 2018 + *------------------------------------------------------------------------- + */ +static herr_t +H5D__use_minimized_dset_headers(H5F_t *file, H5D_t *dset, hbool_t *minimize) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT; + + HDassert(file); + HDassert(dset); + HDassert(minimize); + + /* Get the dataset object header minimize flag for this call */ + if(H5CX_get_dset_min_ohdr_flag(minimize) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset object header minimize flag from API context") + + if(FALSE == *minimize) + *minimize = H5F_get_min_dset_ohdr(file); + +done: + if(FAIL == ret_value) + *minimize = FALSE; + FUNC_LEAVE_NOAPI(ret_value); +} /* H5D__use_minimized_dset_headers */ + + +/*------------------------------------------------------------------------- + * Function: H5D__calculate_minimium_header_size + * + * Purpose: Calculate the size required for the minimized object header. + * + * Return: Success: Positive value > 0 + * Failure: 0 + * + * Programmer: Jacob Smith + * 16 August 2018 + *------------------------------------------------------------------------- + */ +static size_t +H5D__calculate_minimum_header_size(H5F_t *file, H5D_t *dset, H5O_t *ohdr) +{ + H5T_t *type = NULL; + H5O_fill_t *fill_prop = NULL; + hbool_t use_at_least_v18 = FALSE; + const char continuation[1] = ""; /* requred for work-around */ + size_t get_value = 0; + size_t ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT; + + HDassert(file); + HDassert(dset); + HDassert(ohdr); + + type = dset->shared->type; + fill_prop = &(dset->shared->dcpl_cache.fill); + use_at_least_v18 = (H5F_LOW_BOUND(file) >= H5F_LIBVER_V18); + + /* Datatype message size */ + get_value = H5O_msg_size_oh(file, ohdr, H5O_DTYPE_ID, type, 0); + if (get_value == 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "Can't get size of datatype message") + ret_value += get_value; + + /* Shared Dataspace message size */ + get_value = H5O_msg_size_oh(file, ohdr, H5O_SDSPACE_ID, dset->shared->space, 0); + if (get_value == 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "can't get size of dataspace message") + ret_value += get_value; + + /* "Layout" message size */ + get_value = H5O_msg_size_oh(file, ohdr, H5O_LAYOUT_ID, &dset->shared->layout, 0); + if (get_value == 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "can't get size of layout message") + ret_value += get_value; + + /* Fill Value message size */ + get_value = H5O_msg_size_oh(file, ohdr, H5O_FILL_NEW_ID, fill_prop, 0); + if (get_value == 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "can't get size of fill value message") + ret_value += get_value; + + /* "Continuation" message size */ + /* message pointer "continuation" is unused by raw get function, however, + * a null pointer would be intercepted by an assert in H5O_msg_size_oh(). + */ + get_value = H5O_msg_size_oh(file, ohdr, H5O_CONT_ID, continuation, 0); + if (get_value == 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "can't get size of continuation message") + ret_value += get_value; + + /* Fill Value (backwards compatability) message size */ + if(fill_prop->buf && !use_at_least_v18) { + H5O_fill_t old_fill_prop; /* Copy for writing "old" fill value */ + + /* Shallow copy the fill value property */ + /* guards against shared component modification */ + H5MM_memcpy(&old_fill_prop, fill_prop, sizeof(old_fill_prop)); + + if (H5O_msg_reset_share(H5O_FILL_ID, &old_fill_prop) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "can't reset the copied fill property") + + get_value = H5O_msg_size_oh(file, ohdr, H5O_FILL_ID, &old_fill_prop, 0); + if (get_value == 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "can't get size of fill value (backwards compat) message") + ret_value += get_value; + } + + /* Filter/Pipeline message size */ + if(H5D_CHUNKED == dset->shared->layout.type) { + H5O_pline_t *pline = &dset->shared->dcpl_cache.pline; + if(pline->nused > 0) { + get_value = H5O_msg_size_oh(file, ohdr, H5O_PLINE_ID, pline, 0); + if (get_value == 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "can't get size of filter message") + ret_value += get_value; + } + } + + /* External File Link message size */ + if(dset->shared->dcpl_cache.efl.nused > 0) { + get_value = H5O_msg_size_oh(file, ohdr, H5O_EFL_ID, &dset->shared->dcpl_cache.efl, 0); + if (get_value == 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "can't get size of external file link message") + ret_value += get_value; + } + + /* Modification Time message size */ + if(H5O_HDR_STORE_TIMES & H5O_OH_GET_FLAGS(ohdr)) { + HDassert(H5O_OH_GET_VERSION(ohdr) >= 1); /* 1 :: H5O_VERSION_1 (H5Opkg.h) */ + + if(H5O_OH_GET_VERSION(ohdr) == 1) { + /* v1 object headers store modification time as a message */ + time_t mtime; + get_value = H5O_msg_size_oh(file, ohdr, H5O_MTIME_NEW_ID, &mtime, 0); + if (get_value == 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, 0, "can't get size of modification time message") + ret_value += get_value; + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* H5D__calculate_minimum_header_size */ + + +/*------------------------------------------------------------------------- + * Function: H5D__prepare_minimized_oh + * + * Purpose: Create an object header (H5O_t) allocated with the smallest + * possible size. + * + * Return: Success: SUCCEED (0) (non-negative value) + * Failure: FAIL (-1) (negative value) + * + * Programmer: Jacob Smith + * 16 August 2018 + *------------------------------------------------------------------------- + */ +static herr_t +H5D__prepare_minimized_oh(H5F_t *file, H5D_t *dset, H5O_loc_t *oloc) +{ + H5O_t *oh = NULL; + size_t ohdr_size = 0; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT; + + HDassert(file); + HDassert(dset); + HDassert(oloc); + + oh = H5O__create_ohdr(file, dset->shared->dcpl_id); + if(NULL == oh) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "can't instantiate object header") + + ohdr_size = H5D__calculate_minimum_header_size(file, dset, oh); + if (ohdr_size == 0) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "computed header size is invalid") + + /* Special allocation of space for compact datsets is handled by the call here. */ + if(H5O__apply_ohdr(file, oh, dset->shared->dcpl_id, ohdr_size, (size_t)1, oloc) == FAIL) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "can't apply object header to file") + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* H5D__prepare_minimized_oh */ + + +/*------------------------------------------------------------------------- * Function: H5D__update_oh_info * * Purpose: Create and fill object header for dataset @@ -662,17 +883,18 @@ done: static herr_t H5D__update_oh_info(H5F_t *file, H5D_t *dset, hid_t dapl_id) { - H5O_t *oh = NULL; /* Pointer to dataset's object header */ - size_t ohdr_size = H5D_MINHDR_SIZE; /* Size of dataset's object header */ - H5O_loc_t *oloc = NULL; /* Dataset's object location */ - H5O_layout_t *layout; /* Dataset's layout information */ - H5T_t *type; /* Dataset's datatype */ - H5O_fill_t *fill_prop; /* Pointer to dataset's fill value information */ - H5D_fill_value_t fill_status; /* Fill value status */ - hbool_t fill_changed = FALSE; /* Flag indicating the fill value was changed */ - hbool_t layout_init = FALSE; /* Flag to indicate that chunk information was initialized */ - hbool_t use_at_least_v18; /* Flag indicating to use at least v18 format versions */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_t *oh = NULL; /* Pointer to dataset's object header */ + size_t ohdr_size = H5D_MINHDR_SIZE; /* Size of dataset's object header */ + H5O_loc_t *oloc = NULL; /* Dataset's object location */ + H5O_layout_t *layout; /* Dataset's layout information */ + H5T_t *type; /* Dataset's datatype */ + H5O_fill_t *fill_prop; /* Pointer to dataset's fill value information */ + H5D_fill_value_t fill_status; /* Fill value status */ + hbool_t fill_changed = FALSE; /* Flag indicating the fill value was changed */ + hbool_t layout_init = FALSE; /* Flag to indicate that chunk information was initialized */ + hbool_t use_at_least_v18; /* Flag indicating to use at least v18 format versions */ + hbool_t minimize_header = FALSE; + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -740,13 +962,24 @@ H5D__update_oh_info(H5F_t *file, H5D_t *dset, hid_t dapl_id) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set fill value info") } /* end if */ - /* Add the dataset's raw data size to the size of the header, if the raw data will be stored as compact */ - if(layout->type == H5D_COMPACT) - ohdr_size += layout->storage.u.compact.size; + if(H5D__use_minimized_dset_headers(file, dset, &minimize_header) == FAIL) + HGOTO_ERROR(H5E_ARGS, H5E_CANTGET, FAIL, "can't get minimize settings") + + if(TRUE == minimize_header) { + if(H5D__prepare_minimized_oh(file, dset, oloc) == FAIL) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't create minimized dataset object header") + } else { + /* Add the dataset's raw data size to the size of the header, if the + * raw data will be stored as compact + */ + if(H5D_COMPACT == layout->type) + ohdr_size += layout->storage.u.compact.size; + + /* Create an object header for the dataset */ + if(H5O_create(file, ohdr_size, (size_t)1, dset->shared->dcpl_id, oloc/*out*/) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to create dataset object header") + } /* If use minimum/standard object header space */ - /* Create an object header for the dataset */ - if(H5O_create(file, ohdr_size, (size_t)1, dset->shared->dcpl_id, oloc/*out*/) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to create dataset object header") HDassert(file == dset->oloc.file); /* Pin the object header */ @@ -772,7 +1005,7 @@ H5D__update_oh_info(H5F_t *file, H5D_t *dset, hid_t dapl_id) /* Shallow copy the fill value property */ /* (we only want to make certain that the shared component isn't modified) */ - HDmemcpy(&old_fill_prop, fill_prop, sizeof(old_fill_prop)); + H5MM_memcpy(&old_fill_prop, fill_prop, sizeof(old_fill_prop)); /* Reset shared component info */ H5O_msg_reset_share(H5O_FILL_ID, &old_fill_prop); @@ -855,15 +1088,13 @@ done: *-------------------------------------------------------------------------- */ static herr_t -H5D__build_file_prefix(const H5D_t *dset, hid_t dapl_id, const char *prefix_type, - char **file_prefix /*out*/) +H5D__build_file_prefix(const H5D_t *dset, H5F_prefix_open_t prefix_type, char **file_prefix /*out*/) { - char *prefix = NULL; /* prefix used to look for the file */ + const char *prefix = NULL; /* prefix used to look for the file */ char *filepath = NULL; /* absolute path of directory the HDF5 file is in */ size_t filepath_len; /* length of file path */ size_t prefix_len; /* length of prefix */ size_t file_prefix_len; /* length of expanded prefix */ - H5P_genplist_t *plist = NULL; /* Property list pointer */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -877,20 +1108,22 @@ H5D__build_file_prefix(const H5D_t *dset, hid_t dapl_id, const char *prefix_type /* XXX: Future thread-safety note - getenv is not required * to be reentrant. */ - if(HDstrcmp(prefix_type, H5D_ACS_VDS_PREFIX_NAME) == 0) - prefix = HDgetenv("HDF5_VDS_PREFIX"); - else if(HDstrcmp(prefix_type, H5D_ACS_EFILE_PREFIX_NAME) == 0) - prefix = HDgetenv("HDF5_EXTFILE_PREFIX"); - else - HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "prefix name is not sensible") - - if(prefix == NULL || *prefix == '\0') { - /* Set prefix to value of prefix_type property */ - if(NULL == (plist = H5P_object_verify(dapl_id, H5P_DATASET_ACCESS))) - HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") - if(H5P_peek(plist, prefix_type, &prefix) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file prefix") + if(H5F_PREFIX_VDS == prefix_type) { + prefix = H5D_prefix_vds_env; + + if(prefix == NULL || *prefix == '\0') + if(H5CX_get_vds_prefix(&prefix) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get the prefix for vds file") } /* end if */ + else if(H5F_PREFIX_EFILE == prefix_type) { + prefix = H5D_prefix_ext_env; + + if(prefix == NULL || *prefix == '\0') + if(H5CX_get_ext_file_prefix(&prefix) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get the prefix for the external file") + } /* end else-if */ + else + HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "prefix name is not sensible") /* Prefix has to be checked for NULL / empty string again because the * code above might have updated it. @@ -899,11 +1132,10 @@ H5D__build_file_prefix(const H5D_t *dset, hid_t dapl_id, const char *prefix_type /* filename is interpreted as relative to the current directory, * does not need to be expanded */ - if(NULL == (*file_prefix = (char *)H5MM_strdup(""))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + *file_prefix = NULL; } /* end if */ else { - if (HDstrncmp(prefix, "${ORIGIN}", HDstrlen("${ORIGIN}")) == 0) { + if(HDstrncmp(prefix, "${ORIGIN}", HDstrlen("${ORIGIN}")) == 0) { /* Replace ${ORIGIN} at beginning of prefix by directory of HDF5 file */ filepath_len = HDstrlen(filepath); prefix_len = HDstrlen(prefix); @@ -943,7 +1175,8 @@ H5D_t * H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, hid_t dapl_id) { - const H5T_t *type; /* Datatype for dataset */ + H5T_t *type = NULL; /* Datatype for dataset (VOL pointer) */ + H5T_t *dt = NULL; /* Datatype for dataset (non-VOL pointer) */ H5D_t *new_dset = NULL; H5P_genplist_t *dc_plist = NULL; /* New Property list */ hbool_t has_vl_type = FALSE; /* Flag to indicate a VL-type for dataset */ @@ -964,9 +1197,12 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, HDassert(H5I_GENPROP_LST == H5I_get_type(dcpl_id)); /* Get the dataset's datatype */ - if(NULL == (type = (const H5T_t *)H5I_object(type_id))) + if(NULL == (dt = (H5T_t *)H5I_object(type_id))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not a datatype") + /* If this is a named datatype, get the pointer via the VOL plugin */ + type = H5T_get_actual_type(dt); + /* Check if the datatype is "sensible" for use in a dataset */ if(H5T_is_sensible(type) != TRUE) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "datatype is not sensible") @@ -1095,11 +1331,11 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to set up flush append property") /* Set the external file prefix */ - if(H5D__build_file_prefix(new_dset, dapl_id, H5D_ACS_EFILE_PREFIX_NAME, &new_dset->shared->extfile_prefix) < 0) + if(H5D__build_file_prefix(new_dset, H5F_PREFIX_EFILE, &new_dset->shared->extfile_prefix) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize external file prefix") /* Set the VDS file prefix */ - if(H5D__build_file_prefix(new_dset, dapl_id, H5D_ACS_VDS_PREFIX_NAME, &new_dset->shared->vds_prefix) < 0) + if(H5D__build_file_prefix(new_dset, H5F_PREFIX_VDS, &new_dset->shared->vds_prefix) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize VDS prefix") /* Add the dataset to the list of opened objects in the file */ @@ -1254,11 +1490,11 @@ H5D_open(const H5G_loc_t *loc, hid_t dapl_id) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, NULL, "can't copy path") /* Get the external file prefix */ - if(H5D__build_file_prefix(dataset, dapl_id, H5D_ACS_EFILE_PREFIX_NAME, &extfile_prefix) < 0) + if(H5D__build_file_prefix(dataset, H5F_PREFIX_EFILE, &extfile_prefix) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize external file prefix") /* Get the VDS prefix */ - if(H5D__build_file_prefix(dataset, dapl_id, H5D_ACS_VDS_PREFIX_NAME, &vds_prefix) < 0) + if(H5D__build_file_prefix(dataset, H5F_PREFIX_VDS, &vds_prefix) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize VDS prefix") /* Check if dataset was already open */ @@ -1302,8 +1538,13 @@ H5D_open(const H5G_loc_t *loc, hid_t dapl_id) /* Check whether the external file prefix of the already open dataset * matches the new external file prefix */ - if(HDstrcmp(extfile_prefix, dataset->shared->extfile_prefix) != 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, NULL, "new external file prefix does not match external file prefix of already open dataset") + if(extfile_prefix && dataset->shared->extfile_prefix) { + if(HDstrcmp(extfile_prefix, dataset->shared->extfile_prefix) != 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, NULL, "new external file prefix does not match external file prefix of already open dataset") + } else { + if(extfile_prefix || dataset->shared->extfile_prefix) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, NULL, "new external file prefix does not match external file prefix of already open dataset") + } /* Check if the object has been opened through the top file yet */ if(H5FO_top_count(dataset->oloc.file, dataset->oloc.addr) == 0) { @@ -1411,7 +1652,7 @@ H5D__append_flush_setup(H5D_t *dset, hid_t dapl_id) dset->shared->append_flush.ndims = info.ndims; dset->shared->append_flush.func = info.func; dset->shared->append_flush.udata = info.udata; - HDmemcpy(dset->shared->append_flush.boundary, info.boundary, sizeof(info.boundary)); + H5MM_memcpy(dset->shared->append_flush.boundary, info.boundary, sizeof(info.boundary)); } /* end if */ } /* end if */ } /* end if */ @@ -1812,11 +2053,13 @@ H5D_mult_refresh_close(hid_t dset_id) FUNC_ENTER_NOAPI(FAIL) - if(NULL == (dataset = (H5D_t *)H5I_object_verify(dset_id, H5I_DATASET))) + if(NULL == (dataset = (H5D_t *)H5VL_object_verify(dset_id, H5I_DATASET))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset") /* check args */ - HDassert(dataset && dataset->oloc.file && dataset->shared); + HDassert(dataset); + HDassert(dataset->oloc.file); + HDassert(dataset->shared); HDassert(dataset->shared->fo_count > 0); if(dataset->shared->fo_count > 1) { @@ -1949,40 +2192,16 @@ H5D_oloc(H5D_t *dataset) *------------------------------------------------------------------------- */ H5G_name_t * -H5D_nameof(H5D_t *dataset) +H5D_nameof(const H5D_t *dataset) { /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ FUNC_ENTER_NOAPI_NOINIT_NOERR - FUNC_LEAVE_NOAPI(dataset ? &(dataset->path) : (H5G_name_t *)NULL) + FUNC_LEAVE_NOAPI(dataset ? &(dataset->path) : NULL) } /* end H5D_nameof() */ /*------------------------------------------------------------------------- - * Function: H5D_typeof - * - * Purpose: Returns a pointer to the dataset's datatype. The datatype - * is not copied. - * - * Return: Success: Ptr to the dataset's datatype, uncopied. - * Failure: NULL - *------------------------------------------------------------------------- - */ -H5T_t * -H5D_typeof(const H5D_t *dset) -{ - /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ - FUNC_ENTER_NOAPI_NOINIT_NOERR - - HDassert(dset); - HDassert(dset->shared); - HDassert(dset->shared->type); - - FUNC_LEAVE_NOAPI(dset->shared->type) -} /* end H5D_typeof() */ - - -/*------------------------------------------------------------------------- * Function: H5D__alloc_storage * * Purpose: Allocate storage for the raw data of a dataset. @@ -2323,50 +2542,6 @@ done: /*------------------------------------------------------------------------- - * Function: H5D_vlen_reclaim - * - * Purpose: Frees the buffers allocated for storing variable-length data - * in memory. Only frees the VL data in the selection defined in the - * dataspace. - * - * Return: Non-negative on success, negative on failure - *------------------------------------------------------------------------- - */ -herr_t -H5D_vlen_reclaim(hid_t type_id, H5S_t *space, void *buf) -{ - H5T_t *type; /* Datatype */ - H5S_sel_iter_op_t dset_op; /* Operator for iteration */ - H5T_vlen_alloc_info_t vl_alloc_info; /* VL allocation info */ - herr_t ret_value = FAIL; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* Check args */ - HDassert(H5I_DATATYPE == H5I_get_type(type_id)); - HDassert(space); - HDassert(buf); - - if(NULL == (type = (H5T_t *)H5I_object_verify(type_id, H5I_DATATYPE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not an valid base datatype") - - /* Get the allocation info */ - if(H5CX_get_vlen_alloc_info(&vl_alloc_info) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "unable to retrieve VL allocation info") - - /* Call H5S_select_iterate with args, etc. */ - dset_op.op_type = H5S_SEL_ITER_OP_APP; - dset_op.u.app_op.op = H5T_vlen_reclaim; - dset_op.u.app_op.type_id = type_id; - - ret_value = H5S_select_iterate(buf, type, space, &dset_op, &vl_alloc_info); - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_vlen_reclaim() */ - - -/*------------------------------------------------------------------------- * Function: H5D__vlen_get_buf_size_alloc * * Purpose: This routine makes certain there is enough space in the temporary @@ -2422,8 +2597,8 @@ H5D__vlen_get_buf_size(void H5_ATTR_UNUSED *elem, hid_t type_id, unsigned H5_ATTR_UNUSED ndim, const hsize_t *point, void *op_data) { H5D_vlen_bufsize_t *vlen_bufsize = (H5D_vlen_bufsize_t *)op_data; + H5VL_object_t *vol_obj = vlen_bufsize->dset_vol_obj; H5T_t *dt; /* Datatype for operation */ - H5S_t *mspace; /* Memory dataspace for operation */ H5S_t *fspace; /* File dataspace for operation */ herr_t ret_value = SUCCEED; /* Return value */ @@ -2440,10 +2615,6 @@ H5D__vlen_get_buf_size(void H5_ATTR_UNUSED *elem, hid_t type_id, if(NULL == (vlen_bufsize->fl_tbuf = H5FL_BLK_REALLOC(vlen_fl_buf, vlen_bufsize->fl_tbuf, H5T_get_size(dt)))) HGOTO_ERROR(H5E_DATASET, H5E_NOSPACE, FAIL, "can't resize tbuf") - /* Get the memory dataspace from the ID */ - if(NULL == (mspace = (H5S_t *)H5I_object_verify(vlen_bufsize->mspace_id, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") - /* Select point to read in */ if(NULL == (fspace = (H5S_t *)H5I_object_verify(vlen_bufsize->fspace_id, H5I_DATASPACE))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataspace") @@ -2451,7 +2622,7 @@ H5D__vlen_get_buf_size(void H5_ATTR_UNUSED *elem, hid_t type_id, HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, FAIL, "can't select point") /* Read in the point (with the custom VL memory allocator) */ - if(H5D__read(vlen_bufsize->dset, type_id, mspace, fspace, vlen_bufsize->fl_tbuf) < 0) + if(H5VL_dataset_read(vol_obj, type_id, vlen_bufsize->mspace_id, vlen_bufsize->fspace_id, H5P_DATASET_XFER_DEFAULT, vlen_bufsize->fl_tbuf, H5_REQUEST_NULL) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read point") done: @@ -2549,7 +2720,7 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size) /* Keep the current dataspace dimensions for later */ HDcompile_assert(sizeof(curr_dims) == sizeof(dset->shared->curr_dims)); - HDmemcpy(curr_dims, dset->shared->curr_dims, H5S_MAX_RANK * sizeof(curr_dims[0])); + H5MM_memcpy(curr_dims, dset->shared->curr_dims, H5S_MAX_RANK * sizeof(curr_dims[0])); /* Modify the size of the dataspace */ if((changed = H5S_set_extent(dset->shared->space, size)) < 0) @@ -2643,7 +2814,7 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size) dset->shared->layout.storage.u.virt.list[u].virtual_space_status = H5O_VIRTUAL_STATUS_CORRECT; /* Patch sub-source datasets */ - for(v = 0; v < dset->shared->layout.storage.u.virt.list[u].sub_dset_nalloc; v++) + for(v = 0; v < dset->shared->layout.storage.u.virt.list[u].sub_dset_nused; v++) if(H5S_set_extent(dset->shared->layout.storage.u.virt.list[u].sub_dset[v].virtual_select, size) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of dataspace") } /* end for */ @@ -2669,7 +2840,8 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size) *------------------------------------------------------------------------- */ if(H5D_CHUNKED == dset->shared->layout.type) { - if(shrink && (*dset->shared->layout.ops->is_space_alloc)(&dset->shared->layout.storage)) + if(shrink && ((*dset->shared->layout.ops->is_space_alloc)(&dset->shared->layout.storage) + || (dset->shared->layout.ops->is_data_cached && (*dset->shared->layout.ops->is_data_cached)(dset->shared)))) /* Remove excess chunks */ if(H5D__chunk_prune_by_extent(dset, curr_dims) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to remove chunks") @@ -2716,7 +2888,7 @@ H5D__flush_sieve_buf(H5D_t *dataset) HDassert(dataset->shared->layout.type != H5D_COMPACT); /* We should never have a sieve buffer for compact storage */ /* Write dirty data sieve buffer to file */ - if(H5F_block_write(dataset->oloc.file, H5FD_MEM_DRAW, dataset->shared->cache.contig.sieve_loc, + if(H5F_shared_block_write(H5F_SHARED(dataset->oloc.file), H5FD_MEM_DRAW, dataset->shared->cache.contig.sieve_loc, dataset->shared->cache.contig.sieve_size, dataset->shared->cache.contig.sieve_buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "block write failed") @@ -2833,7 +3005,7 @@ H5D__format_convert(H5D_t *dataset) idx_info.storage = &dataset->shared->layout.storage.u.chunk; /* Copy the current layout info to the new layout */ - HDmemcpy(newlayout, &dataset->shared->layout, sizeof(H5O_layout_t)); + H5MM_memcpy(newlayout, &dataset->shared->layout, sizeof(H5O_layout_t)); /* Set up info for version 1 B-tree in the new layout */ newlayout->version = H5O_LAYOUT_VERSION_3; @@ -2884,7 +3056,7 @@ H5D__format_convert(H5D_t *dataset) HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to release chunk index info") /* Copy the new layout to the dataset's layout */ - HDmemcpy(&dataset->shared->layout, newlayout, sizeof(H5O_layout_t)); + H5MM_memcpy(&dataset->shared->layout, newlayout, sizeof(H5O_layout_t)); break; @@ -3048,7 +3220,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5D_flush_all(const H5F_t *f) +H5D_flush_all(H5F_t *f) { herr_t ret_value = SUCCEED; /* Return value */ @@ -3058,7 +3230,7 @@ H5D_flush_all(const H5F_t *f) HDassert(f); /* Iterate over all the open datasets */ - if(H5I_iterate(H5I_DATASET, H5D__flush_all_cb, (void *)f, FALSE) < 0) /* Casting away const OK -QAK */ + if(H5I_iterate(H5I_DATASET, H5D__flush_all_cb, f, FALSE) < 0) /* Casting away const OK -QAK */ HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to flush cached dataset info") done: @@ -3384,8 +3556,18 @@ H5D__get_type(const H5D_t *dset) if(H5T_lock(dt, FALSE) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to lock transient datatype") - if((ret_value = H5I_register(H5I_DATATYPE, dt, TRUE)) < 0) - HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register datatype") + /* Create an atom */ + if(H5T_is_named(dt)) { + /* If this is a committed datatype, we need to recreate the + * two-level IDs, where the VOL object is a copy of the + * returned datatype. + */ + if((ret_value = H5VL_wrap_register(H5I_DATATYPE, dt, TRUE)) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register datatype") + } /* end if */ + else + if((ret_value = H5I_register(H5I_DATATYPE, dt, TRUE)) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register datatype") done: if(ret_value < 0) |