From 59d7db40a72142b9027748e762f4b2e8038aefa9 Mon Sep 17 00:00:00 2001 From: Songyu Lu Date: Fri, 26 Jul 2019 15:01:40 -0500 Subject: HDFFV-10658: setting and getting properties in API context. Porting the changes to the develop branch into the 1.10 branch: mainly the external file prefix and VDS prefix. --- MANIFEST | 6 + configure.ac | 2 + src/H5CX.c | 159 +++++++++++++++++++++ src/H5CXprivate.h | 4 + src/H5Defl.c | 2 - src/H5Dint.c | 65 +++++---- src/H5Fprivate.h | 5 +- src/H5system.c | 6 +- test/CMakeLists.txt | 4 + test/CMakeTests.cmake | 128 +++++++++++++++++ test/Makefile.am | 20 +-- test/external.c | 248 ++------------------------------- test/external_common.c | 121 ++++++++++++++++ test/external_common.h | 45 ++++++ test/external_env.c | 214 +++++++++++++++++++++++++++++ test/testexternal_env.sh.in | 39 ++++++ test/testvds_env.sh.in | 44 ++++++ test/vds.c | 195 +------------------------- test/vds_env.c | 327 ++++++++++++++++++++++++++++++++++++++++++++ 19 files changed, 1165 insertions(+), 469 deletions(-) create mode 100644 test/external_common.c create mode 100644 test/external_common.h create mode 100644 test/external_env.c create mode 100644 test/testexternal_env.sh.in create mode 100644 test/testvds_env.sh.in create mode 100644 test/vds_env.c diff --git a/MANIFEST b/MANIFEST index 3040e1c..1d4ee06 100644 --- a/MANIFEST +++ b/MANIFEST @@ -975,6 +975,9 @@ ./test/evict_on_close.c ./test/extend.c ./test/external.c +./test/external_common.c +./test/external_common.h +./test/external_env.c ./test/error_test.c ./test/err_compat.c ./test/filter_error.h5 @@ -1097,6 +1100,7 @@ ./test/tcoords.c ./test/testabort_fail.sh.in ./test/testcheck_version.sh.in +./test/testexternal_env.sh.in ./test/testerror.sh.in ./test/testlinks_env.sh.in ./test/test_filter_plugin.sh.in @@ -1109,6 +1113,7 @@ ./test/testmeta.c ./test/testswmr.sh.in ./test/testvdsswmr.sh.in +./test/testvds_env.sh.in ./test/tfile.c ./test/tgenprop.c ./test/th5o.c @@ -1153,6 +1158,7 @@ ./test/gen_filters.c ./test/chunk_info.c ./test/vds.c +./test/vds_env.c ./test/vds_swmr.h ./test/vds_swmr_gen.c ./test/vds_swmr_reader.c diff --git a/configure.ac b/configure.ac index 6f6a91d..3c1e600 100644 --- a/configure.ac +++ b/configure.ac @@ -3519,10 +3519,12 @@ AC_CONFIG_FILES([src/libhdf5.settings test/testabort_fail.sh test/testcheck_version.sh test/testerror.sh + test/testexternal_env.sh test/testflushrefresh.sh test/testlibinfo.sh test/testlinks_env.sh test/testswmr.sh + test/testvds_env.sh test/testvdsswmr.sh test/test_filter_plugin.sh test/test_usecases.sh diff --git a/src/H5CX.c b/src/H5CX.c index 7cf283a..01962a1 100644 --- a/src/H5CX.c +++ b/src/H5CX.c @@ -186,6 +186,10 @@ typedef struct H5CX_t { hid_t lapl_id; /* LAPL ID for API operation */ H5P_genplist_t *lapl; /* Link Access Property List */ + /* DAPL */ + hid_t dapl_id; /* DAPL ID for API operation */ + H5P_genplist_t *dapl; /* Dataset Access Property List */ + /* FAPL */ hid_t fapl_id; /* FAPL ID for API operation */ H5P_genplist_t *fapl; /* File Access Property List */ @@ -275,6 +279,12 @@ typedef struct H5CX_t { size_t nlinks; /* Number of soft / UD links to traverse (H5L_ACS_NLINKS_NAME) */ hbool_t nlinks_valid; /* Whether number of soft / UD links to traverse is valid */ + /* Cached DAPL properties */ + char *extfile_prefix; /* Prefix for external file */ + hbool_t extfile_prefix_valid; /* Whether the prefix for external file is valid */ + char *vds_prefix; /* Prefix for VDS */ + hbool_t vds_prefix_valid; /* Whether the prefix for VDS is valid */ + /* Cached FAPL properties */ H5F_libver_t low_bound; /* low_bound property for H5Pset_libver_bounds() */ hbool_t low_bound_valid; /* Whether low_bound property is valid */ @@ -329,6 +339,13 @@ typedef struct H5CX_lapl_cache_t { size_t nlinks; /* Number of soft / UD links to traverse (H5L_ACS_NLINKS_NAME) */ } H5CX_lapl_cache_t; +/* Typedef for cached default dataset access property list information */ +/* (Same as the cached DXPL struct, above, except for the default DXPL) */ +typedef struct H5CX_dapl_cache_t { + char *extfile_prefix; /* Prefix for external file */ + char *vds_prefix; /* Prefix for VDS */ +} H5CX_dapl_cache_t; + /* Typedef for cached default file access property list information */ /* (Same as the cached DXPL struct, above, except for the default DCPL) */ typedef struct H5CX_fapl_cache_t { @@ -367,6 +384,9 @@ static H5CX_dxpl_cache_t H5CX_def_dxpl_cache; /* Define a "default" link access property list cache structure to use for default LAPLs */ static H5CX_lapl_cache_t H5CX_def_lapl_cache; +/* Define a "default" dataset access property list cache structure to use for default DAPLs */ +static H5CX_dapl_cache_t H5CX_def_dapl_cache; + /* Define a "default" file access property list cache structure to use for default FAPLs */ static H5CX_fapl_cache_t H5CX_def_fapl_cache; @@ -390,6 +410,7 @@ H5CX__init_package(void) { H5P_genplist_t *dx_plist; /* Data transfer property list */ H5P_genplist_t *la_plist; /* Link access property list */ + H5P_genplist_t *da_plist; /* Dataset access property list */ H5P_genplist_t *fa_plist; /* File access property list */ herr_t ret_value = SUCCEED; /* Return value */ @@ -490,6 +511,23 @@ H5CX__init_package(void) if(H5P_get(la_plist, H5L_ACS_NLINKS_NAME, &H5CX_def_lapl_cache.nlinks) < 0) HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve number of soft / UD links to traverse") + /* Reset the "default DAPL cache" information */ + HDmemset(&H5CX_def_dapl_cache, 0, sizeof(H5CX_dapl_cache_t)); + + /* Get the default DAPL cache information */ + + /* Get the default dataset access property list */ + if(NULL == (da_plist = (H5P_genplist_t *)H5I_object(H5P_DATASET_ACCESS_DEFAULT))) + HGOTO_ERROR(H5E_CONTEXT, H5E_BADTYPE, FAIL, "not a dataset create property list") + + /* Get the prefix for the external file */ + if(H5P_peek(da_plist, H5D_ACS_EFILE_PREFIX_NAME, &H5CX_def_dapl_cache.extfile_prefix) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve prefix for external file") + + /* Get the prefix for the VDS file */ + if(H5P_peek(da_plist, H5D_ACS_VDS_PREFIX_NAME, &H5CX_def_dapl_cache.vds_prefix) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve prefix for VDS") + /* Reset the "default FAPL cache" information */ HDmemset(&H5CX_def_fapl_cache, 0, sizeof(H5CX_fapl_cache_t)); @@ -630,6 +668,7 @@ H5CX__push_common(H5CX_node_t *cnode) /* Set non-zero context info */ cnode->ctx.dxpl_id = H5P_DATASET_XFER_DEFAULT; cnode->ctx.lapl_id = H5P_LINK_ACCESS_DEFAULT; + cnode->ctx.dapl_id = H5P_DATASET_ACCESS_DEFAULT; cnode->ctx.fapl_id = H5P_FILE_ACCESS_DEFAULT; cnode->ctx.tag = H5AC__INVALID_TAG; cnode->ctx.ring = H5AC_RING_USER; @@ -867,6 +906,7 @@ H5CX_set_apl(hid_t *acspl_id, const H5P_libclass_t *libclass, *acspl_id = *libclass->def_plist_id; else { htri_t is_lapl; /* Whether the access property list is (or is derived from) a link access property list */ + htri_t is_dapl; /* Whether the access property list is (or is derived from) a dataset access property list */ htri_t is_fapl; /* Whether the access property list is (or is derived from) a file access property list */ #ifdef H5CX_DEBUG @@ -881,6 +921,12 @@ H5CX_set_apl(hid_t *acspl_id, const H5P_libclass_t *libclass, else if(is_lapl) (*head)->ctx.lapl_id = *acspl_id; + /* Check for dataset access property and set API context if so */ + if((is_dapl = H5P_class_isa(*libclass->pclass, *H5P_CLS_DACC->pclass)) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "can't check for dataset access class") + else if(is_dapl) + (*head)->ctx.dapl_id = *acspl_id; + /* Check for file access property and set API context if so */ if((is_fapl = H5P_class_isa(*libclass->pclass, *H5P_CLS_FACC->pclass)) < 0) HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "can't check for file access class") @@ -1971,6 +2017,119 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5CX_get_libver_bounds() */ +/*------------------------------------------------------------------------- + * Function: H5CX_get_ext_file_prefix + * + * Purpose: Retrieves the prefix for external file + * + * Return: Non-negative on success / Negative on failure + * + * Programmer: Raymond Lu + * March 6, 2019 + * + *------------------------------------------------------------------------- + */ +herr_t +H5CX_get_ext_file_prefix(char **extfile_prefix) +{ + H5CX_node_t **head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(extfile_prefix); + HDassert(head && *head); + HDassert(H5P_DEFAULT != (*head)->ctx.dapl_id); + + /* Check if the value has been retrieved already */ + if(!(*head)->ctx.extfile_prefix_valid) { + /* Check for default DAPL */ + if((*head)->ctx.dapl_id == H5P_DATASET_ACCESS_DEFAULT) + (*head)->ctx.extfile_prefix = H5CX_def_dapl_cache.extfile_prefix; + else { + /* Check if the property list is already available */ + if(NULL == (*head)->ctx.dapl) + /* Get the dataset access property list pointer */ + if(NULL == ((*head)->ctx.dapl = (H5P_genplist_t *)H5I_object((*head)->ctx.dapl_id))) + HGOTO_ERROR(H5E_CONTEXT, H5E_BADTYPE, FAIL, "can't get default dataset access property list") + + /* Get the prefix for the external file */ + /* (Note: 'peek', not 'get' - if this turns out to be a problem, we may need + * to copy it and free this in the H5CX pop routine. -QAK) + */ + if(H5P_peek((*head)->ctx.dapl, H5D_ACS_EFILE_PREFIX_NAME, &(*head)->ctx.extfile_prefix) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve external file prefix") + } /* end else */ + + /* Mark the value as valid */ + (*head)->ctx.extfile_prefix_valid = TRUE; + } /* end if */ + + /* Get the value */ + *extfile_prefix = (*head)->ctx.extfile_prefix; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5CX_get_ext_file_prefix() */ + + +/*------------------------------------------------------------------------- + * Function: H5CX_get_vds_prefix + * + * Purpose: Retrieves the prefix for VDS + * + * Return: Non-negative on success / Negative on failure + * + * Programmer: Raymond Lu + * March 6, 2019 + * + *------------------------------------------------------------------------- + */ +herr_t +H5CX_get_vds_prefix(char **vds_prefix) +{ + H5CX_node_t **head = H5CX_get_my_context(); /* Get the pointer to the head of the API context, for this thread */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(vds_prefix); + HDassert(head && *head); + HDassert(H5P_DEFAULT != (*head)->ctx.dapl_id); + + /* Check if the value has been retrieved already */ + if(!(*head)->ctx.vds_prefix_valid) { + /* Check for default DAPL */ + if((*head)->ctx.dapl_id == H5P_DATASET_ACCESS_DEFAULT) + (*head)->ctx.vds_prefix = H5CX_def_dapl_cache.vds_prefix; + else { + /* Check if the property list is already available */ + if(NULL == (*head)->ctx.dapl) + /* Get the dataset access property list pointer */ + if(NULL == ((*head)->ctx.dapl = (H5P_genplist_t *)H5I_object((*head)->ctx.dapl_id))) + HGOTO_ERROR(H5E_CONTEXT, H5E_BADTYPE, FAIL, "can't get default dataset access property list") + + /* Get the prefix for the VDS */ + /* (Note: 'peek', not 'get' - if this turns out to be a problem, we may need + * to copy it and free this in the H5CX pop routine. -QAK) + */ + if(H5P_peek((*head)->ctx.dapl, H5D_ACS_VDS_PREFIX_NAME, &(*head)->ctx.vds_prefix) < 0) + HGOTO_ERROR(H5E_CONTEXT, H5E_CANTGET, FAIL, "Can't retrieve VDS prefix") + } /* end else */ + + /* Mark the value as valid */ + (*head)->ctx.vds_prefix_valid = TRUE; + } /* end if */ + + /* Get the value */ + *vds_prefix = (*head)->ctx.vds_prefix; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5CX_get_vds_prefix() */ + /*------------------------------------------------------------------------- * Function: H5CX_set_tag diff --git a/src/H5CXprivate.h b/src/H5CXprivate.h index 6c01119..d67f226 100644 --- a/src/H5CXprivate.h +++ b/src/H5CXprivate.h @@ -102,6 +102,10 @@ H5_DLL herr_t H5CX_get_dt_conv_cb(H5T_conv_cb_t *cb_struct); /* "Getter" routines for LAPL properties cached in API context */ H5_DLL herr_t H5CX_get_nlinks(size_t *nlinks); +/* "Getter" routines for DAPL properties cached in API context */ +H5_DLL herr_t H5CX_get_ext_file_prefix(char **prefix_extfile); +H5_DLL herr_t H5CX_get_vds_prefix(char **prefix_vds); + /* "Getter" routines for FAPL properties cached in API context */ H5_DLL herr_t H5CX_get_libver_bounds(H5F_libver_t *low_bound, H5F_libver_t *high_bound); diff --git a/src/H5Defl.c b/src/H5Defl.c index 42b3947..ce0bb80 100644 --- a/src/H5Defl.c +++ b/src/H5Defl.c @@ -477,7 +477,6 @@ H5D__efl_readvv(const H5D_io_info_t *io_info, HDassert(io_info->u.rbuf); HDassert(io_info->dset); HDassert(io_info->dset->shared); - HDassert(io_info->dset->shared->extfile_prefix); HDassert(dset_curr_seq); HDassert(dset_len_arr); HDassert(dset_off_arr); @@ -561,7 +560,6 @@ H5D__efl_writevv(const H5D_io_info_t *io_info, HDassert(io_info->u.wbuf); HDassert(io_info->dset); HDassert(io_info->dset->shared); - HDassert(io_info->dset->shared->extfile_prefix); HDassert(dset_curr_seq); HDassert(dset_len_arr); HDassert(dset_off_arr); diff --git a/src/H5Dint.c b/src/H5Dint.c index d3cad8f..a54333f 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -54,8 +54,8 @@ static herr_t H5D__init_type(H5F_t *file, const H5D_t *dset, hid_t type_id, static herr_t H5D__cache_dataspace_info(const H5D_t *dset); static herr_t H5D__init_space(H5F_t *file, const H5D_t *dset, const H5S_t *space); static herr_t H5D__update_oh_info(H5F_t *file, H5D_t *dset, hid_t dapl_id); -static herr_t H5D__build_file_prefix(const H5D_t *dset, hid_t dapl_id, - const char *prefix_type, char **file_prefix); +static herr_t H5D__build_file_prefix(const H5D_t *dset, H5F_prefix_open_t prefix_type, + char **file_prefix); static herr_t H5D__open_oid(H5D_t *dataset, hid_t dapl_id); static herr_t H5D__init_storage(const H5D_io_info_t *io_info, hbool_t full_overwrite, hsize_t old_dim[]); @@ -112,6 +112,10 @@ static const H5I_class_t H5I_DATASET_CLS[1] = {{ /* Flag indicating "top" of interface has been initialized */ static hbool_t H5D_top_package_initialize_s = FALSE; +/* Prefixes of VDS and external file from the environment variables + * HDF5_EXTFILE_PREFIX and HDF5_VDS_PREFIX */ +const static char *H5D_prefix_ext_env = NULL; +const static char *H5D_prefix_vds_env = NULL; /*------------------------------------------------------------------------- @@ -188,6 +192,10 @@ H5D__init_package(void) /* Mark "top" of interface as initialized, too */ H5D_top_package_initialize_s = TRUE; + /* Retrieve the prefixes of VDS and external file from the environment variable */ + H5D_prefix_vds_env = HDgetenv("HDF5_VDS_PREFIX"); + H5D_prefix_ext_env = HDgetenv("HDF5_EXTFILE_PREFIX"); + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__init_package() */ @@ -1045,8 +1053,7 @@ done: *-------------------------------------------------------------------------- */ static herr_t -H5D__build_file_prefix(const H5D_t *dset, hid_t dapl_id, const char *prefix_type, - char **file_prefix /*out*/) +H5D__build_file_prefix(const H5D_t *dset, H5F_prefix_open_t prefix_type, char **file_prefix /*out*/) { char *prefix = NULL; /* prefix used to look for the file */ char *filepath = NULL; /* absolute path of directory the HDF5 file is in */ @@ -1067,20 +1074,22 @@ H5D__build_file_prefix(const H5D_t *dset, hid_t dapl_id, const char *prefix_type /* XXX: Future thread-safety note - getenv is not required * to be reentrant. */ - if(HDstrcmp(prefix_type, H5D_ACS_VDS_PREFIX_NAME) == 0) - prefix = HDgetenv("HDF5_VDS_PREFIX"); - else if (HDstrcmp(prefix_type, H5D_ACS_EFILE_PREFIX_NAME) == 0) - prefix = HDgetenv("HDF5_EXTFILE_PREFIX"); - else - HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "prefix name is not sensible") - - if(prefix == NULL || *prefix == '\0') { - /* Set prefix to value of prefix_type property */ - if(NULL == (plist = H5P_object_verify(dapl_id, H5P_DATASET_ACCESS))) - HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") - if(H5P_peek(plist, prefix_type, &prefix) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get file prefix") - } /* end if */ + if(H5F_PREFIX_VDS == prefix_type) { + prefix = (char *)H5D_prefix_vds_env; + + if(prefix == NULL || *prefix == '\0') { + if(H5CX_get_vds_prefix(&prefix) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get the prefix for vds file") + } + } else if(H5F_PREFIX_EFILE == prefix_type) { + prefix = (char *)H5D_prefix_ext_env; + + if(prefix == NULL || *prefix == '\0') { + if(H5CX_get_ext_file_prefix(&prefix) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get the prefix for the external file") + } + } else + HGOTO_ERROR(H5E_DATASET, H5E_BADTYPE, FAIL, "prefix name is not sensible") /* Prefix has to be checked for NULL / empty string again because the * code above might have updated it. @@ -1089,8 +1098,7 @@ H5D__build_file_prefix(const H5D_t *dset, hid_t dapl_id, const char *prefix_type /* filename is interpreted as relative to the current directory, * does not need to be expanded */ - if(NULL == (*file_prefix = (char *)H5MM_strdup(""))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + *file_prefix = NULL; } /* end if */ else { if (HDstrncmp(prefix, "${ORIGIN}", HDstrlen("${ORIGIN}")) == 0) { @@ -1285,11 +1293,11 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to set up flush append property") /* Set the external file prefix */ - if(H5D__build_file_prefix(new_dset, dapl_id, H5D_ACS_EFILE_PREFIX_NAME, &new_dset->shared->extfile_prefix) < 0) + if(H5D__build_file_prefix(new_dset, H5F_PREFIX_EFILE, &new_dset->shared->extfile_prefix) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize external file prefix") /* Set the VDS file prefix */ - if(H5D__build_file_prefix(new_dset, dapl_id, H5D_ACS_VDS_PREFIX_NAME, &new_dset->shared->vds_prefix) < 0) + if(H5D__build_file_prefix(new_dset, H5F_PREFIX_VDS, &new_dset->shared->vds_prefix) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize VDS prefix") /* Add the dataset to the list of opened objects in the file */ @@ -1444,11 +1452,11 @@ H5D_open(const H5G_loc_t *loc, hid_t dapl_id) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, NULL, "can't copy path") /* Get the external file prefix */ - if(H5D__build_file_prefix(dataset, dapl_id, H5D_ACS_EFILE_PREFIX_NAME, &extfile_prefix) < 0) + if(H5D__build_file_prefix(dataset, H5F_PREFIX_EFILE, &extfile_prefix) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize external file prefix") /* Get the VDS prefix */ - if(H5D__build_file_prefix(dataset, dapl_id, H5D_ACS_VDS_PREFIX_NAME, &vds_prefix) < 0) + if(H5D__build_file_prefix(dataset, H5F_PREFIX_VDS, &vds_prefix) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize VDS prefix") /* Check if dataset was already open */ @@ -1492,8 +1500,13 @@ H5D_open(const H5G_loc_t *loc, hid_t dapl_id) /* Check whether the external file prefix of the already open dataset * matches the new external file prefix */ - if(HDstrcmp(extfile_prefix, dataset->shared->extfile_prefix) != 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, NULL, "new external file prefix does not match external file prefix of already open dataset") + if(extfile_prefix && dataset->shared->extfile_prefix) { + if(HDstrcmp(extfile_prefix, dataset->shared->extfile_prefix) != 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, NULL, "new external file prefix does not match external file prefix of already open dataset") + } else { + if(extfile_prefix || dataset->shared->extfile_prefix) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, NULL, "new external file prefix does not match external file prefix of already open dataset") + } /* Check if the object has been opened through the top file yet */ if(H5FO_top_count(dataset->oloc.file, dataset->oloc.addr) == 0) { diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index d60c08f..5e78d9e 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -702,8 +702,9 @@ typedef enum H5F_mem_page_t { /* Type of prefix for opening prefixed files */ typedef enum H5F_prefix_open_t { - H5F_PREFIX_VDS, /* Virtual dataset prefix */ - H5F_PREFIX_ELINK /* External link prefix */ + H5F_PREFIX_VDS = 0, /* Virtual dataset prefix */ + H5F_PREFIX_ELINK = 1, /* External link prefix */ + H5F_PREFIX_EFILE = 2 /* External file prefix */ } H5F_prefix_open_t; diff --git a/src/H5system.c b/src/H5system.c index 186d8fa..41e2a48 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -1125,13 +1125,13 @@ H5_combine_path(const char* path1, const char* path2, char **full_name /*out*/) FUNC_ENTER_NOAPI_NOINIT - HDassert(path1); HDassert(path2); - path1_len = HDstrlen(path1); + if(path1) + path1_len = HDstrlen(path1); path2_len = HDstrlen(path2); - if(*path1 == '\0' || H5_CHECK_ABSOLUTE(path2)) { + if(path1 == NULL || *path1 == '\0' || H5_CHECK_ABSOLUTE(path2)) { /* If path1 is empty or path2 is absolute, simply use path2 */ if(NULL == (*full_name = (char *)H5MM_strdup(path2))) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 5b34ec5..f5a6b60 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -14,12 +14,14 @@ set (TEST_LIB_SOURCES ${HDF5_TEST_SOURCE_DIR}/h5test.c ${HDF5_TEST_SOURCE_DIR}/testframe.c ${HDF5_TEST_SOURCE_DIR}/cache_common.c + ${HDF5_TEST_SOURCE_DIR}/external_common.c ${HDF5_TEST_SOURCE_DIR}/swmr_common.c ) set (TEST_LIB_HEADERS ${HDF5_TEST_SOURCE_DIR}/h5test.h ${HDF5_TEST_SOURCE_DIR}/cache_common.h + ${HDF5_TEST_SOURCE_DIR}/external_common.h ${HDF5_TEST_SOURCE_DIR}/swmr_common.h ) @@ -208,6 +210,7 @@ set (H5_TESTS extend direct_chunk # compression lib link external + external_env efc objcopy links @@ -235,6 +238,7 @@ set (H5_TESTS freespace mf vds + vds_env file_image unregister cache_logging diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index fe6c695..6532536 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -463,8 +463,10 @@ set (H5TEST_SEPARATE_TESTS testhdf5 cache cache_image + external_env flush1 flush2 + vds_env ) foreach (h5_test ${H5_TESTS}) if (NOT h5_test IN_LIST H5TEST_SEPARATE_TESTS) @@ -558,6 +560,71 @@ if (TEST_CACHE_IMAGE) ) endif () +#-- Adding test for external_env +add_test ( + NAME H5TEST-clear-external_env-objects + COMMAND ${CMAKE_COMMAND} + -D "TEST_PROGRAM=$" + -D "TEST_ARGS:STRING=" + -D "TEST_ENV_VAR:STRING=HDF5_EXTFILE_PREFIX" + -D "TEST_ENV_VALUE:STRING=\${ORIGIN}" + -D "TEST_EXPECT=0" + -D "TEST_OUTPUT=external_env.txt" + -D "TEST_REFERENCE=external_env.out" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/H5TEST" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + -E remove + extern_1r.raw + extern_2r.raw + extern_3r.raw + extern_4r.raw + extern_1w.raw + extern_2w.raw + extern_3w.raw + extern_4w.raw + external_env.txt + external_env.out + WORKING_DIRECTORY + ${HDF5_TEST_BINARY_DIR}/H5TEST +) +set_tests_properties (H5TEST-clear-external_env-objects PROPERTIES FIXTURES_SETUP external_env_clear_objects) +add_test (NAME H5TEST-external_env COMMAND $) +set_tests_properties (H5TEST-external_env PROPERTIES + FIXTURES_REQUIRED external_env_clear_objects + ENVIRONMENT "srcdir=${HDF5_TEST_BINARY_DIR}/H5TEST;HDF5TestExpress=${HDF_TEST_EXPRESS}" + WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST +) + +#-- Adding test for vds_env +add_test ( + NAME H5TEST-clear-vds_env-objects + COMMAND ${CMAKE_COMMAND} + -D "TEST_PROGRAM=$" + -D "TEST_ARGS:STRING=" + -D "TEST_ENV_VAR:STRING=HDF5_VDS_PREFIX" + -D "TEST_ENV_VALUE:STRING=\${ORIGIN}/tmp" + -D "TEST_EXPECT=0" + -D "TEST_OUTPUT=vds_env.txt" + -D "TEST_REFERENCE=vds_env.out" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/H5TEST" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + -E remove + vds_virt_0.h5 + vds_virt_3.h5 + vds_src_2.h5 + vds_env.txt + vds_env.out + WORKING_DIRECTORY + ${HDF5_TEST_BINARY_DIR}/H5TEST +) +set_tests_properties (H5TEST-clear-vds_env-objects PROPERTIES FIXTURES_SETUP vds_env_clear_objects) +add_test (NAME H5TEST-vds_env COMMAND $) +set_tests_properties (H5TEST-vds_env PROPERTIES + FIXTURES_REQUIRED vds_env_clear_objects + ENVIRONMENT "srcdir=${HDF5_TEST_BINARY_DIR}/H5TEST;HDF5TestExpress=${HDF_TEST_EXPRESS}" + WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST +) + #-- Adding test for flush1/2 add_test (NAME H5TEST-clear-flush-objects COMMAND ${CMAKE_COMMAND} @@ -787,6 +854,67 @@ set_tests_properties (H5TEST-links_env PROPERTIES WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST ) + #-- Adding test for external_env + add_test (NAME H5TEST-shared-clear-external_env-objects + COMMAND ${CMAKE_COMMAND} + -E remove + extern_1r.raw + extern_2r.raw + extern_3r.raw + extern_4r.raw + extern_1w.raw + extern_2w.raw + extern_3w.raw + extern_4w.raw + WORKING_DIRECTORY + ${HDF5_TEST_BINARY_DIR}/H5TEST-shared + ) + set_tests_properties (H5TEST-shared-clear-external_env-objects PROPERTIES FIXTURES_SETUP shared_external_env_clear_objects) + add_test (NAME H5TEST-shared-external_env COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$" + -D "TEST_ARGS:STRING=" + -D "TEST_ENV_VAR:STRING=HDF5_EXTFILE_PREFIX" + -D "TEST_ENV_VALUE:STRING=\${ORIGIN}" + -D "TEST_EXPECT=0" + -D "TEST_OUTPUT=external_env.txt" + -D "TEST_REFERENCE=external_env.out" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/H5TEST-shared" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5TEST-shared-external_env PROPERTIES + FIXTURES_REQUIRED shared_external_env_clear_objects + ENVIRONMENT "srcdir=${HDF5_TEST_BINARY_DIR}/H5TEST-shared" + WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST-shared + ) + + #-- Adding test for vds_env + add_test (NAME H5TEST-shared-clear-vds_env-objects + COMMAND ${CMAKE_COMMAND} + -E remove + vds_virt_0.h5 + vds_virt_3.h5 + vds_src_2.h5 + WORKING_DIRECTORY + ${HDF5_TEST_BINARY_DIR}/H5TEST-shared + ) + set_tests_properties (H5TEST-shared-clear-vds_env-objects PROPERTIES FIXTURES_SETUP shared_vds_env_clear_objects) + add_test (NAME H5TEST-shared-vds_env COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$" + -D "TEST_ARGS:STRING=" + -D "TEST_ENV_VAR:STRING=HDF5_VDS_PREFIX" + -D "TEST_ENV_VALUE:STRING=\${ORIGIN}/tmp" + -D "TEST_EXPECT=0" + -D "TEST_OUTPUT=vds_env.txt" + -D "TEST_REFERENCE=vds_env.out" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/H5TEST-shared" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5TEST-shared-vds_env PROPERTIES + FIXTURES_REQUIRED shared_vds_env_clear_objects + ENVIRONMENT "srcdir=${HDF5_TEST_BINARY_DIR}/H5TEST-shared" + WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST-shared + ) + if (NOT BUILD_SHARED_LIBS) #-- Adding test for libinfo add_test (NAME H5TEST-testlibinfo diff --git a/test/Makefile.am b/test/Makefile.am index c4fb876..dce2f98 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -26,21 +26,23 @@ AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_builddir)/src # testlibinfo.sh: # testcheck_version.sh: tcheck_version # testlinks_env.sh: links_env +# testexternal_env.sh: external_env # testflushrefresh.sh: flushrefresh +# testvds_env.sh: vds_env # testswmr.sh: swmr* # testvdsswmr.sh: vds_swmr* # testabort_fail.sh: filenotclosed.c and del_many_dense_attrs.c # test_filter_plugin.sh: filter_plugin.c # test_usecases.sh: use_append_chunk, use_append_mchunks, use_disable_mdc_flushes -TEST_SCRIPT = testerror.sh testlibinfo.sh testcheck_version.sh testlinks_env.sh \ - testswmr.sh testvdsswmr.sh testflushrefresh.sh test_usecases.sh testabort_fail.sh +TEST_SCRIPT = testerror.sh testlibinfo.sh testcheck_version.sh testlinks_env.sh testexternal_env.sh \ + testswmr.sh testvds_env.sh testvdsswmr.sh testflushrefresh.sh test_usecases.sh testabort_fail.sh SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) links_env$(EXEEXT) \ filenotclosed$(EXEEXT) del_many_dense_attrs$(EXEEXT) \ - flushrefresh$(EXEEXT) use_append_chunk$(EXEEXT) use_append_mchunks$(EXEEXT) use_disable_mdc_flushes$(EXEEXT) \ + external_env$(EXEEXT) flushrefresh$(EXEEXT) use_append_chunk$(EXEEXT) use_append_mchunks$(EXEEXT) use_disable_mdc_flushes$(EXEEXT) \ swmr_generator$(EXEEXT) swmr_reader$(EXEEXT) swmr_writer$(EXEEXT) \ swmr_remove_reader$(EXEEXT) swmr_remove_writer$(EXEEXT) swmr_addrem_writer$(EXEEXT) \ swmr_sparse_reader$(EXEEXT) swmr_sparse_writer$(EXEEXT) swmr_start_write$(EXEEXT) \ - vds_swmr_gen$(EXEEXT) vds_swmr_reader$(EXEEXT) vds_swmr_writer$(EXEEXT) + vds_env$(EXEEXT) vds_swmr_gen$(EXEEXT) vds_swmr_reader$(EXEEXT) vds_swmr_writer$(EXEEXT) if HAVE_SHARED_CONDITIONAL TEST_SCRIPT += test_filter_plugin.sh SCRIPT_DEPEND += filter_plugin$(EXEEXT) @@ -69,21 +71,23 @@ TEST_PROG= testhdf5 \ # accum_swmr_reader is used by accum.c. # atomic_writer and atomic_reader are standalone programs. # links_env is used by testlinks_env.sh +# external_env is used by testexternal_env.sh # filenotclosed and del_many_dense_attrs are used by testabort_fail.sh # flushrefresh is used by testflushrefresh.sh. # use_append_chunk, use_append_mchunks and use_disable_mdc_flushes are used by test_usecases.sh # swmr_* files (besides swmr.c) are used by testswmr.sh. # vds_swmr_* files are used by testvdsswmr.sh +# vds_env is used by testvds_env.sh # 'make check' doesn't run them directly, so they are not included in TEST_PROG. # Also build testmeta, which is used for timings test. It builds quickly, # and this lets automake keep all its test programs in one place. check_PROGRAMS=$(TEST_PROG) error_test err_compat tcheck_version \ - testmeta accum_swmr_reader atomic_writer atomic_reader \ + testmeta accum_swmr_reader atomic_writer atomic_reader external_env \ links_env filenotclosed del_many_dense_attrs flushrefresh \ use_append_chunk use_append_mchunks use_disable_mdc_flushes \ swmr_generator swmr_start_write swmr_reader swmr_writer swmr_remove_reader \ swmr_remove_writer swmr_addrem_writer swmr_sparse_reader swmr_sparse_writer \ - swmr_check_compat_vfd vds_swmr_gen vds_swmr_reader vds_swmr_writer + swmr_check_compat_vfd vds_swmr_gen vds_env vds_swmr_reader vds_swmr_writer if HAVE_SHARED_CONDITIONAL check_PROGRAMS+= filter_plugin endif @@ -124,7 +128,7 @@ else noinst_LTLIBRARIES=libh5test.la endif -libh5test_la_SOURCES=h5test.c testframe.c cache_common.c swmr_common.c +libh5test_la_SOURCES=h5test.c testframe.c cache_common.c swmr_common.c external_common.c # Use libhd5test.la to compile all of the tests LDADD=libh5test.la $(LIBHDF5) @@ -211,6 +215,6 @@ use_disable_mdc_flushes_SOURCES=use_disable_mdc_flushes.c # Temporary files. DISTCLEANFILES=testerror.sh testlibinfo.sh testcheck_version.sh testlinks_env.sh test_filter_plugin.sh \ - testswmr.sh testvdsswmr.sh test_usecases.sh testflushrefresh.sh testabort_fail.sh + testexternal_env.sh testswmr.sh testvds_env.sh testvdsswmr.sh test_usecases.sh testflushrefresh.sh testabort_fail.sh include $(top_srcdir)/config/conclude.am diff --git a/test/external.c b/test/external.c index 20a9ed8..9644d86 100644 --- a/test/external.c +++ b/test/external.c @@ -18,24 +18,7 @@ * Purpose: Tests datasets stored in external raw files. */ #include "h5test.h" - -const char *FILENAME[] = { - "extern_1", - "extern_2", - "extern_3", - "extern_4", - "extern_dir/file_1", - "extern_5", - NULL -}; - -/* A similar collection of files is used for the tests that - * perform file I/O. - */ -#define N_EXT_FILES 4 -#define PART_SIZE 25 -#define TOTAL_SIZE 100 -#define GARBAGE_PER_FILE 10 +#include "external_common.h" /*------------------------------------------------------------------------- @@ -99,106 +82,6 @@ out: /*------------------------------------------------------------------------- - * Function: reset_raw_data_files - * - * Purpose: Resets the data in the raw data files for tests that - * perform dataset I/O on a set of files. - * - * Return: SUCCEED/FAIL - * - * Programmer: Dana Robinson - * February 2016 - * - *------------------------------------------------------------------------- - */ -static herr_t -reset_raw_data_files(void) -{ - int fd = 0; /* external file descriptor */ - size_t i, j; /* iterators */ - hssize_t n; /* bytes of I/O */ - char filename[1024]; /* file name */ - int data[PART_SIZE]; /* raw data buffer */ - uint8_t *garbage = NULL; /* buffer of garbage data */ - size_t garbage_count; /* size of garbage buffer */ - size_t garbage_bytes; /* # of garbage bytes written to file */ - - /* Set up garbage buffer */ - garbage_count = N_EXT_FILES * GARBAGE_PER_FILE; - if(NULL == (garbage = (uint8_t *)HDcalloc(garbage_count, sizeof(uint8_t)))) - goto error; - for(i = 0; i < garbage_count; i++) - garbage[i] = 0xFF; - - /* The *r files are pre-filled with data and are used to - * verify that read operations work correctly. - */ - for(i = 0; i < N_EXT_FILES; i++) { - - /* Open file */ - HDsprintf(filename, "extern_%lur.raw", (unsigned long)i + 1); - if((fd = HDopen(filename, O_RDWR|O_CREAT|O_TRUNC, H5_POSIX_CREATE_MODE_RW)) < 0) - goto error; - - /* Write garbage data to the file. This allows us to test the - * the ability to set an offset in the raw data file. - */ - garbage_bytes = i * 10; - n = HDwrite(fd, garbage, garbage_bytes); - if(n < 0 || (size_t)n != garbage_bytes) - goto error; - - /* Fill array with data */ - for(j = 0; j < PART_SIZE; j++) { - data[j] = (int)(i * 25 + j); - } /* end for */ - - /* Write raw data to the file. */ - n = HDwrite(fd, data, sizeof(data)); - if(n != sizeof(data)) - goto error; - - /* Close this file */ - HDclose(fd); - - } /* end for */ - - /* The *w files are only pre-filled with the garbage data and are - * used to verify that write operations work correctly. The individual - * tests fill in the actual data. - */ - for(i = 0; i < N_EXT_FILES; i++) { - - /* Open file */ - HDsprintf(filename, "extern_%luw.raw", (unsigned long)i + 1); - if((fd = HDopen(filename, O_RDWR|O_CREAT|O_TRUNC, H5_POSIX_CREATE_MODE_RW)) < 0) - goto error; - - /* Write garbage data to the file. This allows us to test the - * the ability to set an offset in the raw data file. - */ - garbage_bytes = i * 10; - n = HDwrite(fd, garbage, garbage_bytes); - if(n < 0 || (size_t)n != garbage_bytes) - goto error; - - /* Close this file */ - HDclose(fd); - - } /* end for */ - HDfree(garbage); - return SUCCEED; - -error: - if(fd) - HDclose(fd); - if(garbage) - HDfree(garbage); - return FAIL; -} /* end reset_raw_data_files() */ - - -/*------------------------------------------------------------------------- * Function: test_non_extendible * * Purpose: Tests a non-extendible dataset with a single external file. @@ -760,7 +643,7 @@ test_read_file_set(hid_t fapl) * debugging to be emitted before we start playing games with what the * output looks like. */ - h5_fixname(FILENAME[1], fapl, filename, sizeof(filename)); + h5_fixname(EXT_FNAME[1], fapl, filename, sizeof(filename)); if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR if((grp = H5Gcreate2(file, "emit-diagnostics", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) @@ -871,7 +754,7 @@ test_write_file_set(hid_t fapl) TEST_ERROR /* Create another file */ - h5_fixname(FILENAME[2], fapl, filename, sizeof(filename)); + h5_fixname(EXT_FNAME[2], fapl, filename, sizeof(filename)); if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR @@ -989,7 +872,7 @@ test_path_absolute(hid_t fapl) TESTING("absolute filenames for external file"); - h5_fixname(FILENAME[3], fapl, filename, sizeof(filename)); + h5_fixname(EXT_FNAME[3], fapl, filename, sizeof(filename)); if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR @@ -1085,7 +968,7 @@ test_path_relative(hid_t fapl) if (HDmkdir("extern_dir", (mode_t)0755) < 0 && errno != EEXIST) TEST_ERROR; - h5_fixname(FILENAME[4], fapl, filename, sizeof(filename)); + h5_fixname(EXT_FNAME[4], fapl, filename, sizeof(filename)); if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR; @@ -1180,7 +1063,7 @@ test_path_relative_cwd(hid_t fapl) if(HDmkdir("extern_dir", (mode_t)0755) < 0 && errno != EEXIST) TEST_ERROR; - h5_fixname(FILENAME[4], fapl, filename, sizeof(filename)); + h5_fixname(EXT_FNAME[4], fapl, filename, sizeof(filename)); if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR; @@ -1297,116 +1180,6 @@ error: /*------------------------------------------------------------------------- - * Function: test_path_env - * - * Purpose: Test whether the value of HDF5_EXTFILE_PREFIX will overwrite - * the efile_prefix dataset access property. - * This will create an HDF5 file in a subdirectory which will - * refer to ../extern_*a.raw - * The files are then accessed by setting the HDF5_EXTFILE_PREFIX - * environment variable to "${ORIGIN}". - * The efile_prefix dataset access property is set to "someprefix", - * which will cause an error if the value is not overwritten by - * the environment variable. - * - * Return: Success: 0 - * Failure: 1 - * - * Programmer: Steffen Kiess - * March 10, 2015 - * - *------------------------------------------------------------------------- - */ -static int -test_path_env(hid_t fapl) -{ - hid_t file = -1; /* file to write to */ - hid_t dcpl = -1; /* dataset creation properties */ - hid_t space = -1; /* data space */ - hid_t dapl = -1; /* dataset access property list */ - hid_t dset = -1; /* dataset */ - size_t i; /* miscellaneous counters */ - char cwdpath[1024]; /* working directory */ - char filename[1024]; /* file name */ - int part[PART_SIZE]; /* raw data buffer (partial) */ - int whole[TOTAL_SIZE]; /* raw data buffer (total) */ - hsize_t cur_size; /* current data space size */ - char buffer[1024]; /* buffer to read efile_prefix */ - - TESTING("prefix in HDF5_EXTFILE_PREFIX"); - - if(HDsetenv("HDF5_EXTFILE_PREFIX", "${ORIGIN}", 1)) - TEST_ERROR - - if(HDmkdir("extern_dir", (mode_t)0755) < 0 && errno != EEXIST) - TEST_ERROR; - - h5_fixname(FILENAME[4], fapl, filename, sizeof(filename)); - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) - FAIL_STACK_ERROR - - /* Reset the raw data files */ - if(reset_raw_data_files() < 0) - TEST_ERROR - - /* Create the dataset */ - if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) - FAIL_STACK_ERROR - if(NULL == HDgetcwd(cwdpath, sizeof(cwdpath))) - TEST_ERROR - for(i = 0; i < N_EXT_FILES; i++) { - HDsnprintf(filename, sizeof(filename), "..%sextern_%dr.raw", H5_DIR_SEPS, (int) i + 1); - if(H5Pset_external(dcpl, filename, (off_t)(i * GARBAGE_PER_FILE), (hsize_t)sizeof(part)) < 0) - FAIL_STACK_ERROR - } /* end for */ - - cur_size = TOTAL_SIZE; - if((space = H5Screate_simple(1, &cur_size, NULL)) < 0) - FAIL_STACK_ERROR - if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) - FAIL_STACK_ERROR - - /* Set prefix to a nonexistent directory, will be overwritten by environment variable */ - if(H5Pset_efile_prefix(dapl, "someprefix") < 0) - FAIL_STACK_ERROR - if(H5Pget_efile_prefix(dapl, buffer, sizeof(buffer)) < 0) - FAIL_STACK_ERROR - if(HDstrcmp(buffer, "someprefix") != 0) - FAIL_PUTS_ERROR("efile prefix not set correctly"); - - /* Create dataset */ - if((dset = H5Dcreate2(file, "dset1", H5T_NATIVE_INT, space, H5P_DEFAULT, dcpl, dapl)) < 0) - FAIL_STACK_ERROR - - /* Read the entire dataset and compare with the original */ - HDmemset(whole, 0, sizeof(whole)); - if(H5Dread(dset, H5T_NATIVE_INT, space, space, H5P_DEFAULT, whole) < 0) - FAIL_STACK_ERROR - for(i = 0; i < TOTAL_SIZE; i++) - if(whole[i] != (signed)i) - FAIL_PUTS_ERROR("Incorrect value(s) read."); - - if(H5Dclose(dset) < 0) FAIL_STACK_ERROR - if(H5Pclose(dapl) < 0) FAIL_STACK_ERROR - if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR - if(H5Sclose(space) < 0) FAIL_STACK_ERROR - if(H5Fclose(file) < 0) FAIL_STACK_ERROR - PASSED(); - return 0; - -error: - H5E_BEGIN_TRY { - H5Pclose(dapl); - H5Dclose(dset); - H5Pclose(dcpl); - H5Sclose(space); - H5Fclose(file); - } H5E_END_TRY; - return 1; -} /* end test_path_env() */ - - -/*------------------------------------------------------------------------- * Function: test_h5d_get_access_plist * * Purpose: Ensure that H5Dget_access_plist returns correct values. @@ -1441,7 +1214,7 @@ test_h5d_get_access_plist(hid_t fapl_id) TEST_ERROR /* Create the file */ - h5_fixname(FILENAME[5], fapl_id, filename, sizeof(filename)); + h5_fixname(EXT_FNAME[5], fapl_id, filename, sizeof(filename)); if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id)) < 0) FAIL_STACK_ERROR @@ -1532,7 +1305,7 @@ main(void) /* Get a fapl for the old (default) file format */ fapl_id_old = h5_fileaccess(); - h5_fixname(FILENAME[0], fapl_id_old, filename, sizeof(filename)); + h5_fixname(EXT_FNAME[0], fapl_id_old, filename, sizeof(filename)); /* Copy and set up a fapl for the latest file format */ if((fapl_id_new = H5Pcopy(fapl_id_old)) < 0) @@ -1585,10 +1358,9 @@ main(void) nerrors += test_path_absolute(current_fapl_id); nerrors += test_path_relative(current_fapl_id); nerrors += test_path_relative_cwd(current_fapl_id); - nerrors += test_path_env(current_fapl_id); /* Verify symbol table messages are cached */ - nerrors += (h5_verify_cached_stabs(FILENAME, current_fapl_id) < 0 ? 1 : 0); + nerrors += (h5_verify_cached_stabs(EXT_FNAME, current_fapl_id) < 0 ? 1 : 0); /* Close the common file */ if(H5Fclose(fid) < 0) FAIL_STACK_ERROR @@ -1603,7 +1375,7 @@ main(void) HDputs("All external storage tests passed."); /* Clean up files used by file set tests */ - if(h5_cleanup(FILENAME, fapl_id_old)) { + if(h5_cleanup(EXT_FNAME, fapl_id_old)) { HDremove("extern_1r.raw"); HDremove("extern_2r.raw"); HDremove("extern_3r.raw"); diff --git a/test/external_common.c b/test/external_common.c new file mode 100644 index 0000000..e43a713 --- /dev/null +++ b/test/external_common.c @@ -0,0 +1,121 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Raymond Lu + * April, 2019 + * + * Purpose: Private function for external.c and external_env.c + */ + +#include "external_common.h" + + +/*------------------------------------------------------------------------- + * Function: reset_raw_data_files + * + * Purpose: Resets the data in the raw data files for tests that + * perform dataset I/O on a set of files. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * February 2016 + * + *------------------------------------------------------------------------- + */ +herr_t +reset_raw_data_files(void) +{ + int fd = 0; /* external file descriptor */ + size_t i, j; /* iterators */ + hssize_t n; /* bytes of I/O */ + char filename[1024]; /* file name */ + int data[PART_SIZE]; /* raw data buffer */ + uint8_t *garbage = NULL; /* buffer of garbage data */ + size_t garbage_count; /* size of garbage buffer */ + size_t garbage_bytes; /* # of garbage bytes written to file */ + + /* Set up garbage buffer */ + garbage_count = N_EXT_FILES * GARBAGE_PER_FILE; + if(NULL == (garbage = (uint8_t *)HDcalloc(garbage_count, sizeof(uint8_t)))) + goto error; + for(i = 0; i < garbage_count; i++) + garbage[i] = 0xFF; + + /* The *r files are pre-filled with data and are used to + * verify that read operations work correctly. + */ + for(i = 0; i < N_EXT_FILES; i++) { + + /* Open file */ + HDsprintf(filename, "extern_%lur.raw", (unsigned long)i + 1); + if((fd = HDopen(filename, O_RDWR|O_CREAT|O_TRUNC, H5_POSIX_CREATE_MODE_RW)) < 0) + goto error; + + /* Write garbage data to the file. This allows us to test the + * the ability to set an offset in the raw data file. + */ + garbage_bytes = i * 10; + n = HDwrite(fd, garbage, garbage_bytes); + if(n < 0 || (size_t)n != garbage_bytes) + goto error; + + /* Fill array with data */ + for(j = 0; j < PART_SIZE; j++) { + data[j] = (int)(i * 25 + j); + } /* end for */ + + /* Write raw data to the file. */ + n = HDwrite(fd, data, sizeof(data)); + if(n != sizeof(data)) + goto error; + + /* Close this file */ + HDclose(fd); + + } /* end for */ + + /* The *w files are only pre-filled with the garbage data and are + * used to verify that write operations work correctly. The individual + * tests fill in the actual data. + */ + for(i = 0; i < N_EXT_FILES; i++) { + + /* Open file */ + HDsprintf(filename, "extern_%luw.raw", (unsigned long)i + 1); + if((fd = HDopen(filename, O_RDWR|O_CREAT|O_TRUNC, H5_POSIX_CREATE_MODE_RW)) < 0) + goto error; + + /* Write garbage data to the file. This allows us to test the + * the ability to set an offset in the raw data file. + */ + garbage_bytes = i * 10; + n = HDwrite(fd, garbage, garbage_bytes); + if(n < 0 || (size_t)n != garbage_bytes) + goto error; + + /* Close this file */ + HDclose(fd); + + } /* end for */ + HDfree(garbage); + return SUCCEED; + +error: + if(fd) + HDclose(fd); + if(garbage) + HDfree(garbage); + return FAIL; +} diff --git a/test/external_common.h b/test/external_common.h new file mode 100644 index 0000000..f1a15b5 --- /dev/null +++ b/test/external_common.h @@ -0,0 +1,45 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Raymond Lu + * April, 2019 + * + * Purpose: Private function for external.c and external_env.c + */ +#ifndef _EXTERNAL_COMMON_H +#define _EXTERNAL_COMMON_H + +/* Include test header files */ +#include "h5test.h" + +static const char *EXT_FNAME[] = { + "extern_1", + "extern_2", + "extern_3", + "extern_4", + "extern_dir/file_1", + "extern_5", + NULL +}; + +/* A similar collection of files is used for the tests that + * perform file I/O. + */ +#define N_EXT_FILES 4 +#define PART_SIZE 25 +#define TOTAL_SIZE 100 +#define GARBAGE_PER_FILE 10 + +herr_t reset_raw_data_files(void); +#endif /* _EXTERNAL_COMMON_H */ diff --git a/test/external_env.c b/test/external_env.c new file mode 100644 index 0000000..455cf8e --- /dev/null +++ b/test/external_env.c @@ -0,0 +1,214 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Tests datasets stored in external raw files. + */ +#include "external_common.h" + + +/*------------------------------------------------------------------------- + * Function: test_path_env + * + * Purpose: Test whether the value of HDF5_EXTFILE_PREFIX will overwrite + * the efile_prefix dataset access property. + * This will create an HDF5 file in a subdirectory which will + * refer to ../extern_*a.raw + * The files are then accessed by setting the HDF5_EXTFILE_PREFIX + * environment variable to "${ORIGIN}". + * The efile_prefix dataset access property is set to "someprefix", + * which will cause an error if the value is not overwritten by + * the environment variable. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Steffen Kiess + * March 10, 2015 + * + *------------------------------------------------------------------------- + */ +static int +test_path_env(hid_t fapl) +{ + hid_t file = -1; /* file to write to */ + hid_t dcpl = -1; /* dataset creation properties */ + hid_t space = -1; /* data space */ + hid_t dapl = -1; /* dataset access property list */ + hid_t dset = -1; /* dataset */ + size_t i; /* miscellaneous counters */ + char cwdpath[1024]; /* working directory */ + char filename[1024]; /* file name */ + int part[PART_SIZE]; /* raw data buffer (partial) */ + int whole[TOTAL_SIZE]; /* raw data buffer (total) */ + hsize_t cur_size; /* current data space size */ + char buffer[1024]; /* buffer to read efile_prefix */ + + TESTING("prefix in HDF5_EXTFILE_PREFIX"); + + if(HDmkdir("extern_dir", (mode_t)0755) < 0 && errno != EEXIST) + TEST_ERROR; + + h5_fixname(EXT_FNAME[4], fapl, filename, sizeof(filename)); + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR + + /* Reset the raw data files */ + if(reset_raw_data_files() < 0) + TEST_ERROR + + /* Create the dataset */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR + if(NULL == HDgetcwd(cwdpath, sizeof(cwdpath))) + TEST_ERROR + for(i = 0; i < N_EXT_FILES; i++) { + HDsnprintf(filename, sizeof(filename), "..%sextern_%dr.raw", H5_DIR_SEPS, (int) i + 1); + if(H5Pset_external(dcpl, filename, (off_t)(i * GARBAGE_PER_FILE), (hsize_t)sizeof(part)) < 0) + FAIL_STACK_ERROR + } /* end for */ + + cur_size = TOTAL_SIZE; + if((space = H5Screate_simple(1, &cur_size, NULL)) < 0) + FAIL_STACK_ERROR + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR + + /* Set prefix to a nonexistent directory, will be overwritten by environment variable */ + if(H5Pset_efile_prefix(dapl, "someprefix") < 0) + FAIL_STACK_ERROR + if(H5Pget_efile_prefix(dapl, buffer, sizeof(buffer)) < 0) + FAIL_STACK_ERROR + if(HDstrcmp(buffer, "someprefix") != 0) + FAIL_PUTS_ERROR("efile prefix not set correctly"); + + /* Create dataset */ + if((dset = H5Dcreate2(file, "dset1", H5T_NATIVE_INT, space, H5P_DEFAULT, dcpl, dapl)) < 0) + FAIL_STACK_ERROR + + /* Read the entire dataset and compare with the original */ + HDmemset(whole, 0, sizeof(whole)); + if(H5Dread(dset, H5T_NATIVE_INT, space, space, H5P_DEFAULT, whole) < 0) + FAIL_STACK_ERROR + for(i = 0; i < TOTAL_SIZE; i++) + if(whole[i] != (signed)i) + FAIL_PUTS_ERROR("Incorrect value(s) read."); + + if(H5Dclose(dset) < 0) FAIL_STACK_ERROR + if(H5Pclose(dapl) < 0) FAIL_STACK_ERROR + if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR + if(H5Sclose(space) < 0) FAIL_STACK_ERROR + if(H5Fclose(file) < 0) FAIL_STACK_ERROR + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dapl); + H5Dclose(dset); + H5Pclose(dcpl); + H5Sclose(space); + H5Fclose(file); + } H5E_END_TRY; + return 1; +} /* end test_path_env() */ + + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Runs external dataset tests. + * + * Return: EXIT_SUCCESS/EXIT_FAILURE + * + * Programmer: Robb Matzke + * Tuesday, March 3, 1998 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + hid_t fapl_id_old = -1; /* file access properties (old format) */ + hid_t fapl_id_new = -1; /* file access properties (new format) */ + hid_t fid = -1; /* file for test_1* functions */ + hid_t gid = -1; /* group to emit diagnostics */ + char filename[1024]; /* file name for test_1* funcs */ + unsigned latest_format; /* default or latest file format */ + int nerrors = 0; /* number of errors */ + + h5_reset(); + + /* Get a fapl for the old (default) file format */ + fapl_id_old = h5_fileaccess(); + h5_fixname(EXT_FNAME[0], fapl_id_old, filename, sizeof(filename)); + + /* Copy and set up a fapl for the latest file format */ + if((fapl_id_new = H5Pcopy(fapl_id_old)) < 0) + FAIL_STACK_ERROR + if(H5Pset_libver_bounds(fapl_id_new, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + FAIL_STACK_ERROR + + /* Test with old & new format groups */ + for(latest_format = FALSE; latest_format <= TRUE; latest_format++) { + hid_t current_fapl_id = -1; + + /* Set the fapl for different file formats */ + if(latest_format) { + HDputs("\nTesting with the latest file format:"); + current_fapl_id = fapl_id_new; + } /* end if */ + else { + HDputs("Testing with the default file format:"); + current_fapl_id = fapl_id_old; + } /* end else */ + + nerrors += test_path_env(current_fapl_id); + } /* end for */ + + if(nerrors > 0) goto error; + + /* Close the new ff fapl. h5_cleanup will take care of the old ff fapl */ + if(H5Pclose(fapl_id_new) < 0) FAIL_STACK_ERROR + + HDputs("All external storage tests passed."); + + /* Clean up files used by file set tests */ + if(h5_cleanup(EXT_FNAME, fapl_id_old)) { + HDremove("extern_1r.raw"); + HDremove("extern_2r.raw"); + HDremove("extern_3r.raw"); + HDremove("extern_4r.raw"); + + HDremove("extern_1w.raw"); + HDremove("extern_2w.raw"); + HDremove("extern_3w.raw"); + HDremove("extern_4w.raw"); + + HDrmdir("extern_dir"); + } /* end if */ + + return EXIT_SUCCESS; + +error: + H5E_BEGIN_TRY { + H5Fclose(fid); + H5Pclose(fapl_id_old); + H5Pclose(fapl_id_new); + H5Gclose(gid); + } H5E_END_TRY; + nerrors = MAX(1, nerrors); + printf("%d TEST%s FAILED.\n", nerrors, 1 == nerrors ? "" : "s"); + return EXIT_FAILURE; +} /* end main() */ diff --git a/test/testexternal_env.sh.in b/test/testexternal_env.sh.in new file mode 100644 index 0000000..cfd0825 --- /dev/null +++ b/test/testexternal_env.sh.in @@ -0,0 +1,39 @@ +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Test for external file with environment variable: HDF5_EXTFILE_PREFIX + +srcdir=@srcdir@ + +nerrors=0 + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + +# test for external file with HDF5_EXTFILE_PREFIX +echo "Testing external file with HDF5_EXTFILE_PREFIX" +TEST_NAME=external_env # The test name +TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary +ENVCMD="env HDF5_EXTFILE_PREFIX=\${ORIGIN}" # The environment variable & value +# +# Run the test +# echo "$ENVCMD $RUNSERIAL $TEST_BIN" +$ENVCMD $RUNSERIAL $TEST_BIN +exitcode=$? +if [ $exitcode -eq 0 ]; then + echo "Test prefix for HDF5_EXTFILE_PREFIX PASSED" + else + nerrors="`expr $nerrors + 1`" + echo "***Error encountered for HDF5_EXTFILE_PREFIX test***" +fi +exit $nerrors diff --git a/test/testvds_env.sh.in b/test/testvds_env.sh.in new file mode 100644 index 0000000..894e52e --- /dev/null +++ b/test/testvds_env.sh.in @@ -0,0 +1,44 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +# Test for external file with environment variable: HDF5_VDS_PREFIX + +srcdir=@srcdir@ + +nerrors=0 + +############################################################################## +############################################################################## +### T H E T E S T S ### +############################################################################## +############################################################################## + +# test for VDS with HDF5_VDS_PREFIX +echo "Testing basic virtual dataset I/O via H5Pset_vds_prefix(): all selection with ENV prefix" +TEST_NAME=vds_env # The test name +TEST_BIN=`pwd`/$TEST_NAME # The path of the test binary +ENVCMD="env HDF5_VDS_PREFIX=\${ORIGIN}/tmp" # Set the environment variable & value +UNENVCMD="unset HDF5_VDS_PREFIX" # Unset the environment variable & value +# +# Run the test +# echo "$ENVCMD $RUNSERIAL $TEST_BIN" +$ENVCMD $RUNSERIAL $TEST_BIN +exitcode=$? +if [ $exitcode -eq 0 ]; then + echo "Test prefix for HDF5_VDS_PREFIX PASSED" + else + nerrors="`expr $nerrors + 1`" + echo "***Error encountered for HDF5_VDS_PREFIX test***" +fi +$UNENVCMD +exit $nerrors diff --git a/test/vds.c b/test/vds.c index e4f886e..fd5a431 100644 --- a/test/vds.c +++ b/test/vds.c @@ -1121,7 +1121,7 @@ error: } /* end test_api() */ /*------------------------------------------------------------------------- - * Function: vds_link_prefix + * Function: vds_link_prefix_first * * Purpose: Set up vds link prefix via H5Pset_virtual_prefix() to be "tmp" * Should be able to access the target source files in tmp directory via the prefix set @@ -1132,7 +1132,7 @@ error: *------------------------------------------------------------------------- */ static int -test_vds_prefix(unsigned config, hid_t fapl) +test_vds_prefix_first(unsigned config, hid_t fapl) { char srcfilename[FILENAME_BUF_SIZE]; char srcfilename_map[FILENAME_BUF_SIZE]; @@ -1158,7 +1158,7 @@ test_vds_prefix(unsigned config, hid_t fapl) int i, j; char buffer[1024]; /* buffer to read vds_prefix */ - TESTING("basic virtual dataset I/O via H5Pset_vds_prefix()") + TESTING("basic virtual dataset I/O via H5Pset_vds_prefix(): all selection") h5_fixname(FILENAME[0], fapl, vfilename, sizeof vfilename); h5_fixname(FILENAME[7], fapl, vfilename2, sizeof vfilename2); @@ -1191,9 +1191,6 @@ test_vds_prefix(unsigned config, hid_t fapl) if(HDstrcmp(buffer, TMPDIR) != 0) FAIL_PUTS_ERROR("vds prefix not set correctly"); - /* - * Test 1: All - all selection - */ /* Create source dataspace */ if((srcspace[0] = H5Screate_simple(2, dims, NULL)) < 0) TEST_ERROR @@ -1346,187 +1343,6 @@ test_vds_prefix(unsigned config, hid_t fapl) TEST_ERROR dcpl = -1; - /* - * Test 2: All - all selection with ENV prefix - */ - if(HDsetenv("HDF5_VDS_PREFIX", "${ORIGIN}/tmp", 1)) - TEST_ERROR - - /* Create DCPL */ - if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) - TEST_ERROR - - /* Set fill value */ - if(H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fill) < 0) - TEST_ERROR - - /* Set prefix to a nonexistent directory, will be overwritten by environment variable */ - if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) - TEST_ERROR - - if(H5Pset_virtual_prefix(dapl, "someprefix") < 0) - TEST_ERROR - if(H5Pget_virtual_prefix(dapl, buffer, sizeof(buffer)) < 0) - TEST_ERROR - - if(HDstrcmp(buffer, "someprefix") != 0) - FAIL_PUTS_ERROR("vds prefix not set correctly"); - - /* Create source dataspace */ - if((srcspace[0] = H5Screate_simple(2, dims, NULL)) < 0) - TEST_ERROR - - /* Create virtual dataspace */ - if((vspace[0] = H5Screate_simple(2, dims, NULL)) < 0) - TEST_ERROR - - /* Select all (should not be necessary, but just to be sure) */ - if(H5Sselect_all(srcspace[0]) < 0) - TEST_ERROR - if(H5Sselect_all(vspace[0]) < 0) - TEST_ERROR - - /* Add virtual layout mapping */ - if(H5Pset_virtual(dcpl, vspace[0], config & TEST_IO_DIFFERENT_FILE ? srcfilename_map : ".", "src_dset", srcspace[0]) < 0) - TEST_ERROR - - /* Create virtual file */ - if((vfile = H5Fcreate(vfilename2, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) - TEST_ERROR - - /* Create source file if requested */ - if(config & TEST_IO_DIFFERENT_FILE) { - HDgetcwd(buffer, 1024); - HDchdir(TMPDIR); - if((srcfile[0] = H5Fcreate(srcfilename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) - TEST_ERROR - HDchdir(buffer); - } - else { - srcfile[0] = vfile; - if(H5Iinc_ref(srcfile[0]) < 0) - TEST_ERROR - } - - /* Create source dataset */ - if((srcdset[0] = H5Dcreate2(srcfile[0], "src_dset", H5T_NATIVE_INT, srcspace[0], H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) - TEST_ERROR - - /* Create virtual dataset */ - if((vdset = H5Dcreate2(vfile, "v_dset", H5T_NATIVE_INT, vspace[0], H5P_DEFAULT, dcpl, dapl)) < 0) - TEST_ERROR - - /* Populate write buffer */ - for(i = 0; i < (int)(sizeof(buf) / sizeof(buf[0])); i++) - for(j = 0; j < (int)(sizeof(buf[0]) / sizeof(buf[0][0])); j++) - buf[i][j] = (i * (int)(sizeof(buf[0]) / sizeof(buf[0][0]))) + j; - - /* Write data directly to source dataset */ - if(H5Dwrite(srcdset[0], H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf[0]) < 0) - TEST_ERROR - - /* Close srcdset and srcfile if config option specified */ - if(config & TEST_IO_CLOSE_SRC) { - if(H5Dclose(srcdset[0]) < 0) - TEST_ERROR - srcdset[0] = -1; - - if(config & TEST_IO_DIFFERENT_FILE) { - if(H5Fclose(srcfile[0]) < 0) - TEST_ERROR - srcfile[0] = -1; - } - } - - /* Reopen virtual dataset and file if config option specified */ - if(config & TEST_IO_REOPEN_VIRT) { - if(H5Dclose(vdset) < 0) - TEST_ERROR - vdset = -1; - if(H5Fclose(vfile) < 0) - TEST_ERROR - vfile = -1; - if((vfile = H5Fopen(vfilename2, H5F_ACC_RDWR, fapl)) < 0) - TEST_ERROR - if((vdset = H5Dopen2(vfile, "v_dset", dapl)) < 0) - TEST_ERROR - } - - /* Read data through virtual dataset */ - HDmemset(rbuf[0], 0, sizeof(rbuf)); - if(H5Dread(vdset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf[0]) < 0) - TEST_ERROR - - /* Verify read data */ - for(i = 0; i < (int)(sizeof(buf) / sizeof(buf[0])); i++) { - for(j = 0; j < (int)(sizeof(buf[0]) / sizeof(buf[0][0])); j++) - if(rbuf[i][j] != buf[i][j]) { - TEST_ERROR - } - } - - /* Adjust write buffer */ - for(i = 0; i < (int)(sizeof(buf) / sizeof(buf[0])); i++) - for(j = 0; j < (int)(sizeof(buf[0]) / sizeof(buf[0][0])); j++) - buf[i][j] += (int)(sizeof(buf) / sizeof(buf[0][0])); - - /* Write data through virtual dataset */ - if(H5Dwrite(vdset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf[0]) < 0) - TEST_ERROR - - /* Reopen srcdset and srcfile if config option specified */ - if(config & TEST_IO_CLOSE_SRC) { - if(config & TEST_IO_DIFFERENT_FILE) { - HDgetcwd(buffer, 1024); - HDchdir(TMPDIR); - if((srcfile[0] = H5Fopen(srcfilename, H5F_ACC_RDONLY, fapl)) < 0) - TEST_ERROR - HDchdir(buffer); - } - if((srcdset[0] = H5Dopen2(srcfile[0], "src_dset", H5P_DEFAULT)) < 0) - TEST_ERROR - } - - /* Read data directly from source dataset */ - HDmemset(rbuf[0], 0, sizeof(rbuf)); - if(H5Dread(srcdset[0], H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf[0]) < 0) - TEST_ERROR - - /* Verify read data */ - for(i = 0; i < (int)(sizeof(buf) / sizeof(buf[0])); i++) - for(j = 0; j < (int)(sizeof(buf[0]) / sizeof(buf[0][0])); j++) - if(rbuf[i][j] != buf[i][j]) - TEST_ERROR - - /* Close */ - if(H5Dclose(vdset) < 0) - TEST_ERROR - vdset = -1; - if(H5Dclose(srcdset[0]) < 0) - TEST_ERROR - srcdset[0] = -1; - if(H5Fclose(srcfile[0]) < 0) - TEST_ERROR - srcfile[0] = -1; - if(H5Fclose(vfile) < 0) - TEST_ERROR - vfile = -1; - if(H5Sclose(srcspace[0]) < 0) - TEST_ERROR - srcspace[0] = -1; - if(H5Sclose(vspace[0]) < 0) - TEST_ERROR - vspace[0] = -1; - if(H5Pclose(dapl) < 0) - TEST_ERROR - dapl = -1; - if(H5Pclose(dcpl) < 0) - TEST_ERROR - dcpl = -1; - - if(HDsetenv("HDF5_VDS_PREFIX", "", 1) < 0) - TEST_ERROR - PASSED(); return 0; @@ -1538,7 +1354,6 @@ test_vds_prefix(unsigned config, hid_t fapl) for(i = 0; i < (int)(sizeof(srcfile) / sizeof(srcfile[0])); i++) H5Fclose(srcfile[i]); H5Fclose(vfile); - H5Fclose(vfile2); for(i = 0; i < (int)(sizeof(srcspace) / sizeof(srcspace[0])); i++) H5Sclose(srcspace[i]); for(i = 0; i < (int)(sizeof(vspace) / sizeof(vspace[0])); i++) @@ -1552,7 +1367,7 @@ test_vds_prefix(unsigned config, hid_t fapl) TEST_ERROR return 1; -} /* end vds_link_prefix() */ +} /* end test_vds_prefix() */ /*------------------------------------------------------------------------- @@ -11623,7 +11438,7 @@ main(void) for(bit_config = 0; bit_config < TEST_IO_NTESTS; bit_config++) { printf("Config: %s%s%s\n", bit_config & TEST_IO_CLOSE_SRC ? "closed source dataset, " : "", bit_config & TEST_IO_DIFFERENT_FILE ? "different source file" : "same source file", bit_config & TEST_IO_REOPEN_VIRT ? ", reopen virtual file" : ""); nerrors += test_basic_io(bit_config, fapl); - nerrors += test_vds_prefix(bit_config, fapl); + nerrors += test_vds_prefix_first(bit_config, fapl); nerrors += test_unlim(bit_config, fapl); nerrors += test_printf(bit_config, fapl); nerrors += test_all(bit_config, fapl); diff --git a/test/vds_env.c b/test/vds_env.c new file mode 100644 index 0000000..1c2dba9 --- /dev/null +++ b/test/vds_env.c @@ -0,0 +1,327 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Neil Fortner + * Monday, February 16, 2015 + * + * Purpose: Tests datasets with virtual layout. + */ +#include "h5test.h" + +const char *FILENAME[] = { + "vds_env_virt_0", + "vds_env_virt_3", + "vds_env_src_2", + "vds_env_%%_src2", + NULL +}; + +/* I/O test config flags */ +#define TEST_IO_CLOSE_SRC 0x01u +#define TEST_IO_DIFFERENT_FILE 0x02u +#define TEST_IO_REOPEN_VIRT 0x04u +#define TEST_IO_NTESTS 0x08u + +#define FILENAME_BUF_SIZE 1024 + +#define TMPDIR "tmp/" + +/*------------------------------------------------------------------------- + * Function: test_vds_prefix_second + * + * Purpose: Set up vds link prefix via H5Pset_virtual_prefix() to be "tmp" + * Should be able to access the target source files in tmp directory via the prefix set + * by H5Pset_virtual_prefix() + * + * Return: Success: 0 + * Failure: -1 + *------------------------------------------------------------------------- + */ +static int +test_vds_prefix_second(unsigned config, hid_t fapl) +{ + char srcfilename[FILENAME_BUF_SIZE]; + char srcfilename_map[FILENAME_BUF_SIZE]; + char vfilename[FILENAME_BUF_SIZE]; + char vfilename2[FILENAME_BUF_SIZE]; + char srcfilenamepct[FILENAME_BUF_SIZE]; + char srcfilenamepct_map[FILENAME_BUF_SIZE]; + const char *srcfilenamepct_map_orig = "vds%%%%_src"; + hid_t srcfile[4] = {-1, -1, -1, -1}; /* Files with source dsets */ + hid_t vfile = -1; /* File with virtual dset */ + hid_t dcpl = -1; /* Dataset creation property list */ + hid_t dapl = -1; /* Dataset access property list */ + hid_t srcspace[4] = {-1, -1, -1, -1}; /* Source dataspaces */ + hid_t vspace[4] = {-1, -1, -1, -1}; /* Virtual dset dataspaces */ + hid_t memspace = -1; /* Memory dataspace */ + hid_t srcdset[4] = {-1, -1, -1, -1}; /* Source datsets */ + hid_t vdset = -1; /* Virtual dataset */ + hsize_t dims[4] = {10, 26, 0, 0}; /* Data space current size */ + int buf[10][26]; /* Write and expected read buffer */ + int rbuf[10][26]; /* Read buffer */ + int fill = -1; /* Fill value */ + int i, j; + char buffer[1024]; /* buffer to read vds_prefix */ + + TESTING("basic virtual dataset I/O via H5Pset_vds_prefix(): all selection with ENV prefix") + + h5_fixname(FILENAME[0], fapl, vfilename, sizeof vfilename); + h5_fixname(FILENAME[1], fapl, vfilename2, sizeof vfilename2); + h5_fixname(FILENAME[2], fapl, srcfilename, sizeof srcfilename); + h5_fixname_printf(FILENAME[2], fapl, srcfilename_map, sizeof srcfilename_map); + h5_fixname(FILENAME[3], fapl, srcfilenamepct, sizeof srcfilenamepct); + h5_fixname_printf(srcfilenamepct_map_orig, fapl, srcfilenamepct_map, sizeof srcfilenamepct_map); + + /* create tmp directory and get current working directory path */ + if (HDmkdir(TMPDIR, (mode_t)0755) < 0 && errno != EEXIST) + TEST_ERROR + + /* Create DCPL */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR + + /* Set fill value */ + if(H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fill) < 0) + TEST_ERROR + + /* Set prefix to a nonexistent directory, will be overwritten by environment variable */ + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + TEST_ERROR + + if(H5Pset_virtual_prefix(dapl, "someprefix") < 0) + TEST_ERROR + if(H5Pget_virtual_prefix(dapl, buffer, sizeof(buffer)) < 0) + TEST_ERROR + + if(HDstrcmp(buffer, "someprefix") != 0) + FAIL_PUTS_ERROR("vds prefix not set correctly"); + + /* Create source dataspace */ + if((srcspace[0] = H5Screate_simple(2, dims, NULL)) < 0) + TEST_ERROR + + /* Create virtual dataspace */ + if((vspace[0] = H5Screate_simple(2, dims, NULL)) < 0) + TEST_ERROR + + /* Select all (should not be necessary, but just to be sure) */ + if(H5Sselect_all(srcspace[0]) < 0) + TEST_ERROR + if(H5Sselect_all(vspace[0]) < 0) + TEST_ERROR + + /* Add virtual layout mapping */ + if(H5Pset_virtual(dcpl, vspace[0], config & TEST_IO_DIFFERENT_FILE ? srcfilename_map : ".", "src_dset", srcspace[0]) < 0) + TEST_ERROR + + /* Create virtual file */ + if((vfile = H5Fcreate(vfilename2, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + TEST_ERROR + + /* Create source file if requested */ + if(config & TEST_IO_DIFFERENT_FILE) { + HDgetcwd(buffer, 1024); + HDchdir(TMPDIR); + if((srcfile[0] = H5Fcreate(srcfilename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + TEST_ERROR + HDchdir(buffer); + } + else { + srcfile[0] = vfile; + if(H5Iinc_ref(srcfile[0]) < 0) + TEST_ERROR + } + + /* Create source dataset */ + if((srcdset[0] = H5Dcreate2(srcfile[0], "src_dset", H5T_NATIVE_INT, srcspace[0], H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + TEST_ERROR + + /* Create virtual dataset */ + if((vdset = H5Dcreate2(vfile, "v_dset", H5T_NATIVE_INT, vspace[0], H5P_DEFAULT, dcpl, dapl)) < 0) + TEST_ERROR + + /* Populate write buffer */ + for(i = 0; i < (int)(sizeof(buf) / sizeof(buf[0])); i++) + for(j = 0; j < (int)(sizeof(buf[0]) / sizeof(buf[0][0])); j++) + buf[i][j] = (i * (int)(sizeof(buf[0]) / sizeof(buf[0][0]))) + j; + + /* Write data directly to source dataset */ + if(H5Dwrite(srcdset[0], H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf[0]) < 0) + TEST_ERROR + + /* Close srcdset and srcfile if config option specified */ + if(config & TEST_IO_CLOSE_SRC) { + if(H5Dclose(srcdset[0]) < 0) + TEST_ERROR + srcdset[0] = -1; + + if(config & TEST_IO_DIFFERENT_FILE) { + if(H5Fclose(srcfile[0]) < 0) + TEST_ERROR + srcfile[0] = -1; + } + } + + /* Reopen virtual dataset and file if config option specified */ + if(config & TEST_IO_REOPEN_VIRT) { + if(H5Dclose(vdset) < 0) + TEST_ERROR + vdset = -1; + if(H5Fclose(vfile) < 0) + TEST_ERROR + vfile = -1; + if((vfile = H5Fopen(vfilename2, H5F_ACC_RDWR, fapl)) < 0) + TEST_ERROR + if((vdset = H5Dopen2(vfile, "v_dset", dapl)) < 0) + TEST_ERROR + } + + /* Read data through virtual dataset */ + HDmemset(rbuf[0], 0, sizeof(rbuf)); + if(H5Dread(vdset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf[0]) < 0) + TEST_ERROR + + /* Verify read data */ + for(i = 0; i < (int)(sizeof(buf) / sizeof(buf[0])); i++) { + for(j = 0; j < (int)(sizeof(buf[0]) / sizeof(buf[0][0])); j++) + if(rbuf[i][j] != buf[i][j]) { + TEST_ERROR + } + } + + /* Adjust write buffer */ + for(i = 0; i < (int)(sizeof(buf) / sizeof(buf[0])); i++) + for(j = 0; j < (int)(sizeof(buf[0]) / sizeof(buf[0][0])); j++) + buf[i][j] += (int)(sizeof(buf) / sizeof(buf[0][0])); + + /* Write data through virtual dataset */ + if(H5Dwrite(vdset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf[0]) < 0) + TEST_ERROR + + /* Reopen srcdset and srcfile if config option specified */ + if(config & TEST_IO_CLOSE_SRC) { + if(config & TEST_IO_DIFFERENT_FILE) { + HDgetcwd(buffer, 1024); + HDchdir(TMPDIR); + if((srcfile[0] = H5Fopen(srcfilename, H5F_ACC_RDONLY, fapl)) < 0) + TEST_ERROR + HDchdir(buffer); + } + if((srcdset[0] = H5Dopen2(srcfile[0], "src_dset", H5P_DEFAULT)) < 0) + TEST_ERROR + } + + /* Read data directly from source dataset */ + HDmemset(rbuf[0], 0, sizeof(rbuf)); + if(H5Dread(srcdset[0], H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, rbuf[0]) < 0) + TEST_ERROR + + /* Verify read data */ + for(i = 0; i < (int)(sizeof(buf) / sizeof(buf[0])); i++) + for(j = 0; j < (int)(sizeof(buf[0]) / sizeof(buf[0][0])); j++) + if(rbuf[i][j] != buf[i][j]) + TEST_ERROR + + /* Close */ + if(H5Dclose(vdset) < 0) + TEST_ERROR + vdset = -1; + if(H5Dclose(srcdset[0]) < 0) + TEST_ERROR + srcdset[0] = -1; + if(H5Fclose(srcfile[0]) < 0) + TEST_ERROR + srcfile[0] = -1; + if(H5Fclose(vfile) < 0) + TEST_ERROR + vfile = -1; + if(H5Sclose(srcspace[0]) < 0) + TEST_ERROR + srcspace[0] = -1; + if(H5Sclose(vspace[0]) < 0) + TEST_ERROR + vspace[0] = -1; + if(H5Pclose(dapl) < 0) + TEST_ERROR + dapl = -1; + if(H5Pclose(dcpl) < 0) + TEST_ERROR + dcpl = -1; + + PASSED(); + return 0; + + error: + H5E_BEGIN_TRY { + for(i = 0; i < (int)(sizeof(srcdset) / sizeof(srcdset[0])); i++) + H5Dclose(srcdset[i]); + H5Dclose(vdset); + for(i = 0; i < (int)(sizeof(srcfile) / sizeof(srcfile[0])); i++) + H5Fclose(srcfile[i]); + H5Fclose(vfile); + for(i = 0; i < (int)(sizeof(srcspace) / sizeof(srcspace[0])); i++) + H5Sclose(srcspace[i]); + for(i = 0; i < (int)(sizeof(vspace) / sizeof(vspace[0])); i++) + H5Sclose(vspace[i]); + H5Sclose(memspace); + H5Pclose(dapl); + H5Pclose(dcpl); + } H5E_END_TRY; + + return 1; +} /* end test_vds_prefix2 */ + + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Tests datasets with virtual layout + * + * Return: EXIT_SUCCESS/EXIT_FAILURE + *------------------------------------------------------------------------- + */ +int +main(void) +{ + hid_t fapl; + int test_api_config; + unsigned bit_config; + H5F_libver_t low, high; /* Low and high bounds */ + unsigned latest = FALSE; /* Using the latest library version bound */ + int nerrors = 0; + + /* Testing setup */ + h5_reset(); + fapl = h5_fileaccess(); + + for(bit_config = 0; bit_config < TEST_IO_NTESTS; bit_config++) { + HDprintf("Config: %s%s%s\n", bit_config & TEST_IO_CLOSE_SRC ? "closed source dataset, " : "", bit_config & TEST_IO_DIFFERENT_FILE ? "different source file" : "same source file", bit_config & TEST_IO_REOPEN_VIRT ? ", reopen virtual file" : ""); + nerrors += test_vds_prefix_second(bit_config, fapl); + } + + /* Verify symbol table messages are cached */ + nerrors += (h5_verify_cached_stabs(FILENAME, fapl) < 0 ? 1 : 0); + + if(nerrors) + goto error; + HDprintf("All virtual dataset tests passed.\n"); + h5_cleanup(FILENAME, fapl); + + return EXIT_SUCCESS; + +error: + nerrors = MAX(1, nerrors); + HDprintf("***** %d VIRTUAL DATASET TEST%s FAILED! *****\n", + nerrors, 1 == nerrors ? "" : "S"); + return EXIT_FAILURE; +} /* end main() */ -- cgit v0.12