diff options
author | Dana Robinson <derobins@hdfgroup.org> | 2016-04-15 04:18:02 (GMT) |
---|---|---|
committer | Dana Robinson <derobins@hdfgroup.org> | 2016-04-15 04:18:02 (GMT) |
commit | 58382379be519527e74343dade9efc5bed6f81b3 (patch) | |
tree | 31bd36c36fbc363d72a15af8a882c2b87ae204e8 | |
parent | b79b00dbfc603b8a70c0cf50585e07c15da8b7d2 (diff) | |
download | hdf5-58382379be519527e74343dade9efc5bed6f81b3.zip hdf5-58382379be519527e74343dade9efc5bed6f81b3.tar.gz hdf5-58382379be519527e74343dade9efc5bed6f81b3.tar.bz2 |
[svn-r29702] Merged r29182 and 29183 from the trunk.
Fixes HDFFV-8740 (external file storage relative path fix).
Tested on: 64-bit Ubuntu 15.10 (Linux 4.2.0 x86_64) gcc 5.2.1
serial autotools w/ C++ & Fortran
parallel autotools (MPICH 3.1.4) w/ Fortran
serial CMake
-rw-r--r-- | src/H5.c | 3 | ||||
-rw-r--r-- | src/H5Defl.c | 172 | ||||
-rw-r--r-- | src/H5Dint.c | 115 | ||||
-rw-r--r-- | src/H5Dpkg.h | 2 | ||||
-rw-r--r-- | src/H5Dprivate.h | 1 | ||||
-rw-r--r-- | src/H5F.c | 15 | ||||
-rw-r--r-- | src/H5Fquery.c | 2 | ||||
-rw-r--r-- | src/H5Pdapl.c | 276 | ||||
-rw-r--r-- | src/H5Plapl.c | 1 | ||||
-rw-r--r-- | src/H5Ppublic.h | 2 | ||||
-rw-r--r-- | src/H5private.h | 6 | ||||
-rw-r--r-- | src/H5system.c | 129 | ||||
-rw-r--r-- | src/H5win32defs.h | 30 | ||||
-rw-r--r-- | test/external.c | 736 | ||||
-rw-r--r-- | test/tfile.c | 61 |
15 files changed, 1344 insertions, 207 deletions
@@ -188,7 +188,8 @@ H5_init_library(void) /* Normal library termination code */ (void)HDatexit(H5_term_library); - H5_dont_atexit_g = TRUE; + + H5_dont_atexit_g = TRUE; } /* end if */ /* diff --git a/src/H5Defl.c b/src/H5Defl.c index 85ceaf2..6ee9000 100644 --- a/src/H5Defl.c +++ b/src/H5Defl.c @@ -28,12 +28,13 @@ /***********/ /* Headers */ /***********/ -#include "H5private.h" /* Generic Functions */ -#include "H5Dpkg.h" /* Datasets */ -#include "H5Eprivate.h" /* Error handling */ -#include "H5Fprivate.h" /* Files */ -#include "H5HLprivate.h" /* Local Heaps */ -#include "H5VMprivate.h" /* Vector and array functions */ +#include "H5private.h" /* Generic Functions */ +#include "H5Dpkg.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* Files */ +#include "H5HLprivate.h" /* Local Heaps */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5VMprivate.h" /* Vector and array functions */ /****************/ @@ -48,12 +49,14 @@ /* Callback info for readvv operation */ typedef struct H5D_efl_readvv_ud_t { const H5O_efl_t *efl; /* Pointer to efl info */ + const H5D_t *dset; /* The dataset */ unsigned char *rbuf; /* Read buffer */ } H5D_efl_readvv_ud_t; /* Callback info for writevv operation */ typedef struct H5D_efl_writevv_ud_t { const H5O_efl_t *efl; /* Pointer to efl info */ + const H5D_t *dset; /* The dataset */ const unsigned char *wbuf; /* Write buffer */ } H5D_efl_writevv_ud_t; @@ -75,9 +78,9 @@ static ssize_t H5D__efl_writevv(const H5D_io_info_t *io_info, size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[]); /* Helper routines */ -static herr_t H5D__efl_read(const H5O_efl_t *efl, haddr_t addr, size_t size, +static herr_t H5D__efl_read(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t size, uint8_t *buf); -static herr_t H5D__efl_write(const H5O_efl_t *efl, haddr_t addr, size_t size, +static herr_t H5D__efl_write(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t size, const uint8_t *buf); @@ -236,25 +239,22 @@ H5D__efl_io_init(const H5D_io_info_t *io_info, const H5D_type_info_t H5_ATTR_UNU /*------------------------------------------------------------------------- - * Function: H5D__efl_read + * Function: H5D__efl_read * - * Purpose: Reads data from an external file list. It is an error to - * read past the logical end of file, but reading past the end - * of any particular member of the external file list results in - * zeros. + * Purpose: Reads data from an external file list. It is an error to + * read past the logical end of file, but reading past the end + * of any particular member of the external file list results in + * zeros. * - * Return: Non-negative on success/Negative on failure + * Return: SUCCEED/FAIL * - * Programmer: Robb Matzke + * Programmer: Robb Matzke * Wednesday, March 4, 1998 * - * Modifications: - * Robb Matzke, 1999-07-28 - * The ADDR argument is passed by value. *------------------------------------------------------------------------- */ static herr_t -H5D__efl_read(const H5O_efl_t *efl, haddr_t addr, size_t size, uint8_t *buf) +H5D__efl_read(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t size, uint8_t *buf) { int fd = -1; size_t to_read; @@ -265,6 +265,7 @@ H5D__efl_read(const H5O_efl_t *efl, haddr_t addr, size_t size, uint8_t *buf) haddr_t cur; ssize_t n; size_t u; /* Local index variable */ + char *full_name = NULL; /* File name with prefix */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -277,44 +278,49 @@ H5D__efl_read(const H5O_efl_t *efl, haddr_t addr, size_t size, uint8_t *buf) /* Find the first efl member from which to read */ for (u=0, cur=0; u<efl->nused; u++) { - if(H5O_EFL_UNLIMITED == efl->slot[u].size || addr < cur + efl->slot[u].size) { - skip = addr - cur; - break; - } /* end if */ - cur += efl->slot[u].size; + if(H5O_EFL_UNLIMITED == efl->slot[u].size || addr < cur + efl->slot[u].size) { + skip = addr - cur; + break; + } /* end if */ + cur += efl->slot[u].size; } /* end for */ /* Read the data */ while(size) { HDassert(buf); - if(u >= efl->nused) - HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "read past logical end of file") - if(H5F_OVERFLOW_HSIZET2OFFT(efl->slot[u].offset + skip)) - HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed") - if((fd = HDopen(efl->slot[u].name, O_RDONLY, 0)) < 0) - HGOTO_ERROR(H5E_EFL, H5E_CANTOPENFILE, FAIL, "unable to open external raw data file") - if(HDlseek(fd, (off_t)(efl->slot[u].offset + skip), SEEK_SET) < 0) - HGOTO_ERROR(H5E_EFL, H5E_SEEKERROR, FAIL, "unable to seek in external raw data file") + if(u >= efl->nused) + HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "read past logical end of file") + if(H5F_OVERFLOW_HSIZET2OFFT(efl->slot[u].offset + skip)) + HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed") + if(H5_combine_path(dset->shared->extfile_prefix, efl->slot[u].name, &full_name) < 0) + HGOTO_ERROR(H5E_EFL, H5E_NOSPACE, FAIL, "can't build external file name") + if((fd = HDopen(full_name, O_RDONLY, 0)) < 0) + HGOTO_ERROR(H5E_EFL, H5E_CANTOPENFILE, FAIL, "unable to open external raw data file") + if(HDlseek(fd, (HDoff_t)(efl->slot[u].offset + skip), SEEK_SET) < 0) + HGOTO_ERROR(H5E_EFL, H5E_SEEKERROR, FAIL, "unable to seek in external raw data file") #ifndef NDEBUG - tempto_read = MIN(efl->slot[u].size-skip, (hsize_t)size); + tempto_read = MIN((size_t)(efl->slot[u].size-skip), (hsize_t)size); H5_CHECK_OVERFLOW(tempto_read, hsize_t, size_t); - to_read = (size_t)tempto_read; + to_read = (size_t)tempto_read; #else /* NDEBUG */ - to_read = MIN((size_t)(efl->slot[u].size - skip), size); + to_read = MIN((size_t)(efl->slot[u].size - skip), (hsize_t)size); #endif /* NDEBUG */ - if((n = HDread(fd, buf, to_read)) < 0) - HGOTO_ERROR(H5E_EFL, H5E_READERROR, FAIL, "read error in external raw data file") - else if((size_t)n < to_read) - HDmemset(buf + n, 0, to_read - (size_t)n); - HDclose(fd); - fd = -1; - size -= to_read; - buf += to_read; - skip = 0; - u++; + if((n = HDread(fd, buf, to_read)) < 0) + HGOTO_ERROR(H5E_EFL, H5E_READERROR, FAIL, "read error in external raw data file") + else if((size_t)n < to_read) + HDmemset(buf + n, 0, to_read - (size_t)n); + full_name = (char *)H5MM_xfree(full_name); + HDclose(fd); + fd = -1; + size -= to_read; + buf += to_read; + skip = 0; + u++; } /* end while */ done: + if(full_name) + full_name = (char *)H5MM_xfree(full_name); if(fd >= 0) HDclose(fd); @@ -341,7 +347,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D__efl_write(const H5O_efl_t *efl, haddr_t addr, size_t size, const uint8_t *buf) +H5D__efl_write(const H5O_efl_t *efl, const H5D_t *dset, haddr_t addr, size_t size, const uint8_t *buf) { int fd = -1; size_t to_write; @@ -350,7 +356,8 @@ H5D__efl_write(const H5O_efl_t *efl, haddr_t addr, size_t size, const uint8_t *b #endif /* NDEBUG */ haddr_t cur; hsize_t skip = 0; - size_t u; /* Local index variable */ + size_t u; /* Local index variable */ + char *full_name = NULL; /* File name with prefix */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -363,46 +370,51 @@ H5D__efl_write(const H5O_efl_t *efl, haddr_t addr, size_t size, const uint8_t *b /* Find the first efl member in which to write */ for(u = 0, cur = 0; u < efl->nused; u++) { - if(H5O_EFL_UNLIMITED == efl->slot[u].size || addr < cur + efl->slot[u].size) { - skip = addr - cur; - break; - } /* end if */ - cur += efl->slot[u].size; + if(H5O_EFL_UNLIMITED == efl->slot[u].size || addr < cur + efl->slot[u].size) { + skip = addr - cur; + break; + } /* end if */ + cur += efl->slot[u].size; } /* end for */ /* Write the data */ while(size) { HDassert(buf); - if(u >= efl->nused) - HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "write past logical end of file") - if(H5F_OVERFLOW_HSIZET2OFFT(efl->slot[u].offset + skip)) - HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed") - if((fd = HDopen(efl->slot[u].name, O_CREAT | O_RDWR, 0666)) < 0) { - if(HDaccess(efl->slot[u].name, F_OK) < 0) - HGOTO_ERROR(H5E_EFL, H5E_CANTOPENFILE, FAIL, "external raw data file does not exist") - else - HGOTO_ERROR(H5E_EFL, H5E_CANTOPENFILE, FAIL, "unable to open external raw data file") - } /* end if */ - if(HDlseek(fd, (off_t)(efl->slot[u].offset + skip), SEEK_SET) < 0) - HGOTO_ERROR(H5E_EFL, H5E_SEEKERROR, FAIL, "unable to seek in external raw data file") + if(u >= efl->nused) + HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "write past logical end of file") + if(H5F_OVERFLOW_HSIZET2OFFT(efl->slot[u].offset + skip)) + HGOTO_ERROR(H5E_EFL, H5E_OVERFLOW, FAIL, "external file address overflowed") + if(H5_combine_path(dset->shared->extfile_prefix, efl->slot[u].name, &full_name) < 0) + HGOTO_ERROR(H5E_EFL, H5E_NOSPACE, FAIL, "can't build external file name") + if((fd = HDopen(full_name, O_CREAT | O_RDWR, 0666)) < 0) { + if(HDaccess(full_name, F_OK) < 0) + HGOTO_ERROR(H5E_EFL, H5E_CANTOPENFILE, FAIL, "external raw data file does not exist") + else + HGOTO_ERROR(H5E_EFL, H5E_CANTOPENFILE, FAIL, "unable to open external raw data file") + } /* end if */ + if(HDlseek(fd, (HDoff_t)(efl->slot[u].offset + skip), SEEK_SET) < 0) + HGOTO_ERROR(H5E_EFL, H5E_SEEKERROR, FAIL, "unable to seek in external raw data file") #ifndef NDEBUG - tempto_write = MIN(efl->slot[u].size - skip, (hsize_t)size); + tempto_write = MIN(efl->slot[u].size - skip, (hsize_t)size); H5_CHECK_OVERFLOW(tempto_write, hsize_t, size_t); to_write = (size_t)tempto_write; #else /* NDEBUG */ - to_write = MIN((size_t)(efl->slot[u].size - skip), size); + to_write = MIN((size_t)(efl->slot[u].size - skip), size); #endif /* NDEBUG */ - if((size_t)HDwrite(fd, buf, to_write) != to_write) - HGOTO_ERROR(H5E_EFL, H5E_READERROR, FAIL, "write error in external raw data file") - HDclose (fd); - fd = -1; - size -= to_write; - buf += to_write; - skip = 0; - u++; + if((size_t)HDwrite(fd, buf, to_write) != to_write) + HGOTO_ERROR(H5E_EFL, H5E_READERROR, FAIL, "write error in external raw data file") + full_name = (char *)H5MM_xfree(full_name); + HDclose (fd); + fd = -1; + size -= to_write; + buf += to_write; + skip = 0; + u++; } /* end while */ done: + if(full_name) + full_name = (char *)H5MM_xfree(full_name); if(fd >= 0) HDclose(fd); @@ -431,7 +443,7 @@ H5D__efl_readvv_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udata) FUNC_ENTER_STATIC /* Read data */ - if(H5D__efl_read(udata->efl, dst_off, len, (udata->rbuf + src_off)) < 0) + if(H5D__efl_read(udata->efl, udata->dset, dst_off, len, (udata->rbuf + src_off)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "EFL read failed") done: @@ -468,6 +480,9 @@ H5D__efl_readvv(const H5D_io_info_t *io_info, HDassert(io_info); HDassert(io_info->store->efl.nused > 0); HDassert(io_info->u.rbuf); + HDassert(io_info->dset); + HDassert(io_info->dset->shared); + HDassert(io_info->dset->shared->extfile_prefix); HDassert(dset_curr_seq); HDassert(dset_len_arr); HDassert(dset_off_arr); @@ -477,6 +492,7 @@ H5D__efl_readvv(const H5D_io_info_t *io_info, /* Set up user data for H5VM_opvv() */ udata.efl = &(io_info->store->efl); + udata.dset = io_info->dset; udata.rbuf = (unsigned char *)io_info->u.rbuf; /* Call generic sequence operation routine */ @@ -511,7 +527,7 @@ H5D__efl_writevv_cb(hsize_t dst_off, hsize_t src_off, size_t len, void *_udata) FUNC_ENTER_STATIC /* Write data */ - if(H5D__efl_write(udata->efl, dst_off, len, (udata->wbuf + src_off)) < 0) + if(H5D__efl_write(udata->efl, udata->dset, dst_off, len, (udata->wbuf + src_off)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "EFL write failed") done: @@ -548,6 +564,9 @@ H5D__efl_writevv(const H5D_io_info_t *io_info, HDassert(io_info); HDassert(io_info->store->efl.nused > 0); HDassert(io_info->u.wbuf); + HDassert(io_info->dset); + HDassert(io_info->dset->shared); + HDassert(io_info->dset->shared->extfile_prefix); HDassert(dset_curr_seq); HDassert(dset_len_arr); HDassert(dset_off_arr); @@ -557,6 +576,7 @@ H5D__efl_writevv(const H5D_io_info_t *io_info, /* Set up user data for H5VM_opvv() */ udata.efl = &(io_info->store->efl); + udata.dset = io_info->dset; udata.wbuf = (const unsigned char *)io_info->u.wbuf; /* Call generic sequence operation routine */ diff --git a/src/H5Dint.c b/src/H5Dint.c index e48aabc..0e0843f 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -65,6 +65,8 @@ static herr_t H5D__init_type(H5F_t *file, const H5D_t *dset, hid_t type_id, static herr_t H5D__init_space(H5F_t *file, const H5D_t *dset, const H5S_t *space); static herr_t H5D__update_oh_info(H5F_t *file, hid_t dxpl_id, H5D_t *dset, hid_t dapl_id); +static herr_t H5D_build_extfile_prefix(const H5D_t *dset, hid_t dapl_id, + char **extfile_prefix); static herr_t H5D__open_oid(H5D_t *dataset, hid_t dapl_id, hid_t dxpl_id); static herr_t H5D__init_storage(const H5D_t *dataset, hbool_t full_overwrite, hsize_t old_dim[], hid_t dxpl_id); @@ -903,6 +905,84 @@ done: } /* end H5D__update_oh_info() */ +/*-------------------------------------------------------------------------- + * Function: H5D_build_extfile_prefix + * + * Purpose: Determine the external file prefix to be used and store + * it in extfile_prefix. Stores an empty string if no prefix + * should be used. + * + * Return: SUCCEED/FAIL + * + * Programmer: Steffen Kiess + * October 16, 2015 + *-------------------------------------------------------------------------- + */ +static herr_t +H5D_build_extfile_prefix(const H5D_t *dset, hid_t dapl_id, char **extfile_prefix /*out*/) +{ + char *prefix = NULL; /* prefix used to look for the file */ + char *extpath = NULL; /* absolute path of directory the HDF5 file is in */ + size_t extpath_len; /* length of extpath */ + size_t prefix_len; /* length of prefix */ + size_t extfile_prefix_len; /* length of expanded prefix */ + H5P_genplist_t *plist = NULL; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(dset); + HDassert(dset->oloc.file); + + extpath = H5F_EXTPATH(dset->oloc.file); + HDassert(extpath); + + /* XXX: Future thread-safety note - getenv is not required + * to be reentrant. + */ + prefix = HDgetenv("HDF5_EXTFILE_PREFIX"); + + if(prefix == NULL || *prefix == '\0') { + /* Set prefix to value of H5D_ACS_EFILE_PREFIX_NAME property */ + if(NULL == (plist = H5P_object_verify(dapl_id, H5P_DATASET_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + /* No error checking possible here */ + prefix = (char *)H5P_peek_voidp(plist, H5D_ACS_EFILE_PREFIX_NAME); + } /* end if */ + + /* Prefix has to be checked for NULL / empty string again because the + * code above might have updated it. + */ + if(prefix == NULL || *prefix == '\0' || HDstrcmp(prefix, ".") == 0) { + /* filename is interpreted as relative to the current directory, + * does not need to be expanded + */ + if(NULL == (*extfile_prefix = (char *)H5MM_strdup(""))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + } /* end if */ + else { + if (HDstrncmp(prefix, "${ORIGIN}", HDstrlen("${ORIGIN}")) == 0) { + /* Replace ${ORIGIN} at beginning of prefix by directory of HDF5 file */ + extpath_len = HDstrlen(extpath); + prefix_len = HDstrlen(prefix); + extfile_prefix_len = extpath_len + prefix_len - HDstrlen("${ORIGIN}") + 1; + + if(NULL == (*extfile_prefix = (char *)H5MM_malloc(extfile_prefix_len))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate buffer") + HDsnprintf(*extfile_prefix, extfile_prefix_len, "%s%s", extpath, prefix + HDstrlen("${ORIGIN}")); + } /* end if */ + else { + if(NULL == (*extfile_prefix = (char *)H5MM_strdup(prefix))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + } /* end else */ + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D_build_extfile_prefix() */ + + /*------------------------------------------------------------------------- * Function: H5D__create * @@ -1062,6 +1142,10 @@ H5D__create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, /* Indicate that the layout information was initialized */ layout_init = TRUE; + /* Set the external file prefix */ + if(H5D_build_extfile_prefix(new_dset, dapl_id, &new_dset->shared->extfile_prefix) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize external file prefix") + /* Add the dataset to the list of opened objects in the file */ if(H5FO_top_incr(new_dset->oloc.file, new_dset->oloc.addr) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINC, NULL, "can't incr object ref. count") @@ -1095,6 +1179,7 @@ done: } /* end if */ if(new_dset->shared->dcpl_id != 0 && H5I_dec_ref(new_dset->shared->dcpl_id) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTDEC, NULL, "unable to decrement ref count on property list") + new_dset->shared->extfile_prefix = (char *)H5MM_xfree(new_dset->shared->extfile_prefix); new_dset->shared = H5FL_FREE(H5D_shared_t, new_dset->shared); } /* end if */ new_dset->oloc.file = NULL; @@ -1125,7 +1210,8 @@ H5D_open(const H5G_loc_t *loc, hid_t dapl_id, hid_t dxpl_id) { H5D_shared_t *shared_fo = NULL; H5D_t *dataset = NULL; - H5D_t *ret_value; /* Return value */ + char *extfile_prefix = NULL; /* Expanded external file prefix */ + H5D_t *ret_value = NULL; /* Return value */ FUNC_ENTER_NOAPI(NULL) @@ -1144,6 +1230,10 @@ H5D_open(const H5G_loc_t *loc, hid_t dapl_id, hid_t dxpl_id) if(H5G_name_copy(&(dataset->path), loc->path, H5_COPY_SHALLOW) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, NULL, "can't copy path") + /* Get the external file prefix */ + if(H5D_build_extfile_prefix(dataset, dapl_id, &extfile_prefix) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to initialize external file prefix") + /* Check if dataset was already open */ if(NULL == (shared_fo = (H5D_shared_t *)H5FO_opened(dataset->oloc.file, dataset->oloc.addr))) { /* Clear any errors from H5FO_opened() */ @@ -1163,6 +1253,12 @@ H5D_open(const H5G_loc_t *loc, hid_t dapl_id, hid_t dxpl_id) /* We're the first dataset to use the the shared info */ dataset->shared->fo_count = 1; + + /* Set the external file prefix */ + dataset->shared->extfile_prefix = extfile_prefix; + /* Prevent string from being freed during done: */ + extfile_prefix = NULL; + } /* end if */ else { /* Point to shared info */ @@ -1171,6 +1267,12 @@ H5D_open(const H5G_loc_t *loc, hid_t dapl_id, hid_t dxpl_id) /* Increment # of datasets using shared information */ shared_fo->fo_count++; + /* Check whether the external file prefix of the already open dataset + * matches the new external file prefix + */ + if(HDstrcmp(extfile_prefix, dataset->shared->extfile_prefix) != 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, NULL, "new external file prefix does not match external file prefix of already open dataset") + /* Check if the object has been opened through the top file yet */ if(H5FO_top_count(dataset->oloc.file, dataset->oloc.addr) == 0) { /* Open the object through this top file */ @@ -1187,11 +1289,15 @@ H5D_open(const H5G_loc_t *loc, hid_t dapl_id, hid_t dxpl_id) ret_value = dataset; done: + extfile_prefix = (char *)H5MM_xfree(extfile_prefix); + if(ret_value == NULL) { /* Free the location--casting away const*/ if(dataset) { - if(shared_fo == NULL) /* Need to free shared fo */ + if(shared_fo == NULL && dataset->shared) { /* Need to free shared fo */ + dataset->shared->extfile_prefix = (char *)H5MM_xfree(dataset->shared->extfile_prefix); dataset->shared = H5FL_FREE(H5D_shared_t, dataset->shared); + } H5O_loc_free(&(dataset->oloc)); H5G_name_free(&(dataset->path)); @@ -1452,11 +1558,14 @@ H5D_close(H5D_t *dataset) #endif /* NDEBUG */ } /* end switch */ /*lint !e788 All appropriate cases are covered */ + /* Free the external file prefix */ + dataset->shared->extfile_prefix = (char *)H5MM_xfree(dataset->shared->extfile_prefix); + /* * Release datatype, dataspace and creation property list -- there isn't * much we can do if one of these fails, so we just continue. */ - free_failed = (unsigned)(H5I_dec_ref(dataset->shared->type_id) < 0 || H5S_close(dataset->shared->space) < 0 || + free_failed |= (unsigned)(H5I_dec_ref(dataset->shared->type_id) < 0 || H5S_close(dataset->shared->space) < 0 || H5I_dec_ref(dataset->shared->dcpl_id) < 0); /* Remove the dataset from the list of opened objects in the file */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index d0adb51..ff01041 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -428,6 +428,8 @@ typedef struct H5D_shared_t { */ H5D_rdcc_t chunk; /* Information about chunked data */ } cache; + + char *extfile_prefix; /* expanded external file prefix */ } H5D_shared_t; struct H5D_t { diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index d8381e1..deffe89 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -52,6 +52,7 @@ #define H5D_ACS_DATA_CACHE_NUM_SLOTS_NAME "rdcc_nslots" /* Size of raw data chunk cache(slots) */ #define H5D_ACS_DATA_CACHE_BYTE_SIZE_NAME "rdcc_nbytes" /* Size of raw data chunk cache(bytes) */ #define H5D_ACS_PREEMPT_READ_CHUNKS_NAME "rdcc_w0" /* Preemption read chunks first */ +#define H5D_ACS_EFILE_PREFIX_NAME "external file prefix" /* External file prefix */ /* ======== Data transfer properties ======== */ #define H5D_XFER_MAX_TEMP_BUF_NAME "max_temp_buf" /* Maximum temp buffer size */ @@ -815,10 +815,6 @@ done: * Programmer: Robb Matzke * Friday, October 16, 1998 * - * Modifications: - * Quincey Koziol, May 14, 2002 - * Keep old file's read/write intent in reopened file. - * *------------------------------------------------------------------------- */ hid_t @@ -833,26 +829,27 @@ H5Freopen(hid_t file_id) /* Check arguments */ if(NULL == (old_file = (H5F_t *)H5I_object_verify(file_id, H5I_FILE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file") + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a file") /* Get a new "top level" file struct, sharing the same "low level" file struct */ if(NULL == (new_file = H5F_new(old_file->shared, 0, H5P_FILE_CREATE_DEFAULT, H5P_FILE_ACCESS_DEFAULT, NULL))) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to reopen file") + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to reopen file") /* Duplicate old file's names */ new_file->open_name = H5MM_xstrdup(old_file->open_name); new_file->actual_name = H5MM_xstrdup(old_file->actual_name); + new_file->extpath = H5MM_xstrdup(old_file->extpath); if((ret_value = H5I_register(H5I_FILE, new_file, TRUE)) < 0) - HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file handle") + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to atomize file handle") /* Keep this ID in file object structure */ new_file->file_id = ret_value; done: if(ret_value < 0 && new_file) - if(H5F_dest(new_file, H5AC_dxpl_id, FALSE) < 0) - HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close file") + if(H5F_dest(new_file, H5AC_dxpl_id, FALSE) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTCLOSEFILE, FAIL, "can't close file") FUNC_LEAVE_API(ret_value) } /* end H5Freopen() */ diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 6a8c21b..6ce211e 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -158,7 +158,7 @@ H5F_get_actual_name(const H5F_t *f) * Function: H5F_get_extpath * * Purpose: Retrieve the file's 'extpath' flags - * This is used by H5L_extern_traverse() to retrieve the main file's location + * This is used by H5L_extern_traverse() and H5D_build_extfile_prefix() to retrieve the main file's location * when searching the target file. * * Return: 'extpath' on success/abort on failure (shouldn't fail) diff --git a/src/H5Pdapl.c b/src/H5Pdapl.c index 485fcba..2a58c08 100644 --- a/src/H5Pdapl.c +++ b/src/H5Pdapl.c @@ -16,8 +16,8 @@ /*------------------------------------------------------------------------- * * Created: H5Pdapl.c - * October 27, 2008 - * Neil Fortner <nfortne2@hdfgroup.org> + * October 27, 2008 + * Neil Fortner <nfortne2@hdfgroup.org> * * Purpose: Dataset access property list class routines * @@ -38,7 +38,8 @@ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* Files */ #include "H5Iprivate.h" /* IDs */ -#include "H5Ppkg.h" /* Property lists */ +#include "H5Ppkg.h" /* Property lists */ +#include "H5MMprivate.h" /* Memory management */ /****************/ @@ -55,6 +56,15 @@ /* Definition for preemption read chunks first */ #define H5D_ACS_PREEMPT_READ_CHUNKS_SIZE sizeof(double) #define H5D_ACS_PREEMPT_READ_CHUNKS_DEF H5D_CHUNK_CACHE_W0_DEFAULT +/* Definitions for external file prefix */ +#define H5D_ACS_EFILE_PREFIX_SIZE sizeof(char *) +#define H5D_ACS_EFILE_PREFIX_DEF NULL /*default is no prefix */ +#define H5D_ACS_EFILE_PREFIX_SET H5P__dapl_efile_pref_set +#define H5D_ACS_EFILE_PREFIX_GET H5P__dapl_efile_pref_get +#define H5D_ACS_EFILE_PREFIX_DEL H5P__dapl_efile_pref_del +#define H5D_ACS_EFILE_PREFIX_COPY H5P__dapl_efile_pref_copy +#define H5D_ACS_EFILE_PREFIX_CMP H5P__dapl_efile_pref_cmp +#define H5D_ACS_EFILE_PREFIX_CLOSE H5P__dapl_efile_pref_close /******************/ @@ -74,6 +84,14 @@ /* Property class callbacks */ static herr_t H5P__dacc_reg_prop(H5P_genclass_t *pclass); +/* Property list callbacks */ +static herr_t H5P__dapl_efile_pref_set(hid_t prop_id, const char* name, size_t size, void* value); +static herr_t H5P__dapl_efile_pref_get(hid_t prop_id, const char* name, size_t size, void* value); +static herr_t H5P__dapl_efile_pref_del(hid_t prop_id, const char* name, size_t size, void* value); +static herr_t H5P__dapl_efile_pref_copy(const char* name, size_t size, void* value); +static int H5P__dapl_efile_pref_cmp(const void *value1, const void *value2, size_t size); +static herr_t H5P__dapl_efile_pref_close(const char* name, size_t size, void* value); + /*********************/ /* Package Variables */ @@ -108,6 +126,9 @@ const H5P_libclass_t H5P_CLS_DACC[1] = {{ /* Local Variables */ /*******************/ +/* Property value defaults */ +static const char *H5D_def_efile_prefix_g = H5D_ACS_EFILE_PREFIX_DEF; /* Default external file prefix string */ + /*------------------------------------------------------------------------- @@ -144,12 +165,169 @@ H5P__dacc_reg_prop(H5P_genclass_t *pclass) if(H5P_register_real(pclass, H5D_ACS_PREEMPT_READ_CHUNKS_NAME, H5D_ACS_PREEMPT_READ_CHUNKS_SIZE, &rdcc_w0, NULL, NULL, NULL, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register property for external file prefix */ + if(H5P_register_real(pclass, H5D_ACS_EFILE_PREFIX_NAME, H5D_ACS_EFILE_PREFIX_SIZE, &H5D_def_efile_prefix_g, + NULL, H5D_ACS_EFILE_PREFIX_SET, H5D_ACS_EFILE_PREFIX_GET, + H5D_ACS_EFILE_PREFIX_DEL, H5D_ACS_EFILE_PREFIX_COPY, H5D_ACS_EFILE_PREFIX_CMP, H5D_ACS_EFILE_PREFIX_CLOSE) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__dacc_reg_prop() */ /*------------------------------------------------------------------------- + * Function: H5P__dapl_efile_pref_set + * + * Purpose: Copies an external file prefix property when it's set + * for a property list + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dapl_efile_pref_set(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, + size_t H5_ATTR_UNUSED size, void *value) +{ + FUNC_ENTER_STATIC_NOERR + + /* Sanity check */ + HDassert(value); + + /* Copy the prefix */ + *(char **)value = H5MM_xstrdup(*(const char **)value); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dapl_efile_pref_set() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__dapl_efile_pref_get + * + * Purpose: Copies an external file prefix property when it's retrieved + * from a property list + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dapl_efile_pref_get(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, + size_t H5_ATTR_UNUSED size, void *value) +{ + FUNC_ENTER_STATIC_NOERR + + /* Sanity check */ + HDassert(value); + + /* Copy the prefix */ + *(char **)value = H5MM_xstrdup(*(const char **)value); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dapl_efile_pref_get() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__dapl_efile_pref_del + * + * Purpose: Frees memory used to store the external file prefix string + * + * Return: SUCCEED (Can't fail) + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dapl_efile_pref_del(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, + size_t H5_ATTR_UNUSED size, void *value) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(value); + + H5MM_xfree(*(void **)value); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dapl_efile_pref_del() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__dapl_efile_pref_copy + * + * Purpose: Creates a copy of the external file prefix string + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dapl_efile_pref_copy(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *value) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(value); + + *(char **)value = H5MM_xstrdup(*(const char **)value); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dapl_efile_pref_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__dapl_efile_pref_cmp + * + * Purpose: Callback routine which is called whenever the efile prefix + * property in the dataset creation property list is + * compared. + * + * Return: zero if VALUE1 and VALUE2 are equal, non zero otherwise. + * + *------------------------------------------------------------------------- + */ +static int +H5P__dapl_efile_pref_cmp(const void *value1, const void *value2, size_t H5_ATTR_UNUSED size) +{ + const char *pref1 = *(const char * const *)value1; + const char *pref2 = *(const char * const *)value2; + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + if(NULL == pref1 && NULL != pref2) + HGOTO_DONE(1); + if(NULL != pref1 && NULL == pref2) + HGOTO_DONE(-1); + if(NULL != pref1 && NULL != pref2) + ret_value = HDstrcmp(pref1, pref2); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5P__dapl_efile_pref_cmp() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__dapl_efile_pref_close + * + * Purpose: Frees memory used to store the external file prefix string + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dapl_efile_pref_close(const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *value) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(value); + + H5MM_xfree(*(void **)value); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dapl_efile_pref_close() */ + + +/*------------------------------------------------------------------------- * Function: H5Pset_chunk_cache * * Purpose: Set the number of objects in the meta data cache and the @@ -276,3 +454,95 @@ done: FUNC_LEAVE_API(ret_value) } + +/*------------------------------------------------------------------------- + * Function: H5Pset_efile_prefix + * + * Purpose: Set a prefix to be used for any external files. + * + * If the prefix starts with ${ORIGIN}, this will be replaced by + * the absolute path of the directory of the HDF5 file containing + * the dataset. + * + * If the prefix is ".", no prefix will be applied. + * + * This property can be overwritten by the environment variable + * HDF5_EXTFILE_PREFIX. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_efile_prefix(hid_t plist_id, const char *prefix) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*s", plist_id, prefix); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set prefix */ + if(H5P_set(plist, H5D_ACS_EFILE_PREFIX_NAME, &prefix) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set prefix info") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_efile_prefix() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_efile_prefix + * + * Purpose: Gets the prefix to be used for any external files. + * + * If the pointer is not NULL, it points to a user-allocated + * buffer. + * + * Return: Non-negative on success/Negative on failure + * + *------------------------------------------------------------------------- + */ +ssize_t +H5Pget_efile_prefix(hid_t plist_id, char *prefix, size_t size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + char *my_prefix; /* Library's copy of the prefix */ + size_t len; /* Length of prefix string */ + ssize_t ret_value; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE3("Zs", "i*sz", plist_id, prefix, size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get the current prefix */ + /* No error checking possible */ + my_prefix = (char *)H5P_peek_voidp(plist, H5D_ACS_EFILE_PREFIX_NAME); + + /* Check for prefix being set */ + if(my_prefix) { + /* Copy to user's buffer, if given */ + len = HDstrlen(my_prefix); + if(prefix) { + HDstrncpy(prefix, my_prefix, MIN(len + 1, size)); + if(len >= size) + prefix[size - 1] = '\0'; + } /* end if */ + } /* end if */ + else + len = 0; + + /* Set return value */ + ret_value = (ssize_t)len; + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_efile_prefix() */ + diff --git a/src/H5Plapl.c b/src/H5Plapl.c index 24c66145..7628a13 100644 --- a/src/H5Plapl.c +++ b/src/H5Plapl.c @@ -358,7 +358,6 @@ done: * *------------------------------------------------------------------------- */ -/* ARGSUSED */ static herr_t H5P_lacc_elink_pref_del(hid_t H5_ATTR_UNUSED prop_id, const char H5_ATTR_UNUSED *name, size_t H5_ATTR_UNUSED size, void *value) { diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index b47576c..4a703d0 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -381,6 +381,8 @@ H5_DLL herr_t H5Pget_chunk_cache(hid_t dapl_id, size_t *rdcc_nslots/*out*/, size_t *rdcc_nbytes/*out*/, double *rdcc_w0/*out*/); +H5_DLL herr_t H5Pset_efile_prefix(hid_t dapl_id, const char* prefix); +H5_DLL ssize_t H5Pget_efile_prefix(hid_t dapl_id, char* prefix /*out*/, size_t size); /* Dataset xfer property list (DXPL) routines */ H5_DLL herr_t H5Pset_data_transform(hid_t plist_id, const char* expression); diff --git a/src/H5private.h b/src/H5private.h index c046fad..2e2516a 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -1116,6 +1116,9 @@ typedef off_t h5_stat_size_t; #ifndef HDsetbuf #define HDsetbuf(F,S) setbuf(F,S) #endif /* HDsetbuf */ +#ifndef HDsetenv + #define HDsetenv(N,V,O) setenv(N,V,O) +#endif /* HDsetenv */ #ifndef HDsetgid #define HDsetgid(G) setgid(G) #endif /* HDsetgid */ @@ -2108,7 +2111,8 @@ H5_DLL uint32_t H5_checksum_metadata(const void *data, size_t len, uint32_t init H5_DLL uint32_t H5_hash_string(const char *str); /* Functions for building paths, etc. */ -H5_DLL herr_t H5_build_extpath(const char *, char ** /*out*/ ); +H5_DLL herr_t H5_build_extpath(const char *name, char **extpath /*out*/); +H5_DLL herr_t H5_combine_path(const char *path1, const char *path2, char **full_name /*out*/); /* Functions for debugging */ H5_DLL herr_t H5_buffer_dump(FILE *stream, int indent, const uint8_t *buf, diff --git a/src/H5system.c b/src/H5system.c index 5c8027d..3d43a7d 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -613,7 +613,7 @@ void HDsrand(unsigned int seed) int Wgettimeofday(struct timeval *tv, struct timezone *tz) - { +{ union { unsigned long long ns100; /*time since 1 Jan 1601 in 100ns units */ FILETIME ft; @@ -639,7 +639,42 @@ Wgettimeofday(struct timeval *tv, struct timezone *tz) /* Always return 0 as per Open Group Base Specifications Issue 6. Do not set errno on error. */ return 0; -} +} /* end Wgettimeofday() */ + + +/*------------------------------------------------------------------------- + * Function: Wsetenv + * + * Purpose: Wrapper function for setenv on Windows systems. + * Interestingly, getenv *is* available in the Windows + * POSIX layer, just not setenv. + * + * Return: Success: 0 + * Failure: non-zero error code + * + * Programmer: Dana Robinson + * February 2016 + * + *------------------------------------------------------------------------- + */ +int +Wsetenv(const char *name, const char *value, int overwrite) +{ + size_t bufsize; + errno_t err; + + /* If we're not overwriting, check if the environment variable exists. + * If it does (i.e.: the required buffer size to store the variable's + * value is non-zero), then return an error code. + */ + if(!overwrite) { + err = getenv_s(&bufsize, NULL, 0, name); + if (err || bufsize) + return (int)err; + } /* end if */ + + return (int)_putenv_s(name, value); +} /* end Wsetenv() */ #ifdef H5_HAVE_WINSOCK2_H #pragma comment(lib, "advapi32.lib") @@ -689,26 +724,25 @@ int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap) /*------------------------------------------------------------------------- - * Function: H5_build_extpath + * Function: H5_build_extpath * - * Purpose: To build the path for later searching of target file for external - * link. This path can be either: + * Purpose: To build the path for later searching of target file for external + * links and external files. This path can be either: * 1. The absolute path of NAME * or * 2. The current working directory + relative path of NAME * - * Return: Success: 0 - * Failure: -1 + * Return: SUCCEED/FAIL * * Programmer: Vailin Choi - * April 2, 2008 + * April 2, 2008 * *------------------------------------------------------------------------- */ #define MAX_PATH_LEN 1024 herr_t -H5_build_extpath(const char *name, char **extpath/*out*/) +H5_build_extpath(const char *name, char **extpath /*out*/) { char *full_path = NULL; /* Pointer to the full path, as built or passed in */ char *cwdpath = NULL; /* Pointer to the current working directory path */ @@ -806,5 +840,80 @@ done: H5MM_xfree(new_name); FUNC_LEAVE_NOAPI(ret_value) -} /* H5_build_extpath() */ +} /* end H5_build_extpath() */ + + +/*-------------------------------------------------------------------------- + * Function: H5_combine_path + * + * Purpose: If path2 is relative, interpret path2 as relative to path1 + * and store the result in full_name. Otherwise store path2 + * in full_name. + * + * Return: SUCCEED/FAIL + * + * Programmer: Steffen Kiess + * June 22, 2015 + *-------------------------------------------------------------------------- + */ +herr_t +H5_combine_path(const char* path1, const char* path2, char **full_name /*out*/) +{ + size_t path1_len; /* length of path1 */ + size_t path2_len; /* length of path2 */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + HDassert(path1); + HDassert(path2); + + path1_len = HDstrlen(path1); + path2_len = HDstrlen(path2); + + if(*path1 == '\0' || H5_CHECK_ABSOLUTE(path2)) { + + /* If path1 is empty or path2 is absolute, simply use path2 */ + if(NULL == (*full_name = (char *)H5MM_strdup(path2))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + } /* end if */ + else if(H5_CHECK_ABS_PATH(path2)) { + + /* On windows path2 is a path absolute name */ + if (H5_CHECK_ABSOLUTE(path1) || H5_CHECK_ABS_DRIVE(path1)) { + /* path1 is absolute or drive absolute and path2 is path absolute. + * Use the drive letter of path1 + path2 + */ + if(NULL == (*full_name = (char *)H5MM_malloc(path2_len + 3))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate path2 buffer") + HDsnprintf(*full_name, (path2_len + 3), "%c:%s", path1[0], path2); + } /* end if */ + else { + /* On windows path2 is path absolute name ("\foo\bar"), + * path1 does not have a drive letter (i.e. is "a\b" or "\a\b"). + * Use path2. + */ + if(NULL == (*full_name = (char *)H5MM_strdup(path2))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + } /* end else */ + + } /* end else if */ + else { + + /* Relative path2: + * Allocate a buffer to hold path1 + path2 + possibly the delimiter + * + terminating null byte + */ + if(NULL == (*full_name = (char *)H5MM_malloc(path1_len + path2_len + 2))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate filename buffer") + + /* Compose the full file name */ + HDsnprintf(*full_name, (path1_len + path2_len + 2), "%s%s%s", path1, + (H5_CHECK_DELIMITER(path1[path1_len - 1]) ? "" : H5_DIR_SEPS), path2); + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_combine_name() */ diff --git a/src/H5win32defs.h b/src/H5win32defs.h index 58e1412..3dab8de 100644 --- a/src/H5win32defs.h +++ b/src/H5win32defs.h @@ -71,23 +71,27 @@ struct timezone { }; #ifdef __cplusplus - extern "C" { +extern "C" { #endif /* __cplusplus */ - H5_DLL int Wgettimeofday(struct timeval *tv, struct timezone *tz); - H5_DLL char* Wgetlogin(void); - H5_DLL int c99_snprintf(char* str, size_t size, const char* format, ...); - H5_DLL int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap); + H5_DLL int Wgettimeofday(struct timeval *tv, struct timezone *tz); + H5_DLL int Wsetenv(const char *name, const char *value, int overwrite); + H5_DLL char* Wgetlogin(void); + H5_DLL int c99_snprintf(char* str, size_t size, const char* format, ...); + H5_DLL int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap); #ifdef __cplusplus - } +} #endif /* __cplusplus */ + #define HDgettimeofday(V,Z) Wgettimeofday(V,Z) +#define HDsetenv(N,V,O) Wsetenv(N,V,O) #define HDgetlogin() Wgetlogin() #define HDsnprintf c99_snprintf /*varargs*/ -#define HDvsnprintf c99_vsnprintf -#if _MSC_VER >= 1900 // VS 2015 - // In gcc and in Visual Studio prior to VS 2015 'timezone' is a global - // variable declared in time.h. That variable was deprecated and in VS 2015 - // is removed, with _get_timezone replacing it. +#define HDvsnprintf c99_vsnprintf /*varargs*/ +#if _MSC_VER >= 1900 /* VS 2015 */ + /* In gcc and in Visual Studio prior to VS 2015 'timezone' is a global + * variable declared in time.h. That variable was deprecated and in VS 2015 + * is removed, with _get_timezone replacing it. + */ #define HDget_timezone(V) _get_timezone(V); #endif @@ -102,5 +106,7 @@ struct timezone { #ifndef H5_HAVE_MINGW #define HDftruncate(F,L) _chsize_s(F,L) #define HDfseek(F,O,W) _fseeki64(F,O,W) -#endif +#endif /* H5_HAVE_MINGW */ + #endif /* H5_HAVE_WIN32_API */ + diff --git a/test/external.c b/test/external.c index e6be826..c8b874c 100644 --- a/test/external.c +++ b/test/external.c @@ -25,9 +25,19 @@ const char *FILENAME[] = { "extern_1", "extern_2", "extern_3", + "extern_4", + "extern_dir/file_1", NULL }; +/* A similar collection of files is used for the tests that + * perform file I/O. + */ +#define N_EXT_FILES 4 +#define PART_SIZE 25 +#define TOTAL_SIZE 100 +#define GARBAGE_PER_FILE 10 + /*------------------------------------------------------------------------- * Function: files_have_same_contents @@ -90,6 +100,106 @@ out: /*------------------------------------------------------------------------- + * Function: reset_raw_data_files + * + * Purpose: Resets the data in the raw data files for tests that + * perform dataset I/O on a set of files. + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * February 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +reset_raw_data_files(void) +{ + int fd = 0; /* external file descriptor */ + size_t i, j; /* iterators */ + hssize_t n; /* bytes of I/O */ + char filename[1024]; /* file name */ + int data[PART_SIZE]; /* raw data buffer */ + uint8_t *garbage = NULL; /* buffer of garbage data */ + size_t garbage_count; /* size of garbage buffer */ + size_t garbage_bytes; /* # of garbage bytes written to file */ + + /* Set up garbage buffer */ + garbage_count = N_EXT_FILES * GARBAGE_PER_FILE; + if(NULL == (garbage = (uint8_t *)HDcalloc(garbage_count, sizeof(uint8_t)))) + goto error; + for(i = 0; i < garbage_count; i++) + garbage[i] = 0xFF; + + /* The *r files are pre-filled with data and are used to + * verify that read operations work correctly. + */ + for(i = 0; i < N_EXT_FILES; i++) { + + /* Open file */ + HDsprintf(filename, "extern_%lur.raw", (unsigned long)i + 1); + if((fd = HDopen(filename, O_RDWR|O_CREAT|O_TRUNC, 0666)) < 0) + goto error; + + /* Write garbage data to the file. This allows us to test the + * the ability to set an offset in the raw data file. + */ + garbage_bytes = i * 10; + n = HDwrite(fd, garbage, garbage_bytes); + if(n < 0 || (size_t)n != garbage_bytes) + goto error; + + /* Fill array with data */ + for(j = 0; j < PART_SIZE; j++) { + data[j] = (int)(i * 25 + j); + } /* end for */ + + /* Write raw data to the file. */ + n = HDwrite(fd, data, sizeof(data)); + if(n != sizeof(data)) + goto error; + + /* Close this file */ + HDclose(fd); + + } /* end for */ + + /* The *w files are only pre-filled with the garbage data and are + * used to verify that write operations work correctly. The individual + * tests fill in the actual data. + */ + for(i = 0; i < N_EXT_FILES; i++) { + + /* Open file */ + HDsprintf(filename, "extern_%luw.raw", (unsigned long)i + 1); + if((fd = HDopen(filename, O_RDWR|O_CREAT|O_TRUNC, 0666)) < 0) + goto error; + + /* Write garbage data to the file. This allows us to test the + * the ability to set an offset in the raw data file. + */ + garbage_bytes = i * 10; + n = HDwrite(fd, garbage, garbage_bytes); + if(n < 0 || (size_t)n != garbage_bytes) + goto error; + + /* Close this file */ + HDclose(fd); + + } /* end for */ + HDfree(garbage); + return SUCCEED; + +error: + if(fd) + HDclose(fd); + if(garbage) + HDfree(garbage); + return FAIL; +} /* end reset_raw_data_files() */ + + +/*------------------------------------------------------------------------- * Function: test_non_extendible * * Purpose: Tests a non-extendible dataset with a single external file. @@ -361,7 +471,7 @@ test_large_enough_current_not_eventual(hid_t file) /*------------------------------------------------------------------------- - * Function: test_1e + * Function: test_unlimited * * Purpose: Test a single external file of unlimited size and an * unlimited data space. @@ -629,38 +739,25 @@ test_read_file_set(hid_t fapl) hid_t space = -1; /* data space */ hid_t dset = -1; /* dataset */ hid_t grp = -1; /* group to emit diagnostics */ - int fd = -1; /* external file descriptors */ - size_t i, j; /* miscellaneous counters */ - hssize_t n; /* bytes of I/O */ + size_t i; /* miscellaneous counter */ char filename[1024]; /* file names */ - int part[25], whole[100]; /* raw data buffers */ + int part[PART_SIZE]; /* raw data buffer (partial) */ + int whole[TOTAL_SIZE]; /* raw data buffer (total) */ hsize_t cur_size; /* current data space size */ - hid_t hs_space; /* hyperslab data space */ + hid_t hs_space = -1; /* hyperslab data space */ hsize_t hs_start = 30; /* hyperslab starting offset */ hsize_t hs_count = 25; /* hyperslab size */ - int temparray[10] = {0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f}; TESTING("read external dataset"); - /* Write the data to external files directly */ - for(i=0; i<4; i++) { - for(j=0; j<25; j++) { - part[j] = (int)(i*25+j); - } /* end for */ - HDsprintf(filename, "extern_%lua.raw", (unsigned long)i+1); - if((fd = HDopen(filename, O_RDWR|O_CREAT|O_TRUNC, 0666)) < 0) - TEST_ERROR - n = HDwrite(fd, temparray, (size_t)i*10); - if(n < 0 || (size_t)n != i*10) - TEST_ERROR - n = HDwrite(fd, part, sizeof(part)); - if(n != sizeof(part)) + if(HDsetenv("HDF5_EXTFILE_PREFIX", "", 1) < 0) TEST_ERROR - HDclose(fd); - } /* end for */ - /* - * Create the file and an initial group. This causes messages about + /* Reset the raw data files */ + if(reset_raw_data_files() < 0) + TEST_ERROR + + /* Create the file and an initial group. This causes messages about * debugging to be emitted before we start playing games with what the * output looks like. */ @@ -671,50 +768,57 @@ test_read_file_set(hid_t fapl) FAIL_STACK_ERROR if(H5Gclose(grp) < 0) FAIL_STACK_ERROR - /* Create the dataset */ + /* Create the dcpl */ if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) FAIL_STACK_ERROR - if(H5Pset_external(dcpl, "extern_1a.raw", (off_t)0, (hsize_t)sizeof part) < 0 || - H5Pset_external(dcpl, "extern_2a.raw", (off_t)10, (hsize_t)sizeof part) < 0 || - H5Pset_external(dcpl, "extern_3a.raw", (off_t)20, (hsize_t)sizeof part) < 0 || - H5Pset_external(dcpl, "extern_4a.raw", (off_t)30, (hsize_t)sizeof part) < 0) - FAIL_STACK_ERROR - cur_size = 100; + for(i = 0; i < N_EXT_FILES; i++) { + HDsnprintf(filename, sizeof(filename), "extern_%dr.raw", (int) i + 1); + if(H5Pset_external(dcpl, filename, (off_t)(i * GARBAGE_PER_FILE), (hsize_t)sizeof(part)) < 0) + FAIL_STACK_ERROR + } /* end for */ + + /* Create the dataspace */ + cur_size = TOTAL_SIZE; if((space = H5Screate_simple(1, &cur_size, NULL)) < 0) FAIL_STACK_ERROR + + /* Create the dataset */ if((dset = H5Dcreate2(file, "dset1", H5T_NATIVE_INT, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR - /* - * Read the entire dataset and compare with the original - */ + /* Read the entire dataset */ HDmemset(whole, 0, sizeof(whole)); if(H5Dread(dset, H5T_NATIVE_INT, space, space, H5P_DEFAULT, whole) < 0) FAIL_STACK_ERROR - for(i = 0; i < 100; i++) + + /* Compare data */ + for(i = 0; i < TOTAL_SIZE; i++) if(whole[i] != (signed)i) - FAIL_PUTS_ERROR(" Incorrect value(s) read."); + FAIL_PUTS_ERROR("Incorrect value(s) read."); - /* Read the middle of the dataset */ - if((hs_space = H5Scopy(space)) < 0) - FAIL_STACK_ERROR - if(H5Sselect_hyperslab(hs_space, H5S_SELECT_SET, &hs_start, NULL, &hs_count, NULL) < 0) - FAIL_STACK_ERROR + /* Read via a hypserslab in the middle of the dataset */ - HDmemset(whole, 0, sizeof(whole)); - if(H5Dread(dset, H5T_NATIVE_INT, hs_space, hs_space, H5P_DEFAULT, whole) < 0) - FAIL_STACK_ERROR + /* Set up dataspace */ + if((hs_space = H5Scopy(space)) < 0) + FAIL_STACK_ERROR + if(H5Sselect_hyperslab(hs_space, H5S_SELECT_SET, &hs_start, NULL, &hs_count, NULL) < 0) + FAIL_STACK_ERROR - if(H5Sclose(hs_space) < 0) FAIL_STACK_ERROR + /* Read */ + HDmemset(whole, 0, sizeof(whole)); + if(H5Dread(dset, H5T_NATIVE_INT, hs_space, hs_space, H5P_DEFAULT, whole) < 0) + FAIL_STACK_ERROR - for(i = hs_start; i<hs_start+hs_count; i++) { - if(whole[i] != (signed)i) - FAIL_PUTS_ERROR(" Incorrect value(s) read."); - } /* end for */ + /* Verify data */ + for(i = (size_t)hs_start; i < (size_t)(hs_start + hs_count); i++) { + if(whole[i] != (signed)i) + FAIL_PUTS_ERROR("Incorrect value(s) read (hyperslab)."); + } /* end for */ if(H5Dclose(dset) < 0) FAIL_STACK_ERROR if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR if(H5Sclose(space) < 0) FAIL_STACK_ERROR + if(H5Sclose(hs_space) < 0) FAIL_STACK_ERROR if(H5Fclose(file) < 0) FAIL_STACK_ERROR PASSED(); return 0; @@ -724,6 +828,7 @@ test_read_file_set(hid_t fapl) H5Dclose(dset); H5Pclose(dcpl); H5Sclose(space); + H5Sclose(hs_space); H5Fclose(file); } H5E_END_TRY; return 1; @@ -751,49 +856,46 @@ test_write_file_set(hid_t fapl) hid_t mem_space = -1; /* memory data space */ hid_t file_space = -1; /* file data space */ hid_t dset = -1; /* dataset */ - unsigned i; /* miscellaneous counters */ - int fd = -1; /* external file descriptor */ - int part[25], whole[100]; /* raw data buffers */ + unsigned i; /* miscellaneous counter */ + int part[PART_SIZE]; /* raw data buffer (partial) */ + int whole[TOTAL_SIZE]; /* raw data buffer (total) */ hsize_t cur_size = 100; /* current data space size */ hsize_t max_size = 200; /* maximum data space size */ hsize_t hs_start = 100; /* hyperslab starting offset */ hsize_t hs_count = 100; /* hyperslab size */ char filename[1024]; /* file name */ - int temparray[10] = {0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f,0x0f0f0f0f}; TESTING("write external dataset"); + if(HDsetenv("HDF5_EXTFILE_PREFIX", "", 1) < 0) + TEST_ERROR + /* Create another file */ - h5_fixname(FILENAME[2], fapl, filename, sizeof filename); + h5_fixname(FILENAME[2], fapl, filename, sizeof(filename)); if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR - /* Create the external file list */ - if((dcpl=H5Pcreate(H5P_DATASET_CREATE)) < 0) - FAIL_STACK_ERROR - if(H5Pset_external(dcpl, "extern_1b.raw", (off_t)0, (hsize_t)sizeof part) < 0 || - H5Pset_external(dcpl, "extern_2b.raw", (off_t)10, (hsize_t)sizeof part) < 0 || - H5Pset_external(dcpl, "extern_3b.raw", (off_t)20, (hsize_t)sizeof part) < 0 || - H5Pset_external(dcpl, "extern_4b.raw", (off_t)30, H5F_UNLIMITED) < 0) + /* Create the dcpl and external file list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) FAIL_STACK_ERROR + for(i = 0; i < N_EXT_FILES; i++) { + hsize_t size; - /* Make sure the output files are fresh*/ - for(i=1; i<=4; i++) { - HDsprintf(filename, "extern_%db.raw", i); - if((fd= HDopen(filename, O_RDWR|O_CREAT|O_TRUNC, 0666)) < 0) { - H5_FAILED(); - printf(" cannot open %s: %s\n", filename, HDstrerror(errno)); - goto error; - } /* end if */ + HDsnprintf(filename, sizeof(filename), "extern_%dw.raw", (int) i + 1); - if(HDwrite(fd, temparray, (i-1)*10) < 0) { - H5_FAILED(); - printf(" write error to file %s: %s\n", filename, HDstrerror(errno)); - goto error; - } - HDclose(fd); + if(i != N_EXT_FILES -1) + size = (hsize_t)sizeof(part); + else + size = H5F_UNLIMITED; + + if(H5Pset_external(dcpl, filename, (off_t)(i * GARBAGE_PER_FILE), size) < 0) + FAIL_STACK_ERROR } /* end for */ + /* Reset the raw data files */ + if(reset_raw_data_files() < 0) + TEST_ERROR + /* Create the dataset */ if((mem_space = H5Screate_simple(1, &cur_size, &max_size)) < 0) FAIL_STACK_ERROR @@ -807,11 +909,11 @@ test_write_file_set(hid_t fapl) whole[i] = (int)i; if(H5Dwrite(dset, H5T_NATIVE_INT, mem_space, file_space, H5P_DEFAULT, whole) < 0) FAIL_STACK_ERROR - for(i = 0; i < 4; i++) { + for(i = 0; i < N_EXT_FILES; i++) { char name1[64], name2[64]; - HDsprintf(name1, "extern_%da.raw", i + 1); - HDsprintf(name2, "extern_%db.raw", i + 1); + HDsprintf(name1, "extern_%dr.raw", i + 1); + HDsprintf(name2, "extern_%dw.raw", i + 1); if(!files_have_same_contents(name1, name2)) FAIL_PUTS_ERROR(" Output differs from expected value.") } /* end for */ @@ -853,6 +955,456 @@ test_write_file_set(hid_t fapl) return 1; } /* end test_write_file_set() */ + + /*------------------------------------------------------------------------- + * Function: test_path_absolute + * + * Purpose: Test absolute filenames for external files. + * This will create an HDF5 file in a subdirectory which will + * refer to /full/path/extern_*a.raw on unix and to + * c:\full\path\extern_*a.raw and \full\path\extern_*a.raw on + * windows. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Steffen Kiess + * March 10, 2015 + * + *------------------------------------------------------------------------- + */ +static int +test_path_absolute(hid_t fapl) +{ + hid_t file = -1; /* file to write to */ + hid_t dcpl = -1; /* dataset creation properties */ + hid_t space = -1; /* data space */ + hid_t dset = -1; /* dataset */ + size_t i; /* miscellaneous counter */ + char cwdpath[1024]; /* working directory */ + char filename[1024]; /* file name */ + int part[PART_SIZE]; /* raw data buffer (partial) */ + int whole[TOTAL_SIZE]; /* raw data buffer (total) */ + hsize_t cur_size; /* current data space size */ + + TESTING("absolute filenames for external file"); + + h5_fixname(FILENAME[3], fapl, filename, sizeof(filename)); + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR + + /* Reset the raw data files */ + if(reset_raw_data_files() < 0) + TEST_ERROR + + /* Create the dcpl */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR + if(NULL == HDgetcwd(cwdpath, sizeof(cwdpath))) + TEST_ERROR + for(i = 0; i < N_EXT_FILES; i++) { + HDsnprintf(filename, sizeof(filename), "%s%sextern_%dr.raw", cwdpath, H5_DIR_SEPS, (int) i + 1); +#if defined(H5_HAVE_WINDOW_PATH) + /* For windows, test path-absolute case (\dir\file.raw) for the second file */ + if(i == 1) + HDsnprintf(filename, sizeof(filename), "%s%sextern_%dr.raw", cwdpath + 2, H5_DIR_SEPS, i + 1); +#endif + if(H5Pset_external(dcpl, filename, (off_t)(i * GARBAGE_PER_FILE), (hsize_t)sizeof(part)) < 0) + FAIL_STACK_ERROR + } /* end for */ + + /* create the dataspace */ + cur_size = TOTAL_SIZE; + if((space = H5Screate_simple(1, &cur_size, NULL)) < 0) + FAIL_STACK_ERROR + + /* create the dataset */ + if((dset = H5Dcreate2(file, "dset1", H5T_NATIVE_INT, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Read the entire dataset and compare with the original */ + HDmemset(whole, 0, sizeof(whole)); + if(H5Dread(dset, H5T_NATIVE_INT, space, space, H5P_DEFAULT, whole) < 0) + FAIL_STACK_ERROR + for(i = 0; i < TOTAL_SIZE; i++) + if(whole[i] != (signed)i) + FAIL_PUTS_ERROR("Incorrect value(s) read."); + + if(H5Dclose(dset) < 0) FAIL_STACK_ERROR + if(H5Sclose(space) < 0) FAIL_STACK_ERROR + if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR + if(H5Fclose(file) < 0) FAIL_STACK_ERROR + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Sclose(space); + H5Pclose(dcpl); + H5Fclose(file); + } H5E_END_TRY; + return 1; +} /* end test_path_absolute() */ + + +/*------------------------------------------------------------------------- + * Function: test_path_relative + * + * Purpose: Test external files with filename relative to current directory. + * This will create an HDF5 file in a subdirectory which will + * refer to extern_*a.raw + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Steffen Kiess + * March 10, 2015 + * + *------------------------------------------------------------------------- + */ +static int +test_path_relative(hid_t fapl) +{ + hid_t file = -1; /* file to write to */ + hid_t dcpl = -1; /* dataset creation properties */ + hid_t space = -1; /* data space */ + hid_t dset = -1; /* dataset */ + size_t i; /* miscellaneous counters */ + char cwdpath[1024]; /* working directory */ + char filename[1024]; /* file name */ + int part[PART_SIZE]; /* raw data buffer (partial) */ + int whole[TOTAL_SIZE]; /* raw data buffer (total) */ + hsize_t cur_size; /* current data space size */ + + TESTING("filenames relative to current directory for external file"); + + if(HDsetenv("HDF5_EXTFILE_PREFIX", "", 1) < 0) + TEST_ERROR + + if (HDmkdir("extern_dir", (mode_t)0755) < 0 && errno != EEXIST) + TEST_ERROR; + + h5_fixname(FILENAME[4], fapl, filename, sizeof(filename)); + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Reset the raw data files */ + if(reset_raw_data_files() < 0) + TEST_ERROR + + /* Create the dataset */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR + if(NULL == HDgetcwd(cwdpath, sizeof(cwdpath))) + TEST_ERROR + for (i = 0; i < N_EXT_FILES; i++) { + HDsnprintf(filename, sizeof(filename), "extern_%dr.raw", (int)i + 1); + if(H5Pset_external(dcpl, filename, (off_t)(i * GARBAGE_PER_FILE), (hsize_t)sizeof(part)) < 0) + FAIL_STACK_ERROR + } /* end for */ + + cur_size = TOTAL_SIZE; + if((space = H5Screate_simple(1, &cur_size, NULL)) < 0) + FAIL_STACK_ERROR + if((dset = H5Dcreate2(file, "dset1", H5T_NATIVE_INT, space, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Read the entire dataset and compare with the original */ + HDmemset(whole, 0, sizeof(whole)); + if(H5Dread(dset, H5T_NATIVE_INT, space, space, H5P_DEFAULT, whole) < 0) + FAIL_STACK_ERROR + for(i = 0; i < TOTAL_SIZE; i++) + if(whole[i] != (signed)i) + FAIL_PUTS_ERROR("Incorrect value(s) read."); + + if(H5Dclose(dset) < 0) FAIL_STACK_ERROR + if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR + if(H5Sclose(space) < 0) FAIL_STACK_ERROR + if(H5Fclose(file) < 0) FAIL_STACK_ERROR + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(dset); + H5Pclose(dcpl); + H5Sclose(space); + H5Fclose(file); + } H5E_END_TRY; + return 1; +} /* end test_path_relative() */ + + +/*------------------------------------------------------------------------- + * Function: test_path_relative_cwd + * + * Purpose: Test external files with filename relative to current directory. + * This will create an HDF5 file in a subdirectory which will + * refer to ../extern_*a.raw + * The files are then accessed by setting the efile_prefix dataset + * access property to "${ORIGIN}". + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Steffen Kiess + * March 10, 2015 + * + *------------------------------------------------------------------------- + */ +static int +test_path_relative_cwd(hid_t fapl) +{ + hid_t file = -1; /* file to write to */ + hid_t dcpl = -1; /* dataset creation properties */ + hid_t space = -1; /* data space */ + hid_t dapl = -1; /* dataset access property list */ + hid_t dapl2 = -1; /* copy of dapl */ + hid_t dset = -1; /* dataset */ + hid_t dset2 = -1; /* dataset, opened a second time */ + hid_t dset3 = -1; /* dataset, opened with different prefix */ + size_t i; /* miscellaneous counters */ + char cwdpath[1024]; /* working directory */ + char filename[1024]; /* file name */ + int part[PART_SIZE]; /* raw data buffer (partial) */ + int whole[TOTAL_SIZE]; /* raw data buffer (total) */ + hsize_t cur_size; /* current data space size */ + char buffer[1024]; /* buffer to read efile_prefix */ + + TESTING("filenames relative to HDF5 file for external file"); + + if(HDsetenv("HDF5_EXTFILE_PREFIX", "", 1) < 0) + TEST_ERROR + + if(HDmkdir("extern_dir", (mode_t)0755) < 0 && errno != EEXIST) + TEST_ERROR; + + h5_fixname(FILENAME[4], fapl, filename, sizeof(filename)); + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR; + + /* Reset the raw data files */ + if(reset_raw_data_files() < 0) + TEST_ERROR + + /* Create the dataset */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR + if(NULL == HDgetcwd(cwdpath, sizeof(cwdpath))) + TEST_ERROR + for(i = 0; i < N_EXT_FILES; i++) { + HDsnprintf(filename, sizeof(filename), "..%sextern_%dr.raw", H5_DIR_SEPS, (int)i + 1); + if(H5Pset_external(dcpl, filename, (off_t)(i * GARBAGE_PER_FILE), (hsize_t)sizeof(part)) < 0) + FAIL_STACK_ERROR + } /* end for */ + + cur_size = TOTAL_SIZE; + if((space = H5Screate_simple(1, &cur_size, NULL)) < 0) + FAIL_STACK_ERROR + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR + if(H5Pset_efile_prefix(dapl, "${ORIGIN}") < 0) + FAIL_STACK_ERROR + if(H5Pget_efile_prefix(dapl, buffer, sizeof(buffer)) < 0) + FAIL_STACK_ERROR + if(HDstrcmp(buffer, "${ORIGIN}") != 0) + FAIL_PUTS_ERROR("efile prefix not set correctly"); + if((dapl2 = H5Pcopy(dapl)) < 0) + FAIL_STACK_ERROR + + /* Create dataset */ + if((dset = H5Dcreate2(file, "dset1", H5T_NATIVE_INT, space, H5P_DEFAULT, dcpl, dapl2)) < 0) + FAIL_STACK_ERROR + + /* Reopen dataset with same efile_prefix property */ + if((dset2 = H5Dopen2(file, "dset1", dapl2)) < 0) + FAIL_STACK_ERROR + + /* Reopen dataset with different efile_prefix property */ + if(H5Pset_efile_prefix(dapl, "//") < 0) + FAIL_STACK_ERROR + H5E_BEGIN_TRY { + dset3 = H5Dopen2(file, "dset1", dapl); + } H5E_END_TRY; + if(dset3 >= 0) + FAIL_PUTS_ERROR("reopening the dataset with a different efile_prefix succeded"); + + /* Read the entire dataset and compare with the original */ + HDmemset(whole, 0, sizeof(whole)); + if(H5Dread(dset, H5T_NATIVE_INT, space, space, H5P_DEFAULT, whole) < 0) + FAIL_STACK_ERROR + for(i = 0; i < TOTAL_SIZE; i++) + if(whole[i] != (signed)i) + FAIL_PUTS_ERROR("Incorrect value(s) read."); + + /* Close dataset */ + if(H5Dclose(dset2) < 0) FAIL_STACK_ERROR + if(H5Dclose(dset) < 0) FAIL_STACK_ERROR + + /* Open dataset (use a differend prefix than for create. + * This works because the dataset was closed. + */ + if(H5Pset_efile_prefix(dapl2, "${ORIGIN}/.") < 0) + FAIL_STACK_ERROR + if((dset = H5Dopen2(file, "dset1", dapl2)) < 0) + FAIL_STACK_ERROR + + /* Reopen dataset with same efile_prefix property */ + if((dset2 = H5Dopen2(file, "dset1", dapl2)) < 0) + FAIL_STACK_ERROR + + /* Reopen dataset with different efile_prefix property */ + if(H5Pset_efile_prefix(dapl, NULL) < 0) + FAIL_STACK_ERROR + H5E_BEGIN_TRY { + dset3 = H5Dopen2(file, "dset1", dapl); + } H5E_END_TRY; + if(dset3 >= 0) + FAIL_PUTS_ERROR("reopening the dataset with a different efile_prefix succeded"); + + /* Read the entire dataset and compare with the original */ + HDmemset(whole, 0, sizeof(whole)); + if(H5Dread(dset, H5T_NATIVE_INT, space, space, H5P_DEFAULT, whole) < 0) + FAIL_STACK_ERROR + for(i = 0; i < TOTAL_SIZE; i++) + if(whole[i] != (signed)i) + FAIL_PUTS_ERROR("Incorrect value(s) read."); + + if(H5Dclose(dset2) < 0) FAIL_STACK_ERROR + if(H5Dclose(dset) < 0) FAIL_STACK_ERROR + if(H5Pclose(dapl2) < 0) FAIL_STACK_ERROR + if(H5Pclose(dapl) < 0) FAIL_STACK_ERROR + if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR + if(H5Sclose(space) < 0) FAIL_STACK_ERROR + if(H5Fclose(file) < 0) FAIL_STACK_ERROR + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dapl2); + H5Pclose(dapl); + H5Dclose(dset3); + H5Dclose(dset2); + H5Dclose(dset); + H5Pclose(dcpl); + H5Sclose(space); + H5Fclose(file); + } H5E_END_TRY; + return 1; +} /* end test_path_relative_cwd() */ + + +/*------------------------------------------------------------------------- + * Function: test_path_env + * + * Purpose: Test whether the value of HDF5_EXTFILE_PREFIX will overwrite + * the efile_prefix dataset access property. + * This will create an HDF5 file in a subdirectory which will + * refer to ../extern_*a.raw + * The files are then accessed by setting the HDF5_EXTFILE_PREFIX + * environment variable to "${ORIGIN}". + * The efile_prefix dataset access property is set to "someprefix", + * which will cause an error if the value is not overwritten by + * the environment variable. + * + * Return: Success: 0 + * Failure: 1 + * + * Programmer: Steffen Kiess + * March 10, 2015 + * + *------------------------------------------------------------------------- + */ +static int +test_path_env(hid_t fapl) +{ + hid_t file = -1; /* file to write to */ + hid_t dcpl = -1; /* dataset creation properties */ + hid_t space = -1; /* data space */ + hid_t dapl = -1; /* dataset access property list */ + hid_t dset = -1; /* dataset */ + size_t i; /* miscellaneous counters */ + char cwdpath[1024]; /* working directory */ + char filename[1024]; /* file name */ + int part[PART_SIZE]; /* raw data buffer (partial) */ + int whole[TOTAL_SIZE]; /* raw data buffer (total) */ + hsize_t cur_size; /* current data space size */ + char buffer[1024]; /* buffer to read efile_prefix */ + + TESTING("prefix in HDF5_EXTFILE_PREFIX"); + + if(HDsetenv("HDF5_EXTFILE_PREFIX", "${ORIGIN}", 1)) + TEST_ERROR + + if(HDmkdir("extern_dir", (mode_t)0755) < 0 && errno != EEXIST) + TEST_ERROR; + + h5_fixname(FILENAME[4], fapl, filename, sizeof(filename)); + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + FAIL_STACK_ERROR + + /* Reset the raw data files */ + if(reset_raw_data_files() < 0) + TEST_ERROR + + /* Create the dataset */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + FAIL_STACK_ERROR + if(NULL == HDgetcwd(cwdpath, sizeof(cwdpath))) + TEST_ERROR + for(i = 0; i < N_EXT_FILES; i++) { + HDsnprintf(filename, sizeof(filename), "..%sextern_%dr.raw", H5_DIR_SEPS, (int) i + 1); + if(H5Pset_external(dcpl, filename, (off_t)(i * GARBAGE_PER_FILE), (hsize_t)sizeof(part)) < 0) + FAIL_STACK_ERROR + } /* end for */ + + cur_size = TOTAL_SIZE; + if((space = H5Screate_simple(1, &cur_size, NULL)) < 0) + FAIL_STACK_ERROR + if((dapl = H5Pcreate(H5P_DATASET_ACCESS)) < 0) + FAIL_STACK_ERROR + + /* Set prefix to a nonexistent directory, will be overwritten by environment variable */ + if(H5Pset_efile_prefix(dapl, "someprefix") < 0) + FAIL_STACK_ERROR + if(H5Pget_efile_prefix(dapl, buffer, sizeof(buffer)) < 0) + FAIL_STACK_ERROR + if(HDstrcmp(buffer, "someprefix") != 0) + FAIL_PUTS_ERROR("efile prefix not set correctly"); + + /* Create dataset */ + if((dset = H5Dcreate2(file, "dset1", H5T_NATIVE_INT, space, H5P_DEFAULT, dcpl, dapl)) < 0) + FAIL_STACK_ERROR + + /* Read the entire dataset and compare with the original */ + HDmemset(whole, 0, sizeof(whole)); + if(H5Dread(dset, H5T_NATIVE_INT, space, space, H5P_DEFAULT, whole) < 0) + FAIL_STACK_ERROR + for(i = 0; i < TOTAL_SIZE; i++) + if(whole[i] != (signed)i) + FAIL_PUTS_ERROR("Incorrect value(s) read."); + + if(H5Dclose(dset) < 0) FAIL_STACK_ERROR + if(H5Pclose(dapl) < 0) FAIL_STACK_ERROR + if(H5Pclose(dcpl) < 0) FAIL_STACK_ERROR + if(H5Sclose(space) < 0) FAIL_STACK_ERROR + if(H5Fclose(file) < 0) FAIL_STACK_ERROR + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dapl); + H5Dclose(dset); + H5Pclose(dcpl); + H5Sclose(space); + H5Fclose(file); + } H5E_END_TRY; + return 1; +} /* end test_path_env() */ + /*------------------------------------------------------------------------- * Function: main @@ -864,6 +1416,7 @@ test_write_file_set(hid_t fapl) * Failure: exit(non-zero) * * Programmer: Robb Matzke +>>>>>>> .merge-right.r29182 * Tuesday, March 3, 1998 * *------------------------------------------------------------------------- @@ -926,10 +1479,14 @@ main(void) nerrors += test_add_to_unlimited(); nerrors += test_overflow(); - /* These tests use the VFD-aware fapl */ + /* These file set tests use the VFD-aware fapl */ nerrors += test_read_file_set(current_fapl_id); nerrors += test_write_file_set(current_fapl_id); - + nerrors += test_path_absolute(current_fapl_id); + nerrors += test_path_relative(current_fapl_id); + nerrors += test_path_relative_cwd(current_fapl_id); + nerrors += test_path_env(current_fapl_id); + /* Verify symbol table messages are cached */ nerrors += (h5_verify_cached_stabs(FILENAME, current_fapl_id) < 0 ? 1 : 0); @@ -947,14 +1504,17 @@ main(void) /* Clean up files used by file set tests */ if(h5_cleanup(FILENAME, fapl_id_old)) { - HDremove("extern_1a.raw"); - HDremove("extern_1b.raw"); - HDremove("extern_2a.raw"); - HDremove("extern_2b.raw"); - HDremove("extern_3a.raw"); - HDremove("extern_3b.raw"); - HDremove("extern_4a.raw"); - HDremove("extern_4b.raw"); + HDremove("extern_1r.raw"); + HDremove("extern_2r.raw"); + HDremove("extern_3r.raw"); + HDremove("extern_4r.raw"); + + HDremove("extern_1w.raw"); + HDremove("extern_2w.raw"); + HDremove("extern_3w.raw"); + HDremove("extern_4w.raw"); + + HDrmdir("extern_dir"); } /* end if */ return EXIT_SUCCESS; diff --git a/test/tfile.c b/test/tfile.c index 784e4ad..aa20481 100644 --- a/test/tfile.c +++ b/test/tfile.c @@ -52,6 +52,9 @@ #define FILE1 "tfile1.h5" #define SFILE1 "sys_file1" +#define REOPEN_FILE "tfile_reopen.h5" +#define REOPEN_DSET "dset" + #define F2_USERBLOCK_SIZE (hsize_t)512 #define F2_OFFSET_SIZE 8 #define F2_LENGTH_SIZE 8 @@ -504,6 +507,59 @@ test_file_open(void) /**************************************************************** ** +** test_file_reopen(): File reopen test routine. +** +****************************************************************/ +static void +test_file_reopen(void) +{ + hid_t fid = -1; /* file ID from initial open */ + hid_t rfid = -1; /* file ID from reopen */ + hid_t did = -1; /* dataset ID (both opens) */ + hid_t sid = -1; /* dataspace ID for dataset creation */ + hsize_t dims = 6; /* dataspace size */ + herr_t ret; /* Generic return value */ + + /* Output message about test being performed */ + MESSAGE(5, ("Testing File Re-opening\n")); + + /* Create file via first ID */ + fid = H5Fcreate(REOPEN_FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + CHECK_I(fid, "H5Fcreate"); + + /* Create a dataset in the file */ + sid = H5Screate_simple(1, &dims, &dims); + CHECK_I(sid, "H5Screate_simple") + did = H5Dcreate2(fid, REOPEN_DSET, H5T_NATIVE_INT, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + CHECK_I(did, "H5Dcreate2"); + + /* Close dataset and dataspace */ + ret = H5Sclose(sid); + CHECK(ret, FAIL, "H5Sclose"); + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + + /* Reopen the file with a different file ID */ + rfid = H5Freopen(fid); + CHECK_I(rfid, "H5Freopen"); + + /* Reopen the dataset through the reopen file ID */ + did = H5Dopen2(rfid, REOPEN_DSET, H5P_DEFAULT); + CHECK_I(did, "H5Dopen2"); + + /* Close and clean up */ + ret = H5Dclose(did); + CHECK(ret, FAIL, "H5Dclose"); + ret = H5Fclose(fid); + CHECK(ret, FAIL, "H5Fclose"); + ret = H5Fclose(rfid); + CHECK(ret, FAIL, "H5Fclose"); + HDremove(REOPEN_FILE); + +} /* test_file_reopen() */ + +/**************************************************************** +** ** test_file_close(): low-level file close test routine. ** It mainly tests behavior with close degree. ** @@ -3104,8 +3160,9 @@ test_file(void) /* Output message about test being performed */ MESSAGE(5, ("Testing Low-Level File I/O\n")); - test_file_create(); /* Test file creation(also creation templates)*/ - test_file_open(); /* Test file opening */ + test_file_create(); /* Test file creation(also creation templates)*/ + test_file_open(); /* Test file opening */ + test_file_reopen(); /* Test file reopening */ test_file_close(); /* Test file close behavior */ test_get_file_id(); /* Test H5Iget_file_id */ test_get_obj_ids(); /* Test H5Fget_obj_ids for Jira Issue 8528 */ |