diff options
| -rw-r--r-- | src/H5Dint.c | 25 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 6 | ||||
| -rw-r--r-- | src/H5Dvirtual.c | 1348 | ||||
| -rw-r--r-- | src/H5Olayout.c | 41 | ||||
| -rw-r--r-- | src/H5Oprivate.h | 27 | ||||
| -rw-r--r-- | src/H5Pdapl.c | 94 | ||||
| -rw-r--r-- | src/H5Pdcpl.c | 124 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 2 | ||||
| -rw-r--r-- | src/H5Shyper.c | 88 | ||||
| -rw-r--r-- | src/H5Sprivate.h | 2 |
10 files changed, 1356 insertions, 401 deletions
diff --git a/src/H5Dint.c b/src/H5Dint.c index c1e3ad0..c3c13ef 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -1556,18 +1556,30 @@ H5D_close(H5D_t *dataset) case H5D_VIRTUAL: { - size_t i; + size_t i, j; HDassert(dataset->shared->layout.storage.u.virt.list || (dataset->shared->layout.storage.u.virt.list_nused == 0)); /* Close source datasets */ - for(i = 0; i < dataset->shared->layout.storage.u.virt.list_nused; i++) - if(dataset->shared->layout.storage.u.virt.list[i].source_dset) { - HDassert(dataset->shared->layout.storage.u.virt.list[i].source_dset != dataset); - if(H5D_close(dataset->shared->layout.storage.u.virt.list[i].source_dset) < 0) + for(i = 0; i < dataset->shared->layout.storage.u.virt.list_nused; i++) { + /* Close source dataset */ + if(dataset->shared->layout.storage.u.virt.list[i].source_dset.dset) { + HDassert(dataset->shared->layout.storage.u.virt.list[i].source_dset.dset != dataset); + if(H5D_close(dataset->shared->layout.storage.u.virt.list[i].source_dset.dset) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to close source dataset") - dataset->shared->layout.storage.u.virt.list[i].source_dset = NULL; + dataset->shared->layout.storage.u.virt.list[i].source_dset.dset = NULL; } /* end if */ + + /* Close sub datasets */ + for(j = 0; j < dataset->shared->layout.storage.u.virt.list[i].sub_dset_nused; j++) + if(dataset->shared->layout.storage.u.virt.list[i].sub_dset[j].dset) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + HDassert(dataset->shared->layout.storage.u.virt.list[i].sub_dset[j].dset != dataset); + if(H5D_close(dataset->shared->layout.storage.u.virt.list[i].sub_dset[j].dset) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to close source dataset") + dataset->shared->layout.storage.u.virt.list[i].sub_dset[j].dset = NULL; + } /* end if */ + } /* end for */ } /* end block */ break; @@ -2921,3 +2933,4 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_get_type() */ + diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index 35927ff..27585ba 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -53,6 +53,7 @@ #define H5D_ACS_DATA_CACHE_BYTE_SIZE_NAME "rdcc_nbytes" /* Size of raw data chunk cache(bytes) */ #define H5D_ACS_PREEMPT_READ_CHUNKS_NAME "rdcc_w0" /* Preemption read chunks first */ #define H5D_ACS_VDS_VIEW_NAME "vds_view" /* VDS view option */ +#define H5D_ACS_VDS_PRINTF_GAP_NAME "vds_printf_gap" /* VDS printf gap size */ /* ======== Data transfer properties ======== */ #define H5D_XFER_MAX_TEMP_BUF_NAME "max_temp_buf" /* Maximum temp buffer size */ @@ -180,6 +181,11 @@ H5_DLL herr_t H5D_chunk_idx_reset(H5O_storage_chunk_t *storage, hbool_t reset_ad /* Functions that operate on virtual storage */ H5_DLL herr_t H5D_virtual_update_min_dims(H5O_layout_t *layout, size_t idx); +H5_DLL herr_t H5D_virtual_parse_source_name(const char *source_name, + H5O_storage_virtual_name_seg_t **parsed_name, size_t *static_strlen, + size_t *nsubs); +H5_DLL void H5D_virtual_free_parsed_name( + H5O_storage_virtual_name_seg_t *name_seg); /* Functions that operate on indexed storage */ H5_DLL herr_t H5D_btree_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c index 9bc154c..87cbd21 100644 --- a/src/H5Dvirtual.c +++ b/src/H5Dvirtual.c @@ -34,6 +34,7 @@ #include "H5Dpkg.h" /* Dataset functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* Files */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5Gprivate.h" /* Groups */ #include "H5HGprivate.h" /* Global Heaps */ #include "H5Iprivate.h" /* IDs */ @@ -46,6 +47,9 @@ /* Local Macros */ /****************/ +/* Default size for sub_dset array */ +#define H5D_VIRTUAL_DEF_SUB_DSET_SIZE 128 + /******************/ /* Local Typedefs */ @@ -70,7 +74,26 @@ static herr_t H5D__virtual_flush(H5D_t *dset, hid_t dxpl_id); /* Other functions */ static herr_t H5D__virtual_open_source_dset(const H5D_t *vdset, - H5O_storage_virtual_ent_t *virtual_ent, hid_t dxpl_id); + H5O_storage_virtual_ent_t *virtual_ent, + H5O_storage_virtual_srcdset_t *source_dset, hid_t dxpl_id); +static herr_t H5D__virtual_reset_source_dset( + H5O_storage_virtual_ent_t *virtual_ent, + H5O_storage_virtual_srcdset_t *source_dset); +static herr_t H5D__virtual_str_append(const char *src, size_t src_len, char **p, + char **buf, size_t *buf_size); +static herr_t H5D__virtual_copy_parsed_name( + H5O_storage_virtual_name_seg_t **dst, H5O_storage_virtual_name_seg_t *src); +static herr_t H5D__virtual_build_source_name(char *source_name, + const H5O_storage_virtual_name_seg_t *parsed_name, size_t static_strlen, + size_t nsubs, hsize_t blockno, char **built_name); +static herr_t H5D__virtual_read_one(H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, const H5S_t *file_space, + const H5S_t *mem_space, H5O_storage_virtual_ent_t *virtual_ent, + H5O_storage_virtual_srcdset_t *source_dset); +static herr_t H5D__virtual_write_one(H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, const H5S_t *file_space, + const H5S_t *mem_space, H5O_storage_virtual_ent_t *virtual_ent, + H5O_storage_virtual_srcdset_t *source_dset); /*********************/ @@ -100,6 +123,69 @@ const H5D_layout_ops_t H5D_LOPS_VIRTUAL[1] = {{ /* Local Variables */ /*******************/ +/* Declare a free list to manage the H5O_storage_virtual_name_seg_t struct */ +H5FL_DEFINE(H5O_storage_virtual_name_seg_t); + + + +/*------------------------------------------------------------------------- + * Function: H5D_virtual_update_min_dims + * + * Purpose: Updates the virtual layout's "min_dims" field to take into + * account the "idx"th entry in the mapping list. The entry + * must be complete, though top level fields list_nused does + * (and of course min_dims) do not need to take it into + * account. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * February 10, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_virtual_update_min_dims(H5O_layout_t *layout, size_t idx) +{ + H5S_sel_type sel_type; + int rank; + hsize_t bounds_start[H5S_MAX_RANK]; + hsize_t bounds_end[H5S_MAX_RANK]; + int i; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(layout); + HDassert(layout->type == H5D_VIRTUAL); + HDassert(idx < layout->storage.u.virt.list_nalloc); + + /* Get type of selection */ + if(H5S_SEL_ERROR == (sel_type = H5S_GET_SELECT_TYPE(layout->storage.u.virt.list[idx].source_dset.virtual_select))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection type") + + /* Do not update min_dims for "all" or "none" selections */ + if((sel_type == H5S_SEL_ALL) || (sel_type == H5S_SEL_NONE)) + HGOTO_DONE(SUCCEED) + + /* Get rank of vspace */ + if((rank = H5S_GET_EXTENT_NDIMS(layout->storage.u.virt.list[idx].source_dset.virtual_select)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions") + + /* Get selection bounds */ + if(H5S_SELECT_BOUNDS(layout->storage.u.virt.list[idx].source_dset.virtual_select, bounds_start, bounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds") + + /* Update min_dims */ + for(i = 0; i < rank; i++) + /* Don't check unlimited dimensions in the selection */ + if((i != layout->storage.u.virt.list[idx].unlim_dim_virtual) + && (bounds_end[i] >= layout->storage.u.virt.min_dims[i])) + layout->storage.u.virt.min_dims[i] = bounds_end[i] + (hsize_t)1; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_virtual_update_min_dims() */ /*------------------------------------------------------------------------- @@ -140,21 +226,29 @@ H5D__virtual_copy_layout(H5O_layout_t *layout) HGOTO_ERROR(H5E_DATASET, H5E_NOSPACE, FAIL, "unable to allocate memory for virtual dataset entry list") layout->storage.u.virt.list_nalloc = layout->storage.u.virt.list_nused; - /* Copy the list entries, though set source_dset to NULL */ + /* Copy the list entries, though set source_dset.dset and sub_dset to + * NULL */ for(i = 0; i < layout->storage.u.virt.list_nused; i++) { - if(NULL == (layout->storage.u.virt.list[i].source_file_name - = HDstrdup(orig_list[i].source_file_name))) + if(NULL == (layout->storage.u.virt.list[i].source_dset.file_name + = HDstrdup(orig_list[i].source_dset.file_name))) HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source file name") - if(NULL == (layout->storage.u.virt.list[i].source_dset_name - = HDstrdup(orig_list[i].source_dset_name))) + if(NULL == (layout->storage.u.virt.list[i].source_dset.dset_name + = HDstrdup(orig_list[i].source_dset.dset_name))) HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source dataset name") + if(NULL == (layout->storage.u.virt.list[i].source_dset.virtual_select + = H5S_copy(orig_list[i].source_dset.virtual_select, FALSE, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") if(NULL == (layout->storage.u.virt.list[i].source_select = H5S_copy(orig_list[i].source_select, FALSE, TRUE))) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy source selection") - if(NULL == (layout->storage.u.virt.list[i].virtual_select - = H5S_copy(orig_list[i].virtual_select, FALSE, TRUE))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") - layout->storage.u.virt.list[i].source_dset = NULL; + if(H5D__virtual_copy_parsed_name(&layout->storage.u.virt.list[i].parsed_source_file_name, orig_list[i].parsed_source_file_name) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy parsed source file name") + layout->storage.u.virt.list[i].psfn_static_strlen = orig_list[i].psfn_static_strlen; + layout->storage.u.virt.list[i].psfn_nsubs = orig_list[i].psfn_nsubs; + if(H5D__virtual_copy_parsed_name(&layout->storage.u.virt.list[i].parsed_source_dset_name, orig_list[i].parsed_source_dset_name) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy parsed source dataset name") + layout->storage.u.virt.list[i].psdn_static_strlen = orig_list[i].psdn_static_strlen; + layout->storage.u.virt.list[i].psdn_nsubs = orig_list[i].psdn_nsubs; layout->storage.u.virt.list[i].unlim_dim_source = orig_list[i].unlim_dim_source; layout->storage.u.virt.list[i].unlim_dim_virtual = orig_list[i].unlim_dim_virtual; layout->storage.u.virt.list[i].unlim_extent_source = orig_list[i].unlim_extent_source; @@ -184,63 +278,107 @@ done: /*------------------------------------------------------------------------- - * Function: H5D_virtual_update_min_dims + * Function: H5D__virtual_reset_layout * - * Purpose: Updates the virtual layout's "min_dims" field to take into - * account the "idx"th entry in the mapping list. The entry - * must be complete, though top level fields list_nused does - * (and of course min_dims) do not need to take it into - * account. + * Purpose: Frees internal structures in a virtual storage layout + * message in memory. This function is safe to use on + * incomplete structures (for recovery from failure) provided + * the internal structures are initialized with all bytes set + * to 0. * * Return: Non-negative on success/Negative on failure * * Programmer: Neil Fortner - * February 10, 2015 + * February 11, 2015 * *------------------------------------------------------------------------- */ herr_t -H5D_virtual_update_min_dims(H5O_layout_t *layout, size_t idx) +H5D__virtual_reset_layout(H5O_layout_t *layout) { - H5S_sel_type sel_type; - int rank; - hsize_t bounds_start[H5S_MAX_RANK]; - hsize_t bounds_end[H5S_MAX_RANK]; - int i; + size_t i, j; herr_t ret_value = SUCCEED; - FUNC_ENTER_NOAPI(FAIL) + FUNC_ENTER_PACKAGE HDassert(layout); HDassert(layout->type == H5D_VIRTUAL); - HDassert(idx < layout->storage.u.virt.list_nalloc); - /* Get type of selection */ - if(H5S_SEL_ERROR == (sel_type = H5S_GET_SELECT_TYPE(layout->storage.u.virt.list[idx].virtual_select))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection type") + /* Free the list entries. Note we always attempt to free everything even in + * the case of a failure. Because of this, and because we free the list + * afterwards, we do not need to zero out the memory in the list. */ + for(i = 0; i < layout->storage.u.virt.list_nused; i++) { + /* Free source_dset */ + if(H5D__virtual_reset_source_dset(&layout->storage.u.virt.list[i], &layout->storage.u.virt.list[i].source_dset) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to reset source dataset") - /* Do not update min_dims for "all" or "none" selections */ - if((sel_type == H5S_SEL_ALL) || (sel_type == H5S_SEL_NONE)) - HGOTO_DONE(SUCCEED) + /* Free source_select */ + if(layout->storage.u.virt.list[i].source_select) + if(H5S_close(layout->storage.u.virt.list[i].source_select) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release source selection") - /* Get rank of vspace */ - if((rank = H5S_GET_EXTENT_NDIMS(layout->storage.u.virt.list[idx].virtual_select)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions") + /* Free sub_dset */ + for(j = 0; j < layout->storage.u.virt.list[i].sub_dset_nalloc; j++) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + if(H5D__virtual_reset_source_dset(&layout->storage.u.virt.list[i], &layout->storage.u.virt.list[i].sub_dset[j]) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to reset source dataset") + } //VDSINC + layout->storage.u.virt.list[i].sub_dset = (H5O_storage_virtual_srcdset_t *)H5MM_xfree(layout->storage.u.virt.list[i].sub_dset); - /* Get selection bounds */ - if(H5S_SELECT_BOUNDS(layout->storage.u.virt.list[idx].virtual_select, bounds_start, bounds_end) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds") + /* Free parsed_source_file_name */ + H5D_virtual_free_parsed_name(layout->storage.u.virt.list[i].parsed_source_file_name); - /* Update min_dims */ - for(i = 0; i < rank; i++) - /* Don't check unlimited dimensions in the selection */ - if((i != layout->storage.u.virt.list[idx].unlim_dim_virtual) - && (bounds_end[i] >= layout->storage.u.virt.min_dims[i])) - layout->storage.u.virt.min_dims[i] = bounds_end[i] + (hsize_t)1; + /* Free parsed_source_dset_name */ + H5D_virtual_free_parsed_name(layout->storage.u.virt.list[i].parsed_source_dset_name); + } /* end for */ + + /* Free the list */ + layout->storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_xfree(layout->storage.u.virt.list); + layout->storage.u.virt.list_nalloc = (size_t)0; + layout->storage.u.virt.list_nused = (size_t)0; + (void)HDmemset(layout->storage.u.virt.min_dims, 0, sizeof(layout->storage.u.virt.min_dims)); + + /* Note the lack of a done: label. This is because there are no HGOTO_ERROR + * calls. If one is added, a done: label must also be added */ + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_reset_layout() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_delete + * + * Purpose: Delete the file space for a virtual dataset + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * February 6, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__virtual_delete(H5F_t *f, hid_t dxpl_id, H5O_storage_t *storage) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* check args */ + HDassert(f); + HDassert(storage); + HDassert(storage->type == H5D_VIRTUAL); + + /* Delete the global heap block */ + if(H5HG_remove(f, dxpl_id, (H5HG_t *)&(storage->u.virt.serial_list_hobjid)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "Unable to remove heap object") + + /* Clear global heap ID in storage */ + storage->u.virt.serial_list_hobjid.addr = HADDR_UNDEF; + storage->u.virt.serial_list_hobjid.idx = 0; done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D_virtual_update_min_dims() */ +} /* end H5D__virtual_delete */ /*------------------------------------------------------------------------- @@ -257,7 +395,8 @@ done: */ static herr_t H5D__virtual_open_source_dset(const H5D_t *vdset, - H5O_storage_virtual_ent_t *virtual_ent, hid_t dxpl_id) + H5O_storage_virtual_ent_t *virtual_ent, + H5O_storage_virtual_srcdset_t *source_dset, hid_t dxpl_id) { H5F_t *src_file = NULL; /* Source file */ hbool_t src_file_open = FALSE; /* Whether we have opened and need to close src_file */ @@ -267,39 +406,45 @@ H5D__virtual_open_source_dset(const H5D_t *vdset, FUNC_ENTER_STATIC /* Sanity check */ - HDassert(virtual_ent); - HDassert(!virtual_ent->source_dset); + HDassert(vdset); + HDassert(source_dset); + HDassert(!source_dset->dset); + HDassert(source_dset->file_name); + HDassert(source_dset->dset_name); /* Get dapl and fapl from current (virtual dataset) location? VDSINC */ /* Write code to check if these exist and return without opening dset * otherwise VDSINC */ /* Check if we need to open the source file */ - if(HDstrcmp(virtual_ent->source_file_name, ".")) { + if(HDstrcmp(source_dset->file_name, ".")) { /* Open the source file */ - if(NULL == (src_file = H5F_open(virtual_ent->source_file_name, H5F_INTENT(vdset->oloc.file) & H5F_ACC_RDWR, H5P_FILE_CREATE_DEFAULT, H5P_FILE_ACCESS_DEFAULT, dxpl_id))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENFILE, FAIL, "unable to open source file") - src_file_open = TRUE; + if(NULL == (src_file = H5F_open(source_dset->file_name, H5F_INTENT(vdset->oloc.file) & H5F_ACC_RDWR, H5P_FILE_CREATE_DEFAULT, H5P_FILE_ACCESS_DEFAULT, dxpl_id))) + H5E_clear_stack(NULL); //Quick hack until proper support for H5Fopen with missing file is implemented VDSINC HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENFILE, FAIL, "unable to open source file") + else + src_file_open = TRUE; } /* end if */ else /* Source file is ".", use the virtual dataset's file */ src_file = vdset->oloc.file; - /* Set up the root group in the destination file */ - if(NULL == (src_root_loc.oloc = H5G_oloc(H5G_rootof(src_file)))) - HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "unable to get object location for root group") - if(NULL == (src_root_loc.path = H5G_nameof(H5G_rootof(src_file)))) - HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "unable to get path for root group") - - /* Open the source dataset */ - if(NULL == (virtual_ent->source_dset = H5D__open_name(&src_root_loc, virtual_ent->source_dset_name, H5P_DATASET_ACCESS_DEFAULT, dxpl_id))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open dataset") - - /* Patch the source selection if necessary */ - if(virtual_ent->source_space_status != H5O_VIRTUAL_STATUS_CORRECT) { - if(H5S_extent_copy(virtual_ent->source_select, virtual_ent->source_dset->shared->space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy source dataspace extent") - virtual_ent->source_space_status = H5O_VIRTUAL_STATUS_CORRECT; + if(src_file) { + /* Set up the root group in the destination file */ + if(NULL == (src_root_loc.oloc = H5G_oloc(H5G_rootof(src_file)))) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "unable to get object location for root group") + if(NULL == (src_root_loc.path = H5G_nameof(H5G_rootof(src_file)))) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "unable to get path for root group") + + /* Open the source dataset */ + if(NULL == (source_dset->dset = H5D__open_name(&src_root_loc, source_dset->dset_name, H5P_DATASET_ACCESS_DEFAULT, dxpl_id))) + H5E_clear_stack(NULL); //Quick hack until proper support for H5Fopen with missing file is implemented VDSINC HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open dataset") + else + /* Patch the source selection if necessary */ + if(virtual_ent->source_space_status != H5O_VIRTUAL_STATUS_CORRECT) { + if(H5S_extent_copy(virtual_ent->source_select, source_dset->dset->shared->space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy source dataspace extent") + virtual_ent->source_space_status = H5O_VIRTUAL_STATUS_CORRECT; + } /* end if */ } /* end if */ done: @@ -313,6 +458,447 @@ done: /*------------------------------------------------------------------------- + * Function: H5D__virtual_reset_source_dset + * + * Purpose: Frees space reference by a source dataset struct. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 20, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_reset_source_dset(H5O_storage_virtual_ent_t *virtual_ent, + H5O_storage_virtual_srcdset_t *source_dset) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(source_dset); + + /* Free dataset */ + if(source_dset->dset) { + if(H5D_close(source_dset->dset) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to close source dataset") + source_dset->dset = NULL; + } /* end if */ + + /* Free file name */ + if((source_dset == &virtual_ent->source_dset) + || virtual_ent->parsed_source_file_name) + source_dset->file_name = (char *)H5MM_xfree(source_dset->file_name); + + /* Free dataset name */ + if((source_dset == &virtual_ent->source_dset) + || virtual_ent->parsed_source_dset_name) + source_dset->dset_name = (char *)H5MM_xfree(source_dset->dset_name); + + /* Free virtual selection */ + if(source_dset->virtual_select) { + if(H5S_close(source_dset->virtual_select) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release virtual selection") + source_dset->virtual_select = NULL; + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_reset_source_dset() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_str_append + * + * Purpose: Appends src_len bytes of the string src to the position *p + * in the buffer *buf (allocating *buf if necessary). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 19, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_str_append(const char *src, size_t src_len, char **p, char **buf, + size_t *buf_size) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(src); + HDassert(src_len > 0); + HDassert(p); + HDassert(buf); + HDassert(*p >= *buf); + HDassert(buf_size); + + /* Allocate or extend buffer if necessary */ + if(!*buf) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + HDassert(!*p); + HDassert(*buf_size == 0); + + /* Allocate buffer */ + if(NULL == (*buf = (char *)H5MM_malloc(src_len + (size_t)1))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate name segment struct") + *buf_size = src_len + (size_t)1; + *p = *buf; + } /* end if */ + else if(((size_t)(*p - *buf) + src_len + (size_t)1) > *buf_size) { + HDassert(0 && "Checking code coverage..."); //VDSINC + char *tmp_buf; + size_t tmp_buf_size; + + /* Calculate new size of buffer */ + tmp_buf_size = MAX((size_t)(*p - *buf) + src_len + (size_t)1, + *buf_size * (size_t)2); + + /* Reallocate buffer */ + if(NULL == (tmp_buf = (char *)H5MM_realloc(*buf, tmp_buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to reallocate name segment buffer") + *buf = tmp_buf; + *buf_size = tmp_buf_size; + } /* end if */ + + /* Copy string to *p. Note that since src in not NULL terminated, we must + * use memcpy */ + (void)HDmemcpy(*p, src, src_len); + + /* Advance *p */ + *p += src_len; + + /* Add NULL terminator */ + **p = '\0'; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5D__virtual_str_append() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_virtual_parse_source_name + * + * Purpose: Parses a source file or dataset name. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 18, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_virtual_parse_source_name(const char *source_name, + H5O_storage_virtual_name_seg_t **parsed_name, size_t *static_strlen, + size_t *nsubs) +{ + H5O_storage_virtual_name_seg_t *tmp_parsed_name = NULL; + H5O_storage_virtual_name_seg_t **tmp_parsed_name_p = &tmp_parsed_name; + size_t tmp_static_strlen; + size_t tmp_strlen; + size_t tmp_nsubs = 0; + const char *p; + const char *pct; + char *name_seg_p = NULL; + size_t name_seg_size = 0; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(source_name); + HDassert(parsed_name); + HDassert(static_strlen); + HDassert(nsubs); + + /* Initialize p and tmp_static_strlen */ + p = source_name; + tmp_static_strlen = tmp_strlen = HDstrlen(source_name); + + /* Iterate over name */ + /* Note this will not work with UTF-8! We should support this eventually + * -NAF 5/18/2015 */ + while((pct = HDstrchr(p, '%'))) { + HDassert(pct >= p); + + /* Allocate name segment struct if necessary */ + if(!*tmp_parsed_name_p) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + if(NULL == (*tmp_parsed_name_p = H5FL_CALLOC(H5O_storage_virtual_name_seg_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate name segment struct") + } //VDSINC + + /* Check for type of format specifier */ + if(pct[1] == 'b') { + /* Check for blank string before specifier */ + if(pct != p) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + /* Append string to name segment */ + if(H5D__virtual_str_append(p, (size_t)(pct - p), &name_seg_p, &(*tmp_parsed_name_p)->name_segment, + &name_seg_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to append name segment") + } //VDSINC + + /* Update other variables */ + tmp_parsed_name_p = &(*tmp_parsed_name_p)->next; + tmp_static_strlen -= 2; + tmp_nsubs++; + name_seg_p = NULL; + name_seg_size = 0; + } /* end if */ + else if(pct[1] == '%') { + HDassert(0 && "Checking code coverage..."); //VDSINC + /* Append string to name segment (include first '%') */ + if(H5D__virtual_str_append(p, (size_t)(pct - p) + (size_t)1, &name_seg_p, &(*tmp_parsed_name_p)->name_segment, +&name_seg_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to append name segment") + + /* Update other variables */ + tmp_static_strlen -= 1; + } /* end else */ + else + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid format specifier") + + p = pct + 2; + } /* end while */ + + /* Copy last segment of name, if any, unless there were no substitutions */ + if(tmp_nsubs > 0) { + HDassert(p >= source_name); + if(*p == '\0') { + HDassert(0 && "Checking code coverage..."); //VDSINC + HDassert((size_t)(p - source_name) == tmp_strlen); + } //VDSINC + else { + HDassert((size_t)(p - source_name) < tmp_strlen); + + /* Allocate name segment struct if necessary */ + if(!*tmp_parsed_name_p) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + if(NULL == (*tmp_parsed_name_p = H5FL_CALLOC(H5O_storage_virtual_name_seg_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate name segment struct") + } /* end if */ + + /* Append string to name segment */ + if(H5D__virtual_str_append(p, tmp_strlen - (size_t)(p - source_name), &name_seg_p, &(*tmp_parsed_name_p)->name_segment, + &name_seg_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to append name segment") + } /* end else */ + } /* end if */ + + /* Set return values */ + *parsed_name = tmp_parsed_name; + tmp_parsed_name = NULL; + *static_strlen = tmp_static_strlen; + *nsubs = tmp_nsubs; + +done: + if(tmp_parsed_name) { + HDassert(ret_value < 0); + H5D_virtual_free_parsed_name(tmp_parsed_name); + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_virtual_parse_source_name() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_copy_parsed_name + * + * Purpose: Deep copies a parsed source file or dataset name. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 19, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_copy_parsed_name(H5O_storage_virtual_name_seg_t **dst, + H5O_storage_virtual_name_seg_t *src) +{ + H5O_storage_virtual_name_seg_t *tmp_dst = NULL; + H5O_storage_virtual_name_seg_t *p_src = src; + H5O_storage_virtual_name_seg_t **p_dst = &tmp_dst; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(dst); + + /* Walk over parsed name, duplicating it */ + while(p_src) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + /* Allocate name segment struct */ + if(NULL == (*p_dst = H5FL_CALLOC(H5O_storage_virtual_name_seg_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate name segment struct") + + /* Duplicate name segment */ + if(NULL == ((*p_dst)->name_segment = HDstrdup(p_src->name_segment))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to duplicate name segment") + + /* Advance pointers */ + p_src = p_src->next; + p_dst = &(*p_dst)->next; + } /* end while */ + + /* Set dst */ + *dst = tmp_dst; + tmp_dst = NULL; + +done: + if(tmp_dst) { + HDassert(ret_value < 0); + H5D_virtual_free_parsed_name(tmp_dst); + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_copy_parsed_name() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_virtual_free_parsed_name + * + * Purpose: Frees the provided parsed name. + * + * Return: void + * + * Programmer: Neil Fortner + * May 19, 2015 + * + *------------------------------------------------------------------------- + */ +void +H5D_virtual_free_parsed_name(H5O_storage_virtual_name_seg_t *name_seg) +{ + H5O_storage_virtual_name_seg_t *next_seg; + + FUNC_ENTER_NOAPI_NOERR + + /* Walk name segments, freeing them */ + while(name_seg) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + (void)H5MM_xfree(name_seg->name_segment); + next_seg = name_seg->next; + (void)H5FL_FREE(H5O_storage_virtual_name_seg_t, name_seg); + name_seg = next_seg; + } /* end while */ + + FUNC_LEAVE_NOAPI_VOID +} /* end H5D_virtual_free_parsed_name() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_build_source_name + * + * Purpose: Builds a source file or dataset name from a parsed name. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 18, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_build_source_name(char *source_name, + const H5O_storage_virtual_name_seg_t *parsed_name, size_t static_strlen, + size_t nsubs, hsize_t blockno, char **built_name) +{ + char *tmp_name = NULL; /* Name buffer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(source_name); + HDassert(built_name); + + /* Check for static name */ + if(nsubs == 0) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + HDassert(!parsed_name); + *built_name = source_name; + } /* end if */ + else { + const H5O_storage_virtual_name_seg_t *name_seg = parsed_name; + char *p; + hsize_t blockno_down = blockno; + size_t blockno_len = 1; + size_t name_len; + size_t name_len_rem; + size_t seg_len; + size_t nsubs_rem = nsubs; + + HDassert(parsed_name); + + /* Calculate length of printed block number */ + do { + blockno_down /= (hsize_t)10; + if(blockno_down == 0) + break; + blockno_len++; + } while(1); + + /* Calculate length of name buffer */ + name_len_rem = name_len = static_strlen + (nsubs * blockno_len) + (size_t)1; + + /* Allocate name buffer */ + if(NULL == (tmp_name = (char *)H5MM_malloc(name_len))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate name buffer") + p = tmp_name; + + /* Build name */ + do { + /* Add name segment */ + if(name_seg->name_segment) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + seg_len = HDstrlen(name_seg->name_segment); + HDassert(seg_len > 0); + HDassert(seg_len < name_len_rem); + HDstrncpy(p, name_seg->name_segment, name_len_rem); + name_len_rem -= seg_len; + p += seg_len; + } /* end if */ + + /* Add block number */ + if(nsubs_rem > 0) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + HDassert(blockno_len < name_len_rem); + if(HDsnprintf(p, name_len_rem, "%llu", (long long unsigned)blockno) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write block number to string") + name_len_rem -= blockno_len; + p += blockno_len; + nsubs_rem--; + } /* end if */ + + /* Advance name_seg */ + name_seg = name_seg->next; + } while(name_seg); + + /* Assign built_name */ + *built_name = tmp_name; + tmp_name = NULL; + } /* end else */ + +done: + if(tmp_name) { + HDassert(ret_value < 0); + H5MM_free(tmp_name); + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_build_source_name() */ + + +/*------------------------------------------------------------------------- * Function: H5D__virtual_set_extent_unlim * * Purpose: Sets the extent of the virtual dataset by checking the @@ -359,59 +945,173 @@ H5D__virtual_set_extent_unlim(const H5D_t *dset, hid_t dxpl_id) for(i = 0; i < storage->list_nalloc; i++) /* Check for unlimited dimension */ if(storage->list[i].unlim_dim_virtual >= 0) { - HDassert(storage->list[i].unlim_dim_source >= 0); - - /* Open source dataset */ - if(!storage->list[i].source_dset) - if(H5D__virtual_open_source_dset(dset, &storage->list[i], dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") - - /* Check if source dataset is open */ - if(storage->list[i].source_dset) { - /* Retrieve current source dataset extent and patch mapping. - * Note this will clip the source selection to the extent. */ - if(H5S_extent_copy(storage->list[i].source_select, storage->list[i].source_dset->shared->space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy source dataspace extent") + /* Check for "printf" source dataset resolution */ + if(storage->list[i].parsed_source_file_name + || storage->list[i].parsed_source_dset_name) { + hsize_t first_missing = 0; /* First missing dataset in the current block of missing datasets */ + hsize_t j; + + /* Search for source datasets */ + HDassert(storage->printf_gap != HSIZE_UNDEF); + for(j = 0; j <= (storage->printf_gap + first_missing); j++) { + /* Check for running out of space in sub_dset array */ + if(j >= (hsize_t)storage->list[i].sub_dset_nalloc) { + if(storage->list[i].sub_dset_nalloc == 0) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + /* Allocate sub_dset */ + if(NULL == (storage->list[i].sub_dset = (H5O_storage_virtual_srcdset_t *)H5MM_calloc(H5D_VIRTUAL_DEF_SUB_DSET_SIZE * sizeof(H5O_storage_virtual_srcdset_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate sub dataset array") + storage->list[i].sub_dset_nalloc = H5D_VIRTUAL_DEF_SUB_DSET_SIZE; + } /* end if */ + else { + H5O_storage_virtual_srcdset_t *tmp_sub_dset; + + HDassert(0 && "Checking code coverage..."); //VDSINC + /* Extend sub_dset */ + if(NULL == (tmp_sub_dset = (H5O_storage_virtual_srcdset_t *)H5MM_realloc(storage->list[i].sub_dset, 2 * storage->list[i].sub_dset_nalloc * sizeof(H5O_storage_virtual_srcdset_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to extend sub dataset array") + + /* Clear new space in sub_dset */ + (void)HDmemset(&storage->list[i].sub_dset[storage->list[i].sub_dset_nalloc], 0, storage->list[i].sub_dset_nalloc * sizeof(H5O_storage_virtual_srcdset_t)); + + /* Update sub_dset_nalloc */ + storage->list[i].sub_dset_nalloc *= 2; + } /* end else */ + } /* end if */ + + /* Check if the dataset is already open */ + if(storage->list[i].sub_dset[j].dset) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + first_missing = j + 1; + } //VDSINC + else { + /* Resolve file name */ + if(!storage->list[i].sub_dset[j].file_name) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + if(H5D__virtual_build_source_name(storage->list[i].source_dset.file_name, storage->list[i].parsed_source_file_name, storage->list[i].psfn_static_strlen, storage->list[i].psfn_nsubs, j, &storage->list[i].sub_dset[j].file_name) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to build source file name") + } //VDSINC + + /* Resolve dset name */ + if(!storage->list[i].sub_dset[j].dset_name) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + if(H5D__virtual_build_source_name(storage->list[i].source_dset.dset_name, storage->list[i].parsed_source_dset_name, storage->list[i].psdn_static_strlen, storage->list[i].psdn_nsubs, j, &storage->list[i].sub_dset[j].dset_name) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to build source dataset name") + } //VDSINC + + /* Resolve virtual selection for block */ + if(!storage->list[i].sub_dset[j].virtual_select) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + if(NULL == (storage->list[i].sub_dset[j].virtual_select = H5S_hyper_get_unlim_block(storage->list[i].source_dset.virtual_select, j))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get block in unlimited selection") + } //VDSINC + + /* Open source dataset */ + if(H5D__virtual_open_source_dset(dset, &storage->list[i], &storage->list[i].sub_dset[j], dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") + + /* Update first_missing */ + if(storage->list[i].sub_dset[j].dset) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + first_missing = j + 1; + } //VDSINC + } /* end else */ + } /* end for */ + + /* Check if the size changed */ + if(first_missing != (hsize_t)storage->list[i].sub_dset_nused) { + /* Check for no datasets */ + if(first_missing == 0) { + HDassert(0 && "Checking code coverage..."); //VDSINC + /* Set clip size to 0 */ + clip_size = (hsize_t)0; + } //VDSINC + else { + hsize_t bounds_start[H5S_MAX_RANK]; + hsize_t bounds_end[H5S_MAX_RANK]; - /* Get source space dimenstions */ - if(H5S_get_simple_extent_dims(storage->list[i].source_select, curr_dims, NULL) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get source space dimensions") + //HDassert(0 && "Checking code coverage..."); //VDSINC + /* Get bounds from last valid virtual selection */ + if(H5S_SELECT_BOUNDS(storage->list[i].sub_dset[first_missing - (hsize_t)1].virtual_select, bounds_start, bounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds") - /* Check if the source extent in the unlimited dimension - * changed since the last time the VDS extent/mapping - * was updated */ - if(curr_dims[storage->list[i].unlim_dim_source] - == storage->list[i].unlim_extent_source) - /* Use cached result for clip size */ - clip_size = storage->list[i].clip_size_virtual; - else { - /* Get size that virtual selection would be clipped to - * to match size of source selection */ - if(H5S_hyper_get_clip_extent(storage->list[i].virtual_select, storage->list[i].source_select, &clip_size, storage->view == H5D_VDS_FIRST_MISSING) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get hyperslab clip size") - - /* If we are setting the extent by the last available data, - * clip virtual_select. Note that if we used the cached - * clip_size above, the selection will already be clipped to - * the correct size. */ - if(storage->view == H5D_VDS_LAST_AVAILABLE) - if(H5S_hyper_clip_unlim(storage->list[i].virtual_select, clip_size)) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + /* Set clip_size to bounds_end in unlimited dimension */ + clip_size = bounds_end[storage->list[i].unlim_dim_virtual] + + (hsize_t)1; + } /* end else */ - /* Update cached values unlim_extent_source and - * clip_size_virtual */ - storage->list[i].unlim_extent_source = curr_dims[storage->list[i].unlim_dim_source]; + /* Clip entry root virtual select (virtual_select for all + * sub dsets) */ + if(H5S_hyper_clip_unlim(storage->list[i].source_dset.virtual_select, clip_size)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + + /* Set sub_dset_nused and clip_size_virtual */ + storage->list[i].sub_dset_nused = (size_t)first_missing; storage->list[i].clip_size_virtual = clip_size; - } /* end else */ - - /* Update new_dims */ - if((new_dims[storage->list[i].unlim_dim_virtual] == HSIZE_UNDEF) - || (storage->view == H5D_VDS_FIRST_MISSING ? (clip_size - < (hsize_t)new_dims[storage->list[i].unlim_dim_virtual]) - : (clip_size - > (hsize_t)new_dims[storage->list[i].unlim_dim_virtual]))) - new_dims[storage->list[i].unlim_dim_virtual] = clip_size; + } /* end if */ + else { + //HDassert(0 && "Checking code coverage..."); //VDSINC + /* Use cached clip_size */ + clip_size = storage->list[i].clip_size_virtual; + } //VDSINC } /* end if */ + else { + HDassert(storage->list[i].unlim_dim_source >= 0); + + /* Open source dataset */ + if(!storage->list[i].source_dset.dset) + if(H5D__virtual_open_source_dset(dset, &storage->list[i], &storage->list[i].source_dset, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") + + /* Check if source dataset is open */ + if(storage->list[i].source_dset.dset) { + /* Retrieve current source dataset extent and patch mapping. + * Note this will clip the source selection to the extent. */ + if(H5S_extent_copy(storage->list[i].source_select, storage->list[i].source_dset.dset->shared->space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy source dataspace extent") + + /* Get source space dimenstions */ + if(H5S_get_simple_extent_dims(storage->list[i].source_select, curr_dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get source space dimensions") + + /* Check if the source extent in the unlimited dimension + * changed since the last time the VDS extent/mapping + * was updated */ + if(curr_dims[storage->list[i].unlim_dim_source] + == storage->list[i].unlim_extent_source) + /* Use cached result for clip size */ + clip_size = storage->list[i].clip_size_virtual; + else { + /* Get size that virtual selection would be clipped to + * to match size of source selection */ + if(H5S_hyper_get_clip_extent(storage->list[i].source_dset.virtual_select, storage->list[i].source_select, &clip_size, storage->view == H5D_VDS_FIRST_MISSING) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get hyperslab clip size") + + /* If we are setting the extent by the last available data, + * clip virtual_select. Note that if we used the cached + * clip_size above, the selection will already be clipped to + * the correct size. */ + if(storage->view == H5D_VDS_LAST_AVAILABLE) + if(H5S_hyper_clip_unlim(storage->list[i].source_dset.virtual_select, clip_size)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + + /* Update cached values unlim_extent_source and + * clip_size_virtual */ + storage->list[i].unlim_extent_source = curr_dims[storage->list[i].unlim_dim_source]; + storage->list[i].clip_size_virtual = clip_size; + } /* end else */ + } /* end if */ + else + clip_size = 0; + } /* end else */ + + /* Update new_dims */ + if((new_dims[storage->list[i].unlim_dim_virtual] == HSIZE_UNDEF) + || (storage->view == H5D_VDS_FIRST_MISSING ? (clip_size + < (hsize_t)new_dims[storage->list[i].unlim_dim_virtual]) + : (clip_size + > (hsize_t)new_dims[storage->list[i].unlim_dim_virtual]))) + new_dims[storage->list[i].unlim_dim_virtual] = clip_size; } /* end if */ /* Get current VDS dimensions */ @@ -442,14 +1142,13 @@ H5D__virtual_set_extent_unlim(const H5D_t *dset, hid_t dxpl_id) * mapping extents */ for(i = 0; i < storage->list_nalloc; i++) /* Check for unlimited dimension */ - if((storage->list[i].unlim_dim_source >= 0) - && (storage->list[i].source_dset)) { + if(storage->list[i].unlim_dim_source >= 0) { HDassert(storage->list[i].unlim_dim_virtual >= 0); /* Update virtual mapping extent. Note this function does not * clip the selection. */ if(changed) - if(H5S_set_extent(storage->list[i].virtual_select, new_dims) < 0) + if(H5S_set_extent(storage->list[i].source_dset.virtual_select, new_dims) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") /* Check if we are setting extent by the minimum of mappings */ @@ -458,7 +1157,7 @@ H5D__virtual_set_extent_unlim(const H5D_t *dset, hid_t dxpl_id) * extent changed, otherwise it will already be clipped to * the extent) */ if(changed) - if(H5S_hyper_clip_to_extent(storage->list[i].virtual_select)) + if(H5S_hyper_clip_to_extent(storage->list[i].source_dset.virtual_select)) HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") /* Check if the virtual extent in the unlimited dimension @@ -471,7 +1170,7 @@ H5D__virtual_set_extent_unlim(const H5D_t *dset, hid_t dxpl_id) else { /* Get size that source selection will be clipped to to * match size of virtual selection */ - if(H5S_hyper_get_clip_extent(storage->list[i].source_select, storage->list[i].virtual_select, &clip_size, FALSE) < 0) + if(H5S_hyper_get_clip_extent(storage->list[i].source_select, storage->list[i].source_dset.virtual_select, &clip_size, FALSE) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get hyperslab clip size") /* Update cached values unlim_extent_virtual and @@ -495,97 +1194,6 @@ done: /*------------------------------------------------------------------------- - * Function: H5D__virtual_reset_layout - * - * Purpose: Frees internal structures in a virtual storage layout - * message in memory. This function is safe to use on - * incomplete structures (for recovery from failure) provided - * the internal structures are initialized with all bytes set - * to 0. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Neil Fortner - * February 11, 2015 - * - *------------------------------------------------------------------------- - */ -herr_t -H5D__virtual_reset_layout(H5O_layout_t *layout) -{ - size_t i; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_PACKAGE - - HDassert(layout); - HDassert(layout->type == H5D_VIRTUAL); - - /* Free the list entries. Note we always attempt to free everything even in - * the case of a failure. */ - for(i = 0; i < layout->storage.u.virt.list_nused; i++) { - layout->storage.u.virt.list[i].source_file_name = (char *)H5MM_xfree(layout->storage.u.virt.list[i].source_file_name); - layout->storage.u.virt.list[i].source_dset_name = (char *)H5MM_xfree(layout->storage.u.virt.list[i].source_dset_name); - if(layout->storage.u.virt.list[i].source_select - && H5S_close(layout->storage.u.virt.list[i].source_select) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release source selection") - layout->storage.u.virt.list[i].source_select = NULL; - if(layout->storage.u.virt.list[i].virtual_select - && H5S_close(layout->storage.u.virt.list[i].virtual_select) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release virtual selection") - layout->storage.u.virt.list[i].virtual_select = NULL; - /* Close dataset here? VDSINC */ - } /* end for */ - - /* Free the list */ - layout->storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_xfree(layout->storage.u.virt.list); - - /* Note the lack of a done: label. This is because there are no HGOTO_ERROR - * calls. If one is added, a done: label must also be added */ - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__virtual_reset_layout() */ - - -/*------------------------------------------------------------------------- - * Function: H5D__virtual_delete - * - * Purpose: Delete the file space for a virtual dataset - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Neil Fortner - * February 6, 2015 - * - *------------------------------------------------------------------------- - */ -herr_t -H5D__virtual_delete(H5F_t *f, hid_t dxpl_id, H5O_storage_t *storage) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_PACKAGE - - /* check args */ - HDassert(f); - HDassert(storage); - HDassert(storage->type == H5D_VIRTUAL); - - /* Need to add stuff for private data here VDSINC */ - - /* Delete the global heap block */ - if(H5HG_remove(f, dxpl_id, (H5HG_t *)&(storage->u.virt.serial_list_hobjid)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "Unable to remove heap object") - - /* Clear global heap ID in storage */ - storage->u.virt.serial_list_hobjid.addr = HADDR_UNDEF; - storage->u.virt.serial_list_hobjid.idx = 0; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__virtual_delete */ - - -/*------------------------------------------------------------------------- * Function: H5D__virtual_init * * Purpose: Initialize the virtual layout information for a dataset. @@ -615,7 +1223,7 @@ H5D__virtual_init(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const H5D_t *dset, /* Patch the virtual selection dataspaces */ for(i = 0; i < dset->shared->layout.storage.u.virt.list_nused; i++) { if(dset->shared->layout.storage.u.virt.list[i].virtual_space_status != H5O_VIRTUAL_STATUS_CORRECT) { - if(H5S_extent_copy(dset->shared->layout.storage.u.virt.list[i].virtual_select, dset->shared->space) < 0) + if(H5S_extent_copy(dset->shared->layout.storage.u.virt.list[i].source_dset.virtual_select, dset->shared->space) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy virtual dataspace extent") dset->shared->layout.storage.u.virt.list[i].virtual_space_status = H5O_VIRTUAL_STATUS_CORRECT; } /* end if */ @@ -625,10 +1233,14 @@ H5D__virtual_init(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const H5D_t *dset, if(NULL == (dapl = (H5P_genplist_t *)H5I_object(dapl_id))) HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for dapl ID") - /* Get bounds option */ + /* Get view option */ if(H5P_get(dapl, H5D_ACS_VDS_VIEW_NAME, &dset->shared->layout.storage.u.virt.view) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual view option") + /* Get printf gap */ + if(H5P_get(dapl, H5D_ACS_VDS_PRINTF_GAP_NAME, &dset->shared->layout.storage.u.virt.printf_gap) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual printf gap") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_init() */ @@ -688,6 +1300,94 @@ H5D__virtual_io_init(const H5D_io_info_t UNUSED *io_info, const H5D_type_info_t /*------------------------------------------------------------------------- + * Function: H5D__virtual_read_one + * + * Purpose: Read from a singe source dataset in a virtual dataset. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 15, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_read_one(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + const H5S_t *file_space, const H5S_t *mem_space, + H5O_storage_virtual_ent_t *virtual_ent, + H5O_storage_virtual_srcdset_t *source_dset) +{ + H5S_t *projected_mem_space = NULL; /* Memory space for selection in a single mapping */ + H5S_t *projected_src_space = NULL; /* File space for selection in a single source dataset */ + hssize_t select_nelmts; /* Number of elements in selection */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + HDassert(virtual_ent); + HDassert(source_dset); + + /* Project intersection of file space and mapping virtual space onto + * memory space */ + if(H5S_select_project_intersection(file_space, mem_space, source_dset->virtual_select, &projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space") + + /* Get number of elements in projected dataspace */ + if((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS(projected_mem_space)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to get number of elements in selection") + + /* Only perform I/O if there are any elements */ + if(select_nelmts > 0) { + /* Open source dataset */ + if(!source_dset->dset) + /* Try to open dataset */ + if(H5D__virtual_open_source_dset(io_info->dset, virtual_ent, source_dset, io_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") + + /* Check if source dataset is open */ + if(source_dset->dset) { + /* Sanity check that the source space has been patched by now */ + HDassert(virtual_ent->source_space_status == H5O_VIRTUAL_STATUS_CORRECT); + + /* Project intersection of file space and mapping virtual space onto + * mapping source space */ + if(H5S_select_project_intersection(source_dset->virtual_select, virtual_ent->source_select, file_space, &projected_src_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto source space") + + /* Perform read on source dataset */ + if(H5D__read(source_dset->dset, type_info->dst_type_id, projected_mem_space, projected_src_space, io_info->dxpl_id, io_info->u.rbuf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read source dataset") + + /* Close projected_src_space */ + if(H5S_close(projected_src_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected source space") + projected_src_space = NULL; + } /* end if */ + } /* end if */ + + /* Close projected_mem_space */ + if(H5S_close(projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space") + projected_mem_space = NULL; + +done: + /* Release allocated resources on failure */ + if(projected_src_space) { + HDassert(ret_value < 0); + if(H5S_close(projected_src_space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected source space") + } /* end if */ + if(projected_mem_space) { + HDassert(ret_value < 0); + if(H5S_close(projected_mem_space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space") + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_read_one() */ + + +/*------------------------------------------------------------------------- * Function: H5D__virtual_read * * Purpose: Read from a virtual dataset. @@ -701,14 +1401,11 @@ H5D__virtual_io_init(const H5D_io_info_t UNUSED *io_info, const H5D_type_info_t */ static herr_t H5D__virtual_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t UNUSED nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t UNUSED *fm) { H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ - H5S_t *projected_mem_space = NULL; /* Memory space for selection in a single mapping */ - H5S_t *projected_src_space = NULL; /* File space for selection in a single source dataset */ - hssize_t select_nelmts; /* Number of elements in selection */ - size_t i; /* Local index variable */ + size_t i, j; /* Local index variables */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -728,57 +1425,104 @@ H5D__virtual_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, /* Sanity check that the virtual space has been patched by now */ HDassert(storage->list[i].virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); - /* Project intersection of file space and mapping virtual space onto - * memory space */ - if(H5S_select_project_intersection(file_space, mem_space, storage->list[i].virtual_select, &projected_mem_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space") - - /* Get number of elements in projected dataspace */ - if((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS(projected_mem_space)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to get number of elements in selection") - - /* Only perform I/O if there are any elements */ - if(select_nelmts > 0) { - /* Open source dataset */ - if(!storage->list[i].source_dset) - /* Try to open dataset */ - if(H5D__virtual_open_source_dset(io_info->dset, &storage->list[i], io_info->dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") - - /* Check if source dataset is open */ - if(storage->list[i].source_dset) { - /* Sanity check that the source space has been patched by now */ - HDassert(storage->list[i].source_space_status == H5O_VIRTUAL_STATUS_CORRECT); - - /* Project intersection of file space and mapping virtual space onto - * mapping source space */ - if(H5S_select_project_intersection(storage->list[i].virtual_select, storage->list[i].source_select, file_space, &projected_src_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto source space") - - /* Perform read on source dataset */ - if(H5D__read(storage->list[i].source_dset, type_info->dst_type_id, projected_mem_space, projected_src_space, io_info->dxpl_id, io_info->u.rbuf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read source dataset") - - /* Close projected_src_space */ - if(H5S_close(projected_src_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected source space") - projected_src_space = NULL; - } /* end if */ - else - /* We do not have a source dataset open, fill with fill value */ - HDassert(0 && "Not yet implemented...");//VDSINC + /* Check for "printf" source dataset resolution */ + if(storage->list[i].parsed_source_file_name + || storage->list[i].parsed_source_dset_name) { + /* Iterate over sub-source dsets */ + for(j = 0; j < storage->list[i].sub_dset_nused; j++) { + //HDassert(0 && "Checking code coverage..."); //VDSINC + if(H5D__virtual_read_one(io_info, type_info, file_space, mem_space, &storage->list[i], &storage->list[i].sub_dset[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset") + } //VDSINC } /* end if */ - - /* Close projected_mem_space */ - if(H5S_close(projected_mem_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space") - projected_mem_space = NULL; + else + /* Read from source dataset */ + if(H5D__virtual_read_one(io_info, type_info, file_space, mem_space, &storage->list[i], &storage->list[i].source_dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset") } /* end for */ /* Fill unmapped part of buffer with fill value. Keep track of total number * elements written to memory buffer and assert that it == nelmts VDSINC */ done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_write_one + * + * Purpose: Write to a singe source dataset in a virtual dataset. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 15, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_write_one(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + const H5S_t *file_space, const H5S_t *mem_space, + H5O_storage_virtual_ent_t *virtual_ent, + H5O_storage_virtual_srcdset_t *source_dset) +{ + H5S_t *projected_mem_space = NULL; /* Memory space for selection in a single mapping */ + H5S_t *projected_src_space = NULL; /* File space for selection in a single source dataset */ + hssize_t select_nelmts; /* Number of elements in selection */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + HDassert(virtual_ent); + HDassert(source_dset); + + /* Project intersection of file space and mapping virtual space onto + * memory space */ + if(H5S_select_project_intersection(file_space, mem_space, source_dset->virtual_select, &projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space") + + /* Get number of elements in projected dataspace */ + if((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS(projected_mem_space)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to get number of elements in selection") + + /* Only perform I/O if there are any elements */ + if(select_nelmts > 0) { + /* Open source dataset */ + if(!source_dset->dset) { + /* Try to open dataset */ + if(H5D__virtual_open_source_dset(io_info->dset, virtual_ent, source_dset, io_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") + if(!source_dset->dset) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "did not open source dataset") + } /* end if */ + + /* Sanity check that the source space has been patched by now */ + HDassert(virtual_ent->source_space_status == H5O_VIRTUAL_STATUS_CORRECT); + + /* Extend source dataset if necessary and there is an unlimited + * dimension VDSINC */ + /* Project intersection of file space and mapping virtual space onto + * mapping source space */ + if(H5S_select_project_intersection(source_dset->virtual_select, virtual_ent->source_select, file_space, &projected_src_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto source space") + + /* Perform write on source dataset */ + if(H5D__write(source_dset->dset, type_info->dst_type_id, projected_mem_space, projected_src_space, io_info->dxpl_id, io_info->u.wbuf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write to source dataset") + + /* Close projected_src_space */ + if(H5S_close(projected_src_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected source space") + projected_src_space = NULL; + } /* end if */ + + /* Close projected_mem_space */ + if(H5S_close(projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space") + projected_mem_space = NULL; + +done: /* Release allocated resources on failure */ if(projected_src_space) { HDassert(ret_value < 0); @@ -792,7 +1536,7 @@ done: } /* end if */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5D__virtual_read() */ +} /* end H5D__virtual_write_one() */ /*------------------------------------------------------------------------- @@ -809,21 +1553,18 @@ done: */ static herr_t H5D__virtual_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, - hsize_t UNUSED nelmts, const H5S_t *file_space, const H5S_t *mem_space, + hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t UNUSED *fm) { H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ - H5S_t *projected_mem_space = NULL; /* Memory space for selection in a single mapping */ - H5S_t *projected_src_space = NULL; /* File space for selection in a single source dataset */ - hssize_t select_nelmts; /* Number of elements in selection */ - size_t i; /* Local index variable */ + size_t i, j; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Sanity check */ HDassert(io_info); - HDassert(io_info->u.rbuf); + HDassert(io_info->u.wbuf); HDassert(type_info); HDassert(mem_space); HDassert(file_space); @@ -836,65 +1577,26 @@ H5D__virtual_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, /* Sanity check that virtual space has been patched by now */ HDassert(storage->list[i].virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); - /* Project intersection of file space and mapping virtual space onto - * memory space */ - if(H5S_select_project_intersection(file_space, mem_space, storage->list[i].virtual_select, &projected_mem_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space") - - /* Get number of elements in projected dataspace */ - if((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS(projected_mem_space)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to get number of elements in selection") - - /* Only perform I/O if there are any elements */ - if(select_nelmts > 0) { - /* Open source dataset, fail if cannot open */ - if(!storage->list[i].source_dset) { - //VDSINC check all source datasets before any I/O - if(H5D__virtual_open_source_dset(io_info->dset, &storage->list[i], io_info->dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") - if(!storage->list[i].source_dset) - HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "did not open source dataset") - } /* end if */ - - /* Sanity check that source space has been patched by now */ - HDassert(storage->list[i].source_space_status == H5O_VIRTUAL_STATUS_CORRECT); - - /* Extend source dataset if necessary and there is an unlimited - * dimension VDSINC */ - /* Project intersection of file space and mapping virtual space onto - * mapping source space */ - if(H5S_select_project_intersection(storage->list[i].virtual_select, storage->list[i].source_select, file_space, &projected_src_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto source space") - - /* Perform read on source dataset */ - if(H5D__write(storage->list[i].source_dset, type_info->dst_type_id, projected_mem_space, projected_src_space, io_info->dxpl_id, io_info->u.wbuf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write to source dataset") - - /* Close projected_src_space */ - if(H5S_close(projected_src_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected source space") - projected_src_space = NULL; + /* Check for "printf" source dataset resolution */ + if(storage->list[i].parsed_source_file_name + || storage->list[i].parsed_source_dset_name) { + /* Iterate over sub-source dsets */ + for(j = 0; j < storage->list[i].sub_dset_nused; j++) { + HDassert(0 && "Checking code coverage..."); //VDSINC + if(H5D__virtual_write_one(io_info, type_info, file_space, mem_space, &storage->list[i], &storage->list[i].sub_dset[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to write to source dataset") + } //VDSINC } /* end if */ - - /* Close projected_mem_space */ - if(H5S_close(projected_mem_space) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space") - projected_mem_space = NULL; + else + /* Write to source dataset */ + if(H5D__virtual_write_one(io_info, type_info, file_space, mem_space, &storage->list[i], &storage->list[i].source_dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to write to source dataset") } /* end for */ -done: - /* Release allocated resources on failure */ - if(projected_src_space) { - HDassert(ret_value < 0); - if(H5S_close(projected_src_space) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected source space") - } /* end if */ - if(projected_mem_space) { - HDassert(ret_value < 0); - if(H5S_close(projected_mem_space) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space") - } /* end if */ + /* Keep track of total number elements written to disk buffer and issure + * error if it != nelmts VDSINC */ +done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D__virtual_write() */ diff --git a/src/H5Olayout.c b/src/H5Olayout.c index dce41cf..849c072 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -252,6 +252,7 @@ H5O_layout_decode(H5F_t *f, hid_t dxpl_id, H5O_t UNUSED *open_oh, mesg->storage.u.virt.list = NULL; mesg->storage.u.virt.list_nalloc = 0; mesg->storage.u.virt.view = H5D_VDS_ERROR; + mesg->storage.u.virt.printf_gap = HSIZE_UNDEF; /* Decode heap block if it exists */ if(mesg->storage.u.virt.serial_list_hobjid.addr != HADDR_UNDEF) { @@ -282,16 +283,16 @@ H5O_layout_decode(H5F_t *f, hid_t dxpl_id, H5O_t UNUSED *open_oh, for(i = 0; i < mesg->storage.u.virt.list_nused; i++) { /* Source file name */ tmp_size = HDstrlen((const char *)heap_block_p) + 1; - if(NULL == (mesg->storage.u.virt.list[i].source_file_name = (char *)H5MM_malloc(tmp_size))) + if(NULL == (mesg->storage.u.virt.list[i].source_dset.file_name = (char *)H5MM_malloc(tmp_size))) HGOTO_ERROR(H5E_OHDR, H5E_RESOURCE, NULL, "unable to allocate memory for source file name") - (void)HDmemcpy(mesg->storage.u.virt.list[i].source_file_name, heap_block_p, tmp_size); + (void)HDmemcpy(mesg->storage.u.virt.list[i].source_dset.file_name, heap_block_p, tmp_size); heap_block_p += tmp_size; /* Source dataset name */ tmp_size = HDstrlen((const char *)heap_block_p) + 1; - if(NULL == (mesg->storage.u.virt.list[i].source_dset_name = (char *)H5MM_malloc(tmp_size))) + if(NULL == (mesg->storage.u.virt.list[i].source_dset.dset_name = (char *)H5MM_malloc(tmp_size))) HGOTO_ERROR(H5E_OHDR, H5E_RESOURCE, NULL, "unable to allocate memory for source dataset name") - (void)HDmemcpy(mesg->storage.u.virt.list[i].source_dset_name, heap_block_p, tmp_size); + (void)HDmemcpy(mesg->storage.u.virt.list[i].source_dset.dset_name, heap_block_p, tmp_size); heap_block_p += tmp_size; /* Source selection */ @@ -299,12 +300,19 @@ H5O_layout_decode(H5F_t *f, hid_t dxpl_id, H5O_t UNUSED *open_oh, HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, NULL, "can't decode source space selection") /* Virtual selection */ - if(H5S_SELECT_DESERIALIZE(f, &(mesg->storage.u.virt.list[i].virtual_select), &heap_block_p) < 0) + if(H5S_SELECT_DESERIALIZE(f, &(mesg->storage.u.virt.list[i].source_dset.virtual_select), &heap_block_p) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, NULL, "can't decode virtual space selection") + /* Parse source file and dataset names for "printf" + * style format specifiers */ + if(H5D_virtual_parse_source_name(mesg->storage.u.virt.list[i].source_dset.file_name, &mesg->storage.u.virt.list[i].parsed_source_file_name, &mesg->storage.u.virt.list[i].psfn_static_strlen, &mesg->storage.u.virt.list[i].psfn_nsubs) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't parse source file name") + if(H5D_virtual_parse_source_name(mesg->storage.u.virt.list[i].source_dset.dset_name, &mesg->storage.u.virt.list[i].parsed_source_dset_name, &mesg->storage.u.virt.list[i].psdn_static_strlen, &mesg->storage.u.virt.list[i].psdn_nsubs) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't parse source dataset name") + /* unlim_dim fields */ mesg->storage.u.virt.list[i].unlim_dim_source = H5S_get_select_unlim_dim(mesg->storage.u.virt.list[i].source_select); - mesg->storage.u.virt.list[i].unlim_dim_virtual = H5S_get_select_unlim_dim(mesg->storage.u.virt.list[i].virtual_select); + mesg->storage.u.virt.list[i].unlim_dim_virtual = H5S_get_select_unlim_dim(mesg->storage.u.virt.list[i].source_dset.virtual_select); mesg->storage.u.virt.list[i].unlim_extent_source = HSIZE_UNDEF; mesg->storage.u.virt.list[i].unlim_extent_virtual = HSIZE_UNDEF; mesg->storage.u.virt.list[i].clip_size_source = HSIZE_UNDEF; @@ -472,17 +480,17 @@ H5O_layout_encode(H5F_t *f, hbool_t UNUSED disable_shared, uint8_t *p, const voi /* Calculate size of each entry */ for(i = 0; i < mesg->storage.u.virt.list_nused; i++) { - HDassert(mesg->storage.u.virt.list[i].source_file_name); - HDassert(mesg->storage.u.virt.list[i].source_dset_name); + HDassert(mesg->storage.u.virt.list[i].source_dset.file_name); + HDassert(mesg->storage.u.virt.list[i].source_dset.dset_name); HDassert(mesg->storage.u.virt.list[i].source_select); - HDassert(mesg->storage.u.virt.list[i].virtual_select); + HDassert(mesg->storage.u.virt.list[i].source_dset.virtual_select); /* Source file name */ - str_size[2 * i] = HDstrlen(mesg->storage.u.virt.list[i].source_file_name) + (size_t)1; + str_size[2 * i] = HDstrlen(mesg->storage.u.virt.list[i].source_dset.file_name) + (size_t)1; block_size += str_size[2 * i]; /* Source dset name */ - str_size[(2 * i) + 1] = HDstrlen(mesg->storage.u.virt.list[i].source_dset_name) + (size_t)1; + str_size[(2 * i) + 1] = HDstrlen(mesg->storage.u.virt.list[i].source_dset.dset_name) + (size_t)1; block_size += str_size[(2 * i) + 1]; /* Source selection */ @@ -491,7 +499,7 @@ H5O_layout_encode(H5F_t *f, hbool_t UNUSED disable_shared, uint8_t *p, const voi block_size += (size_t)select_serial_size; /* Virtual dataset selection */ - if((select_serial_size = H5S_SELECT_SERIAL_SIZE(f, mesg->storage.u.virt.list[i].virtual_select)) < 0) + if((select_serial_size = H5S_SELECT_SERIAL_SIZE(f, mesg->storage.u.virt.list[i].source_dset.virtual_select)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to check dataspace selection size") block_size += (size_t)select_serial_size; } /* end for */ @@ -515,11 +523,11 @@ H5O_layout_encode(H5F_t *f, hbool_t UNUSED disable_shared, uint8_t *p, const voi /* Encode each entry */ for(i = 0; i < mesg->storage.u.virt.list_nused; i++) { /* Source file name */ - (void)HDstrcpy((char *)heap_block_p, mesg->storage.u.virt.list[i].source_file_name); + (void)HDstrcpy((char *)heap_block_p, mesg->storage.u.virt.list[i].source_dset.file_name); heap_block_p += str_size[2 * i]; /* Source dataset name */ - (void)HDstrcpy((char *)heap_block_p, mesg->storage.u.virt.list[i].source_dset_name); + (void)HDstrcpy((char *)heap_block_p, mesg->storage.u.virt.list[i].source_dset.dset_name); heap_block_p += str_size[(2 * i) + 1]; /* Source selection */ @@ -527,7 +535,7 @@ H5O_layout_encode(H5F_t *f, hbool_t UNUSED disable_shared, uint8_t *p, const voi HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to serialize source selection") /* Virtual selection */ - if(H5S_SELECT_SERIALIZE(f, mesg->storage.u.virt.list[i].virtual_select, &heap_block_p) < 0) + if(H5S_SELECT_SERIALIZE(f, mesg->storage.u.virt.list[i].source_dset.virtual_select, &heap_block_p) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to serialize virtual selection") } /* end for */ @@ -536,8 +544,7 @@ H5O_layout_encode(H5F_t *f, hbool_t UNUSED disable_shared, uint8_t *p, const voi UINT32ENCODE(heap_block_p, chksum) /* Insert block into global heap */ - /* Casting away const OK --NAF */ - if(H5HG_insert(f, H5AC_ind_dxpl_id, block_size, heap_block, &((H5O_layout_t *)mesg)->storage.u.virt.serial_list_hobjid) < 0) + if(H5HG_insert(f, H5AC_ind_dxpl_id, block_size, heap_block, &((H5O_layout_t *)mesg)->storage.u.virt.serial_list_hobjid) < 0) /* Casting away const OK --NAF */ HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "unable to insert virtual dataset heap block") } /* end if */ diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 1059e8b..6351dd5 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -412,6 +412,18 @@ typedef struct H5O_storage_compact_t { void *buf; /* Buffer for compact dataset */ } H5O_storage_compact_t; +typedef struct H5O_storage_virtual_srcdset_t { + struct H5D_t *dset; /* Source dataset */ + char *file_name; /* Source file name used for virtual dataset mapping */ + char *dset_name; /* Source dataset name used for virtual dataset mapping */ + struct H5S_t *virtual_select; /* Selection in the virtual dataset that is mapped to source selection */ +} H5O_storage_virtual_srcdset_t; + +typedef struct H5O_storage_virtual_name_seg_t { + char *name_segment; + struct H5O_storage_virtual_name_seg_t *next; +} H5O_storage_virtual_name_seg_t; + typedef enum H5O_virtual_space_status_t { H5O_VIRTUAL_STATUS_INVALID = 0, /* Space extent is invalid */ H5O_VIRTUAL_STATUS_SEL_BOUNDS, /* Space extent set to bounds of selection */ @@ -421,13 +433,19 @@ typedef enum H5O_virtual_space_status_t { typedef struct H5O_storage_virtual_ent_t { /* Stored */ - char *source_file_name; /* Source file name used for virtual dataset mapping */ - char *source_dset_name; /* Source dataset name used for virtual dataset mapping */ + H5O_storage_virtual_srcdset_t source_dset; /* Information about the source dataset */ struct H5S_t *source_select; /* Selection in the source dataset for mapping */ - struct H5S_t *virtual_select; /* Selection in the virtual dataset that is mapped to source_select */ /* Not stored */ - struct H5D_t *source_dset; /* Source dataset */ + H5O_storage_virtual_srcdset_t *sub_dset; /* Array of sub-source dataset info structs */ + size_t sub_dset_nalloc; + size_t sub_dset_nused; + H5O_storage_virtual_name_seg_t *parsed_source_file_name; + size_t psfn_static_strlen; + size_t psfn_nsubs; + H5O_storage_virtual_name_seg_t *parsed_source_dset_name; + size_t psdn_static_strlen; + size_t psdn_nsubs; int unlim_dim_source; /* Unlimited dimension in source_select */ int unlim_dim_virtual; /* Unlimited dimension in virtual_select */ hsize_t unlim_extent_source; /* Extent of unlimited dimension in source dset last time virtual_select was patched to match selection */ @@ -450,6 +468,7 @@ typedef struct H5O_storage_virtual_t { size_t list_nalloc; /* Number of slots allocated */ hsize_t min_dims[H5S_MAX_RANK]; /* Minimum extent of VDS (maximum of all non-unlimited selection bounds) */ H5D_vds_view_t view; /* Method for calculating the extent of the virtual dataset with unlimited selections */ + hsize_t printf_gap; /* Maximum number of sequential missing source datasets before terminating the search for more */ } H5O_storage_virtual_t; typedef struct H5O_storage_t { diff --git a/src/H5Pdapl.c b/src/H5Pdapl.c index af4b988..c11d22f 100644 --- a/src/H5Pdapl.c +++ b/src/H5Pdapl.c @@ -61,11 +61,16 @@ #define H5D_ACS_PREEMPT_READ_CHUNKS_DEF H5D_CHUNK_CACHE_W0_DEFAULT #define H5D_ACS_PREEMPT_READ_CHUNKS_ENC H5P__encode_double #define H5D_ACS_PREEMPT_READ_CHUNKS_DEC H5P__decode_double -/* Definitions for VDS bounds option */ +/* Definitions for VDS view option */ #define H5D_ACS_VDS_VIEW_SIZE sizeof(H5D_vds_view_t) #define H5D_ACS_VDS_VIEW_DEF H5D_VDS_LAST_AVAILABLE #define H5D_ACS_VDS_VIEW_ENC H5P__dacc_vds_view_enc #define H5D_ACS_VDS_VIEW_DEC H5P__dacc_vds_view_dec +/* Definitions for VDS printf gap */ +#define H5D_ACS_VDS_PRINTF_GAP_SIZE sizeof(hsize_t) +#define H5D_ACS_VDS_PRINTF_GAP_DEF (hsize_t)0 +#define H5D_ACS_VDS_PRINTF_GAP_ENC H5P__encode_hsize_t +#define H5D_ACS_VDS_PRINTF_GAP_DEC H5P__decode_hsize_t /******************/ /* Local Typedefs */ @@ -143,6 +148,7 @@ H5P__dacc_reg_prop(H5P_genclass_t *pclass) size_t rdcc_nbytes = H5D_ACS_DATA_CACHE_BYTE_SIZE_DEF; /* Default raw data chunk cache # of bytes */ double rdcc_w0 = H5D_ACS_PREEMPT_READ_CHUNKS_DEF; /* Default raw data chunk cache dirty ratio */ H5D_vds_view_t virtual_view = H5D_ACS_VDS_VIEW_DEF; /* Default VDS view option */ + hsize_t printf_gap = H5D_ACS_VDS_PRINTF_GAP_DEF; /* Default VDS printf gap */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -168,6 +174,12 @@ H5P__dacc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the VDS printf gap */ + if(H5P_register_real(pclass, H5D_ACS_VDS_PRINTF_GAP_NAME, H5D_ACS_VDS_PRINTF_GAP_SIZE, &printf_gap, + NULL, NULL, NULL, H5D_ACS_VDS_PRINTF_GAP_ENC, H5D_ACS_VDS_PRINTF_GAP_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__dacc_reg_prop() */ @@ -366,7 +378,7 @@ H5Pget_virtual_view(hid_t plist_id) /* Get value from property list */ if(H5P_get(plist, H5D_ACS_VDS_VIEW_NAME, &ret_value) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, H5D_VDS_ERROR, "unable to set value") + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, H5D_VDS_ERROR, "unable to get value") done: FUNC_LEAVE_API(ret_value) @@ -376,7 +388,7 @@ done: /*------------------------------------------------------------------------- * Function: H5P__dacc_vds_view_enc * - * Purpose: Callback routine which is called whenever the vds bounds + * Purpose: Callback routine which is called whenever the vds view * property in the dataset access property list is encoded. * * Return: Success: Non-negative @@ -413,7 +425,7 @@ H5P__dacc_vds_view_enc(const void *value, void **_pp, size_t *size) /*------------------------------------------------------------------------- * Function: H5P__dacc_vds_view_dec * - * Purpose: Callback routine which is called whenever the vds bounds + * Purpose: Callback routine which is called whenever the vds view * property in the dataset access property list is encoded. * * Return: Success: Non-negative @@ -443,3 +455,77 @@ H5P__dacc_vds_view_dec(const void **_pp, void *_value) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5P__dacc_vds_view_dec() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_virtual_printf_gap + * + * Purpose: VDSINC + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 21, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_virtual_printf_gap(hid_t plist_id, hsize_t gap_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + + HDassert(0 && "Checking code coverage..."); //VDSINC + /* Check argument */ + if(gap_size == HSIZE_UNDEF) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid printf gap size") + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Update property list */ + if(H5P_set(plist, H5D_ACS_VDS_PRINTF_GAP_NAME, &gap_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_virtual_printf_gap() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_virtual_printf_gap + * + * Purpose: VDSINC + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 21, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_virtual_printf_gap(hid_t plist_id, hsize_t *gap_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(H5D_VDS_ERROR) + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, H5D_VDS_ERROR, "can't find object for ID") + + /* Get value from property list */ + if(gap_size) { + HDassert(0 && "Checking code coverage..."); //VDSINC + if(H5P_get(plist, H5D_ACS_VDS_PRINTF_GAP_NAME, gap_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, H5D_VDS_ERROR, "unable to get value") + } //VDSINC + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_virtual_printf_gap() */ + diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 575de9c..e647f29 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -57,7 +57,7 @@ #define H5D_DEF_STORAGE_CONTIG_INIT {HADDR_UNDEF, (hsize_t)0} #define H5D_DEF_STORAGE_CHUNK_INIT {H5D_CHUNK_IDX_BTREE, HADDR_UNDEF, NULL, {{HADDR_UNDEF, NULL}}} #define H5D_DEF_LAYOUT_CHUNK_INIT {(unsigned)0, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, (uint32_t)0, (hsize_t)0, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}} -#define H5D_DEF_STORAGE_VIRTUAL_INIT {{HADDR_UNDEF, 0}, 0, NULL, 0, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, TRUE} +#define H5D_DEF_STORAGE_VIRTUAL_INIT {{HADDR_UNDEF, 0}, 0, NULL, 0, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, H5D_VDS_ERROR, HSIZE_UNDEF} #ifdef H5_HAVE_C99_DESIGNATED_INITIALIZER #define H5D_DEF_STORAGE_COMPACT {H5D_COMPACT, { .compact = H5D_DEF_STORAGE_COMPACT_INIT }} #define H5D_DEF_STORAGE_CONTIG {H5D_CONTIGUOUS, { .contig = H5D_DEF_STORAGE_CONTIG_INIT }} @@ -1599,7 +1599,9 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, H5S_t *vspace; /* Virtual dataset space selection */ H5S_t *src_space; /* Source dataset space selection */ hbool_t new_layout = FALSE; /* Whether we are adding a new virtual layout message to plist */ + H5O_storage_virtual_ent_t *ent = NULL; /* Convenience pointer to new VDS entry */ hbool_t adding_entry = FALSE; /* Whether we are in the middle of adding an entry */ + H5O_storage_virtual_ent_t *old_list = NULL; /* List pointer previously on property list */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) @@ -1638,16 +1640,22 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, /* If the layout was not already virtual, Start with default virtual layout. * Otherwise, add the mapping to the current list. */ - if(layout.type != H5D_VIRTUAL) { + if(layout.type == H5D_VIRTUAL) + /* Save old list pointer for error recovery */ + old_list = layout.storage.u.virt.list; + else { new_layout = TRUE; HDmemcpy(&layout, &H5D_def_layout_virtual_g, sizeof(H5D_def_layout_virtual_g)); - } /* end if */ + + HDassert(layout.storage.u.virt.list_nalloc == 0); + } /* end else */ /* Expand list if necessary */ if(layout.storage.u.virt.list_nalloc == 0) { /* Allocate initial entry list */ if(NULL == (layout.storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_calloc(H5D_VIRTUAL_DEF_LIST_SIZE * sizeof(H5O_storage_virtual_ent_t)))) HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't allocate virtual dataset mapping list") + adding_entry = TRUE; layout.storage.u.virt.list_nalloc = H5D_VIRTUAL_DEF_LIST_SIZE; } /* end if */ else if(layout.storage.u.virt.list_nused @@ -1656,28 +1664,34 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, /* Double size of entry list. Make sure to zero out new memory. */ if(NULL == (layout.storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_realloc(layout.storage.u.virt.list, (size_t)2 * layout.storage.u.virt.list_nalloc * sizeof(H5O_storage_virtual_ent_t)))) HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't reallocate virtual dataset mapping list") + adding_entry = TRUE; (void)HDmemset(layout.storage.u.virt.list + layout.storage.u.virt.list_nalloc, 0, layout.storage.u.virt.list_nalloc * sizeof(H5O_storage_virtual_ent_t)); layout.storage.u.virt.list_nalloc *= (size_t)2; } /* end if */ /* Add virtual dataset mapping entry */ - if(NULL == (layout.storage.u.virt.list[layout.storage.u.virt.list_nused].source_file_name = HDstrdup(src_file_name))) + ent = &layout.storage.u.virt.list[layout.storage.u.virt.list_nused]; + if(NULL == (ent->source_dset.file_name = HDstrdup(src_file_name))) HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't duplicate source file name") adding_entry = TRUE; - if(NULL == (layout.storage.u.virt.list[layout.storage.u.virt.list_nused].source_dset_name = HDstrdup(src_dset_name))) + if(NULL == (ent->source_dset.dset_name = HDstrdup(src_dset_name))) HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't duplicate source file name") - if(NULL == (layout.storage.u.virt.list[layout.storage.u.virt.list_nused].source_select = H5S_copy(src_space, FALSE, TRUE))) - HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy source selection") - if(NULL == (layout.storage.u.virt.list[layout.storage.u.virt.list_nused].virtual_select = H5S_copy(vspace, FALSE, TRUE))) + if(NULL == (ent->source_dset.virtual_select = H5S_copy(vspace, FALSE, TRUE))) HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") - layout.storage.u.virt.list[layout.storage.u.virt.list_nused].unlim_dim_source = H5S_get_select_unlim_dim(src_space); - layout.storage.u.virt.list[layout.storage.u.virt.list_nused].unlim_dim_virtual = H5S_get_select_unlim_dim(vspace); - layout.storage.u.virt.list[layout.storage.u.virt.list_nused].unlim_extent_source = HSIZE_UNDEF; - layout.storage.u.virt.list[layout.storage.u.virt.list_nused].unlim_extent_virtual = HSIZE_UNDEF; - layout.storage.u.virt.list[layout.storage.u.virt.list_nused].clip_size_source = HSIZE_UNDEF; - layout.storage.u.virt.list[layout.storage.u.virt.list_nused].clip_size_virtual = HSIZE_UNDEF; - layout.storage.u.virt.list[layout.storage.u.virt.list_nused].source_space_status = H5O_VIRTUAL_STATUS_USER; - layout.storage.u.virt.list[layout.storage.u.virt.list_nused].virtual_space_status = H5O_VIRTUAL_STATUS_USER; + if(NULL == (ent->source_select = H5S_copy(src_space, FALSE, TRUE))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy source selection") + if(H5D_virtual_parse_source_name(ent->source_dset.file_name, &ent->parsed_source_file_name, &ent->psfn_static_strlen, &ent->psfn_nsubs) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't parse source file name") + if(H5D_virtual_parse_source_name(ent->source_dset.dset_name, &ent->parsed_source_dset_name, &ent->psdn_static_strlen, &ent->psdn_nsubs) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't parse source dataset name") + ent->unlim_dim_source = H5S_get_select_unlim_dim(src_space); + ent->unlim_dim_virtual = H5S_get_select_unlim_dim(vspace); + ent->unlim_extent_source = HSIZE_UNDEF; + ent->unlim_extent_virtual = HSIZE_UNDEF; + ent->clip_size_source = HSIZE_UNDEF; + ent->clip_size_virtual = HSIZE_UNDEF; + ent->source_space_status = H5O_VIRTUAL_STATUS_USER; + ent->virtual_space_status = H5O_VIRTUAL_STATUS_USER; /* Update min_dims */ if(H5D_virtual_update_min_dims(&layout, layout.storage.u.virt.list_nused) < 0) @@ -1685,32 +1699,52 @@ H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, /* Finish adding entry */ layout.storage.u.virt.list_nused++; - adding_entry = FALSE; - - /* Set chunk information in property list. If we are adding a new layout, - * use H5P__set_layout so other fields are initialized properly. If we are - * extending the layout, just use H5P_set directly so we don't mess with - * anything else. */ - if(new_layout) { - if(H5P__set_layout(plist, &layout) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set layout") - } /* end if */ - else - if(H5P_set(plist, H5D_CRT_LAYOUT_NAME, &layout) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set layout") done: + /* Only need to do things if we started to add an entry */ if(adding_entry) { - HDassert(ret_value < 0); + hbool_t free_list = FALSE; + + /* Set chunk information in property list. If we are adding a new layout, + * use H5P__set_layout so other fields are initialized properly. If we are + * extending the layout, just use H5P_set directly so we don't mess with + * anything else. */ + if(new_layout) { + if(H5P__set_layout(plist, &layout) < 0) { + HDONE_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set layout") + free_list = TRUE; + } /* end if */ + } /* end if */ + else + if(H5P_set(plist, H5D_CRT_LAYOUT_NAME, &layout) < 0) { + HDONE_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set layout") + if(old_list != layout.storage.u.virt.list) + free_list = TRUE; + } /* end if */ - /* Free incomplete entry */ - layout.storage.u.virt.list[layout.storage.u.virt.list_nused].source_file_name = (char *)H5MM_xfree(layout.storage.u.virt.list[layout.storage.u.virt.list_nused].source_file_name); - layout.storage.u.virt.list[layout.storage.u.virt.list_nused].source_dset_name = (char *)H5MM_xfree(layout.storage.u.virt.list[layout.storage.u.virt.list_nused].source_dset_name); - if(layout.storage.u.virt.list[layout.storage.u.virt.list_nused].source_select - && H5S_close(layout.storage.u.virt.list[layout.storage.u.virt.list_nused].source_select) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release source selection") - layout.storage.u.virt.list[layout.storage.u.virt.list_nused].source_select = NULL; - HDassert(layout.storage.u.virt.list[layout.storage.u.virt.list_nused].virtual_select == NULL); + /* Check if the entry has been partly allocated but not added to the + * property list or not included in list_nused */ + if((ret_value < 0) && ent) { + HDassert(ret_value < 0); + + /* Free incomplete entry */ + ent->source_dset.file_name = (char *)H5MM_xfree(ent->source_dset.file_name); + ent->source_dset.dset_name = (char *)H5MM_xfree(ent->source_dset.dset_name); + if(ent->source_dset.virtual_select && H5S_close(ent->source_dset.virtual_select) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release virtual selection") + ent->source_dset.virtual_select = NULL; + if(ent->source_select && H5S_close(ent->source_select) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release source selection") + ent->source_select = NULL; + H5D_virtual_free_parsed_name(ent->parsed_source_file_name); + ent->parsed_source_file_name = NULL; + H5D_virtual_free_parsed_name(ent->parsed_source_dset_name); + ent->parsed_source_dset_name = NULL; + + /* Free list if necessary */ + if(free_list) + layout.storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_xfree(layout.storage.u.virt.list); + } /* end if */ } /* end if */ FUNC_LEAVE_API(ret_value) @@ -1796,7 +1830,7 @@ H5Pget_virtual_vspace(hid_t dcpl_id, size_t index) if(index >= layout.storage.u.virt.list_nused) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid index (out of range)") HDassert(layout.storage.u.virt.list_nused <= layout.storage.u.virt.list_nalloc); - if(NULL == (space = H5S_copy(layout.storage.u.virt.list[index].virtual_select, FALSE, TRUE))) + if(NULL == (space = H5S_copy(layout.storage.u.virt.list[index].source_dset.virtual_select, FALSE, TRUE))) HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") /* Clip selection to extent */ @@ -1918,10 +1952,10 @@ H5Pget_virtual_filename(hid_t dcpl_id, size_t index, char *name/*out*/, if(index >= layout.storage.u.virt.list_nused) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid index (out of range)") HDassert(layout.storage.u.virt.list_nused <= layout.storage.u.virt.list_nalloc); - HDassert(layout.storage.u.virt.list[index].source_file_name); + HDassert(layout.storage.u.virt.list[index].source_dset.file_name); if(name && (size > 0)) - (void)HDstrncpy(name, layout.storage.u.virt.list[index].source_file_name, size); - ret_value = (ssize_t)HDstrlen(layout.storage.u.virt.list[index].source_file_name); + (void)HDstrncpy(name, layout.storage.u.virt.list[index].source_dset.file_name, size); + ret_value = (ssize_t)HDstrlen(layout.storage.u.virt.list[index].source_dset.file_name); done: FUNC_LEAVE_API(ret_value) @@ -1965,10 +1999,10 @@ H5Pget_virtual_dsetname(hid_t dcpl_id, size_t index, char *name/*out*/, if(index >= layout.storage.u.virt.list_nused) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid index (out of range)") HDassert(layout.storage.u.virt.list_nused <= layout.storage.u.virt.list_nalloc); - HDassert(layout.storage.u.virt.list[index].source_dset_name); + HDassert(layout.storage.u.virt.list[index].source_dset.dset_name); if(name && (size > 0)) - (void)HDstrncpy(name, layout.storage.u.virt.list[index].source_dset_name, size); - ret_value = (ssize_t)HDstrlen(layout.storage.u.virt.list[index].source_dset_name); + (void)HDstrncpy(name, layout.storage.u.virt.list[index].source_dset.dset_name, size); + ret_value = (ssize_t)HDstrlen(layout.storage.u.virt.list[index].source_dset.dset_name); done: FUNC_LEAVE_API(ret_value) diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 5ea857a..f1db483 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -395,6 +395,8 @@ H5_DLL herr_t H5Pget_chunk_cache(hid_t dapl_id, double *rdcc_w0/*out*/); H5_DLL herr_t H5Pset_virtual_view(hid_t plist_id, H5D_vds_view_t view); H5_DLL H5D_vds_view_t H5Pget_virtual_view(hid_t plist_id); +H5_DLL herr_t H5Pset_virtual_printf_gap(hid_t plist_id, hsize_t gap_size); +H5_DLL herr_t H5Pget_virtual_printf_gap(hid_t plist_id, hsize_t *gap_size); /* Dataset xfer property list (DXPL) routines */ H5_DLL herr_t H5Pset_data_transform(hid_t plist_id, const char* expression); diff --git a/src/H5Shyper.c b/src/H5Shyper.c index a483c21..a3e6781 100644 --- a/src/H5Shyper.c +++ b/src/H5Shyper.c @@ -9812,8 +9812,9 @@ H5S_hyper_clip_unlim(H5S_t *space, hsize_t clip_size) block[i] = H5S_MAX_SIZE; /* Generate span tree in selection */ - if(H5S_hyper_generate_spans(space) < 0) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to generate span tree") + if(!hslab->span_lst) + if(H5S_hyper_generate_spans(space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to generate span tree") /* Indicate that the regular dimensions are no longer valid */ hslab->diminfo_valid = FALSE; @@ -9987,6 +9988,89 @@ H5S_hyper_get_clip_extent(const H5S_t *clip_space, const H5S_t *match_space, /*-------------------------------------------------------------------------- NAME + H5S_hyper_get_unlim_block + PURPOSE + VDSINC + USAGE + VDSINC + RETURNS + New space on success/NULL on failure. + DESCRIPTION + VDSINC + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + Note this assumes the offset has been normalized. + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +H5S_t * +H5S_hyper_get_unlim_block(const H5S_t *space, hsize_t block_index) +{ + H5S_hyper_sel_t *hslab; /* Convenience pointer to hyperslab info */ + H5S_t *space_out = NULL; + hsize_t start[H5S_MAX_RANK]; + hsize_t stride[H5S_MAX_RANK]; + hsize_t count[H5S_MAX_RANK]; + hsize_t block[H5S_MAX_RANK]; + unsigned i; + H5S_t *ret_value = NULL; + + FUNC_ENTER_NOAPI(NULL) + + /* Check parameters */ + HDassert(space); + hslab = space->select.sel_info.hslab; + HDassert(hslab); + HDassert(hslab->unlim_dim >= 0); + HDassert(hslab->opt_unlim_diminfo[hslab->unlim_dim].count == H5S_UNLIMITED); + + /* Set block to clip_size in unlimited dimension, H5S_MAX_SIZE in + * others so only unlimited dimension is clipped */ + for(i = 0; i < space->extent.rank; i++) { + if((int)i == hslab->unlim_dim){ + //HDassert(0 && "Checking code coverage..."); //VDSINC + start[i] = hslab->opt_unlim_diminfo[i].start + (block_index + * hslab->opt_unlim_diminfo[i].stride); + count[i] = (hsize_t)1; + } /* end if */ + else { + //HDassert(0 && "Checking code coverage..."); //VDSINC + start[i] = hslab->opt_unlim_diminfo[i].start; + count[i] = hslab->opt_unlim_diminfo[i].count; + } /* end else */ + stride[i] = hslab->opt_unlim_diminfo[i].stride; + block[i] = hslab->opt_unlim_diminfo[i].block; + } /* end for */ + + /* Create output space, copy extent */ + if(NULL == (space_out = H5S_create(H5S_SIMPLE))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, NULL, "unable to create output dataspace") + if(H5S_extent_copy_real(&space_out->extent, &space->extent, TRUE) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOPY, NULL, "unable to copy destination space extent") + + /* Set offset to zeros */ + (void)HDmemset(space_out->select.offset, 0, (size_t)space_out->extent.rank * sizeof(space_out->select.offset[0])); + space_out->select.offset_changed = FALSE; + + /* "And" selection with calculate block to perform clip operation */ + if(H5S_select_hyperslab(space_out, H5S_SELECT_SET, start, stride, count, block) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, NULL, "can't select hyperslab") + + /* Set return value */ + ret_value = space_out; + +done: + /* Free space on error */ + if(!ret_value) + if(space_out && H5S_close(space_out) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, NULL, "unable to release dataspace") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S_hyper_get_unlim_block */ + + +/*-------------------------------------------------------------------------- + NAME H5Sis_regular_hyperslab PURPOSE Determine if a hyperslab selection is regular diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index c796c31..342a9cc 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -270,6 +270,8 @@ H5_DLL herr_t H5S_hyper_clip_unlim(H5S_t *space, hsize_t clip_size); H5_DLL herr_t H5S_hyper_clip_to_extent(H5S_t *space); H5_DLL herr_t H5S_hyper_get_clip_extent(const H5S_t *clip_space, const H5S_t *match_space, hsize_t *clip_size, hbool_t incl_trail); +H5_DLL H5S_t *H5S_hyper_get_unlim_block(const H5S_t *space, + hsize_t block_index); /* Operations on selection iterators */ H5_DLL herr_t H5S_select_iter_init(H5S_sel_iter_t *iter, const H5S_t *space, size_t elmt_size); |