diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | src/H5.c | 12 | ||||
| -rw-r--r-- | src/H5Dint.c | 90 | ||||
| -rw-r--r-- | src/H5Dio.c | 7 | ||||
| -rw-r--r-- | src/H5Dlayout.c | 9 | ||||
| -rw-r--r-- | src/H5Doh.c | 11 | ||||
| -rw-r--r-- | src/H5Dpkg.h | 15 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 17 | ||||
| -rw-r--r-- | src/H5Dpublic.h | 10 | ||||
| -rw-r--r-- | src/H5Dvirtual.c | 2724 | ||||
| -rw-r--r-- | src/H5HG.c | 44 | ||||
| -rw-r--r-- | src/H5HGprivate.h | 1 | ||||
| -rw-r--r-- | src/H5Olayout.c | 314 | ||||
| -rw-r--r-- | src/H5Oprivate.h | 80 | ||||
| -rw-r--r-- | src/H5Pdapl.c | 276 | ||||
| -rw-r--r-- | src/H5Pdcpl.c | 771 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 13 | ||||
| -rw-r--r-- | src/H5S.c | 11 | ||||
| -rw-r--r-- | src/H5Sall.c | 42 | ||||
| -rw-r--r-- | src/H5Shyper.c | 1570 | ||||
| -rw-r--r-- | src/H5Snone.c | 41 | ||||
| -rw-r--r-- | src/H5Spkg.h | 27 | ||||
| -rw-r--r-- | src/H5Spoint.c | 40 | ||||
| -rw-r--r-- | src/H5Sprivate.h | 18 | ||||
| -rw-r--r-- | src/H5Sselect.c | 298 | ||||
| -rw-r--r-- | src/H5trace.c | 34 | ||||
| -rw-r--r-- | src/Makefile.am | 2 |
27 files changed, 6366 insertions, 112 deletions
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6d74939..ef3e760 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -118,6 +118,7 @@ set (H5D_SRCS ${HDF5_SRC_DIR}/H5Dscatgath.c ${HDF5_SRC_DIR}/H5Dselect.c ${HDF5_SRC_DIR}/H5Dtest.c + ${HDF5_SRC_DIR}/H5Dvirtual.c ) set (H5D_HDRS @@ -284,12 +284,12 @@ H5_term_library(void) n) : n)) do { - pending = 0; + pending = 0; /* Try to organize these so the "higher" level components get shut * down before "lower" level components that they might rely on. -QAK */ - pending += DOWN(L); + pending += DOWN(L); /* Close the "top" of various interfaces (IDs, etc) but don't shut * down the whole interface yet, so that the object header messages @@ -300,7 +300,7 @@ H5_term_library(void) pending += DOWN(A_top); pending += DOWN(D_top); pending += DOWN(G_top); - pending += DOWN(R_top); + pending += DOWN(R_top); pending += DOWN(S_top); pending += DOWN(T_top); @@ -308,6 +308,11 @@ H5_term_library(void) if(pending == 0) pending += DOWN(F); + /* Don't shut down the property list code until all objects that might + * use property lists are shut down */ + if(pending == 0) + pending += DOWN(P); + /* Wait to shut down the "bottom" of various interfaces until the * files are closed, so pieces of the file can be serialized * correctly. @@ -333,7 +338,6 @@ H5_term_library(void) if(pending == 0) { pending += DOWN(AC); pending += DOWN(Z); - pending += DOWN(P); pending += DOWN(FD); pending += DOWN(PL); /* Don't shut down the error code until other APIs which use it are shut down */ diff --git a/src/H5Dint.c b/src/H5Dint.c index 01b6dbf..b0e31a6 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -1461,6 +1461,10 @@ H5D__open_oid(H5D_t *dataset, hid_t dapl_id, hid_t dxpl_id) fill_prop->alloc_time = H5D_ALLOC_TIME_INCR; break; + case H5D_VIRTUAL: + fill_prop->alloc_time = H5D_ALLOC_TIME_INCR; + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -1475,7 +1479,8 @@ H5D__open_oid(H5D_t *dataset, hid_t dapl_id, hid_t dxpl_id) alloc_time_state = 0; if((dataset->shared->layout.type == H5D_COMPACT && fill_prop->alloc_time == H5D_ALLOC_TIME_EARLY) || (dataset->shared->layout.type == H5D_CONTIGUOUS && fill_prop->alloc_time == H5D_ALLOC_TIME_LATE) - || (dataset->shared->layout.type == H5D_CHUNKED && fill_prop->alloc_time == H5D_ALLOC_TIME_INCR)) + || (dataset->shared->layout.type == H5D_CHUNKED && fill_prop->alloc_time == H5D_ALLOC_TIME_INCR) + || (dataset->shared->layout.type == H5D_VIRTUAL && fill_prop->alloc_time == H5D_ALLOC_TIME_INCR)) alloc_time_state = 1; /* Set revised fill value properties, if they are different from the defaults */ @@ -1601,6 +1606,34 @@ H5D_close(H5D_t *dataset) /* Nothing special to do (info freed in the layout destroy) */ break; + case H5D_VIRTUAL: + { + size_t i, j; + + HDassert(dataset->shared->layout.storage.u.virt.list || (dataset->shared->layout.storage.u.virt.list_nused == 0)); + + /* Close source datasets */ + for(i = 0; i < dataset->shared->layout.storage.u.virt.list_nused; i++) { + /* Close source dataset */ + if(dataset->shared->layout.storage.u.virt.list[i].source_dset.dset) { + HDassert(dataset->shared->layout.storage.u.virt.list[i].source_dset.dset != dataset); + if(H5D_close(dataset->shared->layout.storage.u.virt.list[i].source_dset.dset) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to close source dataset") + dataset->shared->layout.storage.u.virt.list[i].source_dset.dset = NULL; + } /* end if */ + + /* Close sub datasets */ + for(j = 0; j < dataset->shared->layout.storage.u.virt.list[i].sub_dset_nused; j++) + if(dataset->shared->layout.storage.u.virt.list[i].sub_dset[j].dset) { + HDassert(dataset->shared->layout.storage.u.virt.list[i].sub_dset[j].dset != dataset); + if(H5D_close(dataset->shared->layout.storage.u.virt.list[i].sub_dset[j].dset) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to close source dataset") + dataset->shared->layout.storage.u.virt.list[i].sub_dset[j].dset = NULL; + } /* end if */ + } /* end for */ + } /* end block */ + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -1857,6 +1890,15 @@ H5D__alloc_storage(const H5D_t *dset, hid_t dxpl_id, H5D_time_alloc_t time_alloc } /* end if */ break; + case H5D_VIRTUAL: + /* No-op, as the raw data is stored elsewhere and the global + * heap object containing the mapping information is created + * when the layout message is encoded. We may wish to move the + * creation of the global heap object here at some point, but we + * will have to make sure is it always created before the + * dataset is closed. */ + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -1976,6 +2018,9 @@ H5D__init_storage(const H5D_t *dset, hbool_t full_overwrite, hsize_t old_dim[], break; } /* end block */ + case H5D_VIRTUAL: + /* No-op, as the raw data is stored elsewhere */ + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -2032,6 +2077,12 @@ H5D__get_storage_size(H5D_t *dset, hid_t dxpl_id, hsize_t *storage_size) *storage_size = dset->shared->layout.storage.u.compact.size; break; + case H5D_VIRTUAL: + /* Just set to 0, as virtual datasets do not actually store raw data + */ + *storage_size = 0; + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -2068,6 +2119,7 @@ H5D__get_offset(const H5D_t *dset) HDassert(dset); switch(dset->shared->layout.type) { + case H5D_VIRTUAL: case H5D_CHUNKED: case H5D_COMPACT: break; @@ -2307,7 +2359,7 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size, hid_t dxpl_id) { hsize_t curr_dims[H5S_MAX_RANK]; /* Current dimension sizes */ htri_t changed; /* Whether the dataspace changed size */ - size_t u; /* Local index variable */ + size_t u, v; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_TAG(dxpl_id, dset->oloc.addr, FAIL) @@ -2406,6 +2458,30 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size, hid_t dxpl_id) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to update cached chunk indices") } /* end if */ + /* Operations for virtual datasets */ + if(H5D_VIRTUAL == dset->shared->layout.type) { + /* Check that the dimensions of the VDS are large enough */ + if(H5D_virtual_check_min_dims(dset) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "virtual dataset dimensions not large enough to contain all limited dimensions in all selections") + + /* Patch the virtual selection dataspaces */ + for(u = 0; u < dset->shared->layout.storage.u.virt.list_nused; u++) { + /* Patch extent */ + if(H5S_set_extent(dset->shared->layout.storage.u.virt.list[u].source_dset.virtual_select, size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") + dset->shared->layout.storage.u.virt.list[u].virtual_space_status = H5O_VIRTUAL_STATUS_CORRECT; + + /* Patch sub-source datasets */ + for(v = 0; v < dset->shared->layout.storage.u.virt.list[u].sub_dset_nalloc; v++) + if(H5S_set_extent(dset->shared->layout.storage.u.virt.list[u].sub_dset[v].virtual_select, size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") + } /* end for */ + + /* Mark virtual datasets as not fully initialized so internal + * selections are recalculated (at next I/O operation) */ + dset->shared->layout.storage.u.virt.init = FALSE; + } /* end if */ + /* Allocate space for the new parts of the dataset, if appropriate */ if(expand && dset->shared->dcpl_cache.fill.alloc_time == H5D_ALLOC_TIME_EARLY) if(H5D__alloc_storage(dset, dxpl_id, H5D_ALLOC_EXTEND, FALSE, curr_dims) < 0) @@ -2731,6 +2807,11 @@ H5D_get_create_plist(H5D_t *dset) copied_layout.storage.u.chunk.ops = NULL; break; + case H5D_VIRTUAL: + copied_layout.storage.u.virt.serial_list_hobjid.addr = HADDR_UNDEF; + copied_layout.storage.u.virt.serial_list_hobjid.idx = 0; + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -2913,6 +2994,11 @@ H5D_get_space(H5D_t *dset) FUNC_ENTER_NOAPI_NOINIT + /* If the layout is virtual, update the extent */ + if(dset->shared->layout.type == H5D_VIRTUAL) + if(H5D__virtual_set_extent_unlim(dset, H5AC_ind_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to update virtual dataset extent") + /* Read the data space message and return a data space object */ if(NULL == (space = H5S_copy(dset->shared->space, FALSE, TRUE))) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get data space") diff --git a/src/H5Dio.c b/src/H5Dio.c index d85632a..c0aa83e 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -52,9 +52,6 @@ /********************/ /* Internal I/O routines */ -static herr_t H5D__write(H5D_t *dataset, hid_t mem_type_id, - const H5S_t *mem_space, const H5S_t *file_space, hid_t dset_xfer_plist, - const void *buf); static herr_t H5D__pre_write(H5D_t *dset, hbool_t direct_write, hid_t mem_type_id, const H5S_t *mem_space, const H5S_t *file_space, hid_t dxpl_id, const void *buf); @@ -581,7 +578,7 @@ done: * *------------------------------------------------------------------------- */ -static herr_t +herr_t H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, const H5S_t *file_space, hid_t dxpl_id, const void *buf) { @@ -614,7 +611,7 @@ H5D__write(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, char fake_char; /* Temporary variable for NULL buffer pointers */ herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC_TAG(dxpl_id, dataset->oloc.addr, FAIL) + FUNC_ENTER_PACKAGE_TAG(dxpl_id, dataset->oloc.addr, FAIL) /* check args */ HDassert(dataset && dataset->oloc.file); diff --git a/src/H5Dlayout.c b/src/H5Dlayout.c index c2d547f..ae478b2 100644 --- a/src/H5Dlayout.c +++ b/src/H5Dlayout.c @@ -104,6 +104,10 @@ H5D__layout_set_io_ops(const H5D_t *dataset) dataset->shared->layout.ops = H5D_LOPS_COMPACT; break; + case H5D_VIRTUAL: + dataset->shared->layout.ops = H5D_LOPS_VIRTUAL; + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -169,6 +173,11 @@ H5D__layout_meta_size(const H5F_t *f, const H5O_layout_t *layout, hbool_t includ ret_value += H5F_SIZEOF_ADDR(f); /* Address of data */ break; + case H5D_VIRTUAL: + ret_value += H5F_SIZEOF_ADDR(f); /* Address of global heap */ + ret_value += 4; /* Global heap index */ + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: diff --git a/src/H5Doh.c b/src/H5Doh.c index 057c904..f8a733d 100644 --- a/src/H5Doh.c +++ b/src/H5Doh.c @@ -398,6 +398,17 @@ H5O__dset_bh_info(H5F_t *f, hid_t dxpl_id, H5O_t *oh, H5_ih_info_t *bh_info) if(H5D__chunk_bh_info(f, dxpl_id, &layout, &pline, &(bh_info->index_size)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine chunked dataset btree info") } /* end if */ + else if(layout.type == H5D_VIRTUAL + && (layout.storage.u.virt.serial_list_hobjid.addr != HADDR_UNDEF)) { + size_t virtual_heap_size; + + /* Get size of global heap object for virtual dataset */ + if(H5HG_get_obj_size(f, dxpl_id, &(layout.storage.u.virt.serial_list_hobjid), &virtual_heap_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get global heap size for virtual dataset mapping") + + /* Return heap size */ + bh_info->heap_size = (hsize_t)virtual_heap_size; + } /* end if */ /* Check for External File List message in the object header */ if((exists = H5O_msg_exists_oh(oh, H5O_EFL_ID)) < 0) diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 801800c..552c61b 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -522,6 +522,7 @@ H5_DLLVAR const H5D_layout_ops_t H5D_LOPS_CONTIG[1]; H5_DLLVAR const H5D_layout_ops_t H5D_LOPS_EFL[1]; H5_DLLVAR const H5D_layout_ops_t H5D_LOPS_COMPACT[1]; H5_DLLVAR const H5D_layout_ops_t H5D_LOPS_CHUNK[1]; +H5_DLLVAR const H5D_layout_ops_t H5D_LOPS_VIRTUAL[1]; /* Chunked layout operations */ H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_BTREE[1]; @@ -558,6 +559,9 @@ H5_DLL herr_t H5D__flush_real(H5D_t *dataset, hid_t dxpl_id); H5_DLL herr_t H5D__read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, const H5S_t *file_space, hid_t dset_xfer_plist, void *buf/*out*/); +H5_DLL herr_t H5D__write(H5D_t *dataset, hid_t mem_type_id, + const H5S_t *mem_space, const H5S_t *file_space, hid_t dset_xfer_plist, + const void *buf); /* Functions that perform direct serial I/O operations */ H5_DLL herr_t H5D__select_read(const H5D_io_info_t *io_info, @@ -649,6 +653,17 @@ H5_DLL herr_t H5D__compact_copy(H5F_t *f_src, H5O_storage_compact_t *storage_src H5F_t *f_dst, H5O_storage_compact_t *storage_dst, H5T_t *src_dtype, H5O_copy_t *cpy_info, hid_t dxpl_id); +/* Functions that operate on virtual dataset storage */ +H5_DLL herr_t H5D__virtual_copy_layout(H5O_layout_t *layout); +H5_DLL herr_t H5D__virtual_set_extent_unlim(const H5D_t *dset, hid_t dxpl_id); +H5_DLL herr_t H5D__virtual_reset_layout(H5O_layout_t *layout); +H5_DLL herr_t H5D__virtual_delete(H5F_t *f, hid_t dxpl_id, H5O_storage_t *storage); +H5_DLL herr_t H5D__virtual_copy(H5F_t *f_src, H5O_layout_t *layout_dst, + hid_t dxpl_id); +H5_DLL herr_t H5D__virtual_init(H5F_t *f, hid_t dxpl_id, const H5D_t *dset, + hid_t dapl_id); +H5_DLL hbool_t H5D__virtual_is_space_alloc(const H5O_storage_t *storage); + /* Functions that operate on EFL (External File List)*/ H5_DLL hbool_t H5D__efl_is_space_alloc(const H5O_storage_t *storage); H5_DLL herr_t H5D__efl_bh_info(H5F_t *f, hid_t dxpl_id, H5O_efl_t *efl, diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index 55e5b47..3b43aaf 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -52,6 +52,8 @@ #define H5D_ACS_DATA_CACHE_NUM_SLOTS_NAME "rdcc_nslots" /* Size of raw data chunk cache(slots) */ #define H5D_ACS_DATA_CACHE_BYTE_SIZE_NAME "rdcc_nbytes" /* Size of raw data chunk cache(bytes) */ #define H5D_ACS_PREEMPT_READ_CHUNKS_NAME "rdcc_w0" /* Preemption read chunks first */ +#define H5D_ACS_VDS_VIEW_NAME "vds_view" /* VDS view option */ +#define H5D_ACS_VDS_PRINTF_GAP_NAME "vds_printf_gap" /* VDS printf gap size */ /* ======== Data transfer properties ======== */ #define H5D_XFER_MAX_TEMP_BUF_NAME "max_temp_buf" /* Maximum temp buffer size */ @@ -106,6 +108,9 @@ #define H5D_VLEN_FREE NULL #define H5D_VLEN_FREE_INFO NULL +/* Default virtual dataset list size */ +#define H5D_VIRTUAL_DEF_LIST_SIZE 8 + /****************************/ /* Library Private Typedefs */ @@ -174,6 +179,18 @@ H5_DLL herr_t H5D_vlen_reclaim(hid_t type_id, H5S_t *space, hid_t plist_id, /* Functions that operate on chunked storage */ H5_DLL herr_t H5D_chunk_idx_reset(H5O_storage_chunk_t *storage, hbool_t reset_addr); +/* Functions that operate on virtual storage */ +H5_DLL herr_t H5D_virtual_check_mapping_pre(const H5S_t *vspace, + const H5S_t *src_space, H5O_virtual_space_status_t space_status); +H5_DLL herr_t H5D_virtual_check_mapping_post( + const H5O_storage_virtual_ent_t *ent); +H5_DLL herr_t H5D_virtual_check_min_dims(const H5D_t *dset); +H5_DLL herr_t H5D_virtual_update_min_dims(H5O_layout_t *layout, size_t idx); +H5_DLL herr_t H5D_virtual_parse_source_name(const char *source_name, + H5O_storage_virtual_name_seg_t **parsed_name, size_t *static_strlen, + size_t *nsubs); +H5_DLL herr_t H5D_virtual_free_parsed_name(H5O_storage_virtual_name_seg_t *name_seg); + /* Functions that operate on indexed storage */ H5_DLL herr_t H5D_btree_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int indent, int fwidth, unsigned ndims, const uint32_t *dim); diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 1b5fed7..a1f87e3 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -51,7 +51,8 @@ typedef enum H5D_layout_t { H5D_COMPACT = 0, /*raw data is very small */ H5D_CONTIGUOUS = 1, /*the default */ H5D_CHUNKED = 2, /*slow and fancy */ - H5D_NLAYOUTS = 3 /*this one must be last! */ + H5D_VIRTUAL = 3, /*actual data is stored in other datasets */ + H5D_NLAYOUTS = 4 /*this one must be last! */ } H5D_layout_t; /* Types of chunk index data structures */ @@ -93,6 +94,13 @@ typedef enum H5D_fill_value_t { H5D_FILL_VALUE_USER_DEFINED =2 } H5D_fill_value_t; +/* Values for VDS bounds option */ +typedef enum H5D_vds_view_t { + H5D_VDS_ERROR = -1, + H5D_VDS_FIRST_MISSING = 0, + H5D_VDS_LAST_AVAILABLE = 1 +} H5D_vds_view_t; + /********************/ /* Public Variables */ /********************/ diff --git a/src/H5Dvirtual.c b/src/H5Dvirtual.c new file mode 100644 index 0000000..316fcf7 --- /dev/null +++ b/src/H5Dvirtual.c @@ -0,0 +1,2724 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Programmer: Neil Fortner <nfortne2@hdfgroup.org> + * Wednesday, January 28, 2015 + * + * Purpose: + * Virtual Dataset (VDS) functions. Creates a layout type which allows + * definition of a virtual dataset, where the actual dataset is stored in + * other datasets (called source datasets). The mappings between the + * virtual and source datasets are specified by hyperslab or "all" + * dataspace selections. Point selections are not currently supported. + * Overlaps in the mappings in the virtual dataset result in undefined + * behaviour. + * + * Mapping selections may be unlimited, in which case the size of the + * virtual dataset is determined by the size of the source dataset(s). + * Names for the source datasets may also be generated procedurally, in + * which case the virtual selection should be unlimited with an unlimited + * count and the source selection should be limited with a size equal to + * that of the virtual selection with the unlimited count set to 1. + * + * Source datasets are opened lazily (only when needed for I/O or to + * determine the size of the virtual dataset), and are currently held open + * until the virtual dataset is closed. + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5Dmodule.h" /* This source code file is part of the H5D module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Dpkg.h" /* Dataset functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fprivate.h" /* Files */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Gprivate.h" /* Groups */ +#include "H5HGprivate.h" /* Global Heaps */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Oprivate.h" /* Object headers */ +#include "H5Sprivate.h" /* Dataspaces */ + + +/****************/ +/* Local Macros */ +/****************/ + +/* Default size for sub_dset array */ +#define H5D_VIRTUAL_DEF_SUB_DSET_SIZE 128 + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + +/* Layout operation callbacks */ +static herr_t H5D__virtual_read(H5D_io_info_t *io_info, const H5D_type_info_t + *type_info, hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + H5D_chunk_map_t *fm); +static herr_t H5D__virtual_write(H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, hsize_t nelmts, const H5S_t *file_space, + const H5S_t *mem_space, H5D_chunk_map_t *fm); +static herr_t H5D__virtual_flush(H5D_t *dset, hid_t dxpl_id); + +/* Other functions */ +static herr_t H5D__virtual_open_source_dset(const H5D_t *vdset, + H5O_storage_virtual_ent_t *virtual_ent, + H5O_storage_virtual_srcdset_t *source_dset, hid_t dxpl_id); +static herr_t H5D__virtual_reset_source_dset( + H5O_storage_virtual_ent_t *virtual_ent, + H5O_storage_virtual_srcdset_t *source_dset); +static herr_t H5D__virtual_str_append(const char *src, size_t src_len, char **p, + char **buf, size_t *buf_size); +static herr_t H5D__virtual_copy_parsed_name( + H5O_storage_virtual_name_seg_t **dst, H5O_storage_virtual_name_seg_t *src); +static herr_t H5D__virtual_build_source_name(char *source_name, + const H5O_storage_virtual_name_seg_t *parsed_name, size_t static_strlen, + size_t nsubs, hsize_t blockno, char **built_name); +static herr_t H5D__virtual_init_all(const H5D_t *dset, hid_t dxpl_id); +static herr_t H5D__virtual_pre_io(H5D_io_info_t *io_info, + H5O_storage_virtual_t *storage, const H5S_t *file_space, + const H5S_t *mem_space, hsize_t *tot_nelmts); +static herr_t H5D__virtual_post_io(H5O_storage_virtual_t *storage); +static herr_t H5D__virtual_read_one(H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, const H5S_t *file_space, + H5O_storage_virtual_srcdset_t *source_dset); +static herr_t H5D__virtual_write_one(H5D_io_info_t *io_info, + const H5D_type_info_t *type_info, const H5S_t *file_space, + H5O_storage_virtual_srcdset_t *source_dset); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Contiguous storage layout I/O ops */ +const H5D_layout_ops_t H5D_LOPS_VIRTUAL[1] = {{ + NULL, + H5D__virtual_init, + H5D__virtual_is_space_alloc, + NULL, + H5D__virtual_read, + H5D__virtual_write, +#ifdef H5_HAVE_PARALLEL + NULL, + NULL, +#endif /* H5_HAVE_PARALLEL */ + NULL, + NULL, + H5D__virtual_flush, + NULL +}}; + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Declare a free list to manage the H5O_storage_virtual_name_seg_t struct */ +H5FL_DEFINE(H5O_storage_virtual_name_seg_t); + + + +/*------------------------------------------------------------------------- + * Function: H5D_virtual_check_mapping_pre + * + * Purpose: Checks that the provided virtual and source selections are + * legal for use as a VDS mapping, prior to creating the rest + * of the mapping entry. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * August 12, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_virtual_check_mapping_pre(const H5S_t *vspace, const H5S_t *src_space, + H5O_virtual_space_status_t space_status) +{ + H5S_sel_type select_type; /* Selection type */ + hsize_t nelmts_vs; /* Number of elements in virtual selection */ + hsize_t nelmts_ss; /* Number of elements in source selection */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Check for point selections (currently unsupported) */ + if(H5S_SEL_ERROR == (select_type = H5S_GET_SELECT_TYPE(vspace))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get selection type") + if(select_type == H5S_SEL_POINTS) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "point selections not currently supported with virtual datasets") + if(H5S_SEL_ERROR == (select_type = H5S_GET_SELECT_TYPE(src_space))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get selection type") + if(select_type == H5S_SEL_POINTS) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "point selections not currently supported with virtual datasets") + + /* Get number of elements in spaces */ + nelmts_vs = (hsize_t)H5S_GET_SELECT_NPOINTS(vspace); + nelmts_ss = (hsize_t)H5S_GET_SELECT_NPOINTS(src_space); + + /* Check for unlimited vspace */ + if(nelmts_vs == H5S_UNLIMITED) { + /* Check for unlimited src_space */ + if(nelmts_ss == H5S_UNLIMITED) { + hsize_t nenu_vs; /* Number of elements in the non-unlimited dimensions of vspace */ + hsize_t nenu_ss; /* Number of elements in the non-unlimited dimensions of src_space */ + + /* Non-printf unlimited selection. Make sure both selections have + * the same number of elements in the non-unlimited dimension. Note + * we can always check this even if the space status is invalid + * because unlimited selections are never dependent on the extent. + */ + if(H5S_get_select_num_elem_non_unlim(vspace, &nenu_vs) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "can't get number of elements in non-unlimited dimension") + if(H5S_get_select_num_elem_non_unlim(src_space, &nenu_ss) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "can't get number of elements in non-unlimited dimension") + if(nenu_vs != nenu_ss) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "numbers of elemnts in the non-unlimited dimensions is different for source and virtual spaces") + } /* end if */ + /* We will handle the printf case after parsing the source names */ + } /* end if */ + else if(space_status != H5O_VIRTUAL_STATUS_INVALID) + /* Limited selections. Check number of points is the same. */ + if(nelmts_vs != nelmts_ss) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "virtual and source space selections have different numbers of elements") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_virtual_check_mapping_pre() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_virtual_check_mapping_post + * + * Purpose: Checks that the provided virtual dataset mapping entry is + * legal, after the mapping is otherwise complete. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * August 12, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_virtual_check_mapping_post(const H5O_storage_virtual_ent_t *ent) +{ + hsize_t nelmts_vs; /* Number of elements in virtual selection */ + hsize_t nelmts_ss; /* Number of elements in source selection */ + H5S_t *tmp_space = NULL; /* Temporary dataspace */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Get number of elements in spaces */ + nelmts_vs = (hsize_t)H5S_GET_SELECT_NPOINTS(ent->source_dset.virtual_select); + nelmts_ss = (hsize_t)H5S_GET_SELECT_NPOINTS(ent->source_select); + + /* Check for printf selection */ + if((nelmts_vs == H5S_UNLIMITED) && (nelmts_ss != H5S_UNLIMITED)) { + /* Make sure there at least one %b substitution in the source file or + * dataset name */ + if((ent->psfn_nsubs == 0) && (ent->psdn_nsubs == 0)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "unlimited virtual selection, limited source selection, and no printf specifiers in source names") + + /* Make sure virtual space uses hyperslab selection */ + if(H5S_GET_SELECT_TYPE(ent->source_dset.virtual_select) != H5S_SEL_HYPERSLABS) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "virtual selection with printf mapping must be hyperslab") + + /* Check that the number of elements in one block in the virtual + * selection matches the total number of elements in the source + * selection, if the source space status is not invalid (virtual space + * status does not matter here because it is unlimited) */ + if(ent->source_space_status != H5O_VIRTUAL_STATUS_INVALID) { + /* Get first block in virtual selection */ + if(NULL == (tmp_space = H5S_hyper_get_unlim_block(ent->source_dset.virtual_select, (hsize_t)0))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get first block in virtual selection") + + /* Check number of points */ + nelmts_vs = (hsize_t)H5S_GET_SELECT_NPOINTS(tmp_space); + if(nelmts_vs != nelmts_ss) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "virtual (single block) and source space selections have different numbers of elements") + } /* end if */ + } /* end if */ + else + /* Make sure there are no printf substitutions */ + if((ent->psfn_nsubs > 0) || (ent->psdn_nsubs > 0)) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "printf specifier(s) in source name(s) without an unlimited virtual selection and limited source selection") + +done: + /* Free temporary space */ + if(tmp_space) + if(H5S_close(tmp_space) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CLOSEERROR, FAIL, "can't close dataspace") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_virtual_check_mapping_post() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_virtual_update_min_dims + * + * Purpose: Updates the virtual layout's "min_dims" field to take into + * account the "idx"th entry in the mapping list. The entry + * must be complete, though top level field list_nused (and + * of course min_dims) does not need to take it into account. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * February 10, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_virtual_update_min_dims(H5O_layout_t *layout, size_t idx) +{ + H5S_sel_type sel_type; + int rank; + hsize_t bounds_start[H5S_MAX_RANK]; + hsize_t bounds_end[H5S_MAX_RANK]; + int i; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(layout); + HDassert(layout->type == H5D_VIRTUAL); + HDassert(idx < layout->storage.u.virt.list_nalloc); + + /* Get type of selection */ + if(H5S_SEL_ERROR == (sel_type = H5S_GET_SELECT_TYPE(layout->storage.u.virt.list[idx].source_dset.virtual_select))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection type") + + /* Do not update min_dims for "all" or "none" selections */ + if((sel_type == H5S_SEL_ALL) || (sel_type == H5S_SEL_NONE)) + HGOTO_DONE(SUCCEED) + + /* Get rank of vspace */ + if((rank = H5S_GET_EXTENT_NDIMS(layout->storage.u.virt.list[idx].source_dset.virtual_select)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions") + + /* Get selection bounds */ + if(H5S_SELECT_BOUNDS(layout->storage.u.virt.list[idx].source_dset.virtual_select, bounds_start, bounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds") + + /* Update min_dims */ + for(i = 0; i < rank; i++) + /* Don't check unlimited dimensions in the selection */ + if((i != layout->storage.u.virt.list[idx].unlim_dim_virtual) + && (bounds_end[i] >= layout->storage.u.virt.min_dims[i])) + layout->storage.u.virt.min_dims[i] = bounds_end[i] + (hsize_t)1; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_virtual_update_min_dims() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_virtual_check_min_dims + * + * Purpose: Checks if the dataset's dimensions are at least the + * calculated minimum dimensions from the mappings. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * August 13, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_virtual_check_min_dims(const H5D_t *dset) +{ + int rank; + hsize_t dims[H5S_MAX_RANK]; + int i; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + HDassert(dset); + HDassert(dset->shared); + HDassert(dset->shared->layout.type == H5D_VIRTUAL); + + /* Get rank of dataspace */ + if((rank = H5S_GET_EXTENT_NDIMS(dset->shared->space)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions") + + /* Get VDS dimensions */ + if(H5S_get_simple_extent_dims(dset->shared->space, dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get VDS dimensions") + + /* Verify that dimensions are larger than min_dims */ + for(i = 0; i < rank; i++) + if(dims[i] < dset->shared->layout.storage.u.virt.min_dims[i]) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "virtual dataset dimensions not large enough to contain all limited dimensions in all selections") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_virtual_check_min_dims() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_copy_layout + * + * Purpose: Deep copies virtual storage layout message in memory. + * This function assumes that the top-level struct has + * already been copied (so the source struct retains + * ownership of the fields passed to this function). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * February 10, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__virtual_copy_layout(H5O_layout_t *layout) +{ + H5O_storage_virtual_ent_t *orig_list = NULL; + hid_t orig_source_fapl; + hid_t orig_source_dapl; + H5P_genplist_t *plist; + size_t i; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + HDassert(layout); + HDassert(layout->type == H5D_VIRTUAL); + + /* Save original entry list and top-level property lists and reset in layout + * so the originals aren't closed on error */ + orig_source_fapl = layout->storage.u.virt.source_fapl; + layout->storage.u.virt.source_fapl = -1; + orig_source_dapl = layout->storage.u.virt.source_dapl; + layout->storage.u.virt.source_dapl = -1; + orig_list = layout->storage.u.virt.list; + layout->storage.u.virt.list = NULL; + + /* Copy entry list */ + if(layout->storage.u.virt.list_nused > 0) { + HDassert(orig_list); + + /* Allocate memory for the list */ + if(NULL == (layout->storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_calloc(layout->storage.u.virt.list_nused * sizeof(H5O_storage_virtual_ent_t)))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "unable to allocate memory for virtual dataset entry list") + layout->storage.u.virt.list_nalloc = layout->storage.u.virt.list_nused; + + /* Copy the list entries, though set source_dset.dset and sub_dset to + * NULL */ + for(i = 0; i < layout->storage.u.virt.list_nused; i++) { + /* Copy virtual selection */ + if(NULL == (layout->storage.u.virt.list[i].source_dset.virtual_select + = H5S_copy(orig_list[i].source_dset.virtual_select, FALSE, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") + + /* Copy original source names */ + if(NULL == (layout->storage.u.virt.list[i].source_file_name + = HDstrdup(orig_list[i].source_file_name))) + HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source file name") + if(NULL == (layout->storage.u.virt.list[i].source_dset_name + = HDstrdup(orig_list[i].source_dset_name))) + HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source dataset name") + + /* Copy source selection */ + if(NULL == (layout->storage.u.virt.list[i].source_select + = H5S_copy(orig_list[i].source_select, FALSE, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy source selection") + + /* Initialize clipped selections */ + if(orig_list[i].unlim_dim_virtual < 0) { + layout->storage.u.virt.list[i].source_dset.clipped_source_select = layout->storage.u.virt.list[i].source_select; + layout->storage.u.virt.list[i].source_dset.clipped_virtual_select = layout->storage.u.virt.list[i].source_dset.virtual_select; + } /* end if */ + + /* Copy parsed names */ + if(H5D__virtual_copy_parsed_name(&layout->storage.u.virt.list[i].parsed_source_file_name, orig_list[i].parsed_source_file_name) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy parsed source file name") + layout->storage.u.virt.list[i].psfn_static_strlen = orig_list[i].psfn_static_strlen; + layout->storage.u.virt.list[i].psfn_nsubs = orig_list[i].psfn_nsubs; + if(H5D__virtual_copy_parsed_name(&layout->storage.u.virt.list[i].parsed_source_dset_name, orig_list[i].parsed_source_dset_name) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy parsed source dataset name") + layout->storage.u.virt.list[i].psdn_static_strlen = orig_list[i].psdn_static_strlen; + layout->storage.u.virt.list[i].psdn_nsubs = orig_list[i].psdn_nsubs; + + /* Copy source names in source dset or add reference as appropriate + */ + if(orig_list[i].source_dset.file_name) { + if(orig_list[i].source_dset.file_name + == orig_list[i].source_file_name) + layout->storage.u.virt.list[i].source_dset.file_name = layout->storage.u.virt.list[i].source_file_name; + else if(orig_list[i].parsed_source_file_name + && (orig_list[i].source_dset.file_name + != orig_list[i].parsed_source_file_name->name_segment)) { + HDassert(layout->storage.u.virt.list[i].parsed_source_file_name); + HDassert(layout->storage.u.virt.list[i].parsed_source_file_name->name_segment); + layout->storage.u.virt.list[i].source_dset.file_name = layout->storage.u.virt.list[i].parsed_source_file_name->name_segment; + } /* end if */ + else + if(NULL == (layout->storage.u.virt.list[i].source_dset.file_name + = HDstrdup(orig_list[i].source_dset.file_name))) + HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source file name") + } /* end if */ + if(orig_list[i].source_dset.dset_name) { + if(orig_list[i].source_dset.dset_name + == orig_list[i].source_dset_name) + layout->storage.u.virt.list[i].source_dset.dset_name = layout->storage.u.virt.list[i].source_dset_name; + else if(orig_list[i].parsed_source_dset_name + && (orig_list[i].source_dset.dset_name + != orig_list[i].parsed_source_dset_name->name_segment)) { + HDassert(layout->storage.u.virt.list[i].parsed_source_dset_name); + HDassert(layout->storage.u.virt.list[i].parsed_source_dset_name->name_segment); + layout->storage.u.virt.list[i].source_dset.dset_name = layout->storage.u.virt.list[i].parsed_source_dset_name->name_segment; + } /* end if */ + else + if(NULL == (layout->storage.u.virt.list[i].source_dset.dset_name + = HDstrdup(orig_list[i].source_dset.dset_name))) + HGOTO_ERROR(H5E_DATASET, H5E_RESOURCE, FAIL, "unable to duplicate source dataset name") + } /* end if */ + + /* Copy other fields in entry */ + layout->storage.u.virt.list[i].unlim_dim_source = orig_list[i].unlim_dim_source; + layout->storage.u.virt.list[i].unlim_dim_virtual = orig_list[i].unlim_dim_virtual; + layout->storage.u.virt.list[i].unlim_extent_source = orig_list[i].unlim_extent_source; + layout->storage.u.virt.list[i].unlim_extent_virtual = orig_list[i].unlim_extent_virtual; + layout->storage.u.virt.list[i].clip_size_source = orig_list[i].clip_size_source; + layout->storage.u.virt.list[i].clip_size_virtual = orig_list[i].clip_size_virtual; + layout->storage.u.virt.list[i].source_space_status = orig_list[i].source_space_status; + layout->storage.u.virt.list[i].virtual_space_status = orig_list[i].virtual_space_status; + } /* end for */ + } /* end if */ + else { + /* Zero out other fields related to list, just to be sure */ + layout->storage.u.virt.list = NULL; + layout->storage.u.virt.list_nalloc = 0; + } /* end else */ + + /* Copy property lists */ + if(orig_source_fapl >= 0) { + if(NULL == (plist = (H5P_genplist_t *)H5I_object_verify(orig_source_fapl, H5I_GENPROP_LST))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") + if((layout->storage.u.virt.source_fapl = H5P_copy_plist(plist, FALSE)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy fapl") + } /* end if */ + if(orig_source_dapl >= 0) { + if(NULL == (plist = (H5P_genplist_t *)H5I_object_verify(orig_source_dapl, H5I_GENPROP_LST))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a property list") + if((layout->storage.u.virt.source_dapl = H5P_copy_plist(plist, FALSE)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy dapl") + } /* end if */ + + /* New layout is not fully initialized */ + layout->storage.u.virt.init = FALSE; + +done: + /* Release allocated resources on failure */ + if(ret_value < 0) + if(H5D__virtual_reset_layout(layout) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to reset virtual layout") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_copy_layout() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_reset_layout + * + * Purpose: Frees internal structures in a virtual storage layout + * message in memory. This function is safe to use on + * incomplete structures (for recovery from failure) provided + * the internal structures are initialized with all bytes set + * to 0. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * February 11, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__virtual_reset_layout(H5O_layout_t *layout) +{ + size_t i, j; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + HDassert(layout); + HDassert(layout->type == H5D_VIRTUAL); + + /* Free the list entries. Note we always attempt to free everything even in + * the case of a failure. Because of this, and because we free the list + * afterwards, we do not need to zero out the memory in the list. */ + for(i = 0; i < layout->storage.u.virt.list_nused; i++) { + /* Free source_dset */ + if(H5D__virtual_reset_source_dset(&layout->storage.u.virt.list[i], &layout->storage.u.virt.list[i].source_dset) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to reset source dataset") + + /* Free original source names */ + (void)H5MM_xfree(layout->storage.u.virt.list[i].source_file_name); + (void)H5MM_xfree(layout->storage.u.virt.list[i].source_dset_name); + + /* Free sub_dset */ + for(j = 0; j < layout->storage.u.virt.list[i].sub_dset_nalloc; j++) + if(H5D__virtual_reset_source_dset(&layout->storage.u.virt.list[i], &layout->storage.u.virt.list[i].sub_dset[j]) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to reset source dataset") + layout->storage.u.virt.list[i].sub_dset = (H5O_storage_virtual_srcdset_t *)H5MM_xfree(layout->storage.u.virt.list[i].sub_dset); + + /* Free source_select */ + if(layout->storage.u.virt.list[i].source_select) + if(H5S_close(layout->storage.u.virt.list[i].source_select) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release source selection") + + /* Free parsed_source_file_name */ + H5D_virtual_free_parsed_name(layout->storage.u.virt.list[i].parsed_source_file_name); + + /* Free parsed_source_dset_name */ + H5D_virtual_free_parsed_name(layout->storage.u.virt.list[i].parsed_source_dset_name); + } /* end for */ + + /* Free the list */ + layout->storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_xfree(layout->storage.u.virt.list); + layout->storage.u.virt.list_nalloc = (size_t)0; + layout->storage.u.virt.list_nused = (size_t)0; + (void)HDmemset(layout->storage.u.virt.min_dims, 0, sizeof(layout->storage.u.virt.min_dims)); + + /* Close access property lists */ + if(layout->storage.u.virt.source_fapl >= 0) { + if(H5I_dec_ref(layout->storage.u.virt.source_fapl) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't close source fapl") + layout->storage.u.virt.source_fapl = -1; + } /* end if */ + if(layout->storage.u.virt.source_dapl >= 0) { + if(H5I_dec_ref(layout->storage.u.virt.source_dapl) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "can't close source dapl") + layout->storage.u.virt.source_dapl = -1; + } /* end if */ + + /* The list is no longer initialized */ + layout->storage.u.virt.init = FALSE; + + /* Note the lack of a done: label. This is because there are no HGOTO_ERROR + * calls. If one is added, a done: label must also be added */ + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_reset_layout() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_copy + * + * Purpose: Copy virtual storage raw data from SRC file to DST file. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * February 6, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__virtual_copy(H5F_t H5_ATTR_UNUSED *f_dst, H5O_layout_t *layout_dst, + hid_t H5_ATTR_UNUSED dxpl_id) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Copy message in memory */ + if(H5D__virtual_copy_layout(layout_dst) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual layout") + +#ifdef NOT_YET + /* Check for copy to the same file */ + if(f_dst == f_src) { + /* Increase reference count on global heap object */ + if((heap_rc = H5HG_link(f_dst, dxpl_id, (H5HG_t *)&(layout_dst->u.virt.serial_list_hobjid), 1)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTMODIFY, FAIL, "unable to adjust global heap refence count") + } /* end if */ + else +#endif /* NOT_YET */ + { + /* Reset global heap id so a new heap object is created when the message + * is flushed */ + layout_dst->storage.u.virt.serial_list_hobjid.addr = HADDR_UNDEF; + layout_dst->storage.u.virt.serial_list_hobjid.idx = (size_t)0; + } /* end block/else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_copy() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_delete + * + * Purpose: Delete the file space for a virtual dataset + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * February 6, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__virtual_delete(H5F_t *f, hid_t dxpl_id, H5O_storage_t *storage) +{ +#ifdef NOT_YET + int heap_rc; /* Reference count of global heap object */ +#endif /* NOT_YET */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* check args */ + HDassert(f); + HDassert(storage); + HDassert(storage->type == H5D_VIRTUAL); + + /* Check for global heap block */ + if(storage->u.virt.serial_list_hobjid.addr != HADDR_UNDEF) { +#ifdef NOT_YET + /* Unlink the global heap block */ + if((heap_rc = H5HG_link(f, dxpl_id, (H5HG_t *)&(storage->u.virt.serial_list_hobjid), -1)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTMODIFY, FAIL, "unable to adjust global heap refence count") + if(heap_rc == 0) +#endif /* NOT_YET */ + /* Delete the global heap block */ + if(H5HG_remove(f, dxpl_id, (H5HG_t *)&(storage->u.virt.serial_list_hobjid)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTREMOVE, FAIL, "unable to remove heap object") + } /* end if */ + + /* Clear global heap ID in storage */ + storage->u.virt.serial_list_hobjid.addr = HADDR_UNDEF; + storage->u.virt.serial_list_hobjid.idx = 0; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_delete */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_open_source_dset + * + * Purpose: Attempts to open a source dataset. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * March 6, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_open_source_dset(const H5D_t *vdset, + H5O_storage_virtual_ent_t *virtual_ent, + H5O_storage_virtual_srcdset_t *source_dset, hid_t dxpl_id) +{ + H5F_t *src_file = NULL; /* Source file */ + hbool_t src_file_open = FALSE; /* Whether we have opened and need to close src_file */ + H5G_loc_t src_root_loc; /* Object location of source file root group */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(vdset); + HDassert(source_dset); + HDassert(!source_dset->dset); + HDassert(source_dset->file_name); + HDassert(source_dset->dset_name); + + /* Check if we need to open the source file */ + if(HDstrcmp(source_dset->file_name, ".")) { + /* Open the source file */ + if(NULL == (src_file = H5F_open(source_dset->file_name, H5F_INTENT(vdset->oloc.file) & H5F_ACC_RDWR, H5P_FILE_CREATE_DEFAULT, vdset->shared->layout.storage.u.virt.source_fapl, dxpl_id))) + H5E_clear_stack(NULL); /* Quick hack until proper support for H5Fopen with missing file is implemented */ + else + src_file_open = TRUE; + } /* end if */ + else + /* Source file is ".", use the virtual dataset's file */ + src_file = vdset->oloc.file; + + if(src_file) { + /* Set up the root group in the destination file */ + if(NULL == (src_root_loc.oloc = H5G_oloc(H5G_rootof(src_file)))) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "unable to get object location for root group") + if(NULL == (src_root_loc.path = H5G_nameof(H5G_rootof(src_file)))) + HGOTO_ERROR(H5E_DATASET, H5E_BADVALUE, FAIL, "unable to get path for root group") + + /* Open the source dataset */ + if(NULL == (source_dset->dset = H5D__open_name(&src_root_loc, source_dset->dset_name, vdset->shared->layout.storage.u.virt.source_dapl, dxpl_id))) { + H5E_clear_stack(NULL); /* Quick hack until proper support for H5Dopen with missing file is implemented */ + + /* Dataset does not exist */ + source_dset->dset_exists = FALSE; + } /* end if */ + else { + /* Dataset exists */ + source_dset->dset_exists = TRUE; + + /* Patch the source selection if necessary */ + if(virtual_ent->source_space_status != H5O_VIRTUAL_STATUS_CORRECT) { + if(H5S_extent_copy(virtual_ent->source_select, source_dset->dset->shared->space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy source dataspace extent") + virtual_ent->source_space_status = H5O_VIRTUAL_STATUS_CORRECT; + } /* end if */ + } /* end else */ + } /* end if */ + +done: + /* Close source file */ + if(src_file_open) + if(H5F_try_close(src_file) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTCLOSEFILE, FAIL, "can't close source file") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_open_source_dset() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_reset_source_dset + * + * Purpose: Frees space referenced by a source dataset struct. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 20, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_reset_source_dset(H5O_storage_virtual_ent_t *virtual_ent, + H5O_storage_virtual_srcdset_t *source_dset) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(source_dset); + + /* Free dataset */ + if(source_dset->dset) { + if(H5D_close(source_dset->dset) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to close source dataset") + source_dset->dset = NULL; + } /* end if */ + + /* Free file name */ + if(virtual_ent->parsed_source_file_name + && (source_dset->file_name + != virtual_ent->parsed_source_file_name->name_segment)) + source_dset->file_name = (char *)H5MM_xfree(source_dset->file_name); + else + HDassert((source_dset->file_name == virtual_ent->source_file_name) + || (virtual_ent->parsed_source_file_name + && (source_dset->file_name + == virtual_ent->parsed_source_file_name->name_segment)) + || !source_dset->file_name); + + /* Free dataset name */ + if(virtual_ent->parsed_source_dset_name + && (source_dset->dset_name + != virtual_ent->parsed_source_dset_name->name_segment)) + source_dset->dset_name = (char *)H5MM_xfree(source_dset->dset_name); + else + HDassert((source_dset->dset_name == virtual_ent->source_dset_name) + || (virtual_ent->parsed_source_dset_name + && (source_dset->dset_name + == virtual_ent->parsed_source_dset_name->name_segment)) + || !source_dset->dset_name); + + /* Free clipped virtual selection */ + if(source_dset->clipped_virtual_select) { + if(source_dset->clipped_virtual_select != source_dset->virtual_select) + if(H5S_close(source_dset->clipped_virtual_select) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release clipped virtual selection") + source_dset->clipped_virtual_select = NULL; + } /* end if */ + + /* Free virtual selection */ + if(source_dset->virtual_select) { + if(H5S_close(source_dset->virtual_select) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release virtual selection") + source_dset->virtual_select = NULL; + } /* end if */ + + /* Free clipped source selection */ + if(source_dset->clipped_source_select) { + if(source_dset->clipped_source_select != virtual_ent->source_select) + if(H5S_close(source_dset->clipped_source_select) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release clipped source selection") + source_dset->clipped_source_select = NULL; + } /* end if */ + + /* The projected memory space should never exist when this function is + * called */ + HDassert(!source_dset->projected_mem_space); + + /* Note the lack of a done: label. This is because there are no HGOTO_ERROR + * calls. If one is added, a done: label must also be added */ + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_reset_source_dset() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_str_append + * + * Purpose: Appends src_len bytes of the string src to the position *p + * in the buffer *buf (allocating *buf if necessary). + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 19, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_str_append(const char *src, size_t src_len, char **p, char **buf, + size_t *buf_size) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(src); + HDassert(src_len > 0); + HDassert(p); + HDassert(buf); + HDassert(*p >= *buf); + HDassert(buf_size); + + /* Allocate or extend buffer if necessary */ + if(!*buf) { + HDassert(!*p); + HDassert(*buf_size == 0); + + /* Allocate buffer */ + if(NULL == (*buf = (char *)H5MM_malloc(src_len + (size_t)1))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate name segment struct") + *buf_size = src_len + (size_t)1; + *p = *buf; + } /* end if */ + else { + size_t p_offset = (size_t)(*p - *buf); /* Offset of p within buf */ + + /* Extend buffer if necessary */ + if((p_offset + src_len + (size_t)1) > *buf_size) { + char *tmp_buf; + size_t tmp_buf_size; + + /* Calculate new size of buffer */ + tmp_buf_size = MAX(p_offset + src_len + (size_t)1, + *buf_size * (size_t)2); + + /* Reallocate buffer */ + if(NULL == (tmp_buf = (char *)H5MM_realloc(*buf, tmp_buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to reallocate name segment buffer") + *buf = tmp_buf; + *buf_size = tmp_buf_size; + *p = *buf + p_offset; + } /* end if */ + } /* end else */ + + /* Copy string to *p. Note that since src in not NULL terminated, we must + * use memcpy */ + (void)HDmemcpy(*p, src, src_len); + + /* Advance *p */ + *p += src_len; + + /* Add NULL terminator */ + **p = '\0'; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5D__virtual_str_append() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_virtual_parse_source_name + * + * Purpose: Parses a source file or dataset name. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 18, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_virtual_parse_source_name(const char *source_name, + H5O_storage_virtual_name_seg_t **parsed_name, size_t *static_strlen, + size_t *nsubs) +{ + H5O_storage_virtual_name_seg_t *tmp_parsed_name = NULL; + H5O_storage_virtual_name_seg_t **tmp_parsed_name_p = &tmp_parsed_name; + size_t tmp_static_strlen; + size_t tmp_strlen; + size_t tmp_nsubs = 0; + const char *p; + const char *pct; + char *name_seg_p = NULL; + size_t name_seg_size = 0; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(source_name); + HDassert(parsed_name); + HDassert(static_strlen); + HDassert(nsubs); + + /* Initialize p and tmp_static_strlen */ + p = source_name; + tmp_static_strlen = tmp_strlen = HDstrlen(source_name); + + /* Iterate over name */ + /* Note this will not work with UTF-8! We should support this eventually + * -NAF 5/18/2015 */ + while((pct = HDstrchr(p, '%'))) { + HDassert(pct >= p); + + /* Allocate name segment struct if necessary */ + if(!*tmp_parsed_name_p) + if(NULL == (*tmp_parsed_name_p = H5FL_CALLOC(H5O_storage_virtual_name_seg_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate name segment struct") + + /* Check for type of format specifier */ + if(pct[1] == 'b') { + /* Check for blank string before specifier */ + if(pct != p) + /* Append string to name segment */ + if(H5D__virtual_str_append(p, (size_t)(pct - p), &name_seg_p, &(*tmp_parsed_name_p)->name_segment, &name_seg_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to append name segment") + + /* Update other variables */ + tmp_parsed_name_p = &(*tmp_parsed_name_p)->next; + tmp_static_strlen -= 2; + tmp_nsubs++; + name_seg_p = NULL; + name_seg_size = 0; + } /* end if */ + else if(pct[1] == '%') { + /* Append string to name segment (include first '%') */ + if(H5D__virtual_str_append(p, (size_t)(pct - p) + (size_t)1, &name_seg_p, &(*tmp_parsed_name_p)->name_segment, &name_seg_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to append name segment") + + /* Update other variables */ + tmp_static_strlen -= 1; + } /* end else */ + else + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid format specifier") + + p = pct + 2; + } /* end while */ + + /* Copy last segment of name, if any, unless the parsed name was not + * allocated */ + if(tmp_parsed_name) { + HDassert(p >= source_name); + if(*p == '\0') + HDassert((size_t)(p - source_name) == tmp_strlen); + else { + HDassert((size_t)(p - source_name) < tmp_strlen); + + /* Allocate name segment struct if necessary */ + if(!*tmp_parsed_name_p) + if(NULL == (*tmp_parsed_name_p = H5FL_CALLOC(H5O_storage_virtual_name_seg_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate name segment struct") + + /* Append string to name segment */ + if(H5D__virtual_str_append(p, tmp_strlen - (size_t)(p - source_name), &name_seg_p, &(*tmp_parsed_name_p)->name_segment, &name_seg_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to append name segment") + } /* end else */ + } /* end if */ + + /* Set return values */ + *parsed_name = tmp_parsed_name; + tmp_parsed_name = NULL; + *static_strlen = tmp_static_strlen; + *nsubs = tmp_nsubs; + +done: + if(tmp_parsed_name) { + HDassert(ret_value < 0); + H5D_virtual_free_parsed_name(tmp_parsed_name); + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_virtual_parse_source_name() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_copy_parsed_name + * + * Purpose: Deep copies a parsed source file or dataset name. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 19, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_copy_parsed_name(H5O_storage_virtual_name_seg_t **dst, + H5O_storage_virtual_name_seg_t *src) +{ + H5O_storage_virtual_name_seg_t *tmp_dst = NULL; + H5O_storage_virtual_name_seg_t *p_src = src; + H5O_storage_virtual_name_seg_t **p_dst = &tmp_dst; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(dst); + + /* Walk over parsed name, duplicating it */ + while(p_src) { + /* Allocate name segment struct */ + if(NULL == (*p_dst = H5FL_CALLOC(H5O_storage_virtual_name_seg_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate name segment struct") + + /* Duplicate name segment */ + if(p_src->name_segment) { + if(NULL == ((*p_dst)->name_segment = HDstrdup(p_src->name_segment))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to duplicate name segment") + } /* end if */ + + /* Advance pointers */ + p_src = p_src->next; + p_dst = &(*p_dst)->next; + } /* end while */ + + /* Set dst */ + *dst = tmp_dst; + tmp_dst = NULL; + +done: + if(tmp_dst) { + HDassert(ret_value < 0); + H5D_virtual_free_parsed_name(tmp_dst); + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_copy_parsed_name() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_virtual_free_parsed_name + * + * Purpose: Frees the provided parsed name. + * + * Return: void + * + * Programmer: Neil Fortner + * May 19, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_virtual_free_parsed_name(H5O_storage_virtual_name_seg_t *name_seg) +{ + H5O_storage_virtual_name_seg_t *next_seg; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Walk name segments, freeing them */ + while(name_seg) { + (void)H5MM_xfree(name_seg->name_segment); + next_seg = name_seg->next; + (void)H5FL_FREE(H5O_storage_virtual_name_seg_t, name_seg); + name_seg = next_seg; + } /* end while */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_virtual_free_parsed_name() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_build_source_name + * + * Purpose: Builds a source file or dataset name from a parsed name. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 18, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_build_source_name(char *source_name, + const H5O_storage_virtual_name_seg_t *parsed_name, size_t static_strlen, + size_t nsubs, hsize_t blockno, char **built_name) +{ + char *tmp_name = NULL; /* Name buffer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(source_name); + HDassert(built_name); + + /* Check for static name */ + if(nsubs == 0) { + if(parsed_name) + *built_name = parsed_name->name_segment; + else + *built_name = source_name; + } /* end if */ + else { + const H5O_storage_virtual_name_seg_t *name_seg = parsed_name; + char *p; + hsize_t blockno_down = blockno; + size_t blockno_len = 1; + size_t name_len; + size_t name_len_rem; + size_t seg_len; + size_t nsubs_rem = nsubs; + + HDassert(parsed_name); + + /* Calculate length of printed block number */ + do { + blockno_down /= (hsize_t)10; + if(blockno_down == 0) + break; + blockno_len++; + } while(1); + + /* Calculate length of name buffer */ + name_len_rem = name_len = static_strlen + (nsubs * blockno_len) + (size_t)1; + + /* Allocate name buffer */ + if(NULL == (tmp_name = (char *)H5MM_malloc(name_len))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate name buffer") + p = tmp_name; + + /* Build name */ + do { + /* Add name segment */ + if(name_seg->name_segment) { + seg_len = HDstrlen(name_seg->name_segment); + HDassert(seg_len > 0); + HDassert(seg_len < name_len_rem); + HDstrncpy(p, name_seg->name_segment, name_len_rem); + name_len_rem -= seg_len; + p += seg_len; + } /* end if */ + + /* Add block number */ + if(nsubs_rem > 0) { + HDassert(blockno_len < name_len_rem); + if(HDsnprintf(p, name_len_rem, "%llu", (long long unsigned)blockno) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write block number to string") + name_len_rem -= blockno_len; + p += blockno_len; + nsubs_rem--; + } /* end if */ + + /* Advance name_seg */ + name_seg = name_seg->next; + } while(name_seg); + + /* Assign built_name */ + *built_name = tmp_name; + tmp_name = NULL; + } /* end else */ + +done: + if(tmp_name) { + HDassert(ret_value < 0); + H5MM_free(tmp_name); + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_build_source_name() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_set_extent_unlim + * + * Purpose: Sets the extent of the virtual dataset by checking the + * extents of source datasets where an unlimited selection + * matching. Dimensions that are not unlimited in any + * virtual mapping selections are not affected. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * April 22, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__virtual_set_extent_unlim(const H5D_t *dset, hid_t dxpl_id) +{ + H5O_storage_virtual_t *storage; + hsize_t new_dims[H5S_MAX_RANK]; + hsize_t curr_dims[H5S_MAX_RANK]; + hsize_t clip_size; + int rank; + hbool_t changed = FALSE; /* Whether the VDS extent changed */ + size_t i, j; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(dset); + HDassert(dset->shared->layout.storage.type == H5D_VIRTUAL); + storage = &dset->shared->layout.storage.u.virt; + HDassert((storage->view == H5D_VDS_FIRST_MISSING) || (storage->view == H5D_VDS_LAST_AVAILABLE)); + + /* Get rank of VDS */ + if((rank = H5S_GET_EXTENT_NDIMS(dset->shared->space)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions") + + /* Initialize new_dims to HSIZE_UNDEF */ + for(i = 0; i < (size_t)rank; i++) + new_dims[i] = HSIZE_UNDEF; + + /* Iterate over mappings */ + for(i = 0; i < storage->list_nalloc; i++) + /* Check for unlimited dimension */ + if(storage->list[i].unlim_dim_virtual >= 0) { + /* Check for "printf" source dataset resolution */ + if(storage->list[i].unlim_dim_source >= 0 ) { + /* Non-printf mapping */ + /* Open source dataset */ + if(!storage->list[i].source_dset.dset) + if(H5D__virtual_open_source_dset(dset, &storage->list[i], &storage->list[i].source_dset, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") + + /* Check if source dataset is open */ + if(storage->list[i].source_dset.dset) { + /* Retrieve current source dataset extent and patch mapping + */ + if(H5S_extent_copy(storage->list[i].source_select, storage->list[i].source_dset.dset->shared->space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy source dataspace extent") + + /* Get source space dimenstions */ + if(H5S_get_simple_extent_dims(storage->list[i].source_select, curr_dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get source space dimensions") + + /* Check if the source extent in the unlimited dimension + * changed since the last time the VDS extent/mapping + * was updated */ + if(curr_dims[storage->list[i].unlim_dim_source] + == storage->list[i].unlim_extent_source) + /* Use cached result for clip size */ + clip_size = storage->list[i].clip_size_virtual; + else { + /* Get size that virtual selection would be clipped to + * to match size of source selection within source + * extent */ + clip_size = H5S_hyper_get_clip_extent_match(storage->list[i].source_dset.virtual_select, storage->list[i].source_select, curr_dims[storage->list[i].unlim_dim_source], storage->view == H5D_VDS_FIRST_MISSING); + + /* If we are setting the extent by the last available + * data, clip virtual_select and source_select. Note + * that if we used the cached clip_size above or it + * happens to be the same, the virtual selection will + * already be clipped to the correct size. Likewise, + * if we used the cached clip_size the source selection + * will already be correct. */ + if(storage->view == H5D_VDS_LAST_AVAILABLE) { + if(clip_size != storage->list[i].clip_size_virtual) { + /* Close previous clipped virtual selection, if + * any */ + if(storage->list[i].source_dset.clipped_virtual_select) { + HDassert(storage->list[i].source_dset.clipped_virtual_select + != storage->list[i].source_dset.virtual_select); + if(H5S_close(storage->list[i].source_dset.clipped_virtual_select) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release clipped virtual dataspace") + } /* end if */ + + /* Copy virtual selection */ + if(NULL == (storage->list[i].source_dset.clipped_virtual_select = H5S_copy(storage->list[i].source_dset.virtual_select, FALSE, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") + + /* Clip virtual selection */ + if(H5S_hyper_clip_unlim(storage->list[i].source_dset.clipped_virtual_select, clip_size)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + } /* end if */ + + /* Close previous clipped source selection, if any + */ + if(storage->list[i].source_dset.clipped_source_select) { + HDassert(storage->list[i].source_dset.clipped_source_select + != storage->list[i].source_select); + if(H5S_close(storage->list[i].source_dset.clipped_source_select) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release clipped source dataspace") + } /* end if */ + + /* Copy source selection */ + if(NULL == (storage->list[i].source_dset.clipped_source_select = H5S_copy(storage->list[i].source_select, FALSE, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy source selection") + + /* Clip source selection */ + if(H5S_hyper_clip_unlim(storage->list[i].source_dset.clipped_source_select, curr_dims[storage->list[i].unlim_dim_source])) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + } /* end if */ + + /* Update cached values unlim_extent_source and + * clip_size_virtual */ + storage->list[i].unlim_extent_source = curr_dims[storage->list[i].unlim_dim_source]; + storage->list[i].clip_size_virtual = clip_size; + } /* end else */ + } /* end if */ + else + clip_size = 0; + } /* end if */ + else { + /* printf mapping */ + hsize_t first_missing = 0; /* First missing dataset in the current block of missing datasets */ + + /* Search for source datasets */ + HDassert(storage->printf_gap != HSIZE_UNDEF); + for(j = 0; j <= (storage->printf_gap + first_missing); j++) { + /* Check for running out of space in sub_dset array */ + if(j >= (hsize_t)storage->list[i].sub_dset_nalloc) { + if(storage->list[i].sub_dset_nalloc == 0) { + /* Allocate sub_dset */ + if(NULL == (storage->list[i].sub_dset = (H5O_storage_virtual_srcdset_t *)H5MM_calloc(H5D_VIRTUAL_DEF_SUB_DSET_SIZE * sizeof(H5O_storage_virtual_srcdset_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate sub dataset array") + storage->list[i].sub_dset_nalloc = H5D_VIRTUAL_DEF_SUB_DSET_SIZE; + } /* end if */ + else { + H5O_storage_virtual_srcdset_t *tmp_sub_dset; + + /* Extend sub_dset */ + if(NULL == (tmp_sub_dset = (H5O_storage_virtual_srcdset_t *)H5MM_realloc(storage->list[i].sub_dset, 2 * storage->list[i].sub_dset_nalloc * sizeof(H5O_storage_virtual_srcdset_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to extend sub dataset array") + storage->list[i].sub_dset = tmp_sub_dset; + + /* Clear new space in sub_dset */ + (void)HDmemset(&storage->list[i].sub_dset[storage->list[i].sub_dset_nalloc], 0, storage->list[i].sub_dset_nalloc * sizeof(H5O_storage_virtual_srcdset_t)); + + /* Update sub_dset_nalloc */ + storage->list[i].sub_dset_nalloc *= 2; + } /* end else */ + } /* end if */ + + /* Check if the dataset was already opened */ + if(storage->list[i].sub_dset[j].dset_exists) + first_missing = j + 1; + else { + /* Resolve file name */ + if(!storage->list[i].sub_dset[j].file_name) + if(H5D__virtual_build_source_name(storage->list[i].source_file_name, storage->list[i].parsed_source_file_name, storage->list[i].psfn_static_strlen, storage->list[i].psfn_nsubs, j, &storage->list[i].sub_dset[j].file_name) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to build source file name") + + /* Resolve dset name */ + if(!storage->list[i].sub_dset[j].dset_name) + if(H5D__virtual_build_source_name(storage->list[i].source_dset_name, storage->list[i].parsed_source_dset_name, storage->list[i].psdn_static_strlen, storage->list[i].psdn_nsubs, j, &storage->list[i].sub_dset[j].dset_name) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to build source dataset name") + + /* Resolve virtual selection for block */ + if(!storage->list[i].sub_dset[j].virtual_select) + if(NULL == (storage->list[i].sub_dset[j].virtual_select = H5S_hyper_get_unlim_block(storage->list[i].source_dset.virtual_select, j))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get block in unlimited selection") + + /* Initialize clipped selections */ + if(!storage->list[i].sub_dset[j].clipped_source_select) + storage->list[i].sub_dset[j].clipped_source_select = storage->list[i].source_select; + if(!storage->list[i].sub_dset[j].clipped_virtual_select) + storage->list[i].sub_dset[j].clipped_virtual_select = storage->list[i].sub_dset[j].virtual_select; + + /* Open source dataset */ + if(H5D__virtual_open_source_dset(dset, &storage->list[i], &storage->list[i].sub_dset[j], dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") + + if(storage->list[i].sub_dset[j].dset) { + /* Update first_missing */ + first_missing = j + 1; + + /* Close source dataset so we don't have huge + * numbers of datasets open */ + if(H5D_close(storage->list[i].sub_dset[j].dset) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to close source dataset") + storage->list[i].sub_dset[j].dset = NULL; + } /* end if */ + } /* end else */ + } /* end for */ + + /* Check if the size changed */ + if((first_missing == (hsize_t)storage->list[i].sub_dset_nused) + && (storage->list[i].clip_size_virtual != HSIZE_UNDEF)) + /* Use cached clip_size */ + clip_size = storage->list[i].clip_size_virtual; + else { + /* Check for no datasets */ + if(first_missing == 0) + /* Set clip size to 0 */ + clip_size = (hsize_t)0; + else { + hsize_t bounds_start[H5S_MAX_RANK]; + hsize_t bounds_end[H5S_MAX_RANK]; + + /* Get clip size from selection */ + if(storage->view == H5D_VDS_LAST_AVAILABLE) { + /* Get bounds from last valid virtual selection */ + if(H5S_SELECT_BOUNDS(storage->list[i].sub_dset[first_missing - (hsize_t)1].virtual_select, bounds_start, bounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds") + + /* Set clip_size to bounds_end in unlimited + * dimension */ + clip_size = bounds_end[storage->list[i].unlim_dim_virtual] + (hsize_t)1; + } /* end if */ + else { + /* Get bounds from first missing virtual selection + */ + if(H5S_SELECT_BOUNDS(storage->list[i].sub_dset[first_missing].virtual_select, bounds_start, bounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds") + + /* Set clip_size to bounds_start in unlimited + * dimension */ + clip_size = bounds_start[storage->list[i].unlim_dim_virtual]; + } /* end else */ + } /* end else */ + + /* Set sub_dset_nused and clip_size_virtual */ + storage->list[i].sub_dset_nused = (size_t)first_missing; + storage->list[i].clip_size_virtual = clip_size; + } /* end else */ + } /* end else */ + + /* Update new_dims */ + if((new_dims[storage->list[i].unlim_dim_virtual] == HSIZE_UNDEF) + || (storage->view == H5D_VDS_FIRST_MISSING ? (clip_size < (hsize_t)new_dims[storage->list[i].unlim_dim_virtual]) + : (clip_size > (hsize_t)new_dims[storage->list[i].unlim_dim_virtual]))) + new_dims[storage->list[i].unlim_dim_virtual] = clip_size; + } /* end if */ + + /* Get current VDS dimensions */ + if(H5S_get_simple_extent_dims(dset->shared->space, curr_dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get VDS dimensions") + + /* Calculate new extent */ + for(i = 0; i < (size_t)rank; i++) { + if(new_dims[i] == HSIZE_UNDEF) + new_dims[i] = curr_dims[i]; + else if(new_dims[i] < storage->min_dims[i]) + new_dims[i] = storage->min_dims[i]; + if(new_dims[i] != curr_dims[i]) + changed = TRUE; + } /* end for */ + + /* Update extent if it changed */ + if(changed) { + /* Update VDS extent */ + if(H5S_set_extent(dset->shared->space, new_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") + + /* Mark the space as dirty, for later writing to the file */ + if(H5F_INTENT(dset->oloc.file) & H5F_ACC_RDWR) + if(H5D__mark(dset, dxpl_id, H5D_MARK_SPACE) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to mark dataspace as dirty") + } /* end if */ + + /* If we did not change the VDS dimensions, there is nothing more to update + */ + if(changed || (!storage->init && (storage->view == H5D_VDS_FIRST_MISSING))) { + /* Iterate over mappings again to update source selections and virtual + * mapping extents */ + for(i = 0; i < storage->list_nalloc; i++) { + /* If there is an unlimited dimension, we are setting extent by the + * minimum of mappings, and the virtual extent in the unlimited + * dimension has changed since the last time the VDS extent/mapping + * was updated, we must adjust the selections */ + if((storage->list[i].unlim_dim_virtual >= 0) + && (storage->view == H5D_VDS_FIRST_MISSING) + && (new_dims[storage->list[i].unlim_dim_virtual] + != storage->list[i].unlim_extent_virtual)) { + /* Check for "printf" style mapping */ + if(storage->list[i].unlim_dim_source >= 0) { + /* Non-printf mapping */ + /* Close previous clipped virtual selection, if any */ + if(storage->list[i].source_dset.clipped_virtual_select) { + HDassert(storage->list[i].source_dset.clipped_virtual_select + != storage->list[i].source_dset.virtual_select); + if(H5S_close(storage->list[i].source_dset.clipped_virtual_select) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release clipped virtual dataspace") + } /* end if */ + + /* Copy virtual selection */ + if(NULL == (storage->list[i].source_dset.clipped_virtual_select = H5S_copy(storage->list[i].source_dset.virtual_select, FALSE, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") + + /* Clip space to virtual extent */ + if(H5S_hyper_clip_unlim(storage->list[i].source_dset.clipped_virtual_select, new_dims[storage->list[i].unlim_dim_source])) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + + /* Get size that source selection will be clipped to to + * match size of virtual selection */ + clip_size = H5S_hyper_get_clip_extent(storage->list[i].source_select, storage->list[i].source_dset.clipped_virtual_select, FALSE); + + /* Check if the clip size changed */ + if(clip_size != storage->list[i].clip_size_source) { + /* Close previous clipped source selection, if any */ + if(storage->list[i].source_dset.clipped_source_select) { + HDassert(storage->list[i].source_dset.clipped_source_select + != storage->list[i].source_select); + if(H5S_close(storage->list[i].source_dset.clipped_source_select) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release clipped source dataspace") + } /* end if */ + + /* Copy source selection */ + if(NULL == (storage->list[i].source_dset.clipped_source_select = H5S_copy(storage->list[i].source_select, FALSE, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy source selection") + + /* Clip source selection */ + if(H5S_hyper_clip_unlim(storage->list[i].source_dset.clipped_source_select, clip_size)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + + /* Update cached value clip_size_source */ + storage->list[i].clip_size_source = clip_size; + } /* end if */ + } /* end if */ + else { + /* printf mapping */ + hsize_t first_inc_block; + hbool_t partial_block; + + /* Get index of first incomplete block in virtual + * selection */ + first_inc_block = H5S_hyper_get_first_inc_block(storage->list[i].source_dset.virtual_select, new_dims[storage->list[i].unlim_dim_virtual], &partial_block); + + /* Iterate over sub datasets */ + for(j = 0; j < storage->list[i].sub_dset_nalloc; j++) { + /* Close previous clipped source selection, if any */ + if(storage->list[i].sub_dset[j].clipped_source_select + != storage->list[i].source_select) { + if(storage->list[i].sub_dset[j].clipped_source_select) + if(H5S_close(storage->list[i].sub_dset[j].clipped_source_select) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release clipped source dataspace") + + /* Initialize clipped source selection to point to + * base source selection */ + storage->list[i].sub_dset[j].clipped_source_select = storage->list[i].source_select; + } /* end if */ + + /* Close previous clipped virtual selection, if any */ + if(storage->list[i].sub_dset[j].clipped_virtual_select + != storage->list[i].sub_dset[j].virtual_select) { + if(storage->list[i].sub_dset[j].clipped_virtual_select) + if(H5S_close(storage->list[i].sub_dset[j].clipped_virtual_select) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release clipped virtual dataspace") + + /* Initialize clipped virtual selection to point to + * unclipped virtual selection */ + storage->list[i].sub_dset[j].clipped_virtual_select = storage->list[i].sub_dset[j].virtual_select; + } /* end if */ + + /* Only initialize clipped selections if it is a + * complete block, for incomplete blocks defer to + * H5D__virtual_pre_io() as we may not have a valid + * source extent here. For unused blocks we will never + * need clipped selections (until the extent is + * recalculated in this function). */ + if(j >= (size_t)first_inc_block) { + /* Clear clipped source and virtual selections */ + storage->list[i].sub_dset[j].clipped_source_select = NULL; + storage->list[i].sub_dset[j].clipped_virtual_select = NULL; + } /* end if */ + } /* end for */ + } /* end else */ + + /* Update cached value unlim_extent_virtual */ + storage->list[i].unlim_extent_virtual = new_dims[storage->list[i].unlim_dim_virtual]; + } /* end if */ + + /* Update top level virtual_select and clipped_virtual_select + * extents */ + if(H5S_set_extent(storage->list[i].source_dset.virtual_select, new_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") + if((storage->list[i].source_dset.clipped_virtual_select + != storage->list[i].source_dset.virtual_select) + && storage->list[i].source_dset.clipped_virtual_select) + if(H5S_set_extent(storage->list[i].source_dset.clipped_virtual_select, new_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") + + /* Update sub dataset virtual_select and clipped_virtual_select + * extents */ + for(j = 0; j < storage->list[i].sub_dset_nalloc; j++) + if(storage->list[i].sub_dset[j].virtual_select) { + if(H5S_set_extent(storage->list[i].sub_dset[j].virtual_select, new_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") + if((storage->list[i].sub_dset[j].clipped_virtual_select + != storage->list[i].sub_dset[j].virtual_select) + && storage->list[i].sub_dset[j].clipped_virtual_select) + if(H5S_set_extent(storage->list[i].sub_dset[j].clipped_virtual_select, new_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") + } /* end if */ + else + HDassert(!storage->list[i].sub_dset[j].clipped_virtual_select); + } /* end for */ + } /* end if */ + + /* Mark layout as fully initialized */ + storage->init = TRUE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_set_extent_unlim() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_init_all + * + * Purpose: Finishes initializing layout in preparation for I/O. + * Only necessary if H5D__virtual_set_extent_unlim() has not + * been called yet. Initializes clipped_virtual_select and + * clipped_source_select for all mappings in this layout. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * August 10, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_init_all(const H5D_t *dset, hid_t dxpl_id) +{ + H5O_storage_virtual_t *storage; + hsize_t virtual_dims[H5S_MAX_RANK]; + hsize_t source_dims[H5S_MAX_RANK]; + hsize_t clip_size; + size_t i, j; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(dset); + HDassert(dset->shared->layout.storage.type == H5D_VIRTUAL); + storage = &dset->shared->layout.storage.u.virt; + HDassert((storage->view == H5D_VDS_FIRST_MISSING) || (storage->view == H5D_VDS_LAST_AVAILABLE)); + + /* Get current VDS dimensions */ + if(H5S_get_simple_extent_dims(dset->shared->space, virtual_dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get VDS dimensions") + + /* Iterate over mappings */ + for(i = 0; i < storage->list_nalloc; i++) + /* Check for unlimited dimension */ + if(storage->list[i].unlim_dim_virtual >= 0) { + /* Check for "printf" source dataset resolution */ + if(storage->list[i].unlim_dim_source >= 0 ) { + /* Non-printf mapping */ + /* Open source dataset */ + if(!storage->list[i].source_dset.dset) + if(H5D__virtual_open_source_dset(dset, &storage->list[i], &storage->list[i].source_dset, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") + + /* Check if source dataset is open */ + if(storage->list[i].source_dset.dset) { + /* Retrieve current source dataset extent and patch mapping + */ + if(H5S_extent_copy(storage->list[i].source_select, storage->list[i].source_dset.dset->shared->space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy source dataspace extent") + + /* Get source space dimenstions */ + if(H5S_get_simple_extent_dims(storage->list[i].source_select, source_dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get source space dimensions") + + /* Get size that source selection would be clipped to to + * match size of virtual selection */ + clip_size = H5S_hyper_get_clip_extent_match(storage->list[i].source_select, storage->list[i].source_dset.virtual_select, virtual_dims[storage->list[i].unlim_dim_virtual], FALSE); + + /* Close previous clipped virtual selection, if any */ + if(storage->list[i].source_dset.clipped_virtual_select) { + HDassert(storage->list[i].source_dset.clipped_virtual_select + != storage->list[i].source_dset.virtual_select); + if(H5S_close(storage->list[i].source_dset.clipped_virtual_select) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release clipped virtual dataspace") + } /* end if */ + + /* Copy virtual selection */ + if(NULL == (storage->list[i].source_dset.clipped_virtual_select = H5S_copy(storage->list[i].source_dset.virtual_select, FALSE, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") + + /* Close previous clipped source selection, if any */ + if(storage->list[i].source_dset.clipped_source_select) { + HDassert(storage->list[i].source_dset.clipped_source_select + != storage->list[i].source_select); + if(H5S_close(storage->list[i].source_dset.clipped_source_select) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release clipped source dataspace") + } /* end if */ + + /* Copy source selection */ + if(NULL == (storage->list[i].source_dset.clipped_source_select = H5S_copy(storage->list[i].source_select, FALSE, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy source selection") + + /* Check if the clip size is within the current extent of + * the source dataset */ + if(clip_size <= source_dims[storage->list[i].unlim_dim_source]) { + /* Clip virtual selection to extent */ + if(H5S_hyper_clip_unlim(storage->list[i].source_dset.clipped_virtual_select, virtual_dims[storage->list[i].unlim_dim_virtual])) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + + /* Clip source selection to clip_size */ + if(H5S_hyper_clip_unlim(storage->list[i].source_dset.clipped_source_select, clip_size)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + } /* end if */ + else { + /* Get size that virtual selection will be clipped to to + * match size of source selection within source extent + */ + clip_size = H5S_hyper_get_clip_extent_match(storage->list[i].source_dset.virtual_select, storage->list[i].source_select, source_dims[storage->list[i].unlim_dim_source], FALSE); + + /* Clip virtual selection to clip_size */ + if(H5S_hyper_clip_unlim(storage->list[i].source_dset.clipped_virtual_select, clip_size)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + + /* Clip source selection to extent */ + if(H5S_hyper_clip_unlim(storage->list[i].source_dset.clipped_source_select, source_dims[storage->list[i].unlim_dim_source])) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + } /* end else */ + } /* end if */ + else { + HDassert(!storage->list[i].source_dset.clipped_virtual_select); + HDassert(!storage->list[i].source_dset.clipped_source_select); + } /* end else */ + } /* end if */ + else { + /* printf mapping */ + size_t sub_dset_max; + hbool_t partial_block; + + /* Get number of sub-source datasets in current extent */ + sub_dset_max = (size_t)H5S_hyper_get_first_inc_block(storage->list[i].source_dset.virtual_select, virtual_dims[storage->list[i].unlim_dim_virtual], &partial_block); + if(partial_block) + sub_dset_max++; + + /* Allocate or grow the sub_dset array if necessary */ + if(!storage->list[i].sub_dset) { + /* Allocate sub_dset array */ + if(NULL == (storage->list[i].sub_dset = (H5O_storage_virtual_srcdset_t *)H5MM_calloc(sub_dset_max * sizeof(H5O_storage_virtual_srcdset_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to allocate sub dataset array") + + /* Update sub_dset_nalloc */ + storage->list[i].sub_dset_nalloc = sub_dset_max; + } /* end if */ + else if(sub_dset_max > storage->list[i].sub_dset_nalloc) { + H5O_storage_virtual_srcdset_t *tmp_sub_dset; + + /* Extend sub_dset array */ + if(NULL == (tmp_sub_dset = (H5O_storage_virtual_srcdset_t *)H5MM_realloc(storage->list[i].sub_dset, sub_dset_max * sizeof(H5O_storage_virtual_srcdset_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to extend sub dataset array") + storage->list[i].sub_dset = tmp_sub_dset; + + /* Clear new space in sub_dset */ + (void)HDmemset(&storage->list[i].sub_dset[storage->list[i].sub_dset_nalloc], 0, (sub_dset_max - storage->list[i].sub_dset_nalloc) * sizeof(H5O_storage_virtual_srcdset_t)); + + /* Update sub_dset_nalloc */ + storage->list[i].sub_dset_nalloc = sub_dset_max; + } /* end if */ + + /* Iterate over sub dsets */ + for(j = 0; j < sub_dset_max; j++) { + /* Resolve file name */ + if(!storage->list[i].sub_dset[j].file_name) + if(H5D__virtual_build_source_name(storage->list[i].source_file_name, storage->list[i].parsed_source_file_name, storage->list[i].psfn_static_strlen, storage->list[i].psfn_nsubs, j, &storage->list[i].sub_dset[j].file_name) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to build source file name") + + /* Resolve dset name */ + if(!storage->list[i].sub_dset[j].dset_name) + if(H5D__virtual_build_source_name(storage->list[i].source_dset_name, storage->list[i].parsed_source_dset_name, storage->list[i].psdn_static_strlen, storage->list[i].psdn_nsubs, j, &storage->list[i].sub_dset[j].dset_name) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to build source dataset name") + + /* Resolve virtual selection for block */ + if(!storage->list[i].sub_dset[j].virtual_select) + if(NULL == (storage->list[i].sub_dset[j].virtual_select = H5S_hyper_get_unlim_block(storage->list[i].source_dset.virtual_select, j))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get block in unlimited selection") + + /* Close previous clipped source selection, if any */ + if(storage->list[i].sub_dset[j].clipped_source_select != storage->list[i].source_select) { + if(storage->list[i].sub_dset[j].clipped_source_select) + if(H5S_close(storage->list[i].sub_dset[j].clipped_source_select) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release clipped source dataspace") + + /* Initialize clipped source selection to point to base + * source selection */ + storage->list[i].sub_dset[j].clipped_source_select = storage->list[i].source_select; + } /* end if */ + + /* Close previous clipped virtual selection, if any */ + if(storage->list[i].sub_dset[j].clipped_virtual_select != storage->list[i].sub_dset[j].virtual_select) { + if(storage->list[i].sub_dset[j].clipped_virtual_select) + if(H5S_close(storage->list[i].sub_dset[j].clipped_virtual_select) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release clipped virtual dataspace") + + /* Initialize clipped virtual selection to point to + * unclipped virtual selection */ + storage->list[i].sub_dset[j].clipped_virtual_select = storage->list[i].sub_dset[j].virtual_select; + } /* end if */ + + /* Clear clipped selections if this is a partial block, + * defer calculation of real clipped selections to + * H5D__virtual_pre_io() as we may not have a valid source + * extent here */ + if((j == (sub_dset_max - 1)) && partial_block) { + /* Clear clipped source and virtual selections */ + storage->list[i].sub_dset[j].clipped_source_select = NULL; + storage->list[i].sub_dset[j].clipped_virtual_select = NULL; + } /* end else */ + /* Note we do not need to open the source file, this will + * happen later in H5D__virtual_pre_io() */ + } /* end for */ + + /* Update sub_dset_nused */ + storage->list[i].sub_dset_nused = sub_dset_max; + } /* end else */ + } /* end if */ + else { + /* Limited mapping, just make sure the clipped selections were + * already set. Again, no need to open the source file. */ + HDassert(storage->list[i].source_dset.clipped_virtual_select); + HDassert(storage->list[i].source_dset.clipped_source_select); + } /* end else */ + + /* Mark layout as fully initialized */ + storage->init = TRUE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_init_all() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_init + * + * Purpose: Initialize the virtual layout information for a dataset. + * This is called when the dataset is initialized. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * Thursday, April 30, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D__virtual_init(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, const H5D_t *dset, + hid_t dapl_id) +{ + H5O_storage_virtual_t *storage; /* Convenience pointer */ + H5P_genplist_t *dapl; /* Data access property list object pointer */ + hssize_t old_offset[H5O_LAYOUT_NDIMS]; /* Old selection offset (unused) */ + size_t i; /* Local index variables */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(dset); + storage = &dset->shared->layout.storage.u.virt; + HDassert(storage->list || (storage->list_nused == 0)); + + /* Check that the dimensions of the VDS are large enough */ + if(H5D_virtual_check_min_dims(dset) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "virtual dataset dimensions not large enough to contain all limited dimensions in all selections") + + /* Patch the virtual selection dataspaces. Note we always patch the space + * status because this layout could be from an old version held in the + * object header message code. We cannot update that held message because + * the layout message is constant, so just overwrite the values here (and + * invalidate other fields by setting storage->init to FALSE below). Also + * remove offset from selections. We only have to update + * source_space_status and virtual_space_status because others will be based + * on these and should therefore already have been normalized. */ + for(i = 0; i < storage->list_nused; i++) { + HDassert(storage->list[i].sub_dset_nalloc == 0); + + /* Patch extent */ + if(H5S_extent_copy(storage->list[i].source_dset.virtual_select, dset->shared->space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy virtual dataspace extent") + storage->list[i].virtual_space_status = H5O_VIRTUAL_STATUS_CORRECT; + + /* Mark source extent as invalid */ + storage->list[i].source_space_status = H5O_VIRTUAL_STATUS_INVALID; + + /* Normalize offsets, toss out old offset values */ + if(H5S_hyper_normalize_offset(storage->list[i].source_dset.virtual_select, old_offset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADSELECT, FAIL, "unable to normalize dataspace by offset") + if(H5S_hyper_normalize_offset(storage->list[i].source_select, old_offset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADSELECT, FAIL, "unable to normalize dataspace by offset") + } /* end for */ + + /* Get dataset access property list */ + if(NULL == (dapl = (H5P_genplist_t *)H5I_object(dapl_id))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for dapl ID") + + /* Get view option */ + if(H5P_get(dapl, H5D_ACS_VDS_VIEW_NAME, &storage->view) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual view option") + + /* Get printf gap if view is H5D_VDS_LAST_AVAILABLE, otherwise set to 0 */ + if(storage->view == H5D_VDS_LAST_AVAILABLE) { + if(H5P_get(dapl, H5D_ACS_VDS_PRINTF_GAP_NAME, &storage->printf_gap) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get virtual printf gap") + } /* end if */ + else + storage->printf_gap = (hsize_t)0; + + /* Retrieve VDS file FAPL to layout */ + if(storage->source_fapl <= 0) + if((storage->source_fapl = H5F_get_access_plist(f, FALSE)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get fapl") + + /* Copy DAPL to layout */ + if(storage->source_dapl <= 0) + if((storage->source_dapl = H5P_copy_plist(dapl, FALSE)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "can't copy dapl") + + /* Mark layout as not fully initialized (must be done prior to I/O for + * unlimited/printf selections) */ + storage->init = FALSE; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_init() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_is_space_alloc + * + * Purpose: Query if space is allocated for layout + * + * Return: TRUE if space is allocated + * FALSE if it is not + * Negative on failure + * + * Programmer: Neil Fortner + * February 6, 2015 + * + *------------------------------------------------------------------------- + */ +hbool_t +H5D__virtual_is_space_alloc(const H5O_storage_t H5_ATTR_UNUSED *storage) +{ + hbool_t ret_value; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Just return TRUE, since the global heap object containing the mappings is + * created when the layout message is encoded, and nothing else needs to be + * allocated for virtual datasets. This also ensures that the library never + * assumes (falsely) that no data is present in the dataset, causing errors. + */ + ret_value = TRUE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_is_space_alloc() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_pre_io + * + * Purpose: Project all virtual mappings onto mem_space, with the + * results stored in projected_mem_space for each mapping. + * Opens all source datasets if possible. The total number + * of elements is stored in tot_nelmts. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * June 3, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_pre_io(H5D_io_info_t *io_info, + H5O_storage_virtual_t *storage, const H5S_t *file_space, + const H5S_t *mem_space, hsize_t *tot_nelmts) +{ + hssize_t select_nelmts; /* Number of elements in selection */ + hsize_t bounds_start[H5S_MAX_RANK]; /* Selection bounds start */ + hsize_t bounds_end[H5S_MAX_RANK]; /* Selection bounds end */ + int rank; + hbool_t bounds_init = FALSE; /* Whether bounds_start, bounds_end, and rank are valid */ + size_t i, j, k; /* Local index variables */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(storage); + HDassert(mem_space); + HDassert(file_space); + HDassert(tot_nelmts); + + /* Initialize layout if necessary */ + if(!storage->init) + if(H5D__virtual_init_all(io_info->dset, io_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize virtual layout") + + /* Initialize tot_nelmts */ + *tot_nelmts = 0; + + /* Iterate over mappings */ + for(i = 0; i < storage->list_nused; i++) { + /* Sanity check that the virtual space has been patched by now */ + HDassert(storage->list[i].virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); + + /* Check for "printf" source dataset resolution */ + if(storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { + hbool_t partial_block; + + HDassert(storage->list[i].unlim_dim_virtual >= 0); + + /* Get selection bounds if necessary */ + if(!bounds_init) { + /* Get rank of VDS */ + if((rank = H5S_GET_EXTENT_NDIMS(io_info->dset->shared->space)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get number of dimensions") + + /* Get selection bounds */ + if(H5S_SELECT_BOUNDS(file_space, bounds_start, bounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds") + + /* Adjust bounds_end to represent the extent just enclosing them + * (add 1) */ + for(j = 0; j < (size_t)rank; j++) + bounds_end[j]++; + + /* Bounds are now initialized */ + bounds_init = TRUE; + } /* end if */ + + /* Get index of first block in virtual selection */ + storage->list[i].sub_dset_io_start = (size_t)H5S_hyper_get_first_inc_block(storage->list[i].source_dset.virtual_select, bounds_start[storage->list[i].unlim_dim_virtual], NULL); + + /* Get index of first block outside of virtual selection */ + storage->list[i].sub_dset_io_end = (size_t)H5S_hyper_get_first_inc_block(storage->list[i].source_dset.virtual_select, bounds_end[storage->list[i].unlim_dim_virtual], &partial_block); + if(partial_block) + storage->list[i].sub_dset_io_end++; + if(storage->list[i].sub_dset_io_end > storage->list[i].sub_dset_nused) + storage->list[i].sub_dset_io_end = storage->list[i].sub_dset_nused; + + /* Iterate over sub-source dsets */ + for(j = storage->list[i].sub_dset_io_start; j < storage->list[i].sub_dset_io_end; j++) { + /* Check for clipped virtual selection */ + if(!storage->list[i].sub_dset[j].clipped_virtual_select) { + hsize_t start[H5S_MAX_RANK]; + /* This should only be NULL if this is a partial block */ + HDassert((j == (storage->list[i].sub_dset_io_end - 1)) && partial_block); + + /* If the source space status is not correct, we must try to + * open the source dataset to patch it */ + if(storage->list[i].source_space_status != H5O_VIRTUAL_STATUS_CORRECT) { + HDassert(!storage->list[i].sub_dset[j].dset); + if(H5D__virtual_open_source_dset(io_info->dset, &storage->list[i], &storage->list[i].sub_dset[j], io_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") + } /* end if */ + + /* If we obtained a valid source space, we must create + * clipped source and virtual selections, otherwise we + * cannot do this and we will leave them NULL. This doesn't + * hurt anything because we can't do I/O because the dataset + * must not have been found. */ + if(storage->list[i].source_space_status == H5O_VIRTUAL_STATUS_CORRECT) { + hsize_t tmp_dims[H5S_MAX_RANK]; + hsize_t vbounds_end[H5S_MAX_RANK]; + + /* Get bounds of virtual selection */ + if(H5S_SELECT_BOUNDS(storage->list[i].sub_dset[j].virtual_select, tmp_dims, vbounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get selection bounds") + + HDassert(bounds_init); + + /* Convert bounds to extent (add 1) */ + for(k = 0; k < (size_t)rank; k++) + vbounds_end[k]++; + + /* Temporarily set extent of virtual selection to bounds */ + if(H5S_set_extent(storage->list[i].sub_dset[j].virtual_select, vbounds_end) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") + + /* Get current VDS dimensions */ + if(H5S_get_simple_extent_dims(io_info->dset->shared->space, tmp_dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get VDS dimensions") + + /* Copy virtual selection */ + if(NULL == (storage->list[i].sub_dset[j].clipped_virtual_select = H5S_copy(storage->list[i].sub_dset[j].virtual_select, FALSE, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") + + /* Clip virtual selection to real virtual extent */ + (void)HDmemset(start, 0, sizeof(start)); + if(H5S_select_hyperslab(storage->list[i].sub_dset[j].clipped_virtual_select, H5S_SELECT_AND, start, NULL, tmp_dims, NULL) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSELECT, FAIL, "unable to clip hyperslab") + + /* Project intersection of virtual space and clipped + * virtual space onto source space (create + * clipped_source_select) */ + if(H5S_select_project_intersection(storage->list[i].sub_dset[j].virtual_select, storage->list[i].source_select, storage->list[i].sub_dset[j].clipped_virtual_select, &storage->list[i].sub_dset[j].clipped_source_select) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space") + + /* Set extents of virtual_select and + * clipped_virtual_select to virtual extent */ + if(H5S_set_extent(storage->list[i].sub_dset[j].virtual_select, tmp_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") + if(H5S_set_extent(storage->list[i].sub_dset[j].clipped_virtual_select, tmp_dims) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") + } /* end if */ + } /* end if */ + + /* Only continue if we managed to obtain a + * clipped_virtual_select */ + if(storage->list[i].sub_dset[j].clipped_virtual_select) { + /* Project intersection of file space and mapping virtual space + * onto memory space */ + if(H5S_select_project_intersection(file_space, mem_space, storage->list[i].sub_dset[j].clipped_virtual_select, &storage->list[i].sub_dset[j].projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space") + + /* Check number of elements selected */ + if((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS(storage->list[i].sub_dset[j].projected_mem_space)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to get number of elements in selection") + + /* Check if anything is selected */ + if(select_nelmts > (hssize_t)0) { + /* Open source dataset */ + if(!storage->list[i].sub_dset[j].dset) + /* Try to open dataset */ + if(H5D__virtual_open_source_dset(io_info->dset, &storage->list[i], &storage->list[i].sub_dset[j], io_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") + + /* If the source dataset is not open, mark the selected + * elements as zero so projected_mem_space is freed */ + if(!storage->list[i].sub_dset[j].dset) + select_nelmts = (hssize_t)0; + } /* end if */ + + /* If there are not elements selected in this mapping, free + * projected_mem_space, otherwise update tot_nelmts */ + if(select_nelmts == (hssize_t)0) { + if(H5S_close(storage->list[i].sub_dset[j].projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space") + storage->list[i].sub_dset[j].projected_mem_space = NULL; + } /* end if */ + else + *tot_nelmts += (hsize_t)select_nelmts; + } /* end if */ + } /* end for */ + } /* end if */ + else { + if(storage->list[i].source_dset.clipped_virtual_select) { + /* Project intersection of file space and mapping virtual space onto + * memory space */ + if(H5S_select_project_intersection(file_space, mem_space, storage->list[i].source_dset.clipped_virtual_select, &storage->list[i].source_dset.projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto memory space") + + /* Check number of elements selected, add to tot_nelmts */ + if((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS(storage->list[i].source_dset.projected_mem_space)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to get number of elements in selection") + + /* Check if anything is selected */ + if(select_nelmts > (hssize_t)0) { + /* Open source dataset */ + if(!storage->list[i].source_dset.dset) + /* Try to open dataset */ + if(H5D__virtual_open_source_dset(io_info->dset, &storage->list[i], &storage->list[i].source_dset, io_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "unable to open source dataset") + + /* If the source dataset is not open, mark the selected elements + * as zero so projected_mem_space is freed */ + if(!storage->list[i].source_dset.dset) + select_nelmts = (hssize_t)0; + } /* end if */ + + /* If there are not elements selected in this mapping, free + * projected_mem_space, otherwise update tot_nelmts */ + if(select_nelmts == (hssize_t)0) { + if(H5S_close(storage->list[i].source_dset.projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected memory space") + storage->list[i].source_dset.projected_mem_space = NULL; + } /* end if */ + else + *tot_nelmts += (hsize_t)select_nelmts; + } /* end if */ + else { + /* If there is no clipped_dim_virtual, this must be an unlimited + * selection whose dataset was not found in the last call to + * H5Dget_space(). Do not attempt to open it as this might + * affect the extent and we are not going to recalculate it + * here. */ + HDassert(storage->list[i].unlim_dim_virtual >= 0); + HDassert(!storage->list[i].source_dset.dset); + } /* end else */ + } /* end else */ + } /* end for */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_pre_io() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_post_io + * + * Purpose: Frees memory structures allocated by H5D__virtual_pre_io. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * June 4, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_post_io(H5O_storage_virtual_t *storage) +{ + size_t i, j; /* Local index variables */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(storage); + + /* Iterate over mappings */ + for(i = 0; i < storage->list_nused; i++) + /* Check for "printf" source dataset resolution */ + if(storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { + /* Iterate over sub-source dsets */ + for(j = storage->list[i].sub_dset_io_start; j < storage->list[i].sub_dset_io_end; j++) + /* Close projected memory space */ + if(storage->list[i].sub_dset[j].projected_mem_space) { + if(H5S_close(storage->list[i].sub_dset[j].projected_mem_space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space") + storage->list[i].sub_dset[j].projected_mem_space = NULL; + } /* end if */ + } /* end if */ + else + /* Close projected memory space */ + if(storage->list[i].source_dset.projected_mem_space) { + if(H5S_close(storage->list[i].source_dset.projected_mem_space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close temporary space") + storage->list[i].source_dset.projected_mem_space = NULL; + } /* end if */ + + /* Note the lack of a done: label. This is because there are no HGOTO_ERROR + * calls. If one is added, a done: label must also be added */ + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_post_io() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_read_one + * + * Purpose: Read from a singe source dataset in a virtual dataset. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 15, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_read_one(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + const H5S_t *file_space, H5O_storage_virtual_srcdset_t *source_dset) +{ + H5S_t *projected_src_space = NULL; /* File space for selection in a single source dataset */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + HDassert(source_dset); + + /* Only perform I/O if there is a projected memory space, otherwise there + * were no elements in the projection or the source dataset could not be + * opened */ + if(source_dset->projected_mem_space) { + HDassert(source_dset->dset); + HDassert(source_dset->clipped_source_select); + + /* Project intersection of file space and mapping virtual space onto + * mapping source space */ + if(H5S_select_project_intersection(source_dset->clipped_virtual_select, source_dset->clipped_source_select, file_space, &projected_src_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto source space") + + /* Perform read on source dataset */ + if(H5D__read(source_dset->dset, type_info->dst_type_id, source_dset->projected_mem_space, projected_src_space, io_info->dxpl_id, io_info->u.rbuf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "can't read source dataset") + + /* Close projected_src_space */ + if(H5S_close(projected_src_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected source space") + projected_src_space = NULL; + } /* end if */ + +done: + /* Release allocated resources on failure */ + if(projected_src_space) { + HDassert(ret_value < 0); + if(H5S_close(projected_src_space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected source space") + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_read_one() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_read + * + * Purpose: Read from a virtual dataset. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * February 6, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + H5D_chunk_map_t H5_ATTR_UNUSED *fm) +{ + H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ + hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ + H5S_t *fill_space = NULL; /* Space to fill with fill value */ + size_t i, j; /* Local index variables */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(io_info); + HDassert(io_info->u.rbuf); + HDassert(type_info); + HDassert(mem_space); + HDassert(file_space); + + storage = &io_info->dset->shared->layout.storage.u.virt; + HDassert((storage->view == H5D_VDS_FIRST_MISSING) || (storage->view == H5D_VDS_LAST_AVAILABLE)); + +#ifdef H5_HAVE_PARALLEL + /* Parallel reads are not supported (yet) */ + if(H5F_HAS_FEATURE(io_info->dset->oloc.file, H5FD_FEAT_HAS_MPI)) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "parallel reads not supported on virtual datasets") +#endif /* H5_HAVE_PARALLEL */ + + /* Prepare for I/O operation */ + if(H5D__virtual_pre_io(io_info, storage, file_space, mem_space, &tot_nelmts) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "unable to prepare for I/O operation") + + /* Iterate over mappings */ + for(i = 0; i < storage->list_nused; i++) { + /* Sanity check that the virtual space has been patched by now */ + HDassert(storage->list[i].virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); + + /* Check for "printf" source dataset resolution */ + if(storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { + /* Iterate over sub-source dsets */ + for(j = storage->list[i].sub_dset_io_start; + j < storage->list[i].sub_dset_io_end; j++) + if(H5D__virtual_read_one(io_info, type_info, file_space, &storage->list[i].sub_dset[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset") + } /* end if */ + else + /* Read from source dataset */ + if(H5D__virtual_read_one(io_info, type_info, file_space, &storage->list[i].source_dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read source dataset") + } /* end for */ + + /* Fill unmapped part of buffer with fill value */ + if(tot_nelmts < nelmts) { + H5D_fill_value_t fill_status; /* Fill value status */ + + /* Check the fill value status */ + if(H5P_is_fill_value_defined(&io_info->dset->shared->dcpl_cache.fill, &fill_status) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't tell if fill value defined") + + /* Always write fill value to memory buffer unless it is undefined */ + if(fill_status != H5D_FILL_VALUE_UNDEFINED) { + /* Start with fill space equal to memory space */ + if(NULL == (fill_space = H5S_copy(mem_space, FALSE, TRUE))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOPY, FAIL, "unable to copy memory selection") + + /* Iterate over mappings */ + for(i = 0; i < storage->list_nused; i++) + /* Check for "printf" source dataset resolution */ + if(storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { + /* Iterate over sub-source dsets */ + for(j = storage->list[i].sub_dset_io_start; + j < storage->list[i].sub_dset_io_end; j++) + if(storage->list[i].sub_dset[j].projected_mem_space) + if(H5S_select_subtract(fill_space, storage->list[i].sub_dset[j].projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "unable to clip fill selection") + } /* end if */ + else + if(storage->list[i].source_dset.projected_mem_space) + /* Subtract projected memory space from fill space */ + if(H5S_select_subtract(fill_space, storage->list[i].source_dset.projected_mem_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "unable to clip fill selection") + + /* Write fill values to memory buffer */ + if(H5D__fill(io_info->dset->shared->dcpl_cache.fill.buf, io_info->dset->shared->type, io_info->u.rbuf, type_info->mem_type, fill_space, io_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "filling buf failed") + +#ifndef NDEBUG + /* Make sure the total number of elements written (including fill + * values) >= nelmts */ + { + hssize_t select_nelmts; /* Number of elements in selection */ + + /* Get number of elements in fill dataspace */ + if((select_nelmts = (hssize_t)H5S_GET_SELECT_NPOINTS(fill_space)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCOUNT, FAIL, "unable to get number of elements in selection") + + /* Verify number of elements is correct. Note that since we + * don't check for overlap we can't assert that these are equal + */ + HDassert((tot_nelmts + (hsize_t)select_nelmts) >= nelmts); + } /* end block */ +#endif /* NDEBUG */ + } /* end if */ + } /* end if */ + +done: + /* Cleanup I/O operation */ + if(H5D__virtual_post_io(storage) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't cleanup I/O operation") + + /* Close fill space */ + if(fill_space) + if(H5S_close(fill_space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close fill space") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_write_one + * + * Purpose: Write to a singe source dataset in a virtual dataset. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 15, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_write_one(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + const H5S_t *file_space, H5O_storage_virtual_srcdset_t *source_dset) +{ + H5S_t *projected_src_space = NULL; /* File space for selection in a single source dataset */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + HDassert(source_dset); + + /* Only perform I/O if there is a projected memory space, otherwise there + * were no elements in the projection */ + if(source_dset->projected_mem_space) { + HDassert(source_dset->dset); + HDassert(source_dset->clipped_source_select); + + /* In the future we may wish to extent this implementation to extend + * source datasets if a write to a virtual dataset goes past the current + * extent in the unlimited dimension. -NAF */ + /* Project intersection of file space and mapping virtual space onto + * mapping source space */ + if(H5S_select_project_intersection(source_dset->virtual_select, source_dset->clipped_source_select, file_space, &projected_src_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "can't project virtual intersection onto source space") + + /* Perform write on source dataset */ + if(H5D__write(source_dset->dset, type_info->dst_type_id, source_dset->projected_mem_space, projected_src_space, io_info->dxpl_id, io_info->u.wbuf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "can't write to source dataset") + + /* Close projected_src_space */ + if(H5S_close(projected_src_space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected source space") + projected_src_space = NULL; + } /* end if */ + +done: + /* Release allocated resources on failure */ + if(projected_src_space) { + HDassert(ret_value < 0); + if(H5S_close(projected_src_space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't close projected source space") + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_write_one() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_write + * + * Purpose: Write to a virtual dataset. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * February 6, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_write(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, + hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, + H5D_chunk_map_t H5_ATTR_UNUSED *fm) +{ + H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ + hsize_t tot_nelmts; /* Total number of elements mapped to mem_space */ + size_t i, j; /* Local index variables */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(io_info); + HDassert(io_info->u.wbuf); + HDassert(type_info); + HDassert(mem_space); + HDassert(file_space); + + storage = &io_info->dset->shared->layout.storage.u.virt; + HDassert((storage->view == H5D_VDS_FIRST_MISSING) || (storage->view == H5D_VDS_LAST_AVAILABLE)); + +#ifdef H5_HAVE_PARALLEL + /* Parallel writes are not supported (yet) */ + if(H5F_HAS_FEATURE(io_info->dset->oloc.file, H5FD_FEAT_HAS_MPI)) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "parallel writes not supported on virtual datasets") +#endif /* H5_HAVE_PARALLEL */ + + /* Prepare for I/O operation */ + if(H5D__virtual_pre_io(io_info, storage, file_space, mem_space, &tot_nelmts) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLIP, FAIL, "unable to prepare for I/O operation") + + /* Fail if there are unmapped parts of the selection as they would not be + * written */ + if(tot_nelmts != nelmts) + HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "write requested to unmapped portion of virtual dataset") + + /* Iterate over mappings */ + for(i = 0; i < storage->list_nused; i++) { + /* Sanity check that virtual space has been patched by now */ + HDassert(storage->list[i].virtual_space_status == H5O_VIRTUAL_STATUS_CORRECT); + + /* Check for "printf" source dataset resolution */ + if(storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { + /* Iterate over sub-source dsets */ + for(j = storage->list[i].sub_dset_io_start; + j < storage->list[i].sub_dset_io_end; j++) + if(H5D__virtual_write_one(io_info, type_info, file_space, &storage->list[i].sub_dset[j]) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write to source dataset") + } /* end if */ + else + /* Write to source dataset */ + if(H5D__virtual_write_one(io_info, type_info, file_space, &storage->list[i].source_dset) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write to source dataset") + } /* end for */ + +done: + /* Cleanup I/O operation */ + if(H5D__virtual_post_io(storage) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "can't cleanup I/O operation") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_write() */ + + +/*------------------------------------------------------------------------- + * Function: H5D__virtual_flush + * + * Purpose: Writes all dirty data to disk. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * February 6, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D__virtual_flush(H5D_t *dset, hid_t dxpl_id) +{ + H5O_storage_virtual_t *storage; /* Convenient pointer into layout struct */ + size_t i, j; /* Local index variables */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(dset); + + storage = &dset->shared->layout.storage.u.virt; + + /* Flush only open datasets */ + for(i = 0; i < storage->list_nused; i++) + /* Check for "printf" source dataset resolution */ + if(storage->list[i].psfn_nsubs || storage->list[i].psdn_nsubs) { + /* Iterate over sub-source dsets */ + for(j = 0; j < storage->list[i].sub_dset_nused; j++) + if(storage->list[i].sub_dset[j].dset) + /* Flush source dataset */ + if(H5D__flush_real(storage->list[i].sub_dset[j].dset, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to flush source dataset") + } /* end if */ + else + if(storage->list[i].source_dset.dset) + /* Flush source dataset */ + if(H5D__flush_real(storage->list[i].source_dset.dset, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to write to source dataset") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D__virtual_flush() */ + @@ -724,6 +724,50 @@ done: /*------------------------------------------------------------------------- + * Function: H5HG_get_obj_size + * + * Purpose: Returns the size of a global heap object. + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: Neil Fortner + * Thursday, February 12, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HG_get_obj_size(H5F_t *f, hid_t dxpl_id, H5HG_t *hobj, size_t *obj_size) +{ + H5HG_heap_t *heap = NULL; /* Pointer to global heap object */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__GLOBALHEAP_TAG, FAIL) + + /* Check args */ + HDassert(f); + HDassert(hobj); + HDassert(obj_size); + + /* Load the heap */ + if(NULL == (heap = H5HG_protect(f, dxpl_id, hobj->addr, H5AC__READ_ONLY_FLAG))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTPROTECT, FAIL, "unable to protect global heap") + + HDassert(hobj->idx < heap->nused); + HDassert(heap->obj[hobj->idx].begin); + + /* Set object size */ + *obj_size = heap->obj[hobj->idx].size; + +done: + if(heap && H5AC_unprotect(f, dxpl_id, H5AC_GHEAP, hobj->addr, heap, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_HEAP, H5E_CANTUNPROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL) +} /* end H5HG_get_obj_size() */ + + +/*------------------------------------------------------------------------- * Function: H5HG_remove * * Purpose: Removes the specified object from the global heap. diff --git a/src/H5HGprivate.h b/src/H5HGprivate.h index b090d4c..0c0aa69 100644 --- a/src/H5HGprivate.h +++ b/src/H5HGprivate.h @@ -60,6 +60,7 @@ H5_DLL herr_t H5HG_insert(H5F_t *f, hid_t dxpl_id, size_t size, void *obj, H5HG_t *hobj/*out*/); H5_DLL void *H5HG_read(H5F_t *f, hid_t dxpl_id, H5HG_t *hobj, void *object, size_t *buf_size/*out*/); H5_DLL int H5HG_link(H5F_t *f, hid_t dxpl_id, const H5HG_t *hobj, int adjust); +H5_DLL herr_t H5HG_get_obj_size(H5F_t *f, hid_t dxpl_id, H5HG_t *hobj, size_t *obj_size); H5_DLL herr_t H5HG_remove(H5F_t *f, hid_t dxpl_id, H5HG_t *hobj); /* Support routines */ diff --git a/src/H5Olayout.c b/src/H5Olayout.c index 6f4274f..daffe48 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -35,6 +35,9 @@ /* Local macros */ +/* Version # of encoded virtual dataset global heap blocks */ +#define H5O_LAYOUT_VDS_GH_ENC_VERS 0 + /* PRIVATE PROTOTYPES */ static void *H5O_layout_decode(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, @@ -101,6 +104,7 @@ H5O_layout_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED * unsigned H5_ATTR_UNUSED mesg_flags, unsigned H5_ATTR_UNUSED *ioflags, const uint8_t *p) { H5O_layout_t *mesg = NULL; + uint8_t *heap_block = NULL; unsigned u; void *ret_value = NULL; /* Return value */ @@ -115,7 +119,7 @@ H5O_layout_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED * HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") mesg->version = *p++; - if(mesg->version < H5O_LAYOUT_VERSION_1 || mesg->version > H5O_LAYOUT_VERSION_3) + if(mesg->version < H5O_LAYOUT_VERSION_1 || mesg->version > H5O_LAYOUT_VERSION_4) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad version number for layout message") if(mesg->version < H5O_LAYOUT_VERSION_3) { @@ -242,6 +246,147 @@ H5O_layout_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED * mesg->ops = H5D_LOPS_CHUNK; break; + case H5D_VIRTUAL: + /* Check version */ + if(mesg->version < H5O_LAYOUT_VERSION_4) + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "invalid layout version with virtual layout") + + /* Heap information */ + H5F_addr_decode(f, &p, &(mesg->storage.u.virt.serial_list_hobjid.addr)); + UINT32DECODE(p, mesg->storage.u.virt.serial_list_hobjid.idx); + + /* Initialize other fields */ + mesg->storage.u.virt.list_nused = 0; + mesg->storage.u.virt.list = NULL; + mesg->storage.u.virt.list_nalloc = 0; + mesg->storage.u.virt.view = H5D_VDS_ERROR; + mesg->storage.u.virt.printf_gap = HSIZE_UNDEF; + mesg->storage.u.virt.source_fapl = -1; + mesg->storage.u.virt.source_dapl = -1; + mesg->storage.u.virt.init = FALSE; + + /* Decode heap block if it exists */ + if(mesg->storage.u.virt.serial_list_hobjid.addr != HADDR_UNDEF) { + const uint8_t *heap_block_p; + uint8_t heap_vers; + size_t block_size = 0; + size_t tmp_size; + hsize_t tmp_hsize; + uint32_t stored_chksum; + uint32_t computed_chksum; + size_t i; + + /* Read heap */ + if(NULL == (heap_block = (uint8_t *)H5HG_read(f, dxpl_id, &(mesg->storage.u.virt.serial_list_hobjid), NULL, &block_size))) + HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "Unable to read global heap block") + + heap_block_p = (const uint8_t *)heap_block; + + /* Decode the version number of the heap block encoding */ + heap_vers = (uint8_t)*heap_block_p++; + if((uint8_t)H5O_LAYOUT_VDS_GH_ENC_VERS != heap_vers) + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "bad version # of encoded VDS heap information, expected %u, got %u", (unsigned)H5O_LAYOUT_VDS_GH_ENC_VERS, (unsigned)heap_vers) + + /* Number of entries */ + H5F_DECODE_LENGTH(f, heap_block_p, tmp_hsize) + + /* Allocate entry list */ + if(NULL == (mesg->storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_calloc((size_t)tmp_hsize * sizeof(H5O_storage_virtual_ent_t)))) + HGOTO_ERROR(H5E_OHDR, H5E_RESOURCE, NULL, "unable to allocate heap block") + mesg->storage.u.virt.list_nalloc = (size_t)tmp_hsize; + mesg->storage.u.virt.list_nused = (size_t)tmp_hsize; + + /* Decode each entry */ + for(i = 0; i < mesg->storage.u.virt.list_nused; i++) { + /* Source file name */ + tmp_size = HDstrlen((const char *)heap_block_p) + 1; + if(NULL == (mesg->storage.u.virt.list[i].source_file_name = (char *)H5MM_malloc(tmp_size))) + HGOTO_ERROR(H5E_OHDR, H5E_RESOURCE, NULL, "unable to allocate memory for source file name") + (void)HDmemcpy(mesg->storage.u.virt.list[i].source_file_name, heap_block_p, tmp_size); + heap_block_p += tmp_size; + + /* Source dataset name */ + tmp_size = HDstrlen((const char *)heap_block_p) + 1; + if(NULL == (mesg->storage.u.virt.list[i].source_dset_name = (char *)H5MM_malloc(tmp_size))) + HGOTO_ERROR(H5E_OHDR, H5E_RESOURCE, NULL, "unable to allocate memory for source dataset name") + (void)HDmemcpy(mesg->storage.u.virt.list[i].source_dset_name, heap_block_p, tmp_size); + heap_block_p += tmp_size; + + /* Source selection */ + if(H5S_SELECT_DESERIALIZE(&mesg->storage.u.virt.list[i].source_select, &heap_block_p) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, NULL, "can't decode source space selection") + + /* Virtual selection */ + if(H5S_SELECT_DESERIALIZE(&mesg->storage.u.virt.list[i].source_dset.virtual_select, &heap_block_p) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, NULL, "can't decode virtual space selection") + + /* Parse source file and dataset names for "printf" + * style format specifiers */ + if(H5D_virtual_parse_source_name(mesg->storage.u.virt.list[i].source_file_name, &mesg->storage.u.virt.list[i].parsed_source_file_name, &mesg->storage.u.virt.list[i].psfn_static_strlen, &mesg->storage.u.virt.list[i].psfn_nsubs) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't parse source file name") + if(H5D_virtual_parse_source_name(mesg->storage.u.virt.list[i].source_dset_name, &mesg->storage.u.virt.list[i].parsed_source_dset_name, &mesg->storage.u.virt.list[i].psdn_static_strlen, &mesg->storage.u.virt.list[i].psdn_nsubs) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't parse source dataset name") + + /* Set source names in source_dset struct */ + if((mesg->storage.u.virt.list[i].psfn_nsubs == 0) + && (mesg->storage.u.virt.list[i].psdn_nsubs == 0)) { + if(mesg->storage.u.virt.list[i].parsed_source_file_name) + mesg->storage.u.virt.list[i].source_dset.file_name = mesg->storage.u.virt.list[i].parsed_source_file_name->name_segment; + else + mesg->storage.u.virt.list[i].source_dset.file_name = mesg->storage.u.virt.list[i].source_file_name; + if(mesg->storage.u.virt.list[i].parsed_source_dset_name) + mesg->storage.u.virt.list[i].source_dset.dset_name = mesg->storage.u.virt.list[i].parsed_source_dset_name->name_segment; + else + mesg->storage.u.virt.list[i].source_dset.dset_name = mesg->storage.u.virt.list[i].source_dset_name; + } /* end if */ + + /* unlim_dim fields */ + mesg->storage.u.virt.list[i].unlim_dim_source = H5S_get_select_unlim_dim(mesg->storage.u.virt.list[i].source_select); + mesg->storage.u.virt.list[i].unlim_dim_virtual = H5S_get_select_unlim_dim(mesg->storage.u.virt.list[i].source_dset.virtual_select); + mesg->storage.u.virt.list[i].unlim_extent_source = HSIZE_UNDEF; + mesg->storage.u.virt.list[i].unlim_extent_virtual = HSIZE_UNDEF; + mesg->storage.u.virt.list[i].clip_size_source = HSIZE_UNDEF; + mesg->storage.u.virt.list[i].clip_size_virtual = HSIZE_UNDEF; + + /* Clipped selections */ + if(mesg->storage.u.virt.list[i].unlim_dim_virtual < 0) { + mesg->storage.u.virt.list[i].source_dset.clipped_source_select = mesg->storage.u.virt.list[i].source_select; + mesg->storage.u.virt.list[i].source_dset.clipped_virtual_select = mesg->storage.u.virt.list[i].source_dset.virtual_select; + } /* end if */ + + /* Check mapping for validity (do both pre and post + * checks here, since we had to allocate the entry list + * before decoding the selections anyways) */ + if(H5D_virtual_check_mapping_pre(mesg->storage.u.virt.list[i].source_dset.virtual_select, mesg->storage.u.virt.list[i].source_select, H5O_VIRTUAL_STATUS_INVALID) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "invalid mapping selections") + if(H5D_virtual_check_mapping_post(&mesg->storage.u.virt.list[i]) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "invalid mapping entry") + + /* Update min_dims */ + if(H5D_virtual_update_min_dims(mesg, i) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "unable to update virtual dataset minimum dimensions") + } /* end for */ + + /* Read stored checksum */ + UINT32DECODE(heap_block_p, stored_chksum) + + /* Compute checksum */ + computed_chksum = H5_checksum_metadata(heap_block, block_size - (size_t)4, 0); + + /* Verify checksum */ + if(stored_chksum != computed_chksum) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "incorrect metadata checksum for global heap block") + + /* Verify that the heap block size is correct */ + if((size_t)(heap_block_p - heap_block) != block_size) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "incorrect heap block size") + } /* end if */ + + /* Set the layout operations */ + mesg->ops = H5D_LOPS_VIRTUAL; + + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -254,8 +399,14 @@ H5O_layout_decode(H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, H5O_t H5_ATTR_UNUSED * done: if(ret_value == NULL) - if(mesg) + if(mesg) { + if(mesg->type == H5D_VIRTUAL) + if(H5D__virtual_reset_layout(mesg) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTFREE, NULL, "unable to reset virtual layout") mesg = H5FL_FREE(H5O_layout_t, mesg); + } /* end if */ + + heap_block = (uint8_t *)H5MM_xfree(heap_block); FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_layout_decode() */ @@ -295,6 +446,8 @@ static herr_t H5O_layout_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, const void *_mesg) { const H5O_layout_t *mesg = (const H5O_layout_t *) _mesg; + uint8_t *heap_block = NULL; + size_t *str_size = NULL; unsigned u; herr_t ret_value = SUCCEED; /* Return value */ @@ -306,7 +459,8 @@ H5O_layout_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, c HDassert(p); /* Message version */ - *p++ = (uint8_t)H5O_LAYOUT_VERSION_3; + *p++ = mesg->type == H5D_VIRTUAL ? (uint8_t)H5O_LAYOUT_VERSION_4 + : (uint8_t)H5O_LAYOUT_VERSION_3; /* Layout class */ *p++ = mesg->type; @@ -345,6 +499,109 @@ H5O_layout_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, c UINT32ENCODE(p, mesg->u.chunk.dim[u]); break; + case H5D_VIRTUAL: + /* Create heap block if it has not been created yet */ + /* Note that we assume here that the contents of the heap block + * cannot change! If this ever stops being the case we must change + * this code to allow overwrites of the heap block. -NAF */ + if((mesg->storage.u.virt.serial_list_hobjid.addr == HADDR_UNDEF) + && (mesg->storage.u.virt.list_nused > 0)) { + uint8_t *heap_block_p; + size_t block_size; + hssize_t select_serial_size; + hsize_t tmp_hsize; + uint32_t chksum; + size_t i; + + /* Allocate array for caching results of strlen */ + if(NULL == (str_size = (size_t *)H5MM_malloc(2 * mesg->storage.u.virt.list_nused *sizeof(size_t)))) + HGOTO_ERROR(H5E_OHDR, H5E_RESOURCE, FAIL, "unable to allocate string length array") + + /* + * Calculate heap block size + */ + /* Version and number of entries */ + block_size = (size_t)1 + H5F_SIZEOF_SIZE(f); + + /* Calculate size of each entry */ + for(i = 0; i < mesg->storage.u.virt.list_nused; i++) { + HDassert(mesg->storage.u.virt.list[i].source_file_name); + HDassert(mesg->storage.u.virt.list[i].source_dset_name); + HDassert(mesg->storage.u.virt.list[i].source_select); + HDassert(mesg->storage.u.virt.list[i].source_dset.virtual_select); + + /* Source file name */ + str_size[2 * i] = HDstrlen(mesg->storage.u.virt.list[i].source_file_name) + (size_t)1; + block_size += str_size[2 * i]; + + /* Source dset name */ + str_size[(2 * i) + 1] = HDstrlen(mesg->storage.u.virt.list[i].source_dset_name) + (size_t)1; + block_size += str_size[(2 * i) + 1]; + + /* Source selection */ + if((select_serial_size = H5S_SELECT_SERIAL_SIZE(mesg->storage.u.virt.list[i].source_select)) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to check dataspace selection size") + block_size += (size_t)select_serial_size; + + /* Virtual dataset selection */ + if((select_serial_size = H5S_SELECT_SERIAL_SIZE(mesg->storage.u.virt.list[i].source_dset.virtual_select)) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to check dataspace selection size") + block_size += (size_t)select_serial_size; + } /* end for */ + + /* Checksum */ + block_size += 4; + + /* Allocate heap block */ + if(NULL == (heap_block = (uint8_t *)H5MM_malloc(block_size))) + HGOTO_ERROR(H5E_OHDR, H5E_RESOURCE, FAIL, "unable to allocate heap block") + + /* + * Encode heap block + */ + heap_block_p = heap_block; + + /* Encode heap block encoding version */ + *heap_block_p++ = (uint8_t)H5O_LAYOUT_VDS_GH_ENC_VERS; + + /* Number of entries */ + tmp_hsize = (hsize_t)mesg->storage.u.virt.list_nused; + H5F_ENCODE_LENGTH(f, heap_block_p, tmp_hsize) + + /* Encode each entry */ + for(i = 0; i < mesg->storage.u.virt.list_nused; i++) { + /* Source file name */ + (void)HDmemcpy((char *)heap_block_p, mesg->storage.u.virt.list[i].source_file_name, str_size[2 * i]); + heap_block_p += str_size[2 * i]; + + /* Source dataset name */ + (void)HDmemcpy((char *)heap_block_p, mesg->storage.u.virt.list[i].source_dset_name, str_size[(2 * i) + 1]); + heap_block_p += str_size[(2 * i) + 1]; + + /* Source selection */ + if(H5S_SELECT_SERIALIZE(mesg->storage.u.virt.list[i].source_select, &heap_block_p) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to serialize source selection") + + /* Virtual selection */ + if(H5S_SELECT_SERIALIZE(mesg->storage.u.virt.list[i].source_dset.virtual_select, &heap_block_p) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to serialize virtual selection") + } /* end for */ + + /* Checksum */ + chksum = H5_checksum_metadata(heap_block, block_size - (size_t)4, 0); + UINT32ENCODE(heap_block_p, chksum) + + /* Insert block into global heap */ + if(H5HG_insert(f, H5AC_ind_dxpl_id, block_size, heap_block, &((H5O_layout_t *)mesg)->storage.u.virt.serial_list_hobjid) < 0) /* Casting away const OK --NAF */ + HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "unable to insert virtual dataset heap block") + } /* end if */ + + /* Heap information */ + H5F_addr_encode(f, &p, mesg->storage.u.virt.serial_list_hobjid.addr); + UINT32ENCODE(p, mesg->storage.u.virt.serial_list_hobjid.idx); + + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -352,6 +609,9 @@ H5O_layout_encode(H5F_t *f, hbool_t H5_ATTR_UNUSED disable_shared, uint8_t *p, c } /* end switch */ done: + heap_block = (uint8_t *)H5MM_xfree(heap_block); + str_size = (size_t *)H5MM_xfree(str_size); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_layout_encode() */ @@ -417,6 +677,11 @@ H5O_layout_copy(const void *_mesg, void *_dest) H5D_chunk_idx_reset(&dest->storage.u.chunk, FALSE); break; + case H5D_VIRTUAL: + if(H5D__virtual_copy_layout(dest) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, NULL, "unable to copy virtual layout") + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -488,19 +753,25 @@ static herr_t H5O_layout_reset(void *_mesg) { H5O_layout_t *mesg = (H5O_layout_t *)_mesg; + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_NOERR + FUNC_ENTER_NOAPI_NOINIT if(mesg) { /* Free the compact storage buffer */ if(H5D_COMPACT == mesg->type) mesg->storage.u.compact.buf = H5MM_xfree(mesg->storage.u.compact.buf); + else if(H5D_VIRTUAL == mesg->type) + /* Free the virtual entry list */ + if(H5D__virtual_reset_layout(mesg) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to reset virtual layout") /* Reset the message */ mesg->type = H5D_CONTIGUOUS; } /* end if */ - FUNC_LEAVE_NOAPI(SUCCEED) +done: + FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_layout_reset() */ @@ -580,6 +851,12 @@ H5O_layout_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, void *_mesg) HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free raw data") break; + case H5D_VIRTUAL: /* Virtual dataset */ + /* Free the file space virtual dataset */ + if(H5D__virtual_delete(f, dxpl_id, &mesg->storage) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free raw data") + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -667,6 +944,13 @@ H5O_layout_copy_file(H5F_t *file_src, void *mesg_src, H5F_t *file_dst, } /* end if */ break; + case H5D_VIRTUAL: + /* Copy virtual layout. Always copy so the memory fields get copied + * properly. */ + if(H5D__virtual_copy(file_dst, layout_dst, dxpl_id) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, NULL, "unable to copy virtual storage") + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -766,6 +1050,26 @@ H5O_layout_debug(H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, const vo "Data Size:", mesg->storage.u.compact.size); break; + case H5D_VIRTUAL: + HDfprintf(stream, "%*s%-*s %s\n", indent, "", fwidth, + "Type:", "Virtual"); + HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, + "Global heap address:", mesg->storage.u.virt.serial_list_hobjid.addr); + HDfprintf(stream, "%*s%-*s %Zu\n", indent, "", fwidth, + "Global heap index:", mesg->storage.u.virt.serial_list_hobjid.idx); + for(u = 0; u < mesg->storage.u.virt.list_nused; u++) { + HDfprintf(stream, "%*sMapping %u:\n", indent, "", u); + HDfprintf(stream, "%*s%-*s %s\n", indent + 3, "", fwidth - 3, + "Virtual selection:", "<Not yet implemented>"); + HDfprintf(stream, "%*s%-*s %s\n", indent + 3, "", fwidth - 3, + "Source file name:", mesg->storage.u.virt.list[u].source_file_name); + HDfprintf(stream, "%*s%-*s %s\n", indent + 3, "", fwidth - 3, + "Source dataset name:", mesg->storage.u.virt.list[u].source_dset_name); + HDfprintf(stream, "%*s%-*s %s\n", indent + 3, "", fwidth - 3, + "Source selection:", "<Not yet implemented>"); + } /* end for */ + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index b6667e9..4586adb 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -38,6 +38,7 @@ #include "H5private.h" /* Generic Functions */ #include "H5ACprivate.h" /* Metadata cache */ #include "H5Fprivate.h" /* File access */ +#include "H5HGprivate.h" /* Global Heaps */ #include "H5SLprivate.h" /* Skip lists */ #include "H5Tprivate.h" /* Datatype functions */ #include "H5Zprivate.h" /* I/O pipeline filters */ @@ -376,7 +377,8 @@ typedef struct H5O_efl_t { * for larger chunk dimensions, stores chunk indices into their own * message (the "layout index" message), adds features for compact/dense * storage of elements and/or chunk records, adds features for abbreviating - * the storage used for partial chunks on boundaries, etc. + * the storage used for partial chunks on boundaries, adds the virtual + * layout type, etc. */ #define H5O_LAYOUT_VERSION_4 4 @@ -414,12 +416,88 @@ typedef struct H5O_storage_compact_t { void *buf; /* Buffer for compact dataset */ } H5O_storage_compact_t; +typedef struct H5O_storage_virtual_srcdset_t { + /* Stored */ + struct H5S_t *virtual_select; /* Selection in the virtual dataset that is mapped to source selection */ + + /* Not stored */ + char *file_name; /* Source file name used for virtual dataset mapping */ + char *dset_name; /* Source dataset name used for virtual dataset mapping */ + struct H5S_t *clipped_source_select; /* Clipped version of source_select */ + struct H5S_t *clipped_virtual_select; /* Clipped version of virtual_select */ + struct H5D_t *dset; /* Source dataset */ + hbool_t dset_exists; /* Whether the dataset exists (was opened successfully) */ + + /* Temporary - only used during I/O operation, NULL at all other times */ + struct H5S_t *projected_mem_space; /* Selection within mem_space for this mapping */ +} H5O_storage_virtual_srcdset_t; + +typedef struct H5O_storage_virtual_name_seg_t { + char *name_segment; /* String for this name segment */ + struct H5O_storage_virtual_name_seg_t *next; /* Next name segment */ +} H5O_storage_virtual_name_seg_t; + +typedef enum H5O_virtual_space_status_t { + H5O_VIRTUAL_STATUS_INVALID = 0, /* Space extent is invalid */ + H5O_VIRTUAL_STATUS_SEL_BOUNDS, /* Space extent set to bounds of selection */ + H5O_VIRTUAL_STATUS_USER, /* Space extent provided by application */ + H5O_VIRTUAL_STATUS_CORRECT /* Space extent matches dataset */ +} H5O_virtual_space_status_t; + +typedef struct H5O_storage_virtual_ent_t { + /* Stored */ + H5O_storage_virtual_srcdset_t source_dset; /* Information about the source dataset */ + char *source_file_name; /* Original (unparsed) source file name */ + char *source_dset_name; /* Original (unparsed) source dataset name */ + struct H5S_t *source_select; /* Selection in the source dataset for mapping */ + + /* Not stored */ + H5O_storage_virtual_srcdset_t *sub_dset; /* Array of sub-source dataset info structs */ + size_t sub_dset_nalloc; /* Number of slots allocated in sub_dset */ + size_t sub_dset_nused; /* Number of slots "used" in sub_dset - essentially the farthest sub dataset in the extent */ + size_t sub_dset_io_start; /* First element in sub_dset involved in current I/O op. Field has no meaning and may be uninitialized at all other times */ + size_t sub_dset_io_end; /* First element in sub_dset outside of current I/O op. Field has no meaning and may be uninitialized at all other times */ + H5O_storage_virtual_name_seg_t *parsed_source_file_name; /* Parsed version of source_dset.file_name */ + size_t psfn_static_strlen; /* Length of parsed_source_file_name without block number substitutions */ + size_t psfn_nsubs; /* Number of block number substitutions in parsed_source_file_name */ + H5O_storage_virtual_name_seg_t *parsed_source_dset_name; /* Parsed version of source_dset.dset_name */ + size_t psdn_static_strlen; /* Length of parsed_source_dset_name without block number substitutions */ + size_t psdn_nsubs; /* Number of block number substitutions in parsed_source_dset_name */ + int unlim_dim_source; /* Unlimited dimension in source_select */ + int unlim_dim_virtual; /* Unlimited dimension in virtual_select */ + hsize_t unlim_extent_source; /* Extent of unlimited dimension in source dset last time virtual_select was patched to match selection */ + hsize_t unlim_extent_virtual; /* Extent of unlimited dimension in virtual dset last time source_select was patched to match selection */ + hsize_t clip_size_virtual; /* Size selection would be clipped to in virtual selection, ignoring other mappings, when source extent == unlim_extent_source */ + hsize_t clip_size_source; /* Size selection would be clipped to in source selection when virtual extent == unlim_extent_virtual */ + H5O_virtual_space_status_t source_space_status; /* Extent patching status of source_select */ + H5O_virtual_space_status_t virtual_space_status; /* Extent patching status of virtual_select */ +} H5O_storage_virtual_ent_t; + +typedef struct H5O_storage_virtual_t { + /* Stored in message */ + H5HG_t serial_list_hobjid; /* Global heap ID for the list of virtual mapping entries stored on disk */ + + /* Stored in heap */ + size_t list_nused; /* Number of array elements used in list */ + H5O_storage_virtual_ent_t *list; /* Array of virtual dataset mapping entries */ + + /* Not stored */ + size_t list_nalloc; /* Number of slots allocated */ + hsize_t min_dims[H5S_MAX_RANK]; /* Minimum extent of VDS (maximum of all non-unlimited selection bounds) */ + H5D_vds_view_t view; /* Method for calculating the extent of the virtual dataset with unlimited selections */ + hsize_t printf_gap; /* Maximum number of sequential missing source datasets before terminating the search for more */ + hid_t source_fapl; /* FAPL to use to open source files */ + hid_t source_dapl; /* DAPL to use to open source datasets */ + hbool_t init; /* Whether all information has been completely initialized */ +} H5O_storage_virtual_t; + typedef struct H5O_storage_t { H5D_layout_t type; /* Type of layout */ union { H5O_storage_contig_t contig; /* Information for contiguous storage */ H5O_storage_chunk_t chunk; /* Information for chunked storage */ H5O_storage_compact_t compact; /* Information for compact storage */ + H5O_storage_virtual_t virt; /* Information for virtual storage */ } u; } H5O_storage_t; diff --git a/src/H5Pdapl.c b/src/H5Pdapl.c index dd9c24d..eab9337 100644 --- a/src/H5Pdapl.c +++ b/src/H5Pdapl.c @@ -62,6 +62,16 @@ #define H5D_ACS_PREEMPT_READ_CHUNKS_DEF H5D_CHUNK_CACHE_W0_DEFAULT #define H5D_ACS_PREEMPT_READ_CHUNKS_ENC H5P__encode_double #define H5D_ACS_PREEMPT_READ_CHUNKS_DEC H5P__decode_double +/* Definitions for VDS view option */ +#define H5D_ACS_VDS_VIEW_SIZE sizeof(H5D_vds_view_t) +#define H5D_ACS_VDS_VIEW_DEF H5D_VDS_LAST_AVAILABLE +#define H5D_ACS_VDS_VIEW_ENC H5P__dacc_vds_view_enc +#define H5D_ACS_VDS_VIEW_DEC H5P__dacc_vds_view_dec +/* Definitions for VDS printf gap */ +#define H5D_ACS_VDS_PRINTF_GAP_SIZE sizeof(hsize_t) +#define H5D_ACS_VDS_PRINTF_GAP_DEF (hsize_t)0 +#define H5D_ACS_VDS_PRINTF_GAP_ENC H5P__encode_hsize_t +#define H5D_ACS_VDS_PRINTF_GAP_DEC H5P__decode_hsize_t /******************/ /* Local Typedefs */ @@ -86,6 +96,10 @@ static herr_t H5P__encode_chunk_cache_nbytes(const void *value, void **_pp, size_t *size); static herr_t H5P__decode_chunk_cache_nbytes(const void **_pp, void *_value); +/* Property list callbacks */ +static herr_t H5P__dacc_vds_view_enc(const void *value, void **pp, size_t *size); +static herr_t H5P__dacc_vds_view_dec(const void **pp, void *value); + /*********************/ /* Package Variables */ @@ -140,6 +154,8 @@ H5P__dacc_reg_prop(H5P_genclass_t *pclass) size_t rdcc_nslots = H5D_ACS_DATA_CACHE_NUM_SLOTS_DEF; /* Default raw data chunk cache # of slots */ size_t rdcc_nbytes = H5D_ACS_DATA_CACHE_BYTE_SIZE_DEF; /* Default raw data chunk cache # of bytes */ double rdcc_w0 = H5D_ACS_PREEMPT_READ_CHUNKS_DEF; /* Default raw data chunk cache dirty ratio */ + H5D_vds_view_t virtual_view = H5D_ACS_VDS_VIEW_DEF; /* Default VDS view option */ + hsize_t printf_gap = H5D_ACS_VDS_PRINTF_GAP_DEF; /* Default VDS printf gap */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -159,6 +175,18 @@ H5P__dacc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, H5D_ACS_PREEMPT_READ_CHUNKS_ENC, H5D_ACS_PREEMPT_READ_CHUNKS_DEC, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the VDS view option */ + if(H5P_register_real(pclass, H5D_ACS_VDS_VIEW_NAME, H5D_ACS_VDS_VIEW_SIZE, &virtual_view, + NULL, NULL, NULL, H5D_ACS_VDS_VIEW_ENC, H5D_ACS_VDS_VIEW_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + + /* Register the VDS printf gap */ + if(H5P_register_real(pclass, H5D_ACS_VDS_PRINTF_GAP_NAME, H5D_ACS_VDS_PRINTF_GAP_SIZE, &printf_gap, + NULL, NULL, NULL, H5D_ACS_VDS_PRINTF_GAP_ENC, H5D_ACS_VDS_PRINTF_GAP_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__dacc_reg_prop() */ @@ -511,3 +539,251 @@ H5P__decode_chunk_cache_nbytes(const void **_pp, void *_value) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5P__decode_chunk_cache_nbytes() */ + +/*------------------------------------------------------------------------- + * Function: H5Pset_virtual_view + * + * Purpose: Takes the access property list for the virtual dataset, + * dapl_id, and the flag, view, and sets the VDS view + * according to the flag value. The view will include all + * data before the first missing mapped data found if the + * flag is set to H5D_VDS_FIRST_MISSING or to include all + * available mapped data if the flag is set to + * H5D_VDS_LAST_AVAIALBLE. Missing mapped data will be + * filled with the fill value according to the VDS creation + * property settings. For VDS with unlimited mappings, the + * view defines the extent. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 4, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_virtual_view(hid_t plist_id, H5D_vds_view_t view) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "iDv", plist_id, view); + + /* Check argument */ + if((view != H5D_VDS_FIRST_MISSING) && (view != H5D_VDS_LAST_AVAILABLE)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid bounds option") + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Update property list */ + if(H5P_set(plist, H5D_ACS_VDS_VIEW_NAME, &view) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_virtual_view() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_virtual_view + * + * Purpose: Takes the access property list for the virtual dataset, + * dapl_id, and gets the flag, view, set by the + * H5Pset_virtual_view call. The possible values of view are + * H5D_VDS_FIRST_MISSING or H5D_VDS_LAST_AVAIALBLE. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 4, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_virtual_view(hid_t plist_id, H5D_vds_view_t *view) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*Dv", plist_id, view); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get value from property list */ + if(view) + if(H5P_get(plist, H5D_ACS_VDS_VIEW_NAME, view) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_virtual_view() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__dacc_vds_view_enc + * + * Purpose: Callback routine which is called whenever the vds view + * property in the dataset access property list is encoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Neil Fortner + * Tuesday, May 5, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dacc_vds_view_enc(const void *value, void **_pp, size_t *size) +{ + const H5D_vds_view_t *view = (const H5D_vds_view_t *)value; /* Create local alias for values */ + uint8_t **pp = (uint8_t **)_pp; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity check */ + HDassert(view); + HDassert(size); + + if(NULL != *pp) + /* Encode EDC property */ + *(*pp)++ = (uint8_t)*view; + + /* Size of EDC property */ + (*size)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dacc_vds_view_enc() */ + + +/*------------------------------------------------------------------------- + * Function: H5P__dacc_vds_view_dec + * + * Purpose: Callback routine which is called whenever the vds view + * property in the dataset access property list is encoded. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: Neil Fortner + * Tuesday, May 5, 2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5P__dacc_vds_view_dec(const void **_pp, void *_value) +{ + H5D_vds_view_t *view = (H5D_vds_view_t *)_value; + const uint8_t **pp = (const uint8_t **)_pp; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity checks */ + HDassert(pp); + HDassert(*pp); + HDassert(view); + + /* Decode EDC property */ + *view = (H5D_vds_view_t)*(*pp)++; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5P__dacc_vds_view_dec() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pset_virtual_printf_gap + * + * Purpose: Sets the access property list for the virtual dataset, + * dapl_id, to instruct the library to stop looking for the + * mapped data stored in the files and/or datasets with the + * printf-style names after not finding gap_size files and/or + * datasets. The found source files and datasets will + * determine the extent of the unlimited VDS with the printf + * -style mappings. + * + * For example, if regularly spaced blocks of VDS are mapped + * to datasets with the names d-1, d-2, d-3, ..., d-N, ..., + * and d-2 dataset is missing and gap_size is set to 0, then + * VDS will contain only data found in d-1. If d-2 and d-3 + * are missing and gap_size is set to 2, then VDS will + * contain the data from d-1, d-3, ..., d-N, .... The blocks + * that are mapped to d-2 and d-3 will be filled according to + * the VDS fill value setting. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 21, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_virtual_printf_gap(hid_t plist_id, hsize_t gap_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ih", plist_id, gap_size); + + /* Check argument */ + if(gap_size == HSIZE_UNDEF) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a valid printf gap size") + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Update property list */ + if(H5P_set(plist, H5D_ACS_VDS_PRINTF_GAP_NAME, &gap_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "unable to set value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_virtual_printf_gap() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_virtual_printf_gap + * + * Purpose: Gets the maximum number of missing printf-style files + * and/or datasets for determining the extent of the + * unlimited VDS, gap_size, using the access property list + * for the virtual dataset, dapl_id. The default library + * value for gap_size is 0. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * May 21, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_virtual_printf_gap(hid_t plist_id, hsize_t *gap_size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "i*h", plist_id, gap_size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get value from property list */ + if(gap_size) + if(H5P_get(plist, H5D_ACS_VDS_PRINTF_GAP_NAME, gap_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "unable to get value") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_virtual_printf_gap() */ + diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 5c64ccb..0677517 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -41,7 +41,9 @@ #include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ +#include "H5Oprivate.h" /* Object headers */ #include "H5Ppkg.h" /* Property lists */ +#include "H5Sprivate.h" /* Dataspaces */ #include "H5Tprivate.h" /* Datatypes */ #include "H5Zprivate.h" /* Data filters */ @@ -55,13 +57,16 @@ #define H5D_DEF_STORAGE_CONTIG_INIT {HADDR_UNDEF, (hsize_t)0} #define H5D_DEF_STORAGE_CHUNK_INIT {H5D_CHUNK_IDX_BTREE, HADDR_UNDEF, NULL, {{HADDR_UNDEF, NULL}}} #define H5D_DEF_LAYOUT_CHUNK_INIT {(unsigned)0, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, (uint32_t)0, (hsize_t)0, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}} +#define H5D_DEF_STORAGE_VIRTUAL_INIT {{HADDR_UNDEF, 0}, 0, NULL, 0, {0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0}, H5D_VDS_ERROR, HSIZE_UNDEF, -1, -1, FALSE} #ifdef H5_HAVE_C99_DESIGNATED_INITIALIZER #define H5D_DEF_STORAGE_COMPACT {H5D_COMPACT, { .compact = H5D_DEF_STORAGE_COMPACT_INIT }} #define H5D_DEF_STORAGE_CONTIG {H5D_CONTIGUOUS, { .contig = H5D_DEF_STORAGE_CONTIG_INIT }} #define H5D_DEF_STORAGE_CHUNK {H5D_CHUNKED, { .chunk = H5D_DEF_STORAGE_CHUNK_INIT }} +#define H5D_DEF_STORAGE_VIRTUAL {H5D_VIRTUAL, { .virt = H5D_DEF_STORAGE_VIRTUAL_INIT }} #define H5D_DEF_LAYOUT_COMPACT {H5D_COMPACT, H5O_LAYOUT_VERSION_3, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, H5D_DEF_STORAGE_COMPACT} #define H5D_DEF_LAYOUT_CONTIG {H5D_CONTIGUOUS, H5O_LAYOUT_VERSION_3, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, H5D_DEF_STORAGE_CONTIG} #define H5D_DEF_LAYOUT_CHUNK {H5D_CHUNKED, H5O_LAYOUT_VERSION_3, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, H5D_DEF_STORAGE_CHUNK} +#define H5D_DEF_LAYOUT_VIRTUAL {H5D_VIRTUAL, H5O_LAYOUT_VERSION_3, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, H5D_DEF_STORAGE_VIRTUAL} #else /* H5_HAVE_C99_DESIGNATED_INITIALIZER */ /* Note that the compact & chunked layout initialization values are using the * contiguous layout initialization in the union, because the contiguous @@ -71,6 +76,7 @@ #define H5D_DEF_LAYOUT_COMPACT {H5D_COMPACT, H5O_LAYOUT_VERSION_3, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, {H5D_CONTIGUOUS, H5D_DEF_STORAGE_CONTIG_INIT}} #define H5D_DEF_LAYOUT_CONTIG {H5D_CONTIGUOUS, H5O_LAYOUT_VERSION_3, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, {H5D_CONTIGUOUS, H5D_DEF_STORAGE_CONTIG_INIT}} #define H5D_DEF_LAYOUT_CHUNK {H5D_CHUNKED, H5O_LAYOUT_VERSION_3, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, {H5D_CONTIGUOUS, H5D_DEF_STORAGE_CONTIG_INIT}} +#define H5D_DEF_LAYOUT_VIRTUAL {H5D_VIRTUAL, H5O_LAYOUT_VERSION_3, NULL, {H5D_DEF_LAYOUT_CHUNK_INIT}, {H5D_CONTIGUOUS, H5D_DEF_STORAGE_CONTIG_INIT}} #endif /* H5_HAVE_C99_DESIGNATED_INITIALIZER */ /* ======== Dataset creation properties ======== */ @@ -211,10 +217,12 @@ static const H5O_efl_t H5D_def_efl_g = H5D_CRT_EXT_FILE_LIST_DEF; static const H5O_layout_t H5D_def_layout_compact_g = H5D_DEF_LAYOUT_COMPACT; static const H5O_layout_t H5D_def_layout_contig_g = H5D_DEF_LAYOUT_CONTIG; static const H5O_layout_t H5D_def_layout_chunk_g = H5D_DEF_LAYOUT_CHUNK; +static const H5O_layout_t H5D_def_layout_virtual_g = H5D_DEF_LAYOUT_VIRTUAL; #else /* H5_HAVE_C99_DESIGNATED_INITIALIZER */ static H5O_layout_t H5D_def_layout_compact_g = H5D_DEF_LAYOUT_COMPACT; static H5O_layout_t H5D_def_layout_contig_g = H5D_DEF_LAYOUT_CONTIG; static H5O_layout_t H5D_def_layout_chunk_g = H5D_DEF_LAYOUT_CHUNK; +static H5O_layout_t H5D_def_layout_virtual_g = H5D_DEF_LAYOUT_VIRTUAL; static hbool_t H5P_dcrt_def_layout_init_g = FALSE; #endif /* H5_HAVE_C99_DESIGNATED_INITIALIZER */ @@ -364,8 +372,12 @@ H5P__dcrt_layout_enc(const void *value, void **_pp, size_t *size) { const H5O_layout_t *layout = (const H5O_layout_t *)value; /* Create local aliases for values */ uint8_t **pp = (uint8_t **)_pp; + uint8_t *tmp_p; + size_t tmp_size; + size_t u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_STATIC_NOERR + FUNC_ENTER_STATIC /* Sanity check */ HDassert(layout); @@ -374,31 +386,104 @@ H5P__dcrt_layout_enc(const void *value, void **_pp, size_t *size) if(NULL != *pp) { /* Encode layout type */ *(*pp)++ = (uint8_t)layout->type; + *size += sizeof(uint8_t); /* If layout is chunked, encode chunking structure */ if(H5D_CHUNKED == layout->type) { - unsigned u; /* Local index variable */ - /* Encode rank */ *(*pp)++ = (uint8_t)layout->u.chunk.ndims; + *size += sizeof(uint8_t); /* Encode chunk dims */ HDcompile_assert(sizeof(uint32_t) == sizeof(layout->u.chunk.dim[0])); - for(u = 0; u < layout->u.chunk.ndims; u++) + for(u = 0; u < (size_t)layout->u.chunk.ndims; u++) { UINT32ENCODE(*pp, layout->u.chunk.dim[u]) + *size += sizeof(uint32_t); + } /* end for */ + } /* end if */ + else if(H5D_VIRTUAL == layout->type) { + uint64_t nentries = (uint64_t)layout->storage.u.virt.list_nused; + + /* Encode number of entries */ + UINT64ENCODE(*pp, nentries) + *size += (size_t)8; + + /* Iterate over entries */ + for(u = 0; u < layout->storage.u.virt.list_nused; u++) { + /* Source file name */ + tmp_size = HDstrlen(layout->storage.u.virt.list[u].source_file_name) + (size_t)1; + (void)HDmemcpy(*pp, layout->storage.u.virt.list[u].source_file_name, tmp_size); + *pp += tmp_size; + *size += tmp_size; + + /* Source dataset name */ + tmp_size = HDstrlen(layout->storage.u.virt.list[u].source_dset_name) + (size_t)1; + (void)HDmemcpy(*pp, layout->storage.u.virt.list[u].source_dset_name, tmp_size); + *pp += tmp_size; + *size += tmp_size; + + /* Source selection. Note that we are not passing the real + * allocated size because we do not know it. H5P__encode should + * have verified that the buffer is large enough for the entire + * list before we get here. */ + tmp_size = (size_t)-1; + tmp_p = *pp; + if(H5S_encode(layout->storage.u.virt.list[u].source_select, pp, &tmp_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTENCODE, FAIL, "unable to serialize source selection") + *size += (size_t)(*pp - tmp_p); + + /* Virtual dataset selection. Same notes as above apply. */ + tmp_size = (size_t)-1; + tmp_p = *pp; + if(H5S_encode(layout->storage.u.virt.list[u].source_dset.virtual_select, pp, &tmp_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTENCODE, FAIL, "unable to serialize virtual selection") + *size += (size_t)(*pp - tmp_p); + } /* end for */ } /* end if */ } /* end if */ - - /* Size of layout type */ - *size += sizeof(uint8_t); - - /* Size of chunk info encoding */ - if(H5D_CHUNKED == layout->type) { + else { + /* Size of layout type */ *size += sizeof(uint8_t); - *size += layout->u.chunk.ndims * sizeof(uint32_t); - } /* end if */ - FUNC_LEAVE_NOAPI(SUCCEED) + /* If layout is chunked, calculate chunking structure */ + if(H5D_CHUNKED == layout->type) { + *size += sizeof(uint8_t); + *size += layout->u.chunk.ndims * sizeof(uint32_t); + } /* end if */ + else if(H5D_VIRTUAL == layout->type) { + /* Calculate size of virtual layout info */ + /* number of entries */ + *size += (size_t)8; + + /* Iterate over entries */ + for(u = 0; u < layout->storage.u.virt.list_nused; u++) { + /* Source file name */ + tmp_size = HDstrlen(layout->storage.u.virt.list[u].source_file_name) + (size_t)1; + *size += tmp_size; + + /* Source dataset name */ + tmp_size = HDstrlen(layout->storage.u.virt.list[u].source_dset_name) + (size_t)1; + *size += tmp_size; + + /* Source selection */ + tmp_size = (size_t)0; + tmp_p = NULL; + if(H5S_encode(layout->storage.u.virt.list[u].source_select, &tmp_p, &tmp_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTENCODE, FAIL, "unable to serialize source selection") + *size += tmp_size; + + /* Virtual dataset selection */ + tmp_size = (size_t)0; + tmp_p = NULL; + if(H5S_encode(layout->storage.u.virt.list[u].source_dset.virtual_select, &tmp_p, &tmp_size) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTENCODE, FAIL, "unable to serialize virtual selection") + *size += tmp_size; + } /* end for */ + } /* end if */ + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) } /* end H5P__dcrt_layout_enc() */ @@ -474,6 +559,100 @@ H5P__dcrt_layout_dec(const void **_pp, void *value) } break; + case H5D_VIRTUAL: + { + uint64_t nentries; /* Number of VDS mappings */ + + /* Decode number of entries */ + UINT64DECODE(*pp, nentries) + + if(nentries == (uint64_t)0) + /* Just use the default struct */ + layout = &H5D_def_layout_virtual_g; + else { + size_t tmp_size; + size_t u; /* Local index variable */ + + /* Initialize to default values */ + tmp_layout = H5D_def_layout_virtual_g; + + /* Allocate entry list */ + if(NULL == (tmp_layout.storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_calloc((size_t)nentries * sizeof(H5O_storage_virtual_ent_t)))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "unable to allocate heap block") + tmp_layout.storage.u.virt.list_nalloc = (size_t)nentries; + tmp_layout.storage.u.virt.list_nused = (size_t)nentries; + + /* Decode each entry */ + for(u = 0; u < (size_t)nentries; u++) { + /* Source file name */ + tmp_size = HDstrlen((const char *)*pp) + 1; + if(NULL == (tmp_layout.storage.u.virt.list[u].source_file_name = (char *)H5MM_malloc(tmp_size))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "unable to allocate memory for source file name") + (void)HDmemcpy(tmp_layout.storage.u.virt.list[u].source_file_name, *pp, tmp_size); + *pp += tmp_size; + + /* Source dataset name */ + tmp_size = HDstrlen((const char *)*pp) + 1; + if(NULL == (tmp_layout.storage.u.virt.list[u].source_dset_name = (char *)H5MM_malloc(tmp_size))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTALLOC, FAIL, "unable to allocate memory for source dataset name") + (void)HDmemcpy(tmp_layout.storage.u.virt.list[u].source_dset_name, *pp, tmp_size); + *pp += tmp_size; + + /* Source selection */ + if(NULL == (tmp_layout.storage.u.virt.list[u].source_select = H5S_decode(pp))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTDECODE, FAIL, "can't decode source space selection") + tmp_layout.storage.u.virt.list[u].source_space_status = H5O_VIRTUAL_STATUS_USER; + + /* Virtual selection */ + if(NULL == (tmp_layout.storage.u.virt.list[u].source_dset.virtual_select = H5S_decode(pp))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTDECODE, FAIL, "can't decode virtual space selection") + tmp_layout.storage.u.virt.list[u].virtual_space_status = H5O_VIRTUAL_STATUS_USER; + + /* Parse source file and dataset names for "printf" + * style format specifiers */ + if(H5D_virtual_parse_source_name(tmp_layout.storage.u.virt.list[u].source_file_name, &tmp_layout.storage.u.virt.list[u].parsed_source_file_name, &tmp_layout.storage.u.virt.list[u].psfn_static_strlen, &tmp_layout.storage.u.virt.list[u].psfn_nsubs) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't parse source file name") + if(H5D_virtual_parse_source_name(tmp_layout.storage.u.virt.list[u].source_dset_name, &tmp_layout.storage.u.virt.list[u].parsed_source_dset_name, &tmp_layout.storage.u.virt.list[u].psdn_static_strlen, &tmp_layout.storage.u.virt.list[u].psdn_nsubs) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't parse source dataset name") + + /* Set source names in source_dset struct */ + if((tmp_layout.storage.u.virt.list[u].psfn_nsubs == 0) + && (tmp_layout.storage.u.virt.list[u].psdn_nsubs == 0)) { + if(tmp_layout.storage.u.virt.list[u].parsed_source_file_name) + tmp_layout.storage.u.virt.list[u].source_dset.file_name = tmp_layout.storage.u.virt.list[u].parsed_source_file_name->name_segment; + else + tmp_layout.storage.u.virt.list[u].source_dset.file_name = tmp_layout.storage.u.virt.list[u].source_file_name; + if(tmp_layout.storage.u.virt.list[u].parsed_source_dset_name) + tmp_layout.storage.u.virt.list[u].source_dset.dset_name = tmp_layout.storage.u.virt.list[u].parsed_source_dset_name->name_segment; + else + tmp_layout.storage.u.virt.list[u].source_dset.dset_name = tmp_layout.storage.u.virt.list[u].source_dset_name; + } /* end if */ + + /* unlim_dim fields */ + tmp_layout.storage.u.virt.list[u].unlim_dim_source = H5S_get_select_unlim_dim(tmp_layout.storage.u.virt.list[u].source_select); + tmp_layout.storage.u.virt.list[u].unlim_dim_virtual = H5S_get_select_unlim_dim(tmp_layout.storage.u.virt.list[u].source_dset.virtual_select); + tmp_layout.storage.u.virt.list[u].unlim_extent_source = HSIZE_UNDEF; + tmp_layout.storage.u.virt.list[u].unlim_extent_virtual = HSIZE_UNDEF; + tmp_layout.storage.u.virt.list[u].clip_size_source = HSIZE_UNDEF; + tmp_layout.storage.u.virt.list[u].clip_size_virtual = HSIZE_UNDEF; + + /* Clipped selections */ + if(tmp_layout.storage.u.virt.list[u].unlim_dim_virtual < 0) { + tmp_layout.storage.u.virt.list[u].source_dset.clipped_source_select = tmp_layout.storage.u.virt.list[u].source_select; + tmp_layout.storage.u.virt.list[u].source_dset.clipped_virtual_select = tmp_layout.storage.u.virt.list[u].source_dset.virtual_select; + } /* end if */ + + /* Update min_dims */ + if(H5D_virtual_update_min_dims(&tmp_layout, u) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "unable to update virtual dataset minimum dimensions") + } /* end for */ + + /* Point at the newly set up struct */ + layout = &tmp_layout; + } /* end else */ + } /* end block */ + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -619,6 +798,51 @@ H5P__dcrt_layout_cmp(const void *_layout1, const void *_layout2, } /* end case */ break; + case H5D_VIRTUAL: + { + htri_t equal; + int strcmp_ret; + size_t u; /* Local index variable */ + + /* Compare number of mappings */ + if(layout1->storage.u.virt.list_nused < layout2->storage.u.virt.list_nused) HGOTO_DONE(-1) + if(layout1->storage.u.virt.list_nused > layout2->storage.u.virt.list_nused) HGOTO_DONE(1) + + /* Iterate over mappings */ + for(u = 0; u < layout1->storage.u.virt.list_nused; u++) { + /* Compare virtual spaces. Note we cannot tell which is + * "greater", so just return 1 if different, -1 on failure. + */ + if((equal = H5S_extent_equal(layout1->storage.u.virt.list[u].source_dset.virtual_select, layout2->storage.u.virt.list[u].source_dset.virtual_select)) < 0) HGOTO_DONE(-1) + if(!equal) + HGOTO_DONE(1) + if((equal = H5S_select_shape_same(layout1->storage.u.virt.list[u].source_dset.virtual_select, layout2->storage.u.virt.list[u].source_dset.virtual_select)) < 0) HGOTO_DONE(-1) + if(!equal) + HGOTO_DONE(1) + + /* Compare source file names */ + strcmp_ret = HDstrcmp(layout1->storage.u.virt.list[u].source_file_name, layout2->storage.u.virt.list[u].source_file_name); + if(strcmp_ret < 0) HGOTO_DONE(-1) + if(strcmp_ret > 0) HGOTO_DONE(1) + + /* Compare source dataset names */ + strcmp_ret = HDstrcmp(layout1->storage.u.virt.list[u].source_dset_name, layout2->storage.u.virt.list[u].source_dset_name); + if(strcmp_ret < 0) HGOTO_DONE(-1) + if(strcmp_ret > 0) HGOTO_DONE(1) + + /* Compare source spaces. Note we cannot tell which is + * "greater", so just return 1 if different, -1 on failure. + */ + if((equal = H5S_extent_equal(layout1->storage.u.virt.list[u].source_select, layout2->storage.u.virt.list[u].source_select)) < 0) HGOTO_DONE(-1) + if(!equal) + HGOTO_DONE(1) + if((equal = H5S_select_shape_same(layout1->storage.u.virt.list[u].source_select, layout2->storage.u.virt.list[u].source_select)) < 0) HGOTO_DONE(-1) + if(!equal) + HGOTO_DONE(1) + } /* end for */ + } /* end block */ + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -1545,6 +1769,7 @@ H5P__set_layout(H5P_genplist_t *plist, const H5O_layout_t *layout) break; case H5D_CHUNKED: + case H5D_VIRTUAL: fill.alloc_time = H5D_ALLOC_TIME_INCR; break; @@ -1588,6 +1813,7 @@ H5P__init_def_layout(void) const H5O_layout_chunk_t def_layout_chunk = H5D_DEF_LAYOUT_CHUNK_INIT; const H5O_storage_compact_t def_store_compact = H5D_DEF_STORAGE_COMPACT_INIT; const H5O_storage_chunk_t def_store_chunk = H5D_DEF_STORAGE_CHUNK_INIT; + const H5O_storage_virtual_t def_store_virtual = H5D_DEF_STORAGE_VIRTUAL_INIT; FUNC_ENTER_STATIC_NOERR @@ -1597,6 +1823,8 @@ H5P__init_def_layout(void) H5D_def_layout_chunk_g.u.chunk = def_layout_chunk; H5D_def_layout_chunk_g.storage.type = H5D_CHUNKED; H5D_def_layout_chunk_g.storage.u.chunk = def_store_chunk; + H5D_def_layout_virtual_g.storage.type = H5D_VIRTUAL; + H5D_def_layout_virtual_g.storage.u.virt = def_store_virtual; /* Note that we've initialized the default values */ H5P_dcrt_def_layout_init_g = TRUE; @@ -1666,6 +1894,10 @@ H5Pset_layout(hid_t plist_id, H5D_layout_t layout_type) layout = &H5D_def_layout_chunk_g; break; + case H5D_VIRTUAL: + layout = &H5D_def_layout_virtual_g; + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: @@ -1871,6 +2103,515 @@ done: /*------------------------------------------------------------------------- + * Function: H5Pset_virtual + * + * Purpose: Maps elements of the virtual dataset described by the + * virtual dataspace identifier vspace_id to the elements of + * the source dataset described by the source dataset + * dataspace identifier src_space_id. The source dataset is + * identified by the name of the file where it is located, + * src_file_name, and the name of the dataset, src_dset_name. + * + * As a side effect, the layout method is changed to + * H5D_VIRTUAL. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * Friday, February 13, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, const char *src_file_name, + const char *src_dset_name, hid_t src_space_id) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5O_layout_t virtual_layout; /* Layout information for setting virtual info */ + H5S_t *vspace; /* Virtual dataset space selection */ + H5S_t *src_space; /* Source dataset space selection */ + H5O_storage_virtual_ent_t *old_list = NULL; /* List pointer previously on property list */ + H5O_storage_virtual_ent_t *ent = NULL; /* Convenience pointer to new VDS entry */ + hbool_t retrieved_layout = FALSE; /* Whether the layout has been retrieved */ + hbool_t adding_entry = FALSE; /* Whether we are in the middle of adding an entry */ + hbool_t free_list = FALSE; /* Whether to free the list of virtual entries */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE5("e", "ii*s*si", dcpl_id, vspace_id, src_file_name, src_dset_name, + src_space_id); + + /* Check arguments */ + if(!src_file_name) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "source file name not provided") + if(!src_dset_name) + HGOTO_ERROR(H5E_PLIST, H5E_BADRANGE, FAIL, "source dataset name not provided") + if(NULL == (vspace = (H5S_t *)H5I_object_verify(vspace_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dataspace") + if(NULL == (src_space = (H5S_t *)H5I_object_verify(src_space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_PLIST, H5E_BADTYPE, FAIL, "not a dataspace") + + /* Check selections for validity */ + if(H5D_virtual_check_mapping_pre(vspace, src_space, H5O_VIRTUAL_STATUS_USER) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "invalid mapping selections") + +#ifndef H5_HAVE_C99_DESIGNATED_INITIALIZER + /* If the compiler doesn't support C99 designated initializers, check if + * the default layout structs have been initialized yet or not. *ick* -QAK + */ + if(!H5P_dcrt_def_layout_init_g) + if(H5P__init_def_layout() < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't initialize default layout info") +#endif /* H5_HAVE_C99_DESIGNATED_INITIALIZER */ + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get the current layout */ + if(H5P_peek(plist, H5D_CRT_LAYOUT_NAME, &virtual_layout) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get layout") + retrieved_layout = TRUE; + + /* If the layout was not already virtual, Start with default virtual layout. + * Otherwise, add the mapping to the current list. */ + if(virtual_layout.type == H5D_VIRTUAL) + /* Save old list pointer for error recovery */ + old_list = virtual_layout.storage.u.virt.list; + else { + /* Reset the old layout */ + if(H5O_msg_reset(H5O_LAYOUT_ID, &virtual_layout) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTRESET, FAIL, "can't release layout message") + + /* Copy the default virtual layout */ + HDmemcpy(&virtual_layout, &H5D_def_layout_virtual_g, sizeof(H5D_def_layout_virtual_g)); + + /* Sanity check */ + HDassert(virtual_layout.storage.u.virt.list_nalloc == 0); + } /* end else */ + + /* Expand list if necessary */ + if(virtual_layout.storage.u.virt.list_nused == virtual_layout.storage.u.virt.list_nalloc) { + H5O_storage_virtual_ent_t *x; /* Pointer to the new list */ + size_t new_alloc = MAX(H5D_VIRTUAL_DEF_LIST_SIZE, virtual_layout.storage.u.virt.list_nalloc * 2); + + /* Expand size of entry list */ + if(NULL == (x = (H5O_storage_virtual_ent_t *)H5MM_realloc(virtual_layout.storage.u.virt.list, new_alloc * sizeof(H5O_storage_virtual_ent_t)))) + HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't reallocate virtual dataset mapping list") + virtual_layout.storage.u.virt.list = x; + virtual_layout.storage.u.virt.list_nalloc = new_alloc; + } /* end if */ + + /* Add virtual dataset mapping entry */ + ent = &virtual_layout.storage.u.virt.list[virtual_layout.storage.u.virt.list_nused]; + HDmemset(ent, 0, sizeof(H5O_storage_virtual_ent_t)); /* Clear before starting to set up */ + if(NULL == (ent->source_dset.virtual_select = H5S_copy(vspace, FALSE, TRUE))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") + adding_entry = TRUE; + if(NULL == (ent->source_file_name = HDstrdup(src_file_name))) + HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't duplicate source file name") + if(NULL == (ent->source_dset_name = HDstrdup(src_dset_name))) + HGOTO_ERROR(H5E_PLIST, H5E_RESOURCE, FAIL, "can't duplicate source file name") + if(NULL == (ent->source_select = H5S_copy(src_space, FALSE, TRUE))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy source selection") + if(H5D_virtual_parse_source_name(ent->source_file_name, &ent->parsed_source_file_name, &ent->psfn_static_strlen, &ent->psfn_nsubs) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't parse source file name") + if(H5D_virtual_parse_source_name(ent->source_dset_name, &ent->parsed_source_dset_name, &ent->psdn_static_strlen, &ent->psdn_nsubs) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "can't parse source dataset name") + if((ent->psfn_nsubs == 0) && (ent->psdn_nsubs == 0)) { + if(ent->parsed_source_file_name) + ent->source_dset.file_name = ent->parsed_source_file_name->name_segment; + else + ent->source_dset.file_name = ent->source_file_name; + if(ent->parsed_source_dset_name) + ent->source_dset.dset_name = ent->parsed_source_dset_name->name_segment; + else + ent->source_dset.dset_name = ent->source_dset_name; + } /* end if */ + ent->unlim_dim_source = H5S_get_select_unlim_dim(src_space); + ent->unlim_dim_virtual = H5S_get_select_unlim_dim(vspace); + if(ent->unlim_dim_virtual < 0) { + ent->source_dset.clipped_source_select = ent->source_select; + ent->source_dset.clipped_virtual_select = ent->source_dset.virtual_select; + } /* end if */ + ent->unlim_extent_source = HSIZE_UNDEF; + ent->unlim_extent_virtual = HSIZE_UNDEF; + ent->clip_size_source = HSIZE_UNDEF; + ent->clip_size_virtual = HSIZE_UNDEF; + ent->source_space_status = H5O_VIRTUAL_STATUS_USER; + ent->virtual_space_status = H5O_VIRTUAL_STATUS_USER; + + /* Check entry for validity */ + if(H5D_virtual_check_mapping_post(ent) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid mapping entry") + + /* Update min_dims */ + if(H5D_virtual_update_min_dims(&virtual_layout, virtual_layout.storage.u.virt.list_nused) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINIT, FAIL, "unable to update virtual dataset minimum dimensions") + + /* Finish adding entry */ + virtual_layout.storage.u.virt.list_nused++; + +done: + /* Set VDS layout information in property list */ + /* (Even on faliure, so there's not a mangled layout struct in the list) */ + if(retrieved_layout) { + if(H5P_poke(plist, H5D_CRT_LAYOUT_NAME, &virtual_layout) < 0) { + HDONE_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set layout") + if(old_list != virtual_layout.storage.u.virt.list) + free_list = TRUE; + } /* end if */ + } /* end if */ + + /* Check if the entry has been partly allocated but not added to the + * property list or not included in list_nused */ + if(ret_value < 0) { + /* Free incomplete entry if present */ + if(ent) { + ent->source_file_name = (char *)H5MM_xfree(ent->source_file_name); + ent->source_dset_name = (char *)H5MM_xfree(ent->source_dset_name); + if(ent->source_dset.virtual_select && H5S_close(ent->source_dset.virtual_select) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release virtual selection") + ent->source_dset.virtual_select = NULL; + if(ent->source_select && H5S_close(ent->source_select) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release source selection") + ent->source_select = NULL; + H5D_virtual_free_parsed_name(ent->parsed_source_file_name); + ent->parsed_source_file_name = NULL; + H5D_virtual_free_parsed_name(ent->parsed_source_dset_name); + ent->parsed_source_dset_name = NULL; + } /* end if */ + + /* Free list if necessary */ + if(free_list) + virtual_layout.storage.u.virt.list = (H5O_storage_virtual_ent_t *)H5MM_xfree(virtual_layout.storage.u.virt.list); + } /* end if */ + + FUNC_LEAVE_API(ret_value) +} /* end H5Pset_virtual() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_virtual_count + * + * Purpose: Gets the number of mappings for the virtual dataset that + * has a creation property list specified by the dcpl_id + * parameter. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Neil Fortner + * Friday, February 13, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_virtual_count(hid_t dcpl_id, size_t *count/*out*/) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5O_layout_t layout; /* Layout information */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ix", dcpl_id, count); + + if(count) { + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Retrieve the layout property */ + if(H5P_peek(plist, H5D_CRT_LAYOUT_NAME, &layout) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't get layout") + if(H5D_VIRTUAL != layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a virtual storage layout") + + /* Return the number of mappings */ + *count = layout.storage.u.virt.list_nused; + } /* end if */ + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_virtual_count() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_virtual_vspace + * + * Purpose: Takes the dataset creation property list for the virtual + * dataset, dcpl_id, and the mapping index, index, and + * returns a dataspace identifier for the selection within + * the virtual dataset used in the mapping. + * + * Return: Returns a dataspace identifier if successful; otherwise + * returns a negative value. + * + * Programmer: Neil Fortner + * Friday, February 13, 2015 + * + *------------------------------------------------------------------------- + */ +hid_t +H5Pget_virtual_vspace(hid_t dcpl_id, size_t index) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5O_layout_t layout; /* Layout information */ + H5S_t *space = NULL; /* Dataspace pointer */ + hid_t ret_value; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("i", "iz", dcpl_id, index); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Retrieve the layout property */ + if(H5P_peek(plist, H5D_CRT_LAYOUT_NAME, &layout) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't get layout") + if(H5D_VIRTUAL != layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a virtual storage layout") + + /* Get the virtual space */ + if(index >= layout.storage.u.virt.list_nused) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid index (out of range)") + HDassert(layout.storage.u.virt.list_nused <= layout.storage.u.virt.list_nalloc); + if(NULL == (space = H5S_copy(layout.storage.u.virt.list[index].source_dset.virtual_select, FALSE, TRUE))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy virtual selection") + + /* Register ID */ + if((ret_value = H5I_register(H5I_DATASPACE, space, TRUE)) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register data space") + +done: + /* Free space on failure */ + if((ret_value < 0) && space) + if(H5S_close(space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release source selection") + + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_virtual_vspace() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_virtual_srcspace + * + * Purpose: Takes the dataset creation property list for the virtual + * dataset, dcpl_id, and the mapping index, index, and + * returns a dataspace identifier for the selection within + * the source dataset used in the mapping. + * + * Return: Returns a dataspace identifier if successful; otherwise + * returns a negative value. + * + * Programmer: Neil Fortner + * Saturday, February 14, 2015 + * + *------------------------------------------------------------------------- + */ +hid_t +H5Pget_virtual_srcspace(hid_t dcpl_id, size_t index) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5O_layout_t layout; /* Layout information */ + H5S_t *space = NULL; /* Dataspace pointer */ + hid_t ret_value = FAIL; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("i", "iz", dcpl_id, index); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Retrieve the layout property */ + if(H5P_peek(plist, H5D_CRT_LAYOUT_NAME, &layout) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't get layout") + if(H5D_VIRTUAL != layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a virtual storage layout") + + /* Check index */ + if(index >= layout.storage.u.virt.list_nused) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid index (out of range)") + HDassert(layout.storage.u.virt.list_nused <= layout.storage.u.virt.list_nalloc); + + /* Attempt to open source dataset and patch extent if extent status is not + * H5O_VIRTUAL_STATUS_CORRECT? -NAF */ + /* If source space status is H5O_VIRTUAL_STATUS_INVALID, patch with bounds + * of selection */ + if((H5O_VIRTUAL_STATUS_INVALID == layout.storage.u.virt.list[index].source_space_status) + && (layout.storage.u.virt.list[index].unlim_dim_source < 0)) { + hsize_t bounds_start[H5S_MAX_RANK]; + hsize_t bounds_end[H5S_MAX_RANK]; + int rank; + int i; + + /* Get rank of source space */ + if((rank = H5S_GET_EXTENT_NDIMS(layout.storage.u.virt.list[index].source_select)) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get source space rank") + + /* Get bounds of selection */ + if(H5S_SELECT_BOUNDS(layout.storage.u.virt.list[index].source_select, bounds_start, bounds_end) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get selection bounds") + + /* Adjust bounds to extent */ + for(i = 0; i < rank; i++) + bounds_end[i]++; + + /* Set extent */ + if(H5S_set_extent_simple(layout.storage.u.virt.list[index].source_select, (unsigned)rank, bounds_end, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set source space extent") + + /* Update source space status */ + layout.storage.u.virt.list[index].source_space_status = H5O_VIRTUAL_STATUS_SEL_BOUNDS; + } /* end if */ + + /* Get the source space */ + if(NULL == (space = H5S_copy(layout.storage.u.virt.list[index].source_select, FALSE, TRUE))) + HGOTO_ERROR(H5E_PLIST, H5E_CANTCOPY, FAIL, "unable to copy source selection") + + /* Register ID */ + if((ret_value = H5I_register(H5I_DATASPACE, space, TRUE)) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register data space") + +done: + /* Free space on failure */ + if((ret_value < 0) && space) + if(H5S_close(space) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release source selection") + + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_virtual_srcspace() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_virtual_filename + * + * Purpose: Takes the dataset creation property list for the virtual + * dataset, dcpl_id, and the mapping index, index, and + * retrieves a name of a file for a source dataset used in + * the mapping. + * + * Up to size characters of the filename are returned in + * name; additional characters, if any, are not returned to + * the user application. + * + * If the length of the filename, which determines the + * required value of size, is unknown, a preliminary call to + * H5Pget_virtual_filename with the last two parameters set + * to NULL can be made. The return value of this call will + * be the size in bytes of the filename. That value, plus 1 + * for a NULL terminator, is then assigned to size for a + * second H5Pget_virtual_filename call, which will retrieve + * the actual filename. + * + * Return: Returns the length of the name if successful, otherwise + * returns a negative value. + * + * Programmer: Neil Fortner + * Saturday, February 14, 2015 + * + *------------------------------------------------------------------------- + */ +ssize_t +H5Pget_virtual_filename(hid_t dcpl_id, size_t index, char *name/*out*/, + size_t size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5O_layout_t layout; /* Layout information */ + ssize_t ret_value; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE4("Zs", "izxz", dcpl_id, index, name, size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Retrieve the layout property */ + if(H5P_peek(plist, H5D_CRT_LAYOUT_NAME, &layout) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't get layout") + if(H5D_VIRTUAL != layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a virtual storage layout") + + /* Get the virtual filename */ + if(index >= layout.storage.u.virt.list_nused) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid index (out of range)") + HDassert(layout.storage.u.virt.list_nused <= layout.storage.u.virt.list_nalloc); + HDassert(layout.storage.u.virt.list[index].source_file_name); + if(name && (size > 0)) + (void)HDstrncpy(name, layout.storage.u.virt.list[index].source_file_name, size); + ret_value = (ssize_t)HDstrlen(layout.storage.u.virt.list[index].source_file_name); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_virtual_filename() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_virtual_dsetname + * + * Purpose: Takes the dataset creation property list for the virtual + * dataset, dcpl_id, and the mapping index, index, and + * retrieves the name of a source dataset used in the mapping. + * + * Up to size characters of the name are returned in name; + * additional characters, if any, are not returned to the + * user application. + * + * If the length of the filename, which determines the + * required value of size, is unknown, a preliminary call to + * H5Pget_virtual_dsetname with the last two parameters set + * to NULL can be made. The return value of this call will + * be the size in bytes of the filename. That value, plus 1 + * for a NULL terminator, is then assigned to size for a + * second H5Pget_virtual_dsetname call, which will retrieve + * the actual filename. + * + * Return: Returns the length of the name if successful, otherwise + * returns a negative value. + * + * Programmer: Neil Fortner + * Saturday, February 14, 2015 + * + *------------------------------------------------------------------------- + */ +ssize_t +H5Pget_virtual_dsetname(hid_t dcpl_id, size_t index, char *name/*out*/, + size_t size) +{ + H5P_genplist_t *plist; /* Property list pointer */ + H5O_layout_t layout; /* Layout information */ + ssize_t ret_value; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE4("Zs", "izxz", dcpl_id, index, name, size); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(dcpl_id, H5P_DATASET_CREATE))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Retrieve the layout property */ + if(H5P_peek(plist, H5D_CRT_LAYOUT_NAME, &layout) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_BADVALUE, FAIL, "can't get layout") + if(H5D_VIRTUAL != layout.type) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a virtual storage layout") + + /* Get the virtual filename */ + if(index >= layout.storage.u.virt.list_nused) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid index (out of range)") + HDassert(layout.storage.u.virt.list_nused <= layout.storage.u.virt.list_nalloc); + HDassert(layout.storage.u.virt.list[index].source_dset_name); + if(name && (size > 0)) + (void)HDstrncpy(name, layout.storage.u.virt.list[index].source_dset_name, size); + ret_value = (ssize_t)HDstrlen(layout.storage.u.virt.list[index].source_dset_name); + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_virtual_dsetname() */ + + +/*------------------------------------------------------------------------- * Function: H5Pset_external * * Purpose: Adds an external file to the list of external files. PLIST_ID @@ -2710,6 +3451,10 @@ H5Pset_alloc_time(hid_t plist_id, H5D_alloc_time_t alloc_time) alloc_time = H5D_ALLOC_TIME_INCR; break; + case H5D_VIRTUAL: + alloc_time = H5D_ALLOC_TIME_INCR; + break; + case H5D_LAYOUT_ERROR: case H5D_NLAYOUTS: default: diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index f56692c..4929621 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -354,6 +354,15 @@ H5_DLL herr_t H5Pset_layout(hid_t plist_id, H5D_layout_t layout); H5_DLL H5D_layout_t H5Pget_layout(hid_t plist_id); H5_DLL herr_t H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/]); H5_DLL int H5Pget_chunk(hid_t plist_id, int max_ndims, hsize_t dim[]/*out*/); +H5_DLL herr_t H5Pset_virtual(hid_t dcpl_id, hid_t vspace_id, + const char *src_file_name, const char *src_dset_name, hid_t src_space_id); +H5_DLL herr_t H5Pget_virtual_count(hid_t dcpl_id, size_t *count/*out*/); +H5_DLL hid_t H5Pget_virtual_vspace(hid_t dcpl_id, size_t index); +H5_DLL hid_t H5Pget_virtual_srcspace(hid_t dcpl_id, size_t index); +H5_DLL ssize_t H5Pget_virtual_filename(hid_t dcpl_id, size_t index, + char *name/*out*/, size_t size); +H5_DLL ssize_t H5Pget_virtual_dsetname(hid_t dcpl_id, size_t index, + char *name/*out*/, size_t size); H5_DLL herr_t H5Pset_external(hid_t plist_id, const char *name, off_t offset, hsize_t size); H5_DLL int H5Pget_external_count(hid_t plist_id); @@ -384,6 +393,10 @@ H5_DLL herr_t H5Pget_chunk_cache(hid_t dapl_id, size_t *rdcc_nslots/*out*/, size_t *rdcc_nbytes/*out*/, double *rdcc_w0/*out*/); +H5_DLL herr_t H5Pset_virtual_view(hid_t plist_id, H5D_vds_view_t view); +H5_DLL herr_t H5Pget_virtual_view(hid_t plist_id, H5D_vds_view_t *view); +H5_DLL herr_t H5Pset_virtual_printf_gap(hid_t plist_id, hsize_t gap_size); +H5_DLL herr_t H5Pget_virtual_printf_gap(hid_t plist_id, hsize_t *gap_size); /* Dataset xfer property list (DXPL) routines */ H5_DLL herr_t H5Pset_data_transform(hid_t plist_id, const char* expression); @@ -50,8 +50,6 @@ /* Local Prototypes */ /********************/ static htri_t H5S_is_simple(const H5S_t *sdim); -static herr_t H5S_encode(H5S_t *obj, unsigned char **p, size_t *nalloc); -static H5S_t *H5S_decode(const unsigned char **p); /*********************/ @@ -1575,11 +1573,10 @@ done: * *------------------------------------------------------------------------- */ -static herr_t +herr_t H5S_encode(H5S_t *obj, unsigned char **p, size_t *nalloc) { H5F_t *f = NULL; /* Fake file structure*/ - unsigned char *pp = (*p); /* Local pointer for decoding */ size_t extent_size; /* Size of serialized dataspace extent */ hssize_t sselect_size; /* Signed size of serialized dataspace selection */ size_t select_size; /* Size of serialized dataspace selection */ @@ -1602,9 +1599,11 @@ H5S_encode(H5S_t *obj, unsigned char **p, size_t *nalloc) /* Verify the size of buffer. If it's not big enough, simply return the * right size without filling the buffer. */ - if(!pp || *nalloc < (extent_size + select_size + 1 + 1 + 1 + 4)) + if(!*p || *nalloc < (extent_size + select_size + 1 + 1 + 1 + 4)) *nalloc = extent_size + select_size + 1 + 1 + 1 + 4; else { + unsigned char *pp = (*p); /* Local pointer for decoding */ + /* Encode the type of the information */ *pp++ = H5O_SDSPACE_ID; @@ -1693,7 +1692,7 @@ done: * *------------------------------------------------------------------------- */ -static H5S_t* +H5S_t* H5S_decode(const unsigned char **p) { H5F_t *f = NULL; /* Fake file structure*/ diff --git a/src/H5Sall.c b/src/H5Sall.c index b8a39cc..fb6b45f 100644 --- a/src/H5Sall.c +++ b/src/H5Sall.c @@ -40,9 +40,11 @@ static herr_t H5S_all_release(H5S_t *space); static htri_t H5S_all_is_valid(const H5S_t *space); static hssize_t H5S_all_serial_size(const H5S_t *space); static herr_t H5S_all_serialize(const H5S_t *space, uint8_t **p); -static herr_t H5S_all_deserialize(H5S_t *space, const uint8_t **p); +static herr_t H5S_all_deserialize(H5S_t *space, uint32_t version, uint8_t flags, + const uint8_t **p); static herr_t H5S_all_bounds(const H5S_t *space, hsize_t *start, hsize_t *end); static herr_t H5S_all_offset(const H5S_t *space, hsize_t *off); +static int H5S__all_unlim_dim(const H5S_t *space); static htri_t H5S_all_is_contiguous(const H5S_t *space); static htri_t H5S_all_is_single(const H5S_t *space); static htri_t H5S_all_is_regular(const H5S_t *space); @@ -74,6 +76,8 @@ const H5S_select_class_t H5S_sel_all[1] = {{ H5S_all_deserialize, H5S_all_bounds, H5S_all_offset, + H5S__all_unlim_dim, + NULL, H5S_all_is_contiguous, H5S_all_is_single, H5S_all_is_regular, @@ -545,6 +549,8 @@ H5S_all_serialize(const H5S_t *space, uint8_t **p) herr_t H5S_all_deserialize(space, p) H5S_t *space; IN/OUT: Dataspace pointer to place selection into + uint32_t version IN: Selection version + uint8_t flags IN: Selection flags uint8 **p; OUT: Pointer to buffer holding serialized selection. Will be advanced to end of serialized selection. @@ -559,7 +565,8 @@ H5S_all_serialize(const H5S_t *space, uint8_t **p) REVISION LOG --------------------------------------------------------------------------*/ static herr_t -H5S_all_deserialize(H5S_t *space, const uint8_t H5_ATTR_UNUSED **p) +H5S_all_deserialize(H5S_t *space, uint32_t H5_ATTR_UNUSED version, uint8_t H5_ATTR_UNUSED flags, + const uint8_t H5_ATTR_UNUSED **p) { herr_t ret_value = SUCCEED; /* return value */ @@ -665,6 +672,36 @@ H5S_all_offset(const H5S_t H5_ATTR_UNUSED *space, hsize_t *offset) /*-------------------------------------------------------------------------- NAME + H5S__all_unlim_dim + PURPOSE + Return unlimited dimension of selection, or -1 if none + USAGE + int H5S__all_unlim_dim(space) + H5S_t *space; IN: Dataspace pointer to check + RETURNS + Unlimited dimension of selection, or -1 if none (never fails). + DESCRIPTION + Returns the index of the unlimited dimension in this selection, or -1 + if the selection has no unlimited dimension. "All" selections are + always unlimited in every dimension, though this is not reflected in + other calls, where the selection is "clipped" against the current + extent, so for consistency this function always returns -1. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +static int +H5S__all_unlim_dim(const H5S_t H5_ATTR_UNUSED *space) +{ + FUNC_ENTER_STATIC_NOERR + + FUNC_LEAVE_NOAPI(-1) +} /* end H5S__all_unlim_dim() */ + + +/*-------------------------------------------------------------------------- + NAME H5S_all_is_contiguous PURPOSE Check if a "all" selection is contiguous within the dataspace extent. @@ -982,6 +1019,7 @@ H5S_all_get_seq_list(const H5S_t H5_ATTR_UNUSED *space, unsigned H5_ATTR_UNUSED /* Determine the actual number of elements to use */ H5_CHECK_OVERFLOW(iter->elmt_left,hsize_t,size_t); elem_used=MIN(maxelem,(size_t)iter->elmt_left); + HDassert(elem_used > 0); /* Compute the offset in the dataset */ off[0]=iter->u.all.byte_offset; diff --git a/src/H5Shyper.c b/src/H5Shyper.c index 10bcdd6..28cabf0 100644 --- a/src/H5Shyper.c +++ b/src/H5Shyper.c @@ -45,6 +45,10 @@ static herr_t H5S_hyper_generate_spans(H5S_t *space); #ifdef NEW_HYPERSLAB_API static herr_t H5S_select_select (H5S_t *space1, H5S_seloper_t op, H5S_t *space2); #endif /*NEW_HYPERSLAB_API*/ +static void H5S__hyper_get_clip_diminfo(hsize_t start, hsize_t stride, + hsize_t *count, hsize_t *block, hsize_t clip_size); +static hsize_t H5S__hyper_get_clip_extent_real(const H5S_t *clip_space, + hsize_t num_slices, hbool_t incl_trail); /* Selection callbacks */ static herr_t H5S_hyper_copy(H5S_t *dst, const H5S_t *src, hbool_t share_selection); @@ -55,9 +59,13 @@ static herr_t H5S_hyper_release(H5S_t *space); static htri_t H5S_hyper_is_valid(const H5S_t *space); static hssize_t H5S_hyper_serial_size(const H5S_t *space); static herr_t H5S_hyper_serialize(const H5S_t *space, uint8_t **p); -static herr_t H5S_hyper_deserialize(H5S_t *space, const uint8_t **p); +static herr_t H5S_hyper_deserialize(H5S_t *space, uint32_t version, uint8_t flags, + const uint8_t **p); static herr_t H5S_hyper_bounds(const H5S_t *space, hsize_t *start, hsize_t *end); static herr_t H5S_hyper_offset(const H5S_t *space, hsize_t *offset); +static int H5S__hyper_unlim_dim(const H5S_t *space); +static herr_t H5S_hyper_num_elem_non_unlim(const H5S_t *space, + hsize_t *num_elem_non_unlim); static htri_t H5S_hyper_is_contiguous(const H5S_t *space); static htri_t H5S_hyper_is_single(const H5S_t *space); static htri_t H5S_hyper_is_regular(const H5S_t *space); @@ -94,6 +102,8 @@ const H5S_select_class_t H5S_sel_hyper[1] = {{ H5S_hyper_deserialize, H5S_hyper_bounds, H5S_hyper_offset, + H5S__hyper_unlim_dim, + H5S_hyper_num_elem_non_unlim, H5S_hyper_is_contiguous, H5S_hyper_is_single, H5S_hyper_is_regular, @@ -248,6 +258,7 @@ H5S_hyper_iter_init(H5S_sel_iter_t *iter, const H5S_t *space) /* Check args */ HDassert(space && H5S_SEL_HYPERSLABS == H5S_GET_SELECT_TYPE(space)); HDassert(iter); + HDassert(space->select.sel_info.hslab->unlim_dim < 0); /* Initialize the number of points to iterate over */ iter->elmt_left = space->select.num_elem; @@ -1644,6 +1655,8 @@ H5S_hyper_copy (H5S_t *dst, const H5S_t *src, hbool_t share_selection) dst_hslab->app_diminfo[u]=src_hslab->app_diminfo[u]; } /* end for */ } /* end if */ + dst_hslab->unlim_dim = src_hslab->unlim_dim; + dst_hslab->num_elem_non_unlim = src_hslab->num_elem_non_unlim; dst->select.sel_info.hslab->span_lst=src->select.sel_info.hslab->span_lst; /* Check if there is hyperslab span information to copy */ @@ -1759,6 +1772,10 @@ H5S_hyper_is_valid (const H5S_t *space) HDassert(space); + /* Check for unlimited selection */ + if(space->select.sel_info.hslab->unlim_dim >= 0) + HGOTO_DONE(FALSE) + /* Check for a "regular" hyperslab selection */ if(space->select.sel_info.hslab->diminfo_valid) { const H5S_hyper_dim_t *diminfo=space->select.sel_info.hslab->opt_diminfo; /* local alias for diminfo */ @@ -1863,6 +1880,7 @@ H5S_get_select_hyper_nblocks(H5S_t *space) FUNC_ENTER_NOAPI_NOINIT_NOERR HDassert(space); + HDassert(space->select.sel_info.hslab->unlim_dim < 0); /* Check for a "regular" hyperslab selection */ if(space->select.sel_info.hslab->diminfo_valid) { @@ -1910,6 +1928,8 @@ H5Sget_select_hyper_nblocks(hid_t spaceid) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space") if(H5S_GET_SELECT_TYPE(space) != H5S_SEL_HYPERSLABS) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a hyperslab selection") + if(space->select.sel_info.hslab->unlim_dim >= 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "cannot get number of blocks for unlimited selection") ret_value = (hssize_t)H5S_get_select_hyper_nblocks(space); @@ -1948,24 +1968,41 @@ H5S_hyper_serial_size(const H5S_t *space) HDassert(space); - /* Basic number of bytes required to serialize hyperslab selection: - * <type (4 bytes)> + <version (4 bytes)> + <padding (4 bytes)> + - * <length (4 bytes)> + <rank (4 bytes)> + <# of blocks (4 bytes)> = 24 bytes - */ - ret_value = 24; + /* Check for version (right now, an unlimited dimension is the only thing + * that would bump the version) */ + if(space->select.sel_info.hslab->unlim_dim >= 0) + /* Version 2 */ + /* Size required is always: + * <type (4 bytes)> + <version (4 bytes)> + <flags (1 byte)> + + * <length (4 bytes)> + <rank (4 bytes)> + + * (4 (start/stride/count/block) * <rank> * <value (8 bytes)>) = + * 17 + (4 * rank * 8) bytes + */ + ret_value = (hssize_t)17 + ((hssize_t)4 * (hssize_t)space->extent.rank + * (hssize_t)8); + else { + /* Version 1 */ + /* Basic number of bytes required to serialize hyperslab selection: + * <type (4 bytes)> + <version (4 bytes)> + <padding (4 bytes)> + + * <length (4 bytes)> + <rank (4 bytes)> + <# of blocks (4 bytes)> + * = 24 bytes + */ + ret_value = 24; - /* Check for a "regular" hyperslab selection */ - if(space->select.sel_info.hslab->diminfo_valid) { - /* Check each dimension */ - for(block_count = 1, u = 0; u < space->extent.rank; u++) - block_count *= space->select.sel_info.hslab->opt_diminfo[u].count; - } /* end if */ - else - /* Spin through hyperslab spans, adding 8 * rank bytes for each block */ - block_count = H5S_hyper_span_nblocks(space->select.sel_info.hslab->span_lst); + /* Check for a "regular" hyperslab selection */ + if(space->select.sel_info.hslab->diminfo_valid) { + /* Check each dimension */ + for(block_count = 1, u = 0; u < space->extent.rank; u++) + block_count *= space->select.sel_info.hslab->opt_diminfo[u].count; + } /* end if */ + else + /* Spin through hyperslab spans, adding 8 * rank bytes for each + * block */ + block_count = H5S_hyper_span_nblocks(space->select.sel_info.hslab->span_lst); - H5_CHECK_OVERFLOW((8 * space->extent.rank * block_count), hsize_t, hssize_t); - ret_value += (hssize_t)(8 * block_count * space->extent.rank); + H5_CHECK_OVERFLOW((8 * space->extent.rank * block_count), hsize_t, hssize_t); + ret_value += (hssize_t)(8 * block_count * space->extent.rank); + } /* end else */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5S_hyper_serial_size() */ @@ -2020,8 +2057,7 @@ H5S_hyper_serialize_helper(const H5S_hyper_span_info_t *spans, end[rank]=curr->high; /* Recurse down to the next dimension */ - *p = pp; - H5S_hyper_serialize_helper(curr->down, start, end, rank + 1, p); + H5S_hyper_serialize_helper(curr->down, start, end, rank + 1, &pp); } /* end if */ else { /* Encode all the previous dimensions starting & ending points */ @@ -2079,12 +2115,14 @@ H5S_hyper_serialize(const H5S_t *space, uint8_t **p) const H5S_hyper_dim_t *diminfo; /* Alias for dataspace's diminfo information */ uint8_t *pp = (*p); /* Local pointer for decoding */ hsize_t tmp_count[H5O_LAYOUT_NDIMS]; /* Temporary hyperslab counts */ - hsize_t offset[H5O_LAYOUT_NDIMS]; /* Offset of element in dataspace */ + hsize_t offset[H5O_LAYOUT_NDIMS]; /* Offset of element in dataspace */ hsize_t start[H5O_LAYOUT_NDIMS]; /* Location of start of hyperslab */ hsize_t end[H5O_LAYOUT_NDIMS]; /* Location of end of hyperslab */ - hsize_t temp_off; /* Offset in a given dimension */ + hsize_t temp_off; /* Offset in a given dimension */ uint8_t *lenp; /* pointer to length location for later storage */ uint32_t len = 0; /* number of bytes used */ + uint32_t version; /* Version number */ + uint8_t flags = 0; /* Flags for message */ hsize_t block_count; /* block counter for regular hyperslabs */ unsigned fast_dim; /* Rank of the fastest changing dimension for the dataspace */ unsigned ndims; /* Rank of the dataspace */ @@ -2092,15 +2130,25 @@ H5S_hyper_serialize(const H5S_t *space, uint8_t **p) FUNC_ENTER_NOAPI_NOINIT_NOERR - /* Check args */ HDassert(space); HDassert(p); HDassert(pp); + /* Calculate version */ + if(space->select.sel_info.hslab->unlim_dim >= 0) { + version = 2; + flags |= H5S_SELECT_FLAG_UNLIM; + } /* end if */ + else + version = 1; + /* Store the preamble information */ UINT32ENCODE(pp, (uint32_t)H5S_GET_SELECT_TYPE(space)); /* Store the type of selection */ - UINT32ENCODE(pp, (uint32_t)1); /* Store the version number */ - UINT32ENCODE(pp, (uint32_t)0); /* Store the un-used padding */ + UINT32ENCODE(pp, version); /* Store the version number */ + if(version >= 2) + *(pp)++ = flags; /* Store the flags */ + else + UINT32ENCODE(pp, (uint32_t)0); /* Store the un-used padding */ lenp = pp; /* keep the pointer to the length location for later */ pp += 4; /* skip over space for length */ @@ -2108,8 +2156,23 @@ H5S_hyper_serialize(const H5S_t *space, uint8_t **p) UINT32ENCODE(pp, (uint32_t)space->extent.rank); len += 4; + /* If there is an unlimited dimension, only encode opt_unlim_diminfo */ + if(flags & H5S_SELECT_FLAG_UNLIM) { + unsigned i; + + HDassert(H5S_UNLIMITED == HSIZE_UNDEF); + + /* Iterate over dimensions */ + for(i = 0; i < space->extent.rank; i++) { + /* Encode start/stride/block/count */ + UINT64ENCODE(pp, space->select.sel_info.hslab->opt_diminfo[i].start); + UINT64ENCODE(pp, space->select.sel_info.hslab->opt_diminfo[i].stride); + UINT64ENCODE(pp, space->select.sel_info.hslab->opt_diminfo[i].count); + UINT64ENCODE(pp, space->select.sel_info.hslab->opt_diminfo[i].block); + } /* end for */ + } /* end if */ /* Check for a "regular" hyperslab selection */ - if(space->select.sel_info.hslab->diminfo_valid) { + else if(space->select.sel_info.hslab->diminfo_valid) { unsigned u; /* Local counting variable */ /* Set some convienence values */ @@ -2209,8 +2272,7 @@ H5S_hyper_serialize(const H5S_t *space, uint8_t **p) len += (uint32_t)(8 * space->extent.rank * block_count); /* Encode each hyperslab in selection */ - *p = pp; - H5S_hyper_serialize_helper(space->select.sel_info.hslab->span_lst, start, end, (hsize_t)0, p); + H5S_hyper_serialize_helper(space->select.sel_info.hslab->span_lst, start, end, (hsize_t)0, &pp); } /* end else */ /* Encode length */ @@ -2232,6 +2294,8 @@ H5S_hyper_serialize(const H5S_t *space, uint8_t **p) herr_t H5S_hyper_deserialize(space, p) H5S_t *space; IN/OUT: Dataspace pointer to place selection into + uint32_t version IN: Selection version + uint8_t flags IN: Selection flags uint8 **p; OUT: Pointer to buffer holding serialized selection. Will be advanced to end of serialized selection. @@ -2246,10 +2310,11 @@ H5S_hyper_serialize(const H5S_t *space, uint8_t **p) REVISION LOG --------------------------------------------------------------------------*/ static herr_t -H5S_hyper_deserialize(H5S_t *space, const uint8_t **p) +H5S_hyper_deserialize(H5S_t *space, uint32_t H5_ATTR_UNUSED version, uint8_t flags, + const uint8_t **p) { - const uint8_t *pp = (*p); /* Local pointer for decoding */ unsigned rank; /* rank of points */ + const uint8_t *pp = (*p); /* Local pointer for decoding */ size_t num_elem=0; /* number of elements in selection */ hsize_t start[H5O_LAYOUT_NDIMS]; /* hyperslab start information */ hsize_t end[H5O_LAYOUT_NDIMS]; /* hyperslab end information */ @@ -2274,32 +2339,54 @@ H5S_hyper_deserialize(H5S_t *space, const uint8_t **p) /* Deserialize slabs to select */ /* (The header and rank have already beed decoded) */ rank = space->extent.rank; /* Retrieve rank from space */ - UINT32DECODE(pp,num_elem); /* decode the number of points */ - /* Set the count & stride for all blocks */ - for(tcount=count,tstride=stride,j=0; j<rank; j++,tstride++,tcount++) { - *tcount=1; - *tstride=1; - } /* end for */ + /* If there is an unlimited dimension, only encode opt_unlim_diminfo */ + if(flags & H5S_SELECT_FLAG_UNLIM) { + HDassert(H5S_UNLIMITED == HSIZE_UNDEF); + HDassert(version >= 2); + + /* Iterate over dimensions */ + for(i = 0; i < space->extent.rank; i++) { + /* Decode start/stride/block/count */ + UINT64DECODE(pp, start[i]); + UINT64DECODE(pp, stride[i]); + UINT64DECODE(pp, count[i]); + UINT64DECODE(pp, block[i]); + } /* end for */ - /* Retrieve the coordinates from the buffer */ - for(i=0; i<num_elem; i++) { - /* Decode the starting points */ - for(tstart=start,j=0; j<rank; j++,tstart++) - UINT32DECODE(pp, *tstart); + /* Select the hyperslab to the current selection */ + if((ret_value = H5S_select_hyperslab(space, H5S_SELECT_SET, start, stride, count, block)) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't change selection") + } /* end if */ + else { + /* decode the number of points */ + UINT32DECODE(pp,num_elem); - /* Decode the ending points */ - for(tend=end,j=0; j<rank; j++,tend++) - UINT32DECODE(pp, *tend); + /* Set the count & stride for all blocks */ + for(tcount = count, tstride = stride, j = 0; j < rank; j++, tstride++, tcount++) { + *tcount=1; + *tstride=1; + } /* end for */ - /* Change the ending points into blocks */ - for(tblock=block,tstart=start,tend=end,j=0; j<rank; j++,tstart++,tend++,tblock++) - *tblock=(*tend-*tstart)+1; + /* Retrieve the coordinates from the buffer */ + for(i = 0; i < num_elem; i++) { + /* Decode the starting points */ + for(tstart=start,j=0; j<rank; j++,tstart++) + UINT32DECODE(pp, *tstart); - /* Select or add the hyperslab to the current selection */ - if((ret_value=H5S_select_hyperslab(space,(i==0 ? H5S_SELECT_SET : H5S_SELECT_OR),start,stride,count,block))<0) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't change selection") - } /* end for */ + /* Decode the ending points */ + for(tend = end, j = 0; j < rank; j++, tend++) + UINT32DECODE(pp, *tend); + + /* Change the ending points into blocks */ + for(tblock=block,tstart=start,tend=end,j=0; j<rank; j++,tstart++,tend++,tblock++) + *tblock=(*tend-*tstart)+1; + + /* Select or add the hyperslab to the current selection */ + if((ret_value=H5S_select_hyperslab(space,(i==0 ? H5S_SELECT_SET : H5S_SELECT_OR),start,stride,count,block))<0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't change selection") + } /* end for */ + } /* end else */ /* Update decoding pointer */ *p = pp; @@ -2452,6 +2539,7 @@ H5S_get_select_hyper_blocklist(H5S_t *space, hbool_t internal, hsize_t startbloc HDassert(space); HDassert(buf); + HDassert(space->select.sel_info.hslab->unlim_dim < 0); /* Check for a "regular" hyperslab selection */ if(space->select.sel_info.hslab->diminfo_valid) { @@ -2475,11 +2563,19 @@ H5S_get_select_hyper_blocklist(H5S_t *space, hbool_t internal, hsize_t startbloc */ diminfo = space->select.sel_info.hslab->opt_diminfo; else - /* - * Use the "application dimension information" to pass back to the user - * the blocks they set, not the optimized, internal information. - */ - diminfo = space->select.sel_info.hslab->app_diminfo; + if(space->select.sel_info.hslab->unlim_dim >= 0) + /* + * There is an unlimited dimension so we must use opt_diminfo as + * it has been "clipped" to the current extent. + */ + diminfo = space->select.sel_info.hslab->opt_diminfo; + else + /* + * Use the "application dimension information" to pass back to + * the user the blocks they set, not the optimized, internal + * information. + */ + diminfo = space->select.sel_info.hslab->app_diminfo; /* Build the tables of count sizes as well as the initial offset */ for(u = 0; u < ndims; u++) { @@ -2616,6 +2712,8 @@ H5Sget_select_hyper_blocklist(hid_t spaceid, hsize_t startblock, HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data space") if(H5S_GET_SELECT_TYPE(space)!=H5S_SEL_HYPERSLABS) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a hyperslab selection") + if(space->select.sel_info.hslab->unlim_dim >= 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "cannot get blocklist for unlimited selection") /* Go get the correct number of blocks */ if(numblocks > 0) @@ -2758,7 +2856,10 @@ H5S_hyper_bounds(const H5S_t *space, hsize_t *start, hsize_t *end) start[i] = diminfo[i].start + (hsize_t)space->select.offset[i]; /* Compute the largest location in this dimension */ - end[i] = diminfo[i].start + diminfo[i].stride * (diminfo[i].count - 1) + (diminfo[i].block - 1) + (hsize_t)space->select.offset[i]; + if((int)i == space->select.sel_info.hslab->unlim_dim) + end[i] = H5S_UNLIMITED; + else + end[i] = diminfo[i].start + diminfo[i].stride * (diminfo[i].count - 1) + (diminfo[i].block - 1) + (hsize_t)space->select.offset[i]; } /* end for */ } /* end if */ else { @@ -2881,6 +2982,75 @@ done: /*-------------------------------------------------------------------------- NAME + H5S__hyper_unlim_dim + PURPOSE + Return unlimited dimension of selection, or -1 if none + USAGE + int H5S__hyper_unlim_dim(space) + H5S_t *space; IN: Dataspace pointer to check + RETURNS + Unlimited dimension of selection, or -1 if none (never fails). + DESCRIPTION + Returns the index of the unlimited dimension of the selection, or -1 + if the selection has no unlimited dimension. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +static int +H5S__hyper_unlim_dim(const H5S_t *space) +{ + FUNC_ENTER_STATIC_NOERR + + FUNC_LEAVE_NOAPI(space->select.sel_info.hslab->unlim_dim); +} /* end H5S__hyper_unlim_dim() */ + + +/*-------------------------------------------------------------------------- + NAME + H5S_hyper_num_elem_non_unlim + PURPOSE + Return number of elements in the non-unlimited dimensions + USAGE + herr_t H5S_hyper_num_elem_non_unlim(space,num_elem_non_unlim) + H5S_t *space; IN: Dataspace pointer to check + hsize_t *num_elem_non_unlim; OUT: Number of elements in the non-unlimited dimensions + RETURNS + Non-negative on success/Negative on failure + DESCRIPTION + Returns the number of elements in a slice through the non-unlimited + dimensions of the selection. Fails if the selection has no unlimited + dimension. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +static herr_t +H5S_hyper_num_elem_non_unlim(const H5S_t *space, hsize_t *num_elem_non_unlim) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(space); + HDassert(num_elem_non_unlim); + + /* Get number of elements in the non-unlimited dimensions */ + if(space->select.sel_info.hslab->unlim_dim >= 0) + *num_elem_non_unlim = space->select.sel_info.hslab->num_elem_non_unlim; + else + HGOTO_ERROR(H5E_DATASPACE, H5E_BADVALUE, FAIL, "selection has no unlimited dimension") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S_hyper_num_elem_non_unlim() */ + + +/*-------------------------------------------------------------------------- + NAME H5S_hyper_is_contiguous PURPOSE Check if a hyperslab selection is contiguous within the dataspace extent. @@ -3592,6 +3762,9 @@ H5S_hyper_add_span_element(H5S_t *space, unsigned rank, hsize_t *coords) /* Reset "regular" hyperslab flag */ space->select.sel_info.hslab->diminfo_valid = FALSE; + /* Set unlim_dim */ + space->select.sel_info.hslab->unlim_dim = -1; + /* Set # of elements in selection */ space->select.num_elem = 1; } /* end if */ @@ -4301,6 +4474,9 @@ H5S_hyper_project_simple(const H5S_t *base_space, H5S_t *new_space, hsize_t *off if(NULL == (new_space->select.sel_info.hslab = H5FL_MALLOC(H5S_hyper_sel_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate hyperslab info") + /* Set unlim_dim */ + new_space->select.sel_info.hslab->unlim_dim = -1; + /* Check for a "regular" hyperslab selection */ if(base_space->select.sel_info.hslab->diminfo_valid) { unsigned base_space_dim; /* Current dimension in the base dataspace */ @@ -6047,6 +6223,15 @@ H5S_hyper_generate_spans(H5S_t *space) /* Get the diminfo */ for(u=0; u<space->extent.rank; u++) { + /* Check for unlimited dimension and return error */ + /* These should be able to be converted to assertions once everything + * that calls this function checks for unlimited selections first + * (especially the new hyperslab API) -NAF */ + if(space->select.sel_info.hslab->opt_diminfo[u].count == H5S_UNLIMITED) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "can't generate spans with unlimited count") + if(space->select.sel_info.hslab->opt_diminfo[u].block == H5S_UNLIMITED) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "can't generate spans with unlimited block") + tmp_start[u]=space->select.sel_info.hslab->opt_diminfo[u].start; tmp_stride[u]=space->select.sel_info.hslab->opt_diminfo[u].stride; tmp_count[u]=space->select.sel_info.hslab->opt_diminfo[u].count; @@ -6334,6 +6519,7 @@ H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, const hsize_t *opt_count; /* Optimized count information */ const hsize_t *opt_block; /* Optimized block information */ unsigned u; /* Counters */ + int unlim_dim = -1; /* Unlimited dimension in selection, of -1 if none */ herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -6352,6 +6538,18 @@ H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, if(block==NULL) block = _ones; + /* Check for unlimited dimension */ + for(u = 0; u<space->extent.rank; u++) + if((count[u] == H5S_UNLIMITED) || (block[u] == H5S_UNLIMITED)) { + if(unlim_dim >= 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "cannot have more than one unlimited dimension in selection") + else { + if(count[u] == block[u] /* == H5S_UNLIMITED */) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "count and block cannot both be unlimited") + unlim_dim = (int)u; + } /* end else */ + } /* end if */ + /* * Check new selection. */ @@ -6400,7 +6598,7 @@ H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, opt_block = int_block; for(u=0; u<space->extent.rank; u++) { /* contiguous hyperslabs have the block size equal to the stride */ - if(stride[u]==block[u]) { + if((stride[u] == block[u]) && (count[u] != H5S_UNLIMITED)) { int_count[u]=1; int_stride[u]=1; if(block[u]==1) @@ -6412,7 +6610,8 @@ H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, if(count[u]==1) int_stride[u]=1; else { - HDassert(stride[u] > block[u]); + HDassert((stride[u] > block[u]) || ((stride[u] == block[u]) + && (count[u] == H5S_UNLIMITED))); int_stride[u]=stride[u]; } /* end else */ int_count[u]=count[u]; @@ -6421,6 +6620,32 @@ H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, } /* end for */ } /* end else */ + /* Check for operating on unlimited selection */ + if((H5S_GET_SELECT_TYPE(space) == H5S_SEL_HYPERSLABS) + && (space->select.sel_info.hslab->unlim_dim >= 0) + && (op != H5S_SELECT_SET)) + { + /* Check for invalid operation */ + if(unlim_dim >= 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "cannot modify unlimited selection with another unlimited selection") + if(!((op == H5S_SELECT_AND) || (op == H5S_SELECT_NOTA))) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "unsupported operation on unlimited selection") + HDassert(space->select.sel_info.hslab->diminfo_valid); + + /* Clip unlimited selection to include new selection */ + if(H5S_hyper_clip_unlim(space, + start[space->select.sel_info.hslab->unlim_dim] + + ((opt_count[space->select.sel_info.hslab->unlim_dim] + - (hsize_t)1) + * opt_stride[space->select.sel_info.hslab->unlim_dim]) + + opt_block[space->select.sel_info.hslab->unlim_dim]) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + + /* If an empty space was returned it must be "none" */ + HDassert((space->select.num_elem > (hsize_t)0) + || (space->select.type->type == H5S_SEL_NONE)); + } /* end if */ + /* Fixup operation for non-hyperslab selections */ switch(H5S_GET_SELECT_TYPE(space)) { case H5S_SEL_NONE: /* No elements selected in dataspace */ @@ -6540,16 +6765,64 @@ H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, space->select.num_elem *= (opt_count[u] * opt_block[u]); } /* end for */ + /* Save unlim_dim */ + space->select.sel_info.hslab->unlim_dim = unlim_dim; + /* Indicate that the dimension information is valid */ space->select.sel_info.hslab->diminfo_valid = TRUE; /* Indicate that there's no slab information */ space->select.sel_info.hslab->span_lst = NULL; + + /* Handle unlimited selections */ + if(unlim_dim >= 0) { + /* Calculate num_elem_non_unlim */ + space->select.sel_info.hslab->num_elem_non_unlim = (hsize_t)1; + for(u = 0; u < space->extent.rank; u++) + if((int)u != unlim_dim) + space->select.sel_info.hslab->num_elem_non_unlim *= (opt_count[u] * opt_block[u]); + + /* Set num_elem */ + if(space->select.num_elem != (hsize_t)0) + space->select.num_elem = H5S_UNLIMITED; + } /* end if */ } /* end if */ else if(op >= H5S_SELECT_OR && op <= H5S_SELECT_NOTA) { /* Sanity check */ HDassert(H5S_GET_SELECT_TYPE(space) == H5S_SEL_HYPERSLABS); + /* Handle unlimited selections */ + if(unlim_dim >= 0) { + hsize_t bounds_start[H5S_MAX_RANK]; + hsize_t bounds_end[H5S_MAX_RANK]; + hsize_t tmp_count = opt_count[unlim_dim]; + hsize_t tmp_block = opt_block[unlim_dim]; + + /* Check for invalid operation */ + if(space->select.sel_info.hslab->unlim_dim >= 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "cannot modify unlimited selection with another unlimited selection") + if(!((op == H5S_SELECT_AND) || (op == H5S_SELECT_NOTB))) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "unsupported operation with unlimited selection") + + /* Get bounds of existing selection */ + if(H5S_hyper_bounds(space, bounds_start, bounds_end) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get selection bounds") + + /* Patch count and block to remove unlimited and include the + * existing selection */ + H5S__hyper_get_clip_diminfo(start[unlim_dim], opt_stride[unlim_dim], &tmp_count, &tmp_block, bounds_end[unlim_dim] + (hsize_t)1); + HDassert((tmp_count == 1) || (opt_count != _ones)); + HDassert((tmp_block == 1) || (opt_block != _ones)); + if(opt_count != _ones) { + HDassert(opt_count == int_count); + int_count[unlim_dim] = tmp_count; + } /* end if */ + if(opt_block != _ones) { + HDassert(opt_block == int_block); + int_block[unlim_dim] = tmp_block; + } /* end if */ + } /* end if */ + /* Check if there's no hyperslab span information currently */ if(NULL == space->select.sel_info.hslab->span_lst) if(H5S_hyper_generate_spans(space) < 0) @@ -6896,6 +7169,9 @@ H5S_generate_hyperslab (H5S_t *space, H5S_seloper_t op, /* Allocate space for the hyperslab selection information */ if((space->select.sel_info.hslab=H5FL_MALLOC(H5S_hyper_sel_t))==NULL) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate hyperslab info") + + /* Set unlim_dim */ + space->select.sel_info.hslab->unlim_dim = -1; } /* end if */ /* Combine tmp_space (really space) & new_space, with the result in space */ @@ -6943,6 +7219,7 @@ H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, const hsize_t *opt_count; /* Optimized count information */ const hsize_t *opt_block; /* Optimized block information */ unsigned u; /* Counters */ + int unlim_dim = -1; /* Unlimited dimension in selection, of -1 if none */ herr_t ret_value=SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -6961,6 +7238,18 @@ H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, if(block==NULL) block = _ones; + /* Check for unlimited dimension */ + for(u = 0; u<space->extent.rank; u++) + if((count[u] == H5S_UNLIMITED) || (block[u] == H5S_UNLIMITED)) { + if(unlim_dim >= 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "cannot have more than one unlimited dimension in selection") + else { + if(count[u] == block[u] /* == H5S_UNLIMITED */) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "count and block cannot both be unlimited") + unlim_dim = (int)u; + } /* end else */ + } /* end if */ + /* * Check new selection. */ @@ -7005,7 +7294,7 @@ H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, opt_block = int_block; for(u=0; u<space->extent.rank; u++) { /* contiguous hyperslabs have the block size equal to the stride */ - if(stride[u]==block[u]) { + if((stride[u] == block[u]) && (count[u] != H5S_UNLIMITED)) { int_count[u]=1; int_stride[u]=1; if(block[u]==1) @@ -7017,7 +7306,8 @@ H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, if(count[u]==1) int_stride[u]=1; else { - HDassert(stride[u] > block[u]); + HDassert((stride[u] > block[u]) || ((stride[u] == block[u]) + && (count[u] == H5S_UNLIMITED))); int_stride[u]=stride[u]; } /* end else */ int_count[u]=count[u]; @@ -7026,6 +7316,32 @@ H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, } /* end for */ } /* end else */ + /* Check for operating on unlimited selection */ + if((H5S_GET_SELECT_TYPE(space) == H5S_SEL_HYPERSLABS) + && (space->select.sel_info.hslab->unlim_dim >= 0) + && (op != H5S_SELECT_SET)) + { + /* Check for invalid operation */ + if(unlim_dim >= 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "cannot modify unlimited selection with another unlimited selection") + if(!((op == H5S_SELECT_AND) || (op == H5S_SELECT_NOTA))) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "unsupported operation on unlimited selection") + HDassert(space->select.sel_info.hslab->diminfo_valid); + + /* Clip unlimited selection to include new selection */ + if(H5S_hyper_clip_unlim(space, + start[space->select.sel_info.hslab->unlim_dim] + + ((opt_count[space->select.sel_info.hslab->unlim_dim] + - (hsize_t)1) + * opt_stride[space->select.sel_info.hslab->unlim_dim]) + + opt_block[space->select.sel_info.hslab->unlim_dim]) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCLIP, FAIL, "failed to clip unlimited selection") + + /* If an empty space was returned it must be "none" */ + HDassert((space->select.num_elem > (hsize_t)0) + || (space->select.type->type == H5S_SEL_NONE)); + } /* end if */ + /* Fixup operation for non-hyperslab selections */ switch(H5S_GET_SELECT_TYPE(space)) { case H5S_SEL_NONE: /* No elements selected in dataspace */ @@ -7136,16 +7452,64 @@ H5S_select_hyperslab (H5S_t *space, H5S_seloper_t op, space->select.num_elem*=(opt_count[u]*opt_block[u]); } /* end for */ + /* Save unlim_dim */ + space->select.sel_info.hslab->unlim_dim = unlim_dim; + /* Indicate that the dimension information is valid */ space->select.sel_info.hslab->diminfo_valid = TRUE; /* Indicate that there's no slab information */ space->select.sel_info.hslab->span_lst = NULL; + + /* Handle unlimited selections */ + if(unlim_dim >= 0) { + /* Calculate num_elem_non_unlim */ + space->select.sel_info.hslab->num_elem_non_unlim = (hsize_t)1; + for(u = 0; u < space->extent.rank; u++) + if((int)u != unlim_dim) + space->select.sel_info.hslab->num_elem_non_unlim *= (opt_count[u] * opt_block[u]); + + /* Set num_elem */ + if(space->select.num_elem != (hsize_t)0) + space->select.num_elem = H5S_UNLIMITED; + } /* end if */ } /* end if */ else if(op>=H5S_SELECT_OR && op<=H5S_SELECT_NOTA) { /* Sanity check */ HDassert(H5S_GET_SELECT_TYPE(space) == H5S_SEL_HYPERSLABS); + /* Handle unlimited selections */ + if(unlim_dim >= 0) { + hsize_t bounds_start[H5S_MAX_RANK]; + hsize_t bounds_end[H5S_MAX_RANK]; + hsize_t tmp_count = opt_count[unlim_dim]; + hsize_t tmp_block = opt_block[unlim_dim]; + + /* Check for invalid operation */ + if(space->select.sel_info.hslab->unlim_dim >= 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "cannot modify unlimited selection with another unlimited selection") + if(!((op == H5S_SELECT_AND) || (op == H5S_SELECT_NOTB))) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "unsupported operation with unlimited selection") + + /* Get bounds of existing selection */ + if(H5S_hyper_bounds(space, bounds_start, bounds_end) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTGET, FAIL, "can't get selection bounds") + + /* Patch count and block to remove unlimited and include the + * existing selection */ + H5S__hyper_get_clip_diminfo(start[unlim_dim], opt_stride[unlim_dim], &tmp_count, &tmp_block, bounds_end[unlim_dim] + (hsize_t)1); + HDassert((tmp_count == 1) || (opt_count != _ones)); + HDassert((tmp_block == 1) || (opt_block != _ones)); + if(opt_count != _ones) { + HDassert(opt_count == int_count); + int_count[unlim_dim] = tmp_count; + } /* end if */ + if(opt_block != _ones) { + HDassert(opt_block == int_block); + int_block[unlim_dim] = tmp_block; + } /* end if */ + } /* end if */ + /* Check if there's no hyperslab span information currently */ if(NULL == space->select.sel_info.hslab->span_lst) if(H5S_hyper_generate_spans(space) < 0) @@ -7351,6 +7715,9 @@ H5S_combine_select (H5S_t *space1, H5S_seloper_t op, H5S_t *space2) if((new_space->select.sel_info.hslab=H5FL_CALLOC(H5S_hyper_sel_t))==NULL) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "can't allocate hyperslab info") + /* Set unlim_dim */ + new_space->select.sel_info.hslab->unlim_dim = -1; + /* Combine space1 & space2, with the result in new_space */ if(H5S_operate_hyperslab(new_space,space1->select.sel_info.hslab->span_lst,op,space2->select.sel_info.hslab->span_lst,FALSE,&span2_owned)<0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCLIP, NULL, "can't clip hyperslab information") @@ -7480,6 +7847,9 @@ H5S_select_select (H5S_t *space1, H5S_seloper_t op, H5S_t *space2) if((space1->select.sel_info.hslab=H5FL_CALLOC(H5S_hyper_sel_t))==NULL) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate hyperslab info") + /* Set unlim_dim */ + space1->select.sel_info.hslab->unlim_dim = -1; + /* Combine tmp_spans (from space1) & spans from space2, with the result in space1 */ if(H5S_operate_hyperslab(space1,tmp_spans,op,space2->select.sel_info.hslab->span_lst,FALSE,&span2_owned)<0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCLIP, FAIL, "can't clip hyperslab information") @@ -8752,6 +9122,7 @@ H5S_hyper_get_seq_list(const H5S_t *space, unsigned H5_ATTR_UNUSED flags, H5S_se HDassert(nelem); HDassert(off); HDassert(len); + HDassert(space->select.sel_info.hslab->unlim_dim < 0); /* Check for the special case of just one H5Sselect_hyperslab call made */ if(space->select.sel_info.hslab->diminfo_valid) { @@ -8881,6 +9252,1085 @@ H5S_hyper_get_seq_list(const H5S_t *space, unsigned H5_ATTR_UNUSED flags, H5S_se /*-------------------------------------------------------------------------- NAME + H5S__hyper_project_intersection + PURPOSE + Projects the intersection of of the selections of src_space and + src_intersect_space within the selection of src_space as a selection + within the selection of dst_space + USAGE + herr_t H5S__hyper_project_intersection(src_space,dst_space,src_intersect_space,proj_space) + H5S_t *src_space; IN: Selection that is mapped to dst_space, and intersected with src_intersect_space + H5S_t *dst_space; IN: Selection that is mapped to src_space, and which contains the result + H5S_t *src_intersect_space; IN: Selection whose intersection with src_space is projected to dst_space to obtain the result + H5S_t *proj_space; OUT: Will contain the result (intersection of src_intersect_space and src_space projected from src_space to dst_space) after the operation + RETURNS + Non-negative on success/Negative on failure. + DESCRIPTION + Projects the intersection of of the selections of src_space and + src_intersect_space within the selection of src_space as a selection + within the selection of dst_space. The result is placed in the + selection of proj_space. Note src_space, dst_space, and + src_intersect_space do not need to use hyperslab selections, but they + cannot use point selections. The result is always a hyperslab + selection. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5S__hyper_project_intersection(const H5S_t *src_space, const H5S_t *dst_space, + const H5S_t *src_intersect_space, H5S_t *proj_space) +{ + hsize_t ss_off[H5S_PROJECT_INTERSECT_NSEQS]; /* Offset array for src_space */ + size_t ss_len[H5S_PROJECT_INTERSECT_NSEQS]; /* Length array for src_space */ + size_t ss_nseq; /* Number of sequences for src_space */ + size_t ss_nelem; /* Number of elements for src_space */ + size_t ss_i = (size_t)0; /* Index into offset/length arrays for src_space */ + hbool_t advance_ss = FALSE; /* Whether to advance ss_i on the next iteration */ + H5S_sel_iter_t ss_iter; /* Selection iterator for src_space */ + hbool_t ss_iter_init = FALSE; /* Whether ss_iter is initialized */ + hsize_t ss_sel_off = (hsize_t)0; /* Offset within src_space selection */ + hsize_t ds_off[H5S_PROJECT_INTERSECT_NSEQS]; /* Offset array for dst_space */ + size_t ds_len[H5S_PROJECT_INTERSECT_NSEQS]; /* Length array for dst_space */ + size_t ds_nseq; /* Number of sequences for dst_space */ + size_t ds_nelem; /* Number of elements for dst_space */ + size_t ds_i = (size_t)0; /* Index into offset/length arrays for dst_space */ + H5S_sel_iter_t ds_iter; /* Selection iterator for dst_space */ + hbool_t ds_iter_init = FALSE; /* Whether ds_iter is initialized */ + hsize_t ds_sel_off = (hsize_t)0; /* Offset within dst_space selection */ + hsize_t sis_off[H5S_PROJECT_INTERSECT_NSEQS]; /* Offset array for src_intersect_space */ + size_t sis_len[H5S_PROJECT_INTERSECT_NSEQS]; /* Length array for src_intersect_space */ + size_t sis_nseq; /* Number of sequences for src_intersect_space */ + size_t sis_nelem; /* Number of elements for src_intersect_space */ + size_t sis_i = (size_t)0; /* Index into offset/length arrays for src_intersect_space */ + hbool_t advance_sis = FALSE; /* Whether to advance sis_i on the next iteration */ + H5S_sel_iter_t sis_iter; /* Selection iterator for src_intersect_space */ + hbool_t sis_iter_init = FALSE; /* Whether sis_iter is initialized */ + hsize_t int_sel_off; /* Offset within intersected selections (ss/sis and ds/ps) */ + size_t int_len; /* Length of segment in intersected selections */ + hsize_t proj_off; /* Segment offset in proj_space */ + size_t proj_len; /* Segment length in proj_space */ + size_t proj_len_rem; /* Remaining length in proj_space for segment */ + hsize_t proj_down_dims[H5S_MAX_RANK]; /* "Down" dimensions in proj_space */ + H5S_hyper_span_info_t *curr_span_tree[H5S_MAX_RANK]; /* Current span tree being built (in each dimension) */ + H5S_hyper_span_t *prev_span[H5S_MAX_RANK]; /* Previous span in tree (in each dimension) */ + hsize_t curr_span_up_dim[H5S_MAX_RANK]; /* "Up" dimensions for current span */ + unsigned proj_rank; /* Rank of proj_space */ + hsize_t low; /* Low value of span */ + hsize_t high; /* High value of span */ + size_t span_len; /* Length of span */ + size_t nelem; /* Number of elements returned for get_seq_list op */ + unsigned i; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check parameters */ + HDassert(src_space); + HDassert(dst_space); + HDassert(src_intersect_space); + HDassert(proj_space); + + /* Assert that src_space and src_intersect_space have same extent and there + * are no point selections */ + HDassert(H5S_GET_EXTENT_NDIMS(src_space) + == H5S_GET_EXTENT_NDIMS(src_intersect_space)); + HDassert(!HDmemcmp(src_space->extent.size, src_intersect_space->extent.size, + (size_t)H5S_GET_EXTENT_NDIMS(src_space) + * sizeof(src_space->extent.size[0]))); + HDassert(H5S_GET_SELECT_TYPE(src_space) != H5S_SEL_POINTS); + HDassert(H5S_GET_SELECT_TYPE(dst_space) != H5S_SEL_POINTS); + HDassert(H5S_GET_SELECT_TYPE(src_intersect_space) != H5S_SEL_POINTS); + + /* Initialize prev_space, curr_span_tree, and curr_span_up_dim */ + for(i = 0; i < H5S_MAX_RANK; i++) { + curr_span_tree[i] = NULL; + prev_span[i] = NULL; + curr_span_up_dim[i] = (hsize_t)0; + } /* end for */ + + /* Save rank of projected space */ + proj_rank = proj_space->extent.rank; + HDassert(proj_rank > 0); + + /* Get numbers of elements */ + ss_nelem = (size_t)H5S_GET_SELECT_NPOINTS(src_space); + ds_nelem = (size_t)H5S_GET_SELECT_NPOINTS(dst_space); + sis_nelem = (size_t)H5S_GET_SELECT_NPOINTS(src_intersect_space); + HDassert(ss_nelem == ds_nelem); + + /* Calculate proj_down_dims (note loop relies on unsigned i wrapping around) + */ + if(H5VM_array_down(proj_rank, proj_space->extent.size, proj_down_dims) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSET, FAIL, "can't compute 'down' chunk size value") + + /* Remove current selection from proj_space */ + if(H5S_SELECT_RELEASE(proj_space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't release selection") + + /* If any selections are empty, skip to the end so "none" is selected */ + if((ss_nelem == 0) || (ds_nelem == 0) || (sis_nelem == 0)) + goto loop_end; + + /* Allocate space for the hyperslab selection information (note this sets + * diminfo_valid to FALSE, diminfo arrays to 0, and span list to NULL) */ + if((proj_space->select.sel_info.hslab = H5FL_CALLOC(H5S_hyper_sel_t)) == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate hyperslab info") + + /* Set selection type */ + proj_space->select.type = H5S_sel_hyper; + + /* Set unlim_dim */ + proj_space->select.sel_info.hslab->unlim_dim = -1; + + /* Initialize source space iterator */ + if(H5S_select_iter_init(&ss_iter, src_space, (size_t)1) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator") + ss_iter_init = TRUE; + + /* Get sequence list for source space */ + if(H5S_SELECT_GET_SEQ_LIST(src_space, 0u, &ss_iter, H5S_PROJECT_INTERSECT_NSEQS, ss_nelem, &ss_nseq, &nelem, ss_off, ss_len) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed") + ss_nelem -= nelem; + HDassert(ss_nseq > 0); + + /* Initialize destination space iterator */ + if(H5S_select_iter_init(&ds_iter, dst_space, (size_t)1) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator") + ds_iter_init = TRUE; + + /* Get sequence list for destination space */ + if(H5S_SELECT_GET_SEQ_LIST(dst_space, 0u, &ds_iter, H5S_PROJECT_INTERSECT_NSEQS, ds_nelem, &ds_nseq, &nelem, ds_off, ds_len) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator") + ds_nelem -= nelem; + HDassert(ds_nseq > 0); + + /* Initialize source intersect space iterator */ + if(H5S_select_iter_init(&sis_iter, src_intersect_space, (size_t)1) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to initialize selection iterator") + sis_iter_init = TRUE; + + /* Get sequence list for source intersect space */ + if(H5S_SELECT_GET_SEQ_LIST(src_intersect_space, 0u, &sis_iter, H5S_PROJECT_INTERSECT_NSEQS, sis_nelem, &sis_nseq, &nelem, sis_off, sis_len) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed") + sis_nelem -= nelem; + HDassert(sis_nseq > 0); + + /* Loop until we run out of sequences in either the source or source + * intersect space */ + while(1) { + while(advance_ss || (ss_off[ss_i] + ss_len[ss_i] <= sis_off[sis_i])) { + /* Either we finished the current source sequence or the + * sequences do not intersect. Advance source space. */ + ss_sel_off += (hsize_t)ss_len[ss_i]; + if(++ss_i == ss_nseq) { + if(ss_nelem > 0) { + /* Try to grab more sequences from src_space */ + if(H5S_SELECT_GET_SEQ_LIST(src_space, 0u, &ss_iter, H5S_PROJECT_INTERSECT_NSEQS, ss_nelem, &ss_nseq, &nelem, ss_off, ss_len) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed") + HDassert(ss_len[0] > 0); + + /* Update ss_nelem */ + HDassert(nelem > 0); + HDassert(nelem <= ss_nelem); + ss_nelem -= nelem; + + /* Reset source space index */ + ss_i = 0; + } /* end if */ + else + /* There are no more sequences in src_space, so we can exit + * the loop. Use goto instead of break so we exit the outer + * loop. */ + goto loop_end; + } /* end if */ + + /* Reset advance_ss */ + advance_ss = FALSE; + } /* end if */ + if(advance_sis + || (sis_off[sis_i] + sis_len[sis_i] <= ss_off[ss_i])) { + do { + /* Either we finished the current source intersect sequence or + * the sequences do not intersect. Advance source intersect + * space. */ + if(++sis_i == sis_nseq) { + if(sis_nelem > 0) { + /* Try to grab more sequences from src_intersect_space + */ + if(H5S_SELECT_GET_SEQ_LIST(src_intersect_space, 0u, &sis_iter, H5S_PROJECT_INTERSECT_NSEQS, sis_nelem, &sis_nseq, &nelem, sis_off, sis_len) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed") + HDassert(sis_len[0] > 0); + + /* Update ss_nelem */ + HDassert(nelem > 0); + HDassert(nelem <= sis_nelem); + sis_nelem -= nelem; + + /* Reset source space index */ + sis_i = 0; + } /* end if */ + else + /* There are no more sequences in src_intersect_space, + * so we can exit the loop. Use goto instead of break + * so we exit the outer loop. */ + goto loop_end; + } /* end if */ + } while(sis_off[sis_i] + sis_len[sis_i] <= ss_off[ss_i]); + + /* Reset advance_sis */ + advance_sis = FALSE; + } /* end if */ + else { + /* Sequences intersect, add intersection to projected space */ + /* Calculate intersection sequence in terms of offset within source + * selection and advance any sequences we complete */ + if(ss_off[ss_i] >= sis_off[sis_i]) + int_sel_off = ss_sel_off; + else + int_sel_off = sis_off[sis_i] - ss_off[ss_i] + ss_sel_off; + if((ss_off[ss_i] + (hsize_t)ss_len[ss_i]) <= (sis_off[sis_i] + + (hsize_t)sis_len[sis_i])) { + int_len = (size_t)((hsize_t)ss_len[ss_i] + ss_sel_off - int_sel_off); + advance_ss = TRUE; + } /* end if */ + else + int_len = (size_t)(sis_off[sis_i] + (hsize_t)sis_len[sis_i] - ss_off[ss_i] + ss_sel_off - int_sel_off); + if((ss_off[ss_i] + (hsize_t)ss_len[ss_i]) >= (sis_off[sis_i] + + (hsize_t)sis_len[sis_i])) + advance_sis = TRUE; + + /* Project intersection sequence to destination selection */ + while(int_len > (size_t)0) { + while(ds_sel_off + (hsize_t)ds_len[ds_i] <= int_sel_off) { + /* Intersection is not projected to this destination + * sequence, advance destination space */ + ds_sel_off += (hsize_t)ds_len[ds_i]; + if(++ds_i == ds_nseq) { + HDassert(ds_nelem > 0); + + /* Try to grab more sequences from dst_space */ + if(H5S_SELECT_GET_SEQ_LIST(dst_space, 0u, &ds_iter, H5S_PROJECT_INTERSECT_NSEQS, ds_nelem, &ds_nseq, &nelem, ds_off, ds_len) < 0) + HGOTO_ERROR(H5E_INTERNAL, H5E_UNSUPPORTED, FAIL, "sequence length generation failed") + HDassert(ds_len[0] > 0); + + /* Update ss_nelem */ + HDassert(nelem > 0); + HDassert(nelem <= ds_nelem); + ds_nelem -= nelem; + + /* Reset source space index */ + ds_i = 0; + } /* end if */ + } /* end while */ + + /* Add sequence to projected space */ + HDassert(ds_sel_off <= int_sel_off); + proj_off = ds_off[ds_i] + int_sel_off - ds_sel_off; + proj_len = proj_len_rem = (size_t)MIN(int_len, + (size_t)(ds_sel_off + (hsize_t)ds_len[ds_i] + - int_sel_off)); + + /* Add to span tree */ + while(proj_len_rem > (size_t)0) { + /* Check for more than one full row (in every dim) and + * append multiple spans at once? -NAF */ + /* Append spans in higher dimensions if we're going ouside + * the plane of the span currently being built (i.e. it's + * finished being built) */ + for(i = proj_rank - 1; ((i > 0) + && ((proj_off / proj_down_dims[i - 1]) + != curr_span_up_dim[i - 1])); i--) { + if(curr_span_tree[i]) { + HDassert(prev_span[i]); + + /* Append complete lower dimension span tree to + * current dimension */ + low = curr_span_up_dim[i - 1] % proj_space->extent.size[i - 1]; + if(H5S_hyper_append_span(&prev_span[i - 1], &curr_span_tree[i - 1], low, low, curr_span_tree[i], NULL) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTAPPEND, FAIL, "can't allocate hyperslab span") + + /* Reset lower dimension's span tree and previous + * span since we just committed it and will start + * over with a new one */ + if(H5S_hyper_free_span_info(curr_span_tree[i]) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't free span info") + curr_span_tree[i] = NULL; + prev_span[i] = NULL; + } /* end if */ + + /* Update curr_span_up_dim */ + curr_span_up_dim[i - 1] = proj_off / proj_down_dims[i - 1]; + } /* end for */ + + /* Compute bounds for new span in lowest dimension */ + low = proj_off % proj_space->extent.size[proj_rank - 1]; + span_len = MIN(proj_len_rem, + (size_t)(proj_space->extent.size[proj_rank - 1] + - low)); + HDassert(proj_len_rem >= span_len); + high = low + (hsize_t)span_len - (hsize_t)1; + + /* Append span in lowest dimension */ + if(H5S_hyper_append_span(&prev_span[proj_rank - 1], &curr_span_tree[proj_rank - 1], low, high, NULL, NULL) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTAPPEND, FAIL, "can't allocate hyperslab span") + + /* Update remaining offset and length */ + proj_off += (hsize_t)span_len; + proj_len_rem -= span_len; + } /* end while */ + + /* Update intersection sequence */ + int_sel_off += (hsize_t)proj_len; + int_len -= proj_len; + } /* end while */ + } /* end else */ + } /* end while */ + +loop_end: + /* Add remaining spans to span tree */ + for(i = proj_rank - 1; i > 0; i--) + if(curr_span_tree[i]) { + HDassert(prev_span[i]); + + /* Append remaining span tree to higher dimension */ + low = curr_span_up_dim[i - 1] % proj_space->extent.size[i - 1]; + if(H5S_hyper_append_span(&prev_span[i - 1], &curr_span_tree[i - 1], low, low, curr_span_tree[i], NULL) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTAPPEND, FAIL, "can't allocate hyperslab span") + + /* Reset span tree */ + if(H5S_hyper_free_span_info(curr_span_tree[i]) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't free span info") + curr_span_tree[i] = NULL; + } /* end if */ + + /* Add span tree to proj_space */ + if(curr_span_tree[0]) { + proj_space->select.sel_info.hslab->span_lst = curr_span_tree[0]; + curr_span_tree[0] = NULL; + + /* Set the number of elements in current selection */ + proj_space->select.num_elem = H5S_hyper_spans_nelem(proj_space->select.sel_info.hslab->span_lst); + + /* Attempt to rebuild "optimized" start/stride/count/block information. + * from resulting hyperslab span tree */ + if(H5S_hyper_rebuild(proj_space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOUNT, FAIL, "can't rebuild hyperslab info") + } /* end if */ + else + /* If we did not add anything to proj_space, select none instead */ + if(H5S_select_none(proj_space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't convert selection") + +done: + /* Release source selection iterator */ + if(ss_iter_init) + if(H5S_SELECT_ITER_RELEASE(&ss_iter) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator") + + /* Release destination selection iterator */ + if(ds_iter_init) + if(H5S_SELECT_ITER_RELEASE(&ds_iter) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator") + + /* Release source intersect selection iterator */ + if(sis_iter_init) + if(H5S_SELECT_ITER_RELEASE(&sis_iter) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release selection iterator") + + /* Cleanup on error */ + if(ret_value < 0) { + /* Remove current selection from proj_space */ + if(H5S_SELECT_RELEASE(proj_space) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't release selection") + + /* Free span trees */ + for(i = 0; i < proj_rank; i++) + if(curr_span_tree[i]) { + if(H5S_hyper_free_span_info(curr_span_tree[i]) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTFREE, FAIL, "can't free span info") + curr_span_tree[i] = NULL; + } /* end if */ + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S__hyper_project_intersection() */ + + +/*-------------------------------------------------------------------------- + NAME + H5S__hyper_subtract + PURPOSE + Subtract one hyperslab selection from another + USAGE + herr_t H5S__hyper_subtract(space,subtract_space) + H5S_t *space; IN/OUT: Selection to be operated on + H5S_t *subtract_space; IN: Selection that will be subtracted from space + RETURNS + Non-negative on success/Negative on failure. + DESCRIPTION + Removes any and all portions of space that are also present in + subtract_space. In essence, performs an A_NOT_B operation with the + two selections. + + Note this function basically duplicates a subset of the functionality + of H5S_select_select(). It should probably be removed when that + function is enabled. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5S__hyper_subtract(H5S_t *space, H5S_t *subtract_space) +{ + H5S_hyper_span_info_t *a_not_b = NULL; /* Span tree for hyperslab spans in old span tree and not in new span tree */ + H5S_hyper_span_info_t *a_and_b = NULL; /* Span tree for hyperslab spans in both old and new span trees */ + H5S_hyper_span_info_t *b_not_a = NULL; /* Span tree for hyperslab spans in new span tree and not in old span tree */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Check args */ + HDassert(space); + HDassert(subtract_space); + + /* Check that the space selections both have span trees */ + if(space->select.sel_info.hslab->span_lst == NULL) + if(H5S_hyper_generate_spans(space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNINITIALIZED, FAIL, "dataspace does not have span tree") + if(subtract_space->select.sel_info.hslab->span_lst == NULL) + if(H5S_hyper_generate_spans(subtract_space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNINITIALIZED, FAIL, "dataspace does not have span tree") + + /* Generate lists of spans which overlap and don't overlap */ + if(H5S_hyper_clip_spans(space->select.sel_info.hslab->span_lst, subtract_space->select.sel_info.hslab->span_lst, &a_not_b, &a_and_b, &b_not_a)<0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCLIP, FAIL, "can't clip hyperslab information") + + /* Reset the other dataspace selection information */ + if(H5S_SELECT_RELEASE(space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't release selection") + + /* Allocate space for the hyperslab selection information */ + if((space->select.sel_info.hslab = H5FL_CALLOC(H5S_hyper_sel_t)) == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate hyperslab info") + + /* Set unlim_dim */ + space->select.sel_info.hslab->unlim_dim = -1; + + /* Check for anything returned in a_not_b */ + if(a_not_b) { + /* Update spans in space */ + space->select.sel_info.hslab->span_lst = a_not_b; + a_not_b = NULL; + + /* Update number of elements */ + space->select.num_elem = H5S_hyper_spans_nelem(space->select.sel_info.hslab->span_lst); + + /* Attempt to rebuild "optimized" start/stride/count/block information. + * from resulting hyperslab span tree */ + if(H5S_hyper_rebuild(space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOUNT, FAIL, "can't rebuild hyperslab info") + } /* end if */ + else { + H5S_hyper_span_info_t *spans; /* Empty hyperslab span tree */ + + /* Set number of elements */ + space->select.num_elem = 0; + + /* Allocate a span info node */ + if(NULL == (spans = H5FL_MALLOC(H5S_hyper_span_info_t))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "can't allocate hyperslab span") + + /* Set the reference count */ + spans->count = 1; + + /* Reset the scratch pad space */ + spans->scratch = 0; + + /* Set to empty tree */ + spans->head = NULL; + + /* Set pointer to empty span tree */ + space->select.sel_info.hslab->span_lst = spans; + } /* end if */ + +done: + /* Free span trees */ + if(a_and_b) + H5S_hyper_free_span_info(a_and_b); + if(b_not_a) + H5S_hyper_free_span_info(b_not_a); + if(a_not_b) { + HDassert(ret_value < 0); + H5S_hyper_free_span_info(b_not_a); + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S__hyper_subtract() */ + + +/*-------------------------------------------------------------------------- + NAME + H5S__hyper_get_clip_diminfo + PURPOSE + Calculates the count and block required to clip the specified + unlimited dimension to include clip_size. The returned selection may + extent beyond clip_size. + USAGE + void H5S__hyper_get_clip_diminfo(start,stride,count,block,clip_size) + hsize_t start; IN: Start of hyperslab in unlimited dimension + hsize_t stride; IN: Stride of hyperslab in unlimited dimension + hsize_t *count; IN/OUT: Count of hyperslab in unlimited dimension + hsize_t *block; IN/OUT: Block of hyperslab in unlimited dimension + hsize_t clip_size; IN: Extent that hyperslab will be clipped to + RETURNS + Non-negative on success/Negative on failure. + DESCRIPTION + This function recalculates the internal description of the hyperslab + to make the unlimited dimension extend to the specified extent. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +void +H5S__hyper_get_clip_diminfo(hsize_t start, hsize_t stride, hsize_t *count, + hsize_t *block, hsize_t clip_size) +{ + FUNC_ENTER_PACKAGE_NOERR + + /* Check for selection outside clip size */ + if(start >= clip_size) { + if(*block == H5S_UNLIMITED) + *block = 0; + else + *count = 0; + } /* end if */ + /* Check for single block in unlimited dimension */ + else if((*block == H5S_UNLIMITED) || (*block == stride)) { + /* Calculate actual block size for this clip size */ + *block = clip_size - start; + *count = (hsize_t)1; + } /* end if */ + else { + HDassert(*count == H5S_UNLIMITED); + + /* Calculate initial count (last block may be partial) */ + *count = (clip_size - start + stride - (hsize_t)1) / stride; + HDassert(*count > (hsize_t)0); + } /* end else */ + + FUNC_LEAVE_NOAPI_VOID +} /* end H5S_hyper_get_clip_diminfo() */ + + +/*-------------------------------------------------------------------------- + NAME + H5S_hyper_clip_unlim + PURPOSE + Clips the unlimited dimension of the hyperslab selection to the + specified size + USAGE + void H5S_hyper_clip_unlim(space,clip_size) + H5S_t *space, IN/OUT: Unlimited space to clip + hsize_t clip_size; IN: Extent that hyperslab will be clipped to + RETURNS + Non-negative on success/Negative on failure. + DESCRIPTION + This function changes the unlimited selection into a limited selection + with the extent of the formerly unlimited dimension specified by + * clip_size. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + Note this function does not take the offset into account. + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5S_hyper_clip_unlim(H5S_t *space, hsize_t clip_size) +{ + H5S_hyper_sel_t *hslab; /* Convenience pointer to hyperslab info */ + hsize_t orig_count; /* Original count in unlimited dimension */ + int orig_unlim_dim; /* Original unliminted dimension */ + H5S_hyper_dim_t *diminfo; /* Convenience pointer to opt_diminfo in unlimited dimension */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(FAIL) + + /* Check parameters */ + HDassert(space); + hslab = space->select.sel_info.hslab; + HDassert(hslab); + HDassert(hslab->unlim_dim >= 0); + HDassert(!hslab->span_lst); + + /* Save original unlimited dimension */ + orig_unlim_dim = hslab->unlim_dim; + + diminfo = &hslab->opt_diminfo[orig_unlim_dim]; + + /* Save original count in unlimited dimension */ + orig_count = diminfo->count; + + /* Get initial diminfo */ + H5S__hyper_get_clip_diminfo(diminfo->start, diminfo->stride, &diminfo->count, &diminfo->block, clip_size); + + /* Selection is no longer unlimited */ + space->select.sel_info.hslab->unlim_dim = -1; + + /* Check for nothing returned */ + if((diminfo->block == 0) || (diminfo->count == 0)) { + /* Convert to "none" selection */ + if(H5S_select_none(space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't convert selection") + } /* end if */ + /* Check for single block in unlimited dimension */ + else if(orig_count == (hsize_t)1) { + /* Calculate number of elements */ + space->select.num_elem = diminfo->block * hslab->num_elem_non_unlim; + + /* Mark that opt_diminfo is valid */ + hslab->diminfo_valid = TRUE; + } /* end if */ + else { + /* Calculate number of elements */ + space->select.num_elem = diminfo->count * diminfo->block + * hslab->num_elem_non_unlim; + + /* Check if last block is partial. If superset is set, just keep the + * last block complete to speed computation. */ + HDassert(clip_size > diminfo->start); + if(((diminfo->stride * (diminfo->count - (hsize_t)1)) + diminfo->block) + > (clip_size - diminfo->start)) { + hsize_t start[H5S_MAX_RANK]; + hsize_t block[H5S_MAX_RANK]; + unsigned i; + + /* Last block is partial, need to construct compound selection */ + /* Fill start with zeros */ + HDmemset(start, 0, sizeof(start)); + + /* Set block to clip_size in unlimited dimension, H5S_MAX_SIZE in + * others so only unlimited dimension is clipped */ + for(i = 0; i < space->extent.rank; i++) + if((int)i == orig_unlim_dim) + block[i] = clip_size; + else + block[i] = H5S_MAX_SIZE; + + /* Generate span tree in selection */ + if(!hslab->span_lst) + if(H5S_hyper_generate_spans(space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to generate span tree") + + /* Indicate that the regular dimensions are no longer valid */ + hslab->diminfo_valid = FALSE; + + /* "And" selection with calculated block to perform clip operation + */ + if(H5S_generate_hyperslab(space, H5S_SELECT_AND, start, _ones, _ones, block) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINSERT, FAIL, "can't generate hyperslabs") + } /* end if */ + else + /* Last block is complete, simply mark that opt_diminfo is valid */ + hslab->diminfo_valid = TRUE; + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S_hyper_clip_unlim() */ + + +/*-------------------------------------------------------------------------- + NAME + H5S__hyper_get_clip_extent_real + PURPOSE + Gets the extent a space should be clipped to in order to contain the + specified number of slices in the unlimited dimension + USAGE + hsize_t H5S__hyper_get_clip_extent_real(clip_space,num_slices,incl_trail) + const H5S_t *clip_space, IN: Space that clip size will be calculated based on + hsize_t num_slizes, IN: Number of slices clip_space should contain when clipped + hbool_t incl_trail; IN: Whether to include trailing unselected space + RETURNS + Clip extent to match num_slices (never fails) + DESCRIPTION + Calculates and returns the extent that clip_space should be clipped to + (via H5S_hyper_clip_unlim) in order for it to contain num_slices + slices in the unlimited dimension. If the clipped selection would end + immediately before a section of unselected space (i.e. at the end of a + block), then if incl_trail is TRUE, the returned clip extent is + selected to include that trailing "blank" space, otherwise it is + selected to end at the end before the blank space. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + Note this assumes the offset has been normalized. + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +static hsize_t +H5S__hyper_get_clip_extent_real(const H5S_t *clip_space, hsize_t num_slices, + hbool_t incl_trail) +{ + const H5S_hyper_dim_t *diminfo; /* Convenience pointer to opt_unlim_diminfo in unlimited dimension */ + hsize_t count; + hsize_t rem_slices; + hsize_t ret_value = 0; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check parameters */ + HDassert(clip_space); + HDassert(clip_space->select.sel_info.hslab); + HDassert(clip_space->select.sel_info.hslab->unlim_dim >= 0); + + diminfo = &clip_space->select.sel_info.hslab->opt_diminfo[clip_space->select.sel_info.hslab->unlim_dim]; + + if(num_slices == 0) + ret_value = incl_trail ? diminfo->start : 0; + else if((diminfo->block == H5S_UNLIMITED) + || (diminfo->block == diminfo->stride)) + /* Unlimited block, just set the extent large enough for the block size + * to match num_slices */ + ret_value = diminfo->start + num_slices; + else { + /* Unlimited count, need to match extent so a block (possibly) gets cut + * off so the number of slices matches num_slices */ + HDassert(diminfo->count == H5S_UNLIMITED); + + /* Calculate number of complete blocks in clip_space */ + count = num_slices / diminfo->block; + + /* Calculate slices remaining */ + rem_slices = num_slices - (count * diminfo->block); + + if(rem_slices > 0) + /* Must end extent in middle of partial block (or beginning of empty + * block if include_trailing_space and rem_slices == 0) */ + ret_value = diminfo->start + (count * diminfo->stride) + rem_slices; + else { + if(incl_trail) + /* End extent just before first missing block */ + ret_value = diminfo->start + (count * diminfo->stride); + else + /* End extent at end of last block */ + ret_value = diminfo->start + ((count - (hsize_t)1) + * diminfo->stride) + diminfo->block; + } /* end else */ + } /* end else */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S__hyper_get_clip_extent_real() */ + + +/*-------------------------------------------------------------------------- + NAME + H5S_hyper_get_clip_extent + PURPOSE + Gets the extent a space should be clipped to in order to contain the + same number of elements as another space + USAGE + hsize_t H5S__hyper_get_clip_extent(clip_space,match_space,incl_trail) + const H5S_t *clip_space, IN: Space that clip size will be calculated based on + const H5S_t *match_space, IN: Space containing the same number of elements as clip_space should after clipping + hbool_t incl_trail; IN: Whether to include trailing unselected space + RETURNS + Calculated clip extent (never fails) + DESCRIPTION + Calculates and returns the extent that clip_space should be clipped to + (via H5S_hyper_clip_unlim) in order for it to contain the same number + of elements as match_space. If the clipped selection would end + immediately before a section of unselected space (i.e. at the end of a + block), then if incl_trail is TRUE, the returned clip extent is + selected to include that trailing "blank" space, otherwise it is + selected to end at the end before the blank space. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + Note this assumes the offset has been normalized. + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +hsize_t +H5S_hyper_get_clip_extent(const H5S_t *clip_space, const H5S_t *match_space, + hbool_t incl_trail) +{ + hsize_t num_slices; /* Number of slices in unlimited dimension */ + hsize_t ret_value = 0; /* Return value */ + + FUNC_ENTER_NOAPI(0) + + /* Check parameters */ + HDassert(clip_space); + HDassert(match_space); + HDassert(clip_space->select.sel_info.hslab->unlim_dim >= 0); + + /* Check for "none" match space */ + if(match_space->select.type->type == H5S_SEL_NONE) + num_slices = (hsize_t)0; + else { + HDassert(match_space->select.type->type == H5S_SEL_HYPERSLABS); + HDassert(match_space->select.sel_info.hslab); + + /* Calculate number of slices */ + num_slices = match_space->select.num_elem + / clip_space->select.sel_info.hslab->num_elem_non_unlim; + HDassert((match_space->select.num_elem + % clip_space->select.sel_info.hslab->num_elem_non_unlim) == 0); + } /* end else */ + + /* Call "real" get_clip_extent function */ + ret_value = H5S__hyper_get_clip_extent_real(clip_space, num_slices, incl_trail); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S_hyper_get_clip_extent() */ + + +/*-------------------------------------------------------------------------- + NAME + H5S_hyper_get_clip_extent_match + PURPOSE + Gets the extent a space should be clipped to in order to contain the + same number of elements as another unlimited space that has been + clipped to a different extent + USAGE + hsize_t H5S__hyper_get_clip_extent_match(clip_space,match_space,match_clip_size,incl_trail) + const H5S_t *clip_space, IN: Space that clip size will be calculated based on + const H5S_t *match_space, IN: Space that, after being clipped to match_clip_size, contains the same number of elements as clip_space should after clipping + hsize_t match_clip_size, IN: Extent match_space would be clipped to to match the number of elements in clip_space + hbool_t incl_trail; IN: Whether to include trailing unselected space + RETURNS + Calculated clip extent (never fails) + DESCRIPTION + Calculates and returns the extent that clip_space should be clipped to + (via H5S_hyper_clip_unlim) in order for it to contain the same number + of elements as match_space would have after being clipped to + match_clip_size. If the clipped selection would end immediately + before a section of unselected space (i.e. at the end of a block), + then if incl_trail is TRUE, the returned clip extent is selected to + include that trailing "blank" space, otherwise it is selected to end + at the end before the blank space. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + Note this assumes the offset has been normalized. + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +hsize_t +H5S_hyper_get_clip_extent_match(const H5S_t *clip_space, + const H5S_t *match_space, hsize_t match_clip_size, hbool_t incl_trail) +{ + const H5S_hyper_dim_t *match_diminfo; /* Convenience pointer to opt_unlim_diminfo in unlimited dimension in match_space */ + hsize_t count; /* Temporary count */ + hsize_t block; /* Temporary block */ + hsize_t num_slices; /* Number of slices in unlimited dimension */ + hsize_t ret_value = 0; /* Return value */ + + FUNC_ENTER_NOAPI(0) + + /* Check parameters */ + HDassert(clip_space); + HDassert(match_space); + HDassert(clip_space->select.sel_info.hslab); + HDassert(match_space->select.sel_info.hslab); + HDassert(clip_space->select.sel_info.hslab->unlim_dim >= 0); + HDassert(match_space->select.sel_info.hslab->unlim_dim >= 0); + HDassert(clip_space->select.sel_info.hslab->num_elem_non_unlim + == match_space->select.sel_info.hslab->num_elem_non_unlim); + + match_diminfo = &match_space->select.sel_info.hslab->opt_diminfo[match_space->select.sel_info.hslab->unlim_dim]; + + /* Get initial count and block */ + count = match_diminfo->count; + block = match_diminfo->block; + H5S__hyper_get_clip_diminfo(match_diminfo->start, match_diminfo->stride, &count, &block, match_clip_size); + + /* Calculate number of slices */ + /* Check for nothing returned */ + if((block == 0) || (count == 0)) + num_slices = (hsize_t)0; + /* Check for single block in unlimited dimension */ + else if(count == (hsize_t)1) + num_slices = block; + else { + /* Calculate initial num_slices */ + num_slices = block * count; + + /* Check for partial last block */ + HDassert(match_clip_size >= match_diminfo->start); + if(((match_diminfo->stride * (count - (hsize_t)1)) + block) + > (match_clip_size - match_diminfo->start)) { + /* Subtract slices missing from last block */ + HDassert((((match_diminfo->stride * (count - (hsize_t)1)) + block) + - (match_clip_size - match_diminfo->start)) < num_slices); + num_slices -= ((match_diminfo->stride * (count - (hsize_t)1)) + + block) - (match_clip_size - match_diminfo->start); + } /* end if */ + } /* end else */ + + /* Call "real" get_clip_extent function */ + ret_value = H5S__hyper_get_clip_extent_real(clip_space, num_slices, incl_trail); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S_hyper_get_clip_extent_match() */ + + +/*-------------------------------------------------------------------------- + NAME + H5S_hyper_get_unlim_block + PURPOSE + Get the nth block in the unlimited dimension + USAGE + H5S_t *H5S_hyper_get_unlim_block(space,block_index) + const H5S_t *space, IN: Space with unlimited selection + hsize_t block_index, IN: Index of block to return in unlimited dimension + hbool_t incl_trail; IN: Whether to include trailing unselected space + RETURNS + New space on success/NULL on failure. + DESCRIPTION + Returns a space containing only the block_indexth block in the + unlimited dimension on space. All blocks in all other dimensions are + preserved. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + Note this assumes the offset has been normalized. + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +H5S_t * +H5S_hyper_get_unlim_block(const H5S_t *space, hsize_t block_index) +{ + H5S_hyper_sel_t *hslab; /* Convenience pointer to hyperslab info */ + H5S_t *space_out = NULL; + hsize_t start[H5S_MAX_RANK]; + hsize_t stride[H5S_MAX_RANK]; + hsize_t count[H5S_MAX_RANK]; + hsize_t block[H5S_MAX_RANK]; + unsigned i; + H5S_t *ret_value = NULL; + + FUNC_ENTER_NOAPI(NULL) + + /* Check parameters */ + HDassert(space); + hslab = space->select.sel_info.hslab; + HDassert(hslab); + HDassert(hslab->unlim_dim >= 0); + HDassert(hslab->opt_diminfo[hslab->unlim_dim].count == H5S_UNLIMITED); + + /* Set start to select block_indexth block in unlimited dimension and set + * count to 1 in that dimension to only select that block. Copy all other + * diminfo parameters. */ + for(i = 0; i < space->extent.rank; i++) { + if((int)i == hslab->unlim_dim){ + start[i] = hslab->opt_diminfo[i].start + (block_index + * hslab->opt_diminfo[i].stride); + count[i] = (hsize_t)1; + } /* end if */ + else { + start[i] = hslab->opt_diminfo[i].start; + count[i] = hslab->opt_diminfo[i].count; + } /* end else */ + stride[i] = hslab->opt_diminfo[i].stride; + block[i] = hslab->opt_diminfo[i].block; + } /* end for */ + + /* Create output space, copy extent */ + if(NULL == (space_out = H5S_create(H5S_SIMPLE))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, NULL, "unable to create output dataspace") + if(H5S_extent_copy_real(&space_out->extent, &space->extent, TRUE) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOPY, NULL, "unable to copy destination space extent") + + /* Select block as defined by start/stride/count/block computed above */ + if(H5S_select_hyperslab(space_out, H5S_SELECT_SET, start, stride, count, block) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTINIT, NULL, "can't select hyperslab") + + /* Set return value */ + ret_value = space_out; + +done: + /* Free space on error */ + if(!ret_value) + if(space_out && H5S_close(space_out) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, NULL, "unable to release dataspace") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S_hyper_get_unlim_block */ + + +/*-------------------------------------------------------------------------- + NAME + H5S_hyper_get_first_inc_block + PURPOSE + Get the index of the first incomplete block in the specified extent + USAGE + hsize_t H5S_hyper_get_first_inc_block(space,clip_size,partial) + const H5S_t *space, IN: Space with unlimited selection + hsize_t clip_size, IN: Extent space would be clipped to + hbool_t *partial; OUT: Whether the ret_valueth block (first incomplete block) is partial + RETURNS + Index of first incomplete block in clip_size (never fails). + DESCRIPTION + Calculates and returns the index (as would be passed to + H5S_hyper_get_unlim_block()) of the first block in the unlimited + dimension of space which would be incomplete or missing when space is + clipped to clip_size. partial is set to TRUE if the first incomplete + block is partial, and FALSE if the first incomplete block is missing. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + Note this assumes the offset has been normalized. + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +hsize_t +H5S_hyper_get_first_inc_block(const H5S_t *space, hsize_t clip_size, + hbool_t *partial) +{ + H5S_hyper_sel_t *hslab; /* Convenience pointer to hyperslab info */ + H5S_hyper_dim_t *diminfo; /* Convenience pointer to opt_diminfo in unlimited dimension */ + hsize_t ret_value = 0; + + FUNC_ENTER_NOAPI(0) + + /* Check parameters */ + HDassert(space); + hslab = space->select.sel_info.hslab; + HDassert(hslab); + HDassert(hslab->unlim_dim >= 0); + HDassert(hslab->opt_diminfo[hslab->unlim_dim].count == H5S_UNLIMITED); + + diminfo = &hslab->opt_diminfo[hslab->unlim_dim]; + + /* Check for selection outside of clip_size */ + if(diminfo->start >= clip_size) { + ret_value = 0; + if(partial) + partial = FALSE; + } /* end if */ + else { + /* Calculate index of first incomplete block */ + ret_value = (clip_size - diminfo->start + diminfo->stride + - diminfo->block) / diminfo->stride; + + if(partial) { + /* Check for partial block */ + if((diminfo->stride * ret_value) < (clip_size - diminfo->start)) + *partial = TRUE; + else + *partial = FALSE; + } /* end if */ + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S_hyper_get_first_inc_block */ + + +/*-------------------------------------------------------------------------- + NAME H5Sis_regular_hyperslab PURPOSE Determine if a hyperslab selection is regular diff --git a/src/H5Snone.c b/src/H5Snone.c index 19d5eed..3492325 100644 --- a/src/H5Snone.c +++ b/src/H5Snone.c @@ -41,9 +41,11 @@ static herr_t H5S_none_release(H5S_t *space); static htri_t H5S_none_is_valid(const H5S_t *space); static hssize_t H5S_none_serial_size(const H5S_t *space); static herr_t H5S_none_serialize(const H5S_t *space, uint8_t **p); -static herr_t H5S_none_deserialize(H5S_t *space, const uint8_t **p); +static herr_t H5S_none_deserialize(H5S_t *space, uint32_t version, uint8_t flags, + const uint8_t **p); static herr_t H5S_none_bounds(const H5S_t *space, hsize_t *start, hsize_t *end); static herr_t H5S_none_offset(const H5S_t *space, hsize_t *off); +static int H5S__none_unlim_dim(const H5S_t *space); static htri_t H5S_none_is_contiguous(const H5S_t *space); static htri_t H5S_none_is_single(const H5S_t *space); static htri_t H5S_none_is_regular(const H5S_t *space); @@ -75,6 +77,8 @@ const H5S_select_class_t H5S_sel_none[1] = {{ H5S_none_deserialize, H5S_none_bounds, H5S_none_offset, + H5S__none_unlim_dim, + NULL, H5S_none_is_contiguous, H5S_none_is_single, H5S_none_is_regular, @@ -510,9 +514,11 @@ H5S_none_serialize(const H5S_t *space, uint8_t **p) PURPOSE Deserialize the current selection from a user-provided buffer. USAGE - herr_t H5S_none_deserialize(space, p) + herr_t H5S_none_deserialize(space, version, flags, p) H5S_t *space; IN/OUT: Dataspace pointer to place selection into + uint32_t version IN: Selection version + uint8_t flags IN: Selection flags uint8 **p; OUT: Pointer to buffer holding serialized selection. Will be advanced to end of serialized selection. @@ -527,7 +533,8 @@ H5S_none_serialize(const H5S_t *space, uint8_t **p) REVISION LOG --------------------------------------------------------------------------*/ static herr_t -H5S_none_deserialize(H5S_t *space, const uint8_t H5_ATTR_UNUSED **p) +H5S_none_deserialize(H5S_t *space, uint32_t H5_ATTR_UNUSED version, uint8_t H5_ATTR_UNUSED flags, + const uint8_t H5_ATTR_UNUSED **p) { herr_t ret_value = SUCCEED; /* return value */ @@ -618,6 +625,34 @@ H5S_none_offset(const H5S_t H5_ATTR_UNUSED *space, hsize_t H5_ATTR_UNUSED *offse /*-------------------------------------------------------------------------- NAME + H5S__none_unlim_dim + PURPOSE + Return unlimited dimension of selection, or -1 if none + USAGE + int H5S__none_unlim_dim(space) + H5S_t *space; IN: Dataspace pointer to check + RETURNS + Unlimited dimension of selection, or -1 if none (never fails). + DESCRIPTION + Returns the index of the unlimited dimension in this selection, or -1 + if the selection has no unlimited dimension. "None" selections cannot + have an unlimited dimension, so this function always returns -1. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +static int +H5S__none_unlim_dim(const H5S_t H5_ATTR_UNUSED *space) +{ + FUNC_ENTER_STATIC_NOERR + + FUNC_LEAVE_NOAPI(-1) +} /* end H5S__none_unlim_dim() */ + + +/*-------------------------------------------------------------------------- + NAME H5S_none_is_contiguous PURPOSE Check if a "none" selection is contiguous within the dataspace extent. diff --git a/src/H5Spkg.h b/src/H5Spkg.h index 933de64..e57650a 100644 --- a/src/H5Spkg.h +++ b/src/H5Spkg.h @@ -34,6 +34,13 @@ #define H5S_VALID_MAX 0x01 #define H5S_VALID_PERM 0x02 +/* Flags for serialization of selections */ +#define H5S_SELECT_FLAG_UNLIM 0x01 +#define H5S_SELECT_FLAG_BITS (H5S_SELECT_FLAG_UNLIM) + +/* Length of stack-allocated sequences for "project intersect" routines */ +#define H5S_PROJECT_INTERSECT_NSEQS 256 + /* Initial version of the dataspace information */ #define H5O_SDSPACE_VERSION_1 1 @@ -48,6 +55,10 @@ * and 'size' callbacks for places to change when updating this. */ #define H5O_SDSPACE_VERSION_LATEST H5O_SDSPACE_VERSION_2 +/* Maximum dimension size (highest value that is not a special value e.g. + * H5S_UNLIMITED) */ +#define H5S_MAX_SIZE ((hsize_t)(hssize_t)(-2)) + /* * Dataspace extent information @@ -113,6 +124,8 @@ typedef struct { * are only used for re-gurgitating the original values used to set the * hyperslab to the application when it queries the hyperslab selection * information. */ + int unlim_dim; /* Dimension where selection is unlimited, or -1 if none */ + hsize_t num_elem_non_unlim; /* # of elements in a "slice" excluding the unlimited dimension */ H5S_hyper_span_info_t *span_lst; /* List of hyperslab span information */ } H5S_hyper_sel_t; @@ -132,11 +145,17 @@ typedef hssize_t (*H5S_sel_serial_size_func_t)(const H5S_t *space); /* Method to store current selection in "serialized" form (a byte sequence suitable for storing on disk) */ typedef herr_t (*H5S_sel_serialize_func_t)(const H5S_t *space, uint8_t **p); /* Method to create selection from "serialized" form (a byte sequence suitable for storing on disk) */ -typedef herr_t (*H5S_sel_deserialize_func_t)(H5S_t *space, const uint8_t **p); +typedef herr_t (*H5S_sel_deserialize_func_t)(H5S_t *space, uint32_t version, uint8_t flags, + const uint8_t **p); /* Method to determine smallest n-D bounding box containing the current selection */ typedef herr_t (*H5S_sel_bounds_func_t)(const H5S_t *space, hsize_t *start, hsize_t *end); /* Method to determine linear offset of initial element in selection within dataspace */ typedef herr_t (*H5S_sel_offset_func_t)(const H5S_t *space, hsize_t *offset); +/* Method to get unlimited dimension of selection (or -1 for none) */ +typedef int (*H5S_sel_unlim_dim_func_t)(const H5S_t *space); +/* Method to get the number of elements in a slice through the unlimited dimension */ +typedef herr_t (*H5S_sel_num_elem_non_unlim_func_t)(const H5S_t *space, + hsize_t *num_elem_non_unlim); /* Method to determine if current selection is contiguous */ typedef htri_t (*H5S_sel_is_contiguous_func_t)(const H5S_t *space); /* Method to determine if current selection is a single block */ @@ -166,6 +185,8 @@ typedef struct { H5S_sel_deserialize_func_t deserialize; /* Method to store create selection from "serialized" form (a byte sequence suitable for storing on disk) */ H5S_sel_bounds_func_t bounds; /* Method to determine to smallest n-D bounding box containing the current selection */ H5S_sel_offset_func_t offset; /* Method to determine linear offset of initial element in selection within dataspace */ + H5S_sel_unlim_dim_func_t unlim_dim; /* Method to get unlimited dimension of selection (or -1 for none) */ + H5S_sel_num_elem_non_unlim_func_t num_elem_non_unlim; /* Method to get the number of elements in a slice through the unlimited dimension */ H5S_sel_is_contiguous_func_t is_contiguous; /* Method to determine if current selection is contiguous */ H5S_sel_is_single_func_t is_single; /* Method to determine if current selection is a single block */ H5S_sel_is_regular_func_t is_regular; /* Method to determine if current selection is "regular" */ @@ -249,6 +270,10 @@ H5_DLL herr_t H5S_extent_copy_real(H5S_extent_t *dst, const H5S_extent_t *src, hbool_t copy_max); /* Operations on selections */ +H5_DLL herr_t H5S__hyper_project_intersection(const H5S_t *src_space, + const H5S_t *dst_space, const H5S_t *src_intersect_space, + H5S_t *proj_space); +H5_DLL herr_t H5S__hyper_subtract(H5S_t *space, H5S_t *subtract_space); /* Testing functions */ #ifdef H5S_TESTING diff --git a/src/H5Spoint.c b/src/H5Spoint.c index daee160..0f85110 100644 --- a/src/H5Spoint.c +++ b/src/H5Spoint.c @@ -42,9 +42,11 @@ static herr_t H5S_point_release(H5S_t *space); static htri_t H5S_point_is_valid(const H5S_t *space); static hssize_t H5S_point_serial_size(const H5S_t *space); static herr_t H5S_point_serialize(const H5S_t *space, uint8_t **p); -static herr_t H5S_point_deserialize(H5S_t *space, const uint8_t **p); +static herr_t H5S_point_deserialize(H5S_t *space, uint32_t version, uint8_t flags, + const uint8_t **p); static herr_t H5S_point_bounds(const H5S_t *space, hsize_t *start, hsize_t *end); static herr_t H5S_point_offset(const H5S_t *space, hsize_t *off); +static int H5S__point_unlim_dim(const H5S_t *space); static htri_t H5S_point_is_contiguous(const H5S_t *space); static htri_t H5S_point_is_single(const H5S_t *space); static htri_t H5S_point_is_regular(const H5S_t *space); @@ -76,6 +78,8 @@ const H5S_select_class_t H5S_sel_point[1] = {{ H5S_point_deserialize, H5S_point_bounds, H5S_point_offset, + H5S__point_unlim_dim, + NULL, H5S_point_is_contiguous, H5S_point_is_single, H5S_point_is_regular, @@ -882,6 +886,8 @@ H5S_point_serialize (const H5S_t *space, uint8_t **p) herr_t H5S_point_deserialize(space, p) H5S_t *space; IN/OUT: Dataspace pointer to place selection into + uint32_t version IN: Selection version + uint8_t flags IN: Selection flags uint8 **p; OUT: Pointer to buffer holding serialized selection. Will be advanced to end of serialized selection. @@ -896,7 +902,8 @@ H5S_point_serialize (const H5S_t *space, uint8_t **p) REVISION LOG --------------------------------------------------------------------------*/ static herr_t -H5S_point_deserialize (H5S_t *space, const uint8_t **p) +H5S_point_deserialize(H5S_t *space, uint32_t H5_ATTR_UNUSED version, uint8_t H5_ATTR_UNUSED flags, + const uint8_t **p) { H5S_seloper_t op = H5S_SELECT_SET; /* Selection operation */ hsize_t *coord = NULL, *tcoord; /* Pointer to array of elements */ @@ -1197,6 +1204,35 @@ done: /*-------------------------------------------------------------------------- NAME + H5S__point_unlim_dim + PURPOSE + Return unlimited dimension of selection, or -1 if none + USAGE + int H5S__point_unlim_dim(space) + H5S_t *space; IN: Dataspace pointer to check + RETURNS + Unlimited dimension of selection, or -1 if none (never fails). + DESCRIPTION + Returns the index of the unlimited dimension in this selection, or -1 + if the selection has no unlimited dimension. Currently point + selections cannot have an unlimited dimension, so this function always + returns -1. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +static int +H5S__point_unlim_dim(const H5S_t H5_ATTR_UNUSED *space) +{ + FUNC_ENTER_STATIC_NOERR + + FUNC_LEAVE_NOAPI(-1) +} /* end H5S__point_unlim_dim() */ + + +/*-------------------------------------------------------------------------- + NAME H5S_point_is_contiguous PURPOSE Check if a point selection is contiguous within the dataspace extent. diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index ff6955c..b00dc6b 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -220,6 +220,8 @@ H5_DLL H5S_t *H5S_create(H5S_class_t type); H5_DLL H5S_t *H5S_create_simple(unsigned rank, const hsize_t dims[/*rank*/], const hsize_t maxdims[/*rank*/]); H5_DLL herr_t H5S_set_latest_version(H5S_t *ds); +H5_DLL herr_t H5S_encode(H5S_t *obj, unsigned char **p, size_t *nalloc); +H5_DLL H5S_t *H5S_decode(const unsigned char **p); H5_DLL herr_t H5S_debug(H5F_t *f, hid_t dxpl_id, const void *_mesg, FILE *stream, int indent, int fwidth); #ifndef H5_NO_DEPRECATED_SYMBOLS @@ -243,6 +245,9 @@ H5_DLL htri_t H5S_select_valid(const H5S_t *space); H5_DLL hssize_t H5S_get_select_npoints(const H5S_t *space); H5_DLL herr_t H5S_get_select_bounds(const H5S_t *space, hsize_t *start, hsize_t *end); H5_DLL herr_t H5S_get_select_offset(const H5S_t *space, hsize_t *offset); +H5_DLL int H5S_get_select_unlim_dim(const H5S_t *space); +H5_DLL herr_t H5S_get_select_num_elem_non_unlim(const H5S_t *space, + hsize_t *num_elem_non_unlim); H5_DLL herr_t H5S_select_offset(H5S_t *space, const hssize_t *offset); H5_DLL herr_t H5S_select_copy(H5S_t *dst, const H5S_t *src, hbool_t share_selection); H5_DLL htri_t H5S_select_shape_same(const H5S_t *space1, const H5S_t *space2); @@ -261,6 +266,10 @@ H5_DLL htri_t H5S_select_is_regular(const H5S_t *space); H5_DLL herr_t H5S_select_adjust_u(H5S_t *space, const hsize_t *offset); H5_DLL herr_t H5S_select_project_scalar(const H5S_t *space, hsize_t *offset); H5_DLL herr_t H5S_select_project_simple(const H5S_t *space, H5S_t *new_space, hsize_t *offset); +H5_DLL herr_t H5S_select_project_intersection(const H5S_t *src_space, + const H5S_t *dst_space, const H5S_t *src_intersect_space, + H5S_t **new_space_ptr); +H5_DLL herr_t H5S_select_subtract(H5S_t *space, H5S_t *subtract_space); /* Operations on all selections */ H5_DLL herr_t H5S_select_all(H5S_t *space, hbool_t rel_prev); @@ -286,6 +295,15 @@ H5_DLL htri_t H5S_hyper_intersect_block (H5S_t *space, hsize_t *start, hsize_t * H5_DLL herr_t H5S_hyper_adjust_s(H5S_t *space, const hssize_t *offset); H5_DLL htri_t H5S_hyper_normalize_offset(H5S_t *space, hssize_t *old_offset); H5_DLL herr_t H5S_hyper_denormalize_offset(H5S_t *space, const hssize_t *old_offset); +H5_DLL herr_t H5S_hyper_clip_unlim(H5S_t *space, hsize_t clip_size); +H5_DLL hsize_t H5S_hyper_get_clip_extent(const H5S_t *clip_space, + const H5S_t *match_space, hbool_t incl_trail); +H5_DLL hsize_t H5S_hyper_get_clip_extent_match(const H5S_t *clip_space, + const H5S_t *match_space, hsize_t match_clip_size, hbool_t incl_trail); +H5_DLL H5S_t *H5S_hyper_get_unlim_block(const H5S_t *space, + hsize_t block_index); +H5_DLL hsize_t H5S_hyper_get_first_inc_block(const H5S_t *space, + hsize_t clip_size, hbool_t *partial); /* Operations on selection iterators */ H5_DLL herr_t H5S_select_iter_init(H5S_sel_iter_t *iter, const H5S_t *space, size_t elmt_size); diff --git a/src/H5Sselect.c b/src/H5Sselect.c index d7a2340..d4a4d69 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -455,6 +455,8 @@ H5S_select_deserialize(H5S_t **space, const uint8_t **p) H5S_t *tmp_space = NULL; /* Pointer to actual dataspace to use, either *space or a newly allocated one */ uint32_t sel_type; /* Pointer to the selection type */ + uint32_t version; /* Version number */ + uint8_t flags = 0; /* Flags */ herr_t ret_value = FAIL; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -472,8 +474,23 @@ H5S_select_deserialize(H5S_t **space, const uint8_t **p) /* Decode selection type */ UINT32DECODE(*p, sel_type); - /* Skip over the remainder of the header */ - *p += 12; + /* Decode version */ + UINT32DECODE(*p, version); + + if(version >= (uint32_t)2) { + /* Decode flags */ + flags = *(*p)++; + + /* Check for unknown flags */ + if(flags & ~H5S_SELECT_FLAG_BITS) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTLOAD, FAIL, "unknown flag for selection") + + /* Skip over the remainder of the header */ + *p += 4; + } /* end if */ + else + /* Skip over the remainder of the header */ + *p += 8; /* Decode and check or patch rank for point and hyperslab selections */ if((sel_type == H5S_SEL_POINTS) || (sel_type == H5S_SEL_HYPERSLABS)) { @@ -499,19 +516,19 @@ H5S_select_deserialize(H5S_t **space, const uint8_t **p) /* Make routine for selection type */ switch(sel_type) { case H5S_SEL_POINTS: /* Sequence of points selected */ - ret_value = (*H5S_sel_point->deserialize)(tmp_space, p); + ret_value = (*H5S_sel_point->deserialize)(tmp_space, version, flags, p); break; case H5S_SEL_HYPERSLABS: /* Hyperslab selection defined */ - ret_value = (*H5S_sel_hyper->deserialize)(tmp_space, p); + ret_value = (*H5S_sel_hyper->deserialize)(tmp_space, version, flags, p); break; case H5S_SEL_ALL: /* Entire extent selected */ - ret_value = (*H5S_sel_all->deserialize)(tmp_space, p); + ret_value = (*H5S_sel_all->deserialize)(tmp_space, version, flags, p); break; case H5S_SEL_NONE: /* Nothing selected */ - ret_value = (*H5S_sel_none->deserialize)(tmp_space, p); + ret_value = (*H5S_sel_none->deserialize)(tmp_space, version, flags, p); break; default: @@ -673,6 +690,89 @@ H5S_get_select_offset(const H5S_t *space, hsize_t *offset) /*-------------------------------------------------------------------------- NAME + H5S_get_select_unlim_dim + PURPOSE + Gets the unlimited dimension in the selection, or -1 if there is no + unlimited dimension. + USAGE + int H5S_get_select_unlim_dim(space) + const H5S_t *space; IN: Dataspace pointer of selection to query + RETURNS + Unlimited dimension in the selection, or -1 if there is no unlimited + dimension (never fails) + DESCRIPTION + Gets the unlimited dimension in the selection, or -1 if there is no + unlimited dimension. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + Currently only implemented for hyperslab selections, all others + simply return -1. + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +int +H5S_get_select_unlim_dim(const H5S_t *space) +{ + herr_t ret_value = FAIL; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check args */ + HDassert(space); + + ret_value = (*space->select.type->unlim_dim)(space); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5S_get_select_unlim_dim() */ + + +/*-------------------------------------------------------------------------- + NAME + H5S_get_select_num_elem_non_unlim + PURPOSE + Gets the number of elements in the non-unlimited dimensions + USAGE + herr_t H5S_get_select_num_elem_non_unlim(space,num_elem_non_unlim) + H5S_t *space; IN: Dataspace pointer to check + hsize_t *num_elem_non_unlim; OUT: Number of elements in the non-unlimited dimensions + RETURNS + Non-negative on success/Negative on failure + DESCRIPTION + Returns the number of elements in a slice through the non-unlimited + dimensions of the selection. Fails if the selection has no unlimited + dimension. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5S_get_select_num_elem_non_unlim(const H5S_t *space, + hsize_t *num_elem_non_unlim) +{ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Check args */ + HDassert(space); + HDassert(num_elem_non_unlim); + + /* Check for selection callback */ + if(!space->select.type->num_elem_non_unlim) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "selection type has no num_elem_non_unlim callback") + + /* Make selection callback */ + if((*space->select.type->num_elem_non_unlim)(space, num_elem_non_unlim) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOUNT, FAIL, "can't get number of elements in non-unlimited dimension") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5S_get_select_unlim_dim() */ + + +/*-------------------------------------------------------------------------- + NAME H5S_select_is_contiguous PURPOSE Determines if a selection is contiguous in the dataspace @@ -2114,3 +2214,189 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5S_select_fill() */ + +/*-------------------------------------------------------------------------- + NAME + H5S_select_project_intersection + + PURPOSE + Projects the intersection of of the selections of src_space and + src_intersect_space within the selection of src_space as a selection + within the selection of dst_space + + USAGE + herr_t H5S_select_project_intersection(src_space,dst_space,src_intersect_space,proj_space) + H5S_t *src_space; IN: Selection that is mapped to dst_space, and intersected with src_intersect_space + H5S_t *dst_space; IN: Selection that is mapped to src_space, and which contains the result + H5S_t *src_intersect_space; IN: Selection whose intersection with src_space is projected to dst_space to obtain the result + H5S_t *proj_space; OUT: Will contain the result (intersection of src_intersect_space and src_space projected from src_space to dst_space) after the operation + + RETURNS + Non-negative on success/Negative on failure. + + DESCRIPTION + Projects the intersection of of the selections of src_space and + src_intersect_space within the selection of src_space as a selection + within the selection of dst_space. The result is placed in the + selection of proj_space. + + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5S_select_project_intersection(const H5S_t *src_space, const H5S_t *dst_space, + const H5S_t *src_intersect_space, H5S_t **new_space_ptr) +{ + H5S_t *new_space = NULL; /* New dataspace constructed */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(src_space); + HDassert(dst_space); + HDassert(src_intersect_space); + HDassert(new_space_ptr); + + /* Create new space, using dst extent. Start with "all" selection. */ + if(NULL == (new_space = H5S_create(H5S_SIMPLE))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, "unable to create output dataspace") + if(H5S_extent_copy_real(&new_space->extent, &dst_space->extent, TRUE) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOPY, FAIL, "unable to copy destination space extent") + + /* If the intersecting space is "all", the intersection must be equal to the + * source space and the projection must be equal to the destination space */ + if(src_intersect_space->select.type->type == H5S_SEL_ALL) { + /* Copy the destination selection. */ + if(H5S_select_copy(new_space, dst_space, FALSE) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCOPY, FAIL, "can't copy destination space selection") + } /* end if */ + /* If any of the spaces are "none", the projection must also be "none" */ + else if((src_intersect_space->select.type->type == H5S_SEL_NONE) + || (src_space->select.type->type == H5S_SEL_NONE) + || (dst_space->select.type->type == H5S_SEL_NONE)) { + /* Change to "none" selection */ + if(H5S_select_none(new_space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't change selection") + } /* end if */ + /* If any of the spaces use point selection, fall back to general algorithm + */ + else if((src_intersect_space->select.type->type == H5S_SEL_POINTS) + || (src_space->select.type->type == H5S_SEL_POINTS) + || (dst_space->select.type->type == H5S_SEL_POINTS)) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "point selections not currently supported") + else { + HDassert(src_intersect_space->select.type->type == H5S_SEL_HYPERSLABS); + /* Intersecting space is hyperslab selection. Call the hyperslab + * routine to project to another hyperslab selection. */ + if(H5S__hyper_project_intersection(src_space, dst_space, src_intersect_space, new_space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCLIP, FAIL, "can't project hyperslab ondot destination selection") + } /* end else */ + + /* load the address of the new space into *new_space_ptr */ + *new_space_ptr = new_space; + +done: + /* Cleanup on error */ + if(ret_value < 0) { + if(new_space && H5S_close(new_space) < 0) + HDONE_ERROR(H5E_DATASPACE, H5E_CANTRELEASE, FAIL, "unable to release dataspace") + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5S_select_project_intersection() */ + + +/*-------------------------------------------------------------------------- + NAME + H5S_select_subtract + + PURPOSE + Subtract one selection from another + + USAGE + herr_t H5S_select_subtract(space,subtract_space) + H5S_t *space; IN/OUT: Selection to be operated on + H5S_t *subtract_space; IN: Selection that will be subtracted from space + + RETURNS + Non-negative on success/Negative on failure. + + DESCRIPTION + Removes any and all portions of space that are also present in + subtract_space. In essence, performs an A_NOT_B operation with the + two selections. + + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5S_select_subtract(H5S_t *space, H5S_t *subtract_space) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(space); + HDassert(subtract_space); + + /* If either space is using the none selection, then we do not need to do + * anything */ + if((space->select.type->type != H5S_SEL_NONE) + && (subtract_space->select.type->type != H5S_SEL_NONE)) { + /* If subtract_space is using the all selection, set space to none */ + if(subtract_space->select.type->type == H5S_SEL_ALL) { + /* Change to "none" selection */ + if(H5S_select_none(space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTDELETE, FAIL, "can't change selection") + } /* end if */ + else { + /* Check for point selection in subtract_space, convert to + * hyperslab */ + if(subtract_space->select.type->type == H5S_SEL_POINTS) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "point selections not currently supported") + + /* Check for point or all selection in space, convert to hyperslab + */ + if(space->select.type->type == H5S_SEL_ALL) { + /* Convert current "all" selection to "real" hyperslab selection */ + /* Then allow operation to proceed */ + hsize_t tmp_start[H5O_LAYOUT_NDIMS]; /* Temporary start information */ + hsize_t tmp_stride[H5O_LAYOUT_NDIMS]; /* Temporary stride information */ + hsize_t tmp_count[H5O_LAYOUT_NDIMS]; /* Temporary count information */ + hsize_t tmp_block[H5O_LAYOUT_NDIMS]; /* Temporary block information */ + unsigned i; /* Local index variable */ + + /* Fill in temporary information for the dimensions */ + for(i = 0; i < space->extent.rank; i++) { + tmp_start[i] = 0; + tmp_stride[i] = 1; + tmp_count[i] = 1; + tmp_block[i] = space->extent.size[i]; + } /* end for */ + + /* Convert to hyperslab selection */ + if(H5S_select_hyperslab(space, H5S_SELECT_SET, tmp_start, tmp_stride, tmp_count, tmp_block) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSELECT, FAIL, "can't convert selection") + } /* end if */ + else if(space->select.type->type == H5S_SEL_POINTS) + HGOTO_ERROR(H5E_DATASPACE, H5E_UNSUPPORTED, FAIL, "point selections not currently supported") + + HDassert(space->select.type->type == H5S_SEL_HYPERSLABS); + HDassert(subtract_space->select.type->type == H5S_SEL_HYPERSLABS); + + /* Both spaces are now hyperslabs, perform the operation */ + if(H5S__hyper_subtract(space, subtract_space) < 0) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCLIP, FAIL, "can't subtract hyperslab") + } /* end else */ + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5S_select_subtract() */ + diff --git a/src/H5trace.c b/src/H5trace.c index 41f3418..f212bef 100644 --- a/src/H5trace.c +++ b/src/H5trace.c @@ -523,6 +523,10 @@ H5_trace(const double *returning, const char *func, const char *type, ...) fprintf(out, "H5D_CHUNKED"); break; + case H5D_VIRTUAL: + fprintf(out, "H5D_VIRTUAL"); + break; + case H5D_NLAYOUTS: fprintf(out, "H5D_NLAYOUTS"); break; @@ -675,6 +679,36 @@ H5_trace(const double *returning, const char *func, const char *type, ...) } /* end else */ break; + case 'v': + if(ptr) { + if(vp) + fprintf(out, "0x%lx", (unsigned long)vp); + else + fprintf(out, "NULL"); + } /* end if */ + else { + H5D_vds_view_t view = (H5D_vds_view_t)va_arg(ap, int); + + switch(view) { + case H5D_VDS_ERROR: + fprintf(out, "H5D_VDS_ERROR"); + break; + + case H5D_VDS_FIRST_MISSING: + fprintf(out, "H5D_VDS_FIRST_MISSING"); + break; + + case H5D_VDS_LAST_AVAILABLE: + fprintf(out, "H5D_VDS_LAST_AVAILABLE"); + break; + + default: + fprintf(out, "%ld", (long)view); + break; + } /* end switch */ + } /* end else */ + break; + default: fprintf (out, "BADTYPE(D%c)", type[1]); goto error; diff --git a/src/Makefile.am b/src/Makefile.am index f63a88e..86dad23 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -50,7 +50,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5D.c H5Dbtree.c H5Dchunk.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \ H5Ddeprec.c H5Defl.c H5Dfill.c H5Dint.c \ H5Dio.c H5Dlayout.c \ - H5Doh.c H5Dscatgath.c H5Dselect.c H5Dtest.c \ + H5Doh.c H5Dscatgath.c H5Dselect.c H5Dtest.c H5Dvirtual.c \ H5E.c H5Edeprec.c H5Eint.c \ H5EA.c H5EAcache.c H5EAdbg.c H5EAdblkpage.c H5EAdblock.c H5EAhdr.c \ H5EAiblock.c H5EAint.c H5EAsblock.c H5EAstat.c H5EAtest.c \ |