diff options
Diffstat (limited to 'src/H5Dpkg.h')
| -rw-r--r-- | src/H5Dpkg.h | 104 |
1 files changed, 66 insertions, 38 deletions
diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index a242b8e..8eaf265 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -33,6 +33,7 @@ /* Other private headers needed by this file */ #include "H5ACprivate.h" /* Metadata cache */ +#include "H5B2private.h" /* v2 B-trees */ #include "H5Fprivate.h" /* File access */ #include "H5Gprivate.h" /* Groups */ #include "H5SLprivate.h" /* Skip lists */ @@ -46,17 +47,19 @@ #define H5D_MINHDR_SIZE 256 /* [Simple] Macro to construct a H5D_io_info_t from it's components */ -#define H5D_BUILD_IO_INFO_WRT(io_info, ds, dxpl_c, dxpl_i, str, buf) \ +#define H5D_BUILD_IO_INFO_WRT(io_info, ds, dxpl_c, dxpl_m, dxpl_r, str, buf) \ (io_info)->dset = ds; \ (io_info)->dxpl_cache = dxpl_c; \ - (io_info)->dxpl_id = dxpl_i; \ + (io_info)->raw_dxpl_id = dxpl_r; \ + (io_info)->md_dxpl_id = dxpl_m; \ (io_info)->store = str; \ (io_info)->op_type = H5D_IO_OP_WRITE; \ (io_info)->u.wbuf = buf -#define H5D_BUILD_IO_INFO_RD(io_info, ds, dxpl_c, dxpl_i, str, buf) \ +#define H5D_BUILD_IO_INFO_RD(io_info, ds, dxpl_c, dxpl_m, dxpl_r, str, buf) \ (io_info)->dset = ds; \ (io_info)->dxpl_cache = dxpl_c; \ - (io_info)->dxpl_id = dxpl_i; \ + (io_info)->raw_dxpl_id = dxpl_r; \ + (io_info)->md_dxpl_id = dxpl_m; \ (io_info)->store = str; \ (io_info)->op_type = H5D_IO_OP_READ; \ (io_info)->u.rbuf = buf @@ -65,6 +68,27 @@ #define H5D_MARK_SPACE 0x01 #define H5D_MARK_LAYOUT 0x02 +/* Default creation parameters for chunk index data structures */ +/* See H5O_layout_chunk_t */ + +/* Fixed array creation values */ +#define H5D_FARRAY_CREATE_PARAM_SIZE 1 /* Size of the creation parameters in bytes */ +#define H5D_FARRAY_MAX_DBLK_PAGE_NELMTS_BITS 10 /* i.e. 1024 elements per data block page */ + +/* Extensible array creation values */ +#define H5D_EARRAY_CREATE_PARAM_SIZE 5 /* Size of the creation parameters in bytes */ +#define H5D_EARRAY_MAX_NELMTS_BITS 32 /* i.e. 4 giga-elements */ +#define H5D_EARRAY_IDX_BLK_ELMTS 4 +#define H5D_EARRAY_SUP_BLK_MIN_DATA_PTRS 4 +#define H5D_EARRAY_DATA_BLK_MIN_ELMTS 16 +#define H5D_EARRAY_MAX_DBLOCK_PAGE_NELMTS_BITS 10 /* i.e. 1024 elements per data block page */ + +/* v2 B-tree creation values for raw meta_size */ +#define H5D_BT2_CREATE_PARAM_SIZE 6 /* Size of the creation parameters in bytes */ +#define H5D_BT2_NODE_SIZE 2048 +#define H5D_BT2_SPLIT_PERC 100 +#define H5D_BT2_MERGE_PERC 40 + /****************************/ /* Package Private Typedefs */ @@ -192,7 +216,8 @@ typedef struct H5D_io_info_t { const #endif /* H5_HAVE_PARALLEL */ H5D_dxpl_cache_t *dxpl_cache; /* Pointer to cached DXPL info */ - hid_t dxpl_id; /* Original DXPL ID */ + hid_t raw_dxpl_id; /* Original DXPL ID */ + hid_t md_dxpl_id; /* metadata dxpl needed for parallel HDF5 */ #ifdef H5_HAVE_PARALLEL MPI_Comm comm; /* MPI communicator for file */ hbool_t using_mpi_vfd; /* Whether the file is using an MPI-based VFD */ @@ -240,7 +265,7 @@ typedef struct H5D_chk_idx_info_t { typedef struct H5D_chunk_rec_t { hsize_t scaled[H5O_LAYOUT_NDIMS]; /* Logical offset to start */ uint32_t nbytes; /* Size of stored data */ - unsigned filter_mask; /* Excluded filters */ + uint32_t filter_mask; /* Excluded filters */ haddr_t chunk_addr; /* Address of chunk in file */ } H5D_chunk_rec_t; @@ -250,7 +275,6 @@ typedef struct H5D_chunk_rec_t { * to which the index points. */ typedef struct H5D_chunk_common_ud_t { - /* downward */ const H5O_layout_chunk_t *layout; /* Chunk layout description */ const H5O_storage_chunk_t *storage; /* Chunk storage description */ const hsize_t *scaled; /* Scaled coordinates for a chunk */ @@ -258,12 +282,15 @@ typedef struct H5D_chunk_common_ud_t { /* B-tree callback info for various operations */ typedef struct H5D_chunk_ud_t { + /* Downward */ H5D_chunk_common_ud_t common; /* Common info for B-tree user data (must be first) */ /* Upward */ - unsigned idx_hint; /*index of chunk in cache, if present */ - H5F_block_t chunk_block; /*offset/length of chunk in file */ - unsigned filter_mask; /*excluded filters */ + unsigned idx_hint; /* Index of chunk in cache, if present */ + H5F_block_t chunk_block; /* Offset/length of chunk in file */ + unsigned filter_mask; /* Excluded filters */ + hbool_t new_unfilt_chunk; /* Whether the chunk just became unfiltered */ + hsize_t chunk_idx; /* Chunk index for EA, FA indexing */ } H5D_chunk_ud_t; /* Typedef for "generic" chunk callbacks */ @@ -363,10 +390,12 @@ typedef struct H5D_chunk_cached_t { hsize_t scaled[H5O_LAYOUT_NDIMS]; /*scaled offset of chunk*/ haddr_t addr; /*file address of chunk */ uint32_t nbytes; /*size of stored data */ + hsize_t chunk_idx; /*index of chunk in dataset */ unsigned filter_mask; /*excluded filters */ } H5D_chunk_cached_t; /* The raw data chunk cache */ +struct H5D_rdcc_ent_t; /* Forward declaration of struct used below */ typedef struct H5D_rdcc_t { struct { unsigned ninits; /* Number of chunk creations */ @@ -379,6 +408,7 @@ typedef struct H5D_rdcc_t { double w0; /* Chunk preemption policy */ struct H5D_rdcc_ent_t *head; /* Head of doubly linked list */ struct H5D_rdcc_ent_t *tail; /* Tail of doubly linked list */ + struct H5D_rdcc_ent_t *tmp_head; /* Head of temporary doubly linked list. Chunks on this list are not in the hash table (slot). The head entry is a sentinel (does not refer to an actual chunk). */ size_t nbytes_used; /* Current cached raw data in bytes */ int nused; /* Number of chunk slots in use */ H5D_chunk_cached_t last; /* Cached copy of last chunk information */ @@ -422,6 +452,7 @@ typedef struct H5D_shared_t { /* Cached dataspace info */ unsigned ndims; /* The dataset's dataspace rank */ hsize_t curr_dims[H5S_MAX_RANK]; /* The curr. size of dataset dimensions */ + hsize_t curr_power2up[H5S_MAX_RANK]; /* The curr. dim sizes, rounded up to next power of 2 */ hsize_t max_dims[H5S_MAX_RANK]; /* The max. size of dataset dimensions */ /* Buffered/cached information for types of raw data storage*/ @@ -433,6 +464,8 @@ typedef struct H5D_shared_t { */ H5D_rdcc_t chunk; /* Information about chunked data */ } cache; + + char *extfile_prefix; /* expanded external file prefix */ } H5D_shared_t; struct H5D_t { @@ -493,22 +526,6 @@ typedef struct { hsize_t size; /* Accumulated number of bytes for the selection */ } H5D_vlen_bufsize_t; -/* Raw data chunks are cached. Each entry in the cache is: */ -typedef struct H5D_rdcc_ent_t { - hbool_t locked; /*entry is locked in cache */ - hbool_t dirty; /*needs to be written to disk? */ - hbool_t deleted; /*chunk about to be deleted */ - hsize_t scaled[H5O_LAYOUT_NDIMS]; /*scaled chunk 'name' (coordinates) */ - uint32_t rd_count; /*bytes remaining to be read */ - uint32_t wr_count; /*bytes remaining to be written */ - H5F_block_t chunk_block; /*offset/length of chunk in file */ - uint8_t *chunk; /*the unfiltered chunk data */ - unsigned idx; /*index in hash table */ - struct H5D_rdcc_ent_t *next;/*next item in doubly-linked list */ - struct H5D_rdcc_ent_t *prev;/*previous item in doubly-linked list */ -} H5D_rdcc_ent_t; -typedef H5D_rdcc_ent_t *H5D_rdcc_ent_ptr_t; /* For free lists */ - /*****************************/ /* Package Private Variables */ @@ -524,6 +541,13 @@ H5_DLLVAR const H5D_layout_ops_t H5D_LOPS_VIRTUAL[1]; /* Chunked layout operations */ H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_BTREE[1]; +H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_EARRAY[1]; +H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_FARRAY[1]; +H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_BT2[1]; + +/* The v2 B-tree class for indexing chunked datasets with >1 unlimited dimensions */ +H5_DLLVAR const H5B2_class_t H5D_BT2[1]; +H5_DLLVAR const H5B2_class_t H5D_BT2_FILT[1]; /******************************/ @@ -539,7 +563,7 @@ H5_DLL H5D_t *H5D__open_name(const H5G_loc_t *loc, const char *name, hid_t dapl_id, hid_t dxpl_id); H5_DLL herr_t H5D__get_space_status(H5D_t *dset, H5D_space_status_t *allocation, hid_t dxpl_id); -H5_DLL herr_t H5D__alloc_storage(const H5D_t *dset, hid_t dxpl_id, H5D_time_alloc_t time_alloc, +H5_DLL herr_t H5D__alloc_storage(const H5D_io_info_t *io_info, H5D_time_alloc_t time_alloc, hbool_t full_overwrite, hsize_t old_dim[]); H5_DLL herr_t H5D__get_storage_size(H5D_t *dset, hid_t dxpl_id, hsize_t *storage_size); H5_DLL haddr_t H5D__get_offset(const H5D_t *dset); @@ -550,8 +574,11 @@ H5_DLL herr_t H5D__check_filters(H5D_t *dataset); H5_DLL herr_t H5D__set_extent(H5D_t *dataset, const hsize_t *size, hid_t dxpl_id); H5_DLL herr_t H5D__get_dxpl_cache(hid_t dxpl_id, H5D_dxpl_cache_t **cache); H5_DLL herr_t H5D__flush_sieve_buf(H5D_t *dataset, hid_t dxpl_id); -H5_DLL herr_t H5D__mark(const H5D_t *dataset, hid_t dxpl_id, unsigned flags); H5_DLL herr_t H5D__flush_real(H5D_t *dataset, hid_t dxpl_id); +H5_DLL herr_t H5D__mark(const H5D_t *dataset, hid_t dxpl_id, unsigned flags); +#ifdef H5_DEBUG_BUILD +H5_DLL herr_t H5D_set_io_info_dxpls(H5D_io_info_t *io_info, hid_t dxpl_id); +#endif /* H5_DEBUG_BUILD */ /* Internal I/O routines */ H5_DLL herr_t H5D__pre_write(H5D_t *dset, hbool_t direct_write, hid_t mem_type_id, @@ -584,6 +611,10 @@ H5_DLL herr_t H5D__scatgath_write(const H5D_io_info_t *io_info, H5_DLL herr_t H5D__layout_set_io_ops(const H5D_t *dataset); H5_DLL size_t H5D__layout_meta_size(const H5F_t *f, const H5O_layout_t *layout, hbool_t include_compact_data); +H5_DLL herr_t H5D__layout_set_latest_version(H5O_layout_t *layout, + const H5S_t *space, const H5D_dcpl_cache_t *dcpl_cache); +H5_DLL herr_t H5D__layout_set_latest_indexing(H5O_layout_t *layout, + const H5S_t *space, const H5D_dcpl_cache_t *dcpl_cache); H5_DLL herr_t H5D__layout_oh_create(H5F_t *file, hid_t dxpl_id, H5O_t *oh, H5D_t *dset, hid_t dapl_id); H5_DLL herr_t H5D__layout_oh_read(H5D_t *dset, hid_t dxpl_id, hid_t dapl_id, @@ -595,7 +626,7 @@ H5_DLL herr_t H5D__layout_oh_write(H5D_t *dataset, hid_t dxpl_id, H5O_t *oh, H5_DLL herr_t H5D__contig_alloc(H5F_t *f, hid_t dxpl_id, H5O_storage_contig_t *storage); H5_DLL hbool_t H5D__contig_is_space_alloc(const H5O_storage_t *storage); -H5_DLL herr_t H5D__contig_fill(const H5D_t *dset, hid_t dxpl_id); +H5_DLL herr_t H5D__contig_fill(const H5D_io_info_t *io_info); H5_DLL herr_t H5D__contig_read(H5D_io_info_t *io_info, const H5D_type_info_t *type_info, hsize_t nelmts, const H5S_t *file_space, const H5S_t *mem_space, H5D_chunk_map_t *fm); @@ -616,16 +647,13 @@ H5_DLL herr_t H5D__chunk_set_info(const H5D_t *dset); H5_DLL hbool_t H5D__chunk_is_space_alloc(const H5O_storage_t *storage); H5_DLL herr_t H5D__chunk_lookup(const H5D_t *dset, hid_t dxpl_id, const hsize_t *scaled, H5D_chunk_ud_t *udata); -H5_DLL void *H5D__chunk_lock(const H5D_io_info_t *io_info, - H5D_chunk_ud_t *udata, hbool_t relax); -H5_DLL herr_t H5D__chunk_unlock(const H5D_io_info_t *io_info, - const H5D_chunk_ud_t *udata, hbool_t dirty, void *chunk, - uint32_t naccessed); H5_DLL herr_t H5D__chunk_allocated(H5D_t *dset, hid_t dxpl_id, hsize_t *nbytes); -H5_DLL herr_t H5D__chunk_allocate(const H5D_t *dset, hid_t dxpl_id, - hbool_t full_overwrite, hsize_t old_dim[]); +H5_DLL herr_t H5D__chunk_allocate(const H5D_io_info_t *io_info, hbool_t full_overwrite, hsize_t old_dim[]); +H5_DLL herr_t H5D__chunk_update_old_edge_chunks(H5D_t *dset, hid_t dxpl_id, + hsize_t old_dim[]); H5_DLL herr_t H5D__chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, const hsize_t *old_dim); +H5_DLL herr_t H5D__chunk_set_sizes(H5D_t *dset); #ifdef H5_HAVE_PARALLEL H5_DLL herr_t H5D__chunk_addrmap(const H5D_io_info_t *io_info, haddr_t chunk_addr[]); #endif /* H5_HAVE_PARALLEL */ @@ -675,8 +703,7 @@ H5_DLL herr_t H5D__fill_init(H5D_fill_buf_info_t *fb_info, void *caller_fill_buf H5MM_free_t free_func, void *free_info, const H5O_fill_t *fill, const H5T_t *dset_type, hid_t dset_type_id, size_t nelmts, size_t min_buf_size, hid_t dxpl_id); -H5_DLL herr_t H5D__fill_refill_vl(H5D_fill_buf_info_t *fb_info, size_t nelmts, - hid_t dxpl_id); +H5_DLL herr_t H5D__fill_refill_vl(H5D_fill_buf_info_t *fb_info, size_t nelmts, hid_t dxpl_id); H5_DLL herr_t H5D__fill_term(H5D_fill_buf_info_t *fb_info); #ifdef H5_HAVE_PARALLEL @@ -725,6 +752,7 @@ H5_DLL htri_t H5D__mpio_opt_possible(const H5D_io_info_t *io_info, #ifdef H5D_TESTING H5_DLL herr_t H5D__layout_version_test(hid_t did, unsigned *version); H5_DLL herr_t H5D__layout_contig_size_test(hid_t did, hsize_t *size); +H5_DLL herr_t H5D__layout_idx_type_test(hid_t did, H5D_chunk_index_t *idx_type); H5_DLL herr_t H5D__current_cache_size_test(hid_t did, size_t *nbytes_used, int *nused); #endif /* H5D_TESTING */ |