From 2aa344d77a31f8fb692d25fc3adc6ed668184d72 Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Sat, 31 Jan 2009 15:21:43 -0500 Subject: [svn-r16393] Description: Connect extensible array as index for 1-D unlimited datasets (although datasets w/filters are not yet supported). Tested on: FreeBSD/32 6.3 (duty) in debug mode FreeBSD/64 6.3 (liberty) w/C++ & FORTRAN, in debug mode Linux/32 2.6 (jam) w/PGI compilers, w/C++ & FORTRAN, w/threadsafe, in debug mode Linux/64-amd64 2.6 (smirom) w/Intel compilers w/default API=1.6.x, w/C++ & FORTRAN, in production mode Solaris/32 2.10 (linew) w/deprecated symbols disabled, w/C++ & FORTRAN, w/szip filter, in production mode Linux/64-ia64 2.6 (cobalt) w/Intel compilers, w/C++ & FORTRAN, in production mode Linux/64-ia64 2.4 (tg-login3) w/parallel, w/FORTRAN, in production mode Linux/64-amd64 2.6 (abe) w/parallel, w/FORTRAN, in production mode Mac OS X/32 10.5.6 (amazon) in debug mode Mac OS X/32 10.5.6 (amazon) w/C++ & FORTRAN, w/threadsafe, in production mode --- MANIFEST | 1 + src/H5Dbtree.c | 46 +-- src/H5Dchunk.c | 106 ++++- src/H5Dearray.c | 1111 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/H5Dint.c | 23 +- src/H5Dpkg.h | 3 +- src/H5Dpublic.h | 3 +- src/H5EAprivate.h | 4 +- src/H5EAtest.c | 2 + src/H5Olayout.c | 225 +++++++++-- src/H5Oprivate.h | 12 + src/Makefile.am | 3 +- src/Makefile.in | 5 +- 13 files changed, 1467 insertions(+), 77 deletions(-) create mode 100644 src/H5Dearray.c diff --git a/MANIFEST b/MANIFEST index 5913ba4..b523b18 100644 --- a/MANIFEST +++ b/MANIFEST @@ -475,6 +475,7 @@ ./src/H5Dcontig.c ./src/H5Ddbg.c ./src/H5Ddeprec.c +./src/H5Dearray.c ./src/H5Defl.c ./src/H5Dfill.c ./src/H5Dint.c diff --git a/src/H5Dbtree.c b/src/H5Dbtree.c index b0988c1..86a86cb 100644 --- a/src/H5Dbtree.c +++ b/src/H5Dbtree.c @@ -16,29 +16,10 @@ /* Programmer: Robb Matzke * Wednesday, October 8, 1997 * - * Purpose: v1 B-tree indexed (chunked) I/O functions. The logical - * multi-dimensional data space is regularly partitioned into - * same-sized "chunks", the first of which is aligned with the - * logical origin. The chunks are given a multi-dimensional - * index which is used as a lookup key in a B-tree that maps - * chunk index to disk address. Each chunk can be compressed - * independently and the chunks may move around in the file as - * their storage requirements change. - * - * Cache: Disk I/O is performed in units of chunks and H5MF_alloc() - * contains code to optionally align chunks on disk block - * boundaries for performance. - * - * The chunk cache is an extendible hash indexed by a function - * of storage B-tree address and chunk N-dimensional offset - * within the dataset. Collisions are not resolved -- one of - * the two chunks competing for the hash slot must be preempted - * from the cache. All entries in the hash also participate in - * a doubly-linked list and entries are penalized by moving them - * toward the front of the list. When a new chunk is about to - * be added to the cache the heap is pruned by preempting - * entries near the front of the list to make room for the new - * entry which is added to the end of the list. + * Purpose: v1 B-tree indexed (chunked) I/O functions. The chunks are + * given a multi-dimensional index which is used as a lookup key + * in a B-tree that maps chunk index to disk address. + * */ /****************/ @@ -169,7 +150,7 @@ static herr_t H5D_btree_idx_delete(const H5D_chk_idx_info_t *idx_info); static herr_t H5D_btree_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk_idx_info_t *idx_info_dst); static herr_t H5D_btree_idx_copy_shutdown(H5O_layout_t *layout_src, - H5O_layout_t *layout_dst); + H5O_layout_t *layout_dst, hid_t dxpl_id); static herr_t H5D_btree_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *size); static herr_t H5D_btree_idx_reset(H5O_layout_t *layout); @@ -1079,7 +1060,7 @@ H5D_btree_idx_iterate_cb(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, /*------------------------------------------------------------------------- * Function: H5D_btree_idx_iterate * - * Purpose: Iterate over the chunks in the B-tree index, making a callback + * Purpose: Iterate over the chunks in an index, making a callback * for each one. * * Return: Non-negative on success/Negative on failure @@ -1122,7 +1103,7 @@ H5D_btree_idx_iterate(const H5D_chk_idx_info_t *idx_info, /*------------------------------------------------------------------------- * Function: H5D_btree_idx_remove * - * Purpose: Remove chunk from v1 B-tree index. + * Purpose: Remove chunk from index. * * Return: Non-negative on success/Negative on failure * @@ -1158,7 +1139,7 @@ done: /*------------------------------------------------------------------------- * Function: H5D_btree_idx_delete * - * Purpose: Delete v1 B-tree index and raw data storage for entire dataset + * Purpose: Delete index and raw data storage for entire dataset * (i.e. all chunks) * * Return: Success: Non-negative @@ -1268,7 +1249,8 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D_btree_idx_copy_shutdown(H5O_layout_t *layout_src, H5O_layout_t *layout_dst) +H5D_btree_idx_copy_shutdown(H5O_layout_t *layout_src, H5O_layout_t *layout_dst, + hid_t UNUSED dxpl_id) { herr_t ret_value = SUCCEED; /* Return value */ @@ -1291,7 +1273,7 @@ done: /*------------------------------------------------------------------------- * Function: H5D_btree_idx_size * - * Purpose: Retrieve the amount of B-tree storage for chunked dataset + * Purpose: Retrieve the amount of index storage for chunked dataset * * Return: Success: Non-negative * Failure: negative @@ -1301,10 +1283,10 @@ done: * *------------------------------------------------------------------------- */ -herr_t +static herr_t H5D_btree_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size) { - H5D_btree_ud0_t udata; /* User-data for loading btree nodes */ + H5D_btree_ud0_t udata; /* User-data for loading B-tree nodes */ H5B_info_t bt_info; /* B-tree info */ hbool_t shared_init = FALSE; /* Whether shared B-tree info is initialized */ herr_t ret_value = SUCCEED; /* Return value */ @@ -1322,7 +1304,7 @@ H5D_btree_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") shared_init = TRUE; - /* Initialize btree node user-data */ + /* Initialize B-tree node user-data */ HDmemset(&udata, 0, sizeof udata); udata.mesg = idx_info->layout; diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index afcdc6d..1437d58 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -13,6 +13,33 @@ * access to either file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +/* Programmer: Quincey Koziol + * Thursday, April 24, 2008 + * + * Purpose: Abstract indexed (chunked) I/O functions. The logical + * multi-dimensional data space is regularly partitioned into + * same-sized "chunks", the first of which is aligned with the + * logical origin. The chunks are indexed by different methods, + * that map a chunk index to disk address. Each chunk can be + * compressed independently and the chunks may move around in the + * file as their storage requirements change. + * + * Cache: Disk I/O is performed in units of chunks and H5MF_alloc() + * contains code to optionally align chunks on disk block + * boundaries for performance. + * + * The chunk cache is an extendible hash indexed by a function + * of storage B-tree address and chunk N-dimensional offset + * within the dataset. Collisions are not resolved -- one of + * the two chunks competing for the hash slot must be preempted + * from the cache. All entries in the hash also participate in + * a doubly-linked list and entries are penalized by moving them + * toward the front of the list. When a new chunk is about to + * be added to the cache the heap is pruned by preempting + * entries near the front of the list to make room for the new + * entry which is added to the end of the list. + */ + /****************/ /* Module Setup */ /****************/ @@ -319,6 +346,10 @@ H5D_chunk_new(H5F_t *f, hid_t dapl_id, hid_t dxpl_id, H5D_t *dset, /* Retain computed chunk size */ H5_ASSIGN_OVERFLOW(dset->shared->layout.u.chunk.size, chunk_size, uint64_t, uint32_t); + /* Reset index address */ + if(H5D_chunk_idx_reset(&dset->shared->layout) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "unable to reset chunked storage index") + /* Initialize the chunk cache for the dataset */ if(H5D_chunk_init(f, dapl_id, dxpl_id, dset) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize chunk cache") @@ -349,6 +380,10 @@ H5D_chunk_is_space_alloc(const H5O_layout_t *layout) /* Sanity checks */ HDassert(layout); + HDassert((H5D_CHUNK_EARRAY == layout->u.chunk.idx_type && + H5D_COPS_EARRAY == layout->u.chunk.ops) || + (H5D_CHUNK_BTREE == layout->u.chunk.idx_type && + H5D_COPS_BTREE == layout->u.chunk.ops)); /* Query index layer */ ret_value = (layout->u.chunk.ops->is_space_alloc)(layout); @@ -1778,6 +1813,10 @@ H5D_chunk_init(H5F_t *f, hid_t dapl_id, hid_t dxpl_id, const H5D_t *dset) /* Sanity check */ HDassert(f); HDassert(dset); + HDassert((H5D_CHUNK_EARRAY == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_EARRAY == dset->shared->layout.u.chunk.ops) || + (H5D_CHUNK_BTREE == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_BTREE == dset->shared->layout.u.chunk.ops)); if(NULL == (dapl = (H5P_genplist_t *)H5I_object(dapl_id))) HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for fapl ID"); @@ -1816,7 +1855,7 @@ H5D_chunk_init(H5F_t *f, hid_t dapl_id, hid_t dxpl_id, const H5D_t *dset) idx_info.layout = &dset->shared->layout; /* Allocate any indexing structures */ - if((dset->shared->layout.u.chunk.ops->init)(&idx_info) < 0) + if(dset->shared->layout.u.chunk.ops->init && (dset->shared->layout.u.chunk.ops->init)(&idx_info) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize indexing information") done: @@ -1846,6 +1885,10 @@ H5D_chunk_idx_reset(H5O_layout_t *layout) /* Sanity checks */ HDassert(layout); HDassert(layout->u.chunk.ops); + HDassert((H5D_CHUNK_EARRAY == layout->u.chunk.idx_type && + H5D_COPS_EARRAY == layout->u.chunk.ops) || + (H5D_CHUNK_BTREE == layout->u.chunk.idx_type && + H5D_COPS_BTREE == layout->u.chunk.ops)); /* Reset index structures */ if((layout->u.chunk.ops->reset)(layout) < 0) @@ -1997,6 +2040,10 @@ H5D_chunk_create(H5D_t *dset /*in,out*/, hid_t dxpl_id) HDassert(dset); HDassert(H5D_CHUNKED == dset->shared->layout.type); HDassert(dset->shared->layout.u.chunk.ndims > 0 && dset->shared->layout.u.chunk.ndims <= H5O_LAYOUT_NDIMS); + HDassert((H5D_CHUNK_EARRAY == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_EARRAY == dset->shared->layout.u.chunk.ops) || + (H5D_CHUNK_BTREE == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_BTREE == dset->shared->layout.u.chunk.ops)); #ifndef NDEBUG { unsigned u; /* Local index variable */ @@ -2044,6 +2091,10 @@ H5D_chunk_get_info(const H5D_t *dset, hid_t dxpl_id, const hsize_t *chunk_offset HDassert(dset); HDassert(dset->shared->layout.u.chunk.ndims > 0); + HDassert((H5D_CHUNK_EARRAY == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_EARRAY == dset->shared->layout.u.chunk.ops) || + (H5D_CHUNK_BTREE == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_BTREE == dset->shared->layout.u.chunk.ops)); HDassert(chunk_offset); HDassert(udata); @@ -2104,6 +2155,11 @@ H5D_chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t * FUNC_ENTER_NOAPI_NOINIT(H5D_chunk_flush_entry) HDassert(dset); + HDassert(dset->shared); + HDassert((H5D_CHUNK_EARRAY == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_EARRAY == dset->shared->layout.u.chunk.ops) || + (H5D_CHUNK_BTREE == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_BTREE == dset->shared->layout.u.chunk.ops)); HDassert(dxpl_cache); HDassert(ent); HDassert(!ent->locked); @@ -2863,6 +2919,11 @@ H5D_chunk_allocated(H5D_t *dset, hid_t dxpl_id, hsize_t *nbytes) FUNC_ENTER_NOAPI(H5D_chunk_allocated, FAIL) HDassert(dset); + HDassert(dset->shared); + HDassert((H5D_CHUNK_EARRAY == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_EARRAY == dset->shared->layout.u.chunk.ops) || + (H5D_CHUNK_BTREE == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_BTREE == dset->shared->layout.u.chunk.ops)); /* Fill the DXPL cache values for later use */ if(H5D_get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) @@ -2880,7 +2941,7 @@ H5D_chunk_allocated(H5D_t *dset, hid_t dxpl_id, hsize_t *nbytes) idx_info.dxpl_id = dxpl_id; idx_info.layout = &dset->shared->layout; - /* Call the index-specific "get all the allocated chunks sizes" routine */ + /* Iterate over the chunks */ if((dset->shared->layout.u.chunk.ops->iterate)(&idx_info, H5D_chunk_allocated_cb, &chunk_bytes) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to retrieve allocated chunk information from index") @@ -2941,6 +3002,10 @@ H5D_chunk_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) /* Check args */ HDassert(dset && H5D_CHUNKED == layout->type); HDassert(layout->u.chunk.ndims > 0 && layout->u.chunk.ndims <= H5O_LAYOUT_NDIMS); + HDassert((H5D_CHUNK_EARRAY == layout->u.chunk.idx_type && + H5D_COPS_EARRAY == layout->u.chunk.ops) || + (H5D_CHUNK_BTREE == layout->u.chunk.idx_type && + H5D_COPS_BTREE == layout->u.chunk.ops)); HDassert(TRUE == H5P_isa_class(dxpl_id, H5P_DATASET_XFER)); /* Retrieve the dataset dimensions */ @@ -3512,6 +3577,10 @@ H5D_chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, const hsize_t *old_dims) /* Check args */ HDassert(dset && H5D_CHUNKED == layout->type); HDassert(layout->u.chunk.ndims > 0 && layout->u.chunk.ndims <= H5O_LAYOUT_NDIMS); + HDassert((H5D_CHUNK_EARRAY == layout->u.chunk.idx_type && + H5D_COPS_EARRAY == layout->u.chunk.ops) || + (H5D_CHUNK_BTREE == layout->u.chunk.idx_type && + H5D_COPS_BTREE == layout->u.chunk.ops)); HDassert(dxpl_cache); /* Fill the DXPL cache values for later use */ @@ -3680,6 +3749,11 @@ H5D_chunk_addrmap(const H5D_io_info_t *io_info, haddr_t chunk_addr[], FUNC_ENTER_NOAPI(H5D_chunk_addrmap, FAIL) HDassert(dset); + HDassert(dset->shared); + HDassert((H5D_CHUNK_EARRAY == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_EARRAY == dset->shared->layout.u.chunk.ops) || + (H5D_CHUNK_BTREE == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_BTREE == dset->shared->layout.u.chunk.ops)); HDassert(chunk_addr); HDassert(down_chunks); @@ -3725,6 +3799,12 @@ H5D_chunk_delete(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout) FUNC_ENTER_NOAPI(H5D_chunk_delete, FAIL) + /* Sanity check */ + HDassert((H5D_CHUNK_EARRAY == layout->u.chunk.idx_type && + H5D_COPS_EARRAY == layout->u.chunk.ops) || + (H5D_CHUNK_BTREE == layout->u.chunk.idx_type && + H5D_COPS_BTREE == layout->u.chunk.ops)); + /* Compose chunked index info struct */ idx_info.f = f; idx_info.dxpl_id = dxpl_id; @@ -4052,7 +4132,15 @@ H5D_chunk_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, HDassert(f_src); HDassert(f_dst); HDassert(layout_src && H5D_CHUNKED == layout_src->type); + HDassert((H5D_CHUNK_EARRAY == layout_src->u.chunk.idx_type && + H5D_COPS_EARRAY == layout_src->u.chunk.ops) || + (H5D_CHUNK_BTREE == layout_src->u.chunk.idx_type && + H5D_COPS_BTREE == layout_src->u.chunk.ops)); HDassert(layout_dst && H5D_CHUNKED == layout_dst->type); + HDassert((H5D_CHUNK_EARRAY == layout_dst->u.chunk.idx_type && + H5D_COPS_EARRAY == layout_dst->u.chunk.ops) || + (H5D_CHUNK_BTREE == layout_dst->u.chunk.idx_type && + H5D_COPS_BTREE == layout_dst->u.chunk.ops)); HDassert(dt_src); /* Compose source & dest chunked index info structs */ @@ -4218,7 +4306,7 @@ done: /* Clean up any index information */ if(copy_setup_done) - if((layout_src->u.chunk.ops->copy_shutdown)(layout_src, layout_dst) < 0) + if((layout_src->u.chunk.ops->copy_shutdown)(layout_src, layout_dst, dxpl_id) < 0) HDONE_ERROR(H5E_DATASET, H5E_CANTRELEASE, FAIL, "unable to shut down index copying info") FUNC_LEAVE_NOAPI(ret_value) @@ -4250,6 +4338,10 @@ H5D_chunk_bh_info(H5F_t *f, hid_t dxpl_id, H5O_layout_t *layout, /* Check args */ HDassert(f); HDassert(layout); + HDassert((H5D_CHUNK_EARRAY == layout->u.chunk.idx_type && + H5D_COPS_EARRAY == layout->u.chunk.ops) || + (H5D_CHUNK_BTREE == layout->u.chunk.idx_type && + H5D_COPS_BTREE == layout->u.chunk.ops)); HDassert(index_size); /* Compose chunked index info struct */ @@ -4335,6 +4427,10 @@ H5D_chunk_dump_index(H5D_t *dset, hid_t dxpl_id, FILE *stream) /* Sanity check */ HDassert(dset); + HDassert((H5D_CHUNK_EARRAY == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_EARRAY == dset->shared->layout.u.chunk.ops) || + (H5D_CHUNK_BTREE == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_BTREE == dset->shared->layout.u.chunk.ops)); /* Only display info if stream is defined */ if(stream) { @@ -4393,6 +4489,10 @@ H5D_chunk_dest(H5F_t *f, hid_t dxpl_id, H5D_t *dset) HDassert(f); HDassert(dset); + HDassert((H5D_CHUNK_EARRAY == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_EARRAY == dset->shared->layout.u.chunk.ops) || + (H5D_CHUNK_BTREE == dset->shared->layout.u.chunk.idx_type && + H5D_COPS_BTREE == dset->shared->layout.u.chunk.ops)); /* Fill the DXPL cache values for later use */ if(H5D_get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) diff --git a/src/H5Dearray.c b/src/H5Dearray.c new file mode 100644 index 0000000..02747c5 --- /dev/null +++ b/src/H5Dearray.c @@ -0,0 +1,1111 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Programmer: Quincey Koziol + * Tuesday, January 27, 2009 + * + * Purpose: Extensible array indexed (chunked) I/O functions. The chunks + * are given a single-dimensional index which is used as the + * offset in an extensible array that maps a chunk coordinate to + * a disk address. + * + */ + +/****************/ +/* Module Setup */ +/****************/ + +#define H5D_PACKAGE /*suppress error about including H5Dpkg */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Dpkg.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5EAprivate.h" /* Extensible arrays */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5MFprivate.h" /* File space management */ +#include "H5Vprivate.h" /* Vector functions */ + + +/****************/ +/* Local Macros */ +/****************/ + +/* Value to fill unset array elements with */ +#define H5D_EARRAY_FILL HADDR_UNDEF + +/* Extensible array creation values */ +#define H5D_EARRAY_MAX_NELMTS_BITS 32 /* i.e. 4 giga-elements */ +#define H5D_EARRAY_IDX_BLK_ELMTS 4 +#define H5D_EARRAY_SUP_BLK_MIN_DATA_PTRS 4 +#define H5D_EARRAY_DATA_BLK_MIN_ELMTS 16 +#define H5D_EARRAY_MAX_DBLOCK_PAGE_NELMTS_BITS 10 /* i.e. 1024 elements per data block page */ + + +/******************/ +/* Local Typedefs */ +/******************/ + +/* Extensible array callback context */ +typedef struct H5D_earray_ctx_t { + size_t file_addr_len; /* Size of addresses in the file */ +} H5D_earray_ctx_t; + +/* User data for chunk callbacks */ +typedef struct H5D_earray_ud_t { + H5F_t *f; /* File pointer for operation */ + hid_t dxpl_id; /* DXPL ID for operation */ +} H5D_earray_ud_t; + + +/********************/ +/* Local Prototypes */ +/********************/ + +/* Extensible array class callbacks */ +static void *H5D_earray_crt_context(const H5F_t *f); +static herr_t H5D_earray_dst_context(void *ctx); +static herr_t H5D_earray_fill(void *nat_blk, size_t nelmts); +static herr_t H5D_earray_encode(void *raw, const void *elmt, size_t nelmts, + void *ctx); +static herr_t H5D_earray_decode(const void *raw, void *elmt, size_t nelmts, + void *ctx); +static herr_t H5D_earray_debug(FILE *stream, int indent, int fwidth, + hsize_t idx, const void *elmt); + +/* Chunked layout indexing callbacks */ +static herr_t H5D_earray_idx_create(const H5D_chk_idx_info_t *idx_info); +static hbool_t H5D_earray_idx_is_space_alloc(const H5O_layout_t *layout); +static herr_t H5D_earray_idx_insert(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_ud_t *udata); +static herr_t H5D_earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_ud_t *udata); +static int H5D_earray_idx_iterate(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_cb_func_t chunk_cb, void *chunk_udata); +static herr_t H5D_earray_idx_remove(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_common_ud_t *udata); +static herr_t H5D_earray_idx_delete(const H5D_chk_idx_info_t *idx_info); +static herr_t H5D_earray_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, + const H5D_chk_idx_info_t *idx_info_dst); +static herr_t H5D_earray_idx_copy_shutdown(H5O_layout_t *layout_src, + H5O_layout_t *layout_dst, hid_t dxpl_id); +static herr_t H5D_earray_idx_size(const H5D_chk_idx_info_t *idx_info, + hsize_t *size); +static herr_t H5D_earray_idx_reset(H5O_layout_t *layout); +static herr_t H5D_earray_idx_dump(const H5D_chk_idx_info_t *idx_info, + FILE *stream); +static herr_t H5D_earray_idx_dest(const H5D_chk_idx_info_t *idx_info); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Extensible array indexed chunk I/O ops */ +const H5D_chunk_ops_t H5D_COPS_EARRAY[1] = {{ + NULL, + H5D_earray_idx_create, + H5D_earray_idx_is_space_alloc, + H5D_earray_idx_insert, + H5D_earray_idx_get_addr, + H5D_earray_idx_iterate, + H5D_earray_idx_remove, + H5D_earray_idx_delete, + H5D_earray_idx_copy_setup, + H5D_earray_idx_copy_shutdown, + H5D_earray_idx_size, + H5D_earray_idx_reset, + H5D_earray_idx_dump, + H5D_earray_idx_dest +}}; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Extensible array class callbacks for dataset chunks */ +const H5EA_class_t H5EA_CLS_CHUNK[1]={{ + H5EA_CLS_CHUNK_ID, /* Type of extensible array */ + sizeof(haddr_t), /* Size of native element */ + H5D_earray_crt_context, /* Create context */ + H5D_earray_dst_context, /* Destroy context */ + H5D_earray_fill, /* Fill block of missing elements callback */ + H5D_earray_encode, /* Element encoding callback */ + H5D_earray_decode, /* Element decoding callback */ + H5D_earray_debug /* Element debugging callback */ +}}; + +/* Declare a free list to manage the H5D_earray_ctx_t struct */ +H5FL_DEFINE_STATIC(H5D_earray_ctx_t); + + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_crt_context + * + * Purpose: Create context for callbacks + * + * Return: Success: non-NULL + * Failure: NULL + * + * Programmer: Quincey Koziol + * Thursday, January 29, 2009 + * + *------------------------------------------------------------------------- + */ +static void * +H5D_earray_crt_context(const H5F_t *f) +{ + H5D_earray_ctx_t *ctx; /* Extensible array callback context */ + void *ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_crt_context) + + /* Sanity checks */ + HDassert(f); + + /* Allocate new context structure */ + if(NULL == (ctx = H5FL_MALLOC(H5D_earray_ctx_t))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, NULL, "can't allocate extensible array client callback context") + + /* Initialize the context */ + ctx->file_addr_len = H5F_SIZEOF_ADDR(f); + + /* Set return value */ + ret_value = ctx; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_crt_context() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_dst_context + * + * Purpose: Destroy context for callbacks + * + * Return: Success: non-NULL + * Failure: NULL + * + * Programmer: Quincey Koziol + * Thursday, January 29, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_dst_context(void *_ctx) +{ + H5D_earray_ctx_t *ctx = (H5D_earray_ctx_t *)_ctx; /* Extensible array callback context */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_earray_dst_context) + + /* Sanity checks */ + HDassert(ctx); + + /* Release context structure */ + ctx = H5FL_FREE(H5D_earray_ctx_t, ctx); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_earray_dst_context() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_fill + * + * Purpose: Fill "missing elements" in block of elements + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Tuesday, January 27, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_fill(void *nat_blk, size_t nelmts) +{ + haddr_t fill_val = H5D_EARRAY_FILL; /* Value to fill elements with */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_earray_fill) + + /* Sanity checks */ + HDassert(nat_blk); + HDassert(nelmts); + + H5V_array_fill(nat_blk, &fill_val, H5EA_CLS_CHUNK->nat_elmt_size, nelmts); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_earray_fill() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_encode + * + * Purpose: Encode an element from "native" to "raw" form + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Tuesday, January 27, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_encode(void *raw, const void *_elmt, size_t nelmts, void *_ctx) +{ + H5D_earray_ctx_t *ctx = (H5D_earray_ctx_t *)_ctx; /* Extensible array callback context */ + const haddr_t *elmt = (const haddr_t *)_elmt; /* Convenience pointer to native elements */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_earray_encode) + + /* Sanity checks */ + HDassert(raw); + HDassert(elmt); + HDassert(nelmts); + HDassert(ctx); + + /* Encode native elements into raw elements */ + while(nelmts) { + /* Encode element */ + /* (advances 'raw' pointer */ + H5F_addr_encode_len(ctx->file_addr_len, (uint8_t **)&raw, *elmt); + + /* Advance native element pointer */ + elmt++; + + /* Decrement # of elements to encode */ + nelmts--; + } /* end while */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_earray_encode() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_decode + * + * Purpose: Decode an element from "raw" to "native" form + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Thursday, January 29, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_decode(const void *_raw, void *_elmt, size_t nelmts, void *_ctx) +{ + H5D_earray_ctx_t *ctx = (H5D_earray_ctx_t *)_ctx; /* Extensible array callback context */ + haddr_t *elmt = (haddr_t *)_elmt; /* Convenience pointer to native elements */ + const uint8_t *raw = (const uint8_t *)_raw; /* Convenience pointer to raw elements */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_earray_decode) + + /* Sanity checks */ + HDassert(raw); + HDassert(elmt); + HDassert(nelmts); + + /* Decode raw elements into native elements */ + while(nelmts) { + /* Decode element */ + /* (advances 'raw' pointer */ + H5F_addr_decode_len(ctx->file_addr_len, &raw, elmt); + + /* Advance native element pointer */ + elmt++; + + /* Decrement # of elements to decode */ + nelmts--; + } /* end while */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_earray_decode() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_debug + * + * Purpose: Display an element for debugging + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Thursday, January 29, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_debug(FILE *stream, int indent, int fwidth, hsize_t idx, + const void *elmt) +{ + char temp_str[128]; /* Temporary string, for formatting */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_earray_debug) + + /* Sanity checks */ + HDassert(stream); + HDassert(elmt); + + /* Print element */ + sprintf(temp_str, "Element #%llu:", (unsigned long_long)idx); + HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, temp_str, + *(const haddr_t *)elmt); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_earray_debug() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_open + * + * Purpose: Opens an existing extensible array and initializes + * the layout struct with information about the storage. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Thursday, January 29, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_open(const H5D_chk_idx_info_t *idx_info) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_open) + + /* Check args */ + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(H5D_CHUNK_EARRAY == idx_info->layout->u.chunk.idx_type); + HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); + HDassert(NULL == idx_info->layout->u.chunk.u.earray.ea); + + /* Open the extensible array for the chunk index */ + if(NULL == (idx_info->layout->u.chunk.u.earray.ea = H5EA_open(idx_info->f, idx_info->dxpl_id, idx_info->layout->u.chunk.u.earray.addr, H5EA_CLS_CHUNK))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open extensible array") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_idx_open() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_create + * + * Purpose: Creates a new indexed-storage extensible array and initializes + * the layout struct with information about the storage. The + * struct should be immediately written to the object header. + * + * This function must be called before passing LAYOUT to any of + * the other indexed storage functions! + * + * Return: Non-negative on success (with the LAYOUT argument initialized + * and ready to write to an object header). Negative on failure. + * + * Programmer: Quincey Koziol + * Tuesday, January 27, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_create(const H5D_chk_idx_info_t *idx_info) +{ + H5EA_create_t cparam; /* Extensible array creation parameters */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_create) + + /* Check args */ + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(!H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); + HDassert(NULL == idx_info->layout->u.chunk.u.earray.ea); + + /* General parameters */ + cparam.cls = H5EA_CLS_CHUNK; + cparam.raw_elmt_size = (uint8_t)H5F_SIZEOF_ADDR(idx_info->f); + cparam.max_nelmts_bits = H5D_EARRAY_MAX_NELMTS_BITS; + cparam.idx_blk_elmts = H5D_EARRAY_IDX_BLK_ELMTS; + cparam.sup_blk_min_data_ptrs = H5D_EARRAY_SUP_BLK_MIN_DATA_PTRS; + cparam.data_blk_min_elmts = H5D_EARRAY_DATA_BLK_MIN_ELMTS; + cparam.max_dblk_page_nelmts_bits = H5D_EARRAY_MAX_DBLOCK_PAGE_NELMTS_BITS; + + /* Create the extensible array for the chunk index */ + if(NULL == (idx_info->layout->u.chunk.u.earray.ea = H5EA_create(idx_info->f, idx_info->dxpl_id, &cparam))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't create extensible array") + + /* Get the address of the extensible array in file */ + if(H5EA_get_addr(idx_info->layout->u.chunk.u.earray.ea, &(idx_info->layout->u.chunk.u.earray.addr)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query extensible array address") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_idx_create() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_is_space_alloc + * + * Purpose: Query if space is allocated for index method + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, January 29, 2009 + * + *------------------------------------------------------------------------- + */ +static hbool_t +H5D_earray_idx_is_space_alloc(const H5O_layout_t *layout) +{ + hbool_t ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_earray_idx_is_space_alloc) + + /* Check args */ + HDassert(layout); + + /* Set return value */ + ret_value = (hbool_t)H5F_addr_defined(layout->u.chunk.u.earray.addr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_idx_is_space_alloc() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_insert + * + * Purpose: Create the chunk it if it doesn't exist, or reallocate the + * chunk if its size changed. + * + * Note: Doesn't support reallocating chunks that change size yet + * (as chunks w/filters aren't supported yet) + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, January 29, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_insert(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) +{ + H5EA_t *ea; /* Pointer to extensible array structure */ + haddr_t addr = HADDR_UNDEF; /* Address of chunk in file */ + hsize_t idx; /* Array index of chunk */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_insert) + + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(idx_info->layout->u.chunk.ndims == 2); /* (for now) */ + HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); + HDassert(udata); + + /* Check if the extensible array is open yet */ + if(NULL == idx_info->layout->u.chunk.u.earray.ea) { + /* Open the extensible array in file */ + if(H5D_earray_idx_open(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array") + } /* end if */ + + /* Set convenience pointer to extensible array structure */ + ea = idx_info->layout->u.chunk.u.earray.ea; + + /* Compute array index for chunk offset */ + idx = udata->common.offset[0] / idx_info->layout->u.chunk.dim[0]; + +#ifndef NDEBUG + /* Get the address for the chunk */ + if(H5EA_get(ea, idx_info->dxpl_id, idx, &addr) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + HDassert(!H5F_addr_defined(addr)); + addr = HADDR_UNDEF; +#endif /* NDEBUG */ + + /* + * Create the chunk it if it doesn't exist, or reallocate the chunk if + * its size changed. + */ + /* (not yet) */ + HDassert(!H5F_addr_defined(udata->addr)); + HDassert(udata->nbytes == idx_info->layout->u.chunk.size); + + /* + * Allocate storage for the new chunk + */ + H5_CHECK_OVERFLOW(udata->nbytes, /*From: */uint32_t, /*To: */hsize_t); + udata->addr = H5MF_alloc(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, (hsize_t)udata->nbytes); + if(!H5F_addr_defined(udata->addr)) + HGOTO_ERROR(H5E_DATASET, H5E_CANTALLOC, FAIL, "file allocation failed") + + /* Set the address for the chunk */ + if(H5EA_set(ea, idx_info->dxpl_id, idx, &udata->addr) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D_earray_idx_insert() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earraye_idx_get_addr + * + * Purpose: Get the file address of a chunk if file space has been + * assigned. Save the retrieved information in the udata + * supplied. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, January 29, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_get_addr(const H5D_chk_idx_info_t *idx_info, H5D_chunk_ud_t *udata) +{ + H5EA_t *ea; /* Pointer to extensible array structure */ + hsize_t idx; /* Array index of chunk */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_get_addr) + + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(idx_info->layout->u.chunk.ndims == 2); /* (for now) */ + HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); + HDassert(udata); + + /* Check if the extensible array is open yet */ + if(NULL == idx_info->layout->u.chunk.u.earray.ea) { + /* Open the extensible array in file */ + if(H5D_earray_idx_open(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array") + } /* end if */ + + /* Set convenience pointer to extensible array structure */ + ea = idx_info->layout->u.chunk.u.earray.ea; + + /* Compute array index for chunk offset */ + idx = udata->common.offset[0] / idx_info->layout->u.chunk.dim[0]; + + /* Get the address for the chunk */ + if(H5EA_get(ea, idx_info->dxpl_id, idx, &udata->addr) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D_earray_idx_get_addr() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_iterate + * + * Purpose: Iterate over the chunks in an index, making a callback + * for each one. + * + * Note: This implementation is slow, particularly for sparse + * extensible arrays, replace it with call to H5EA_iterate() + * when that's available. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, January 29, 2009 + * + *------------------------------------------------------------------------- + */ +static int +H5D_earray_idx_iterate(const H5D_chk_idx_info_t *idx_info, + H5D_chunk_cb_func_t chunk_cb, void *chunk_udata) +{ + H5EA_t *ea; /* Pointer to extensible array structure */ + H5EA_stat_t ea_stat; /* Extensible array statistics */ + int ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_iterate) + + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); + HDassert(chunk_cb); + HDassert(chunk_udata); + + /* Check if the extensible array is open yet */ + if(NULL == idx_info->layout->u.chunk.u.earray.ea) { + /* Open the extensible array in file */ + if(H5D_earray_idx_open(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array") + } /* end if */ + + /* Set convenience pointer to extensible array structure */ + ea = idx_info->layout->u.chunk.u.earray.ea; + + /* Get the extensible array statistics */ + if(H5EA_get_stats(ea, &ea_stat) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query extensible array statistics") + + /* Check if there are any array elements */ + if(ea_stat.max_idx_set > 0) { + H5D_chunk_rec_t chunk_rec; /* Generic chunk record for callback */ + hsize_t u; /* Local index variable */ + + /* Prepare common fields of chunk record for callback */ + /* (Note: these are not common for chunks w/filters) */ + HDmemset(&chunk_rec, 0, sizeof(chunk_rec)); + chunk_rec.nbytes = idx_info->layout->u.chunk.size; + chunk_rec.filter_mask = 0; + + /* Loop over array elements */ + /* (Note: this is too simple for datasets with >1 dimension) */ + for(u = 0; u < ea_stat.max_idx_set; u++, chunk_rec.offset[0] += idx_info->layout->u.chunk.dim[0]) { + haddr_t addr; /* Chunk address */ + + /* Get the address of the chunk for the index */ + if(H5EA_get(ea, idx_info->dxpl_id, u, &addr) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + + /* Check if chunk exists */ + if(H5F_addr_defined(addr)) { + /* Set chunk address for callback record */ + chunk_rec.chunk_addr = addr; + + /* Make chunk callback */ + if((ret_value = (*chunk_cb)(&chunk_rec, chunk_udata)) < 0) + HERROR(H5E_DATASET, H5E_CALLBACK, "failure in generic chunk iterator callback"); + } /* end if */ + } /* end for */ + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_idx_iterate() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_remove + * + * Purpose: Remove chunk from index. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, January 29, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_remove(const H5D_chk_idx_info_t *idx_info, H5D_chunk_common_ud_t *udata) +{ + H5EA_t *ea; /* Pointer to extensible array structure */ + haddr_t addr = HADDR_UNDEF; /* Chunk address */ + hsize_t idx; /* Array index of chunk */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_remove) + + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); + HDassert(udata); + + /* Check if the extensible array is open yet */ + if(NULL == idx_info->layout->u.chunk.u.earray.ea) { + /* Open the extensible array in file */ + if(H5D_earray_idx_open(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array") + } /* end if */ + + /* Set convenience pointer to extensible array structure */ + ea = idx_info->layout->u.chunk.u.earray.ea; + + /* Compute array index for chunk offset */ + idx = udata->offset[0] / idx_info->layout->u.chunk.dim[0]; + + /* Get the address of the chunk for the index */ + if(H5EA_get(ea, idx_info->dxpl_id, idx, &addr) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get chunk address") + + /* Remove raw data chunk from file */ + H5_CHECK_OVERFLOW(idx_info->layout->u.chunk.size, /*From: */uint32_t, /*To: */hsize_t); + if(H5MF_xfree(idx_info->f, H5FD_MEM_DRAW, idx_info->dxpl_id, addr, (hsize_t)idx_info->layout->u.chunk.size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "unable to free chunk") + + /* Reset the address of the chunk for the index */ + addr = HADDR_UNDEF; + if(H5EA_set(ea, idx_info->dxpl_id, idx, &addr) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, FAIL, "unable to reset chunk address") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D_earray_idx_remove() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_delete_cb + * + * Purpose: Delete space for chunk in file + * + * Return: Success: Non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Saturday, January 31, 2009 + * + *------------------------------------------------------------------------- + */ +static int +H5D_earray_idx_delete_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) +{ + H5D_earray_ud_t *udata = (H5D_earray_ud_t *)_udata; /* User data for callback */ + int ret_value = H5_ITER_CONT; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_delete_cb) + + /* Sanity checks */ + HDassert(chunk_rec); + HDassert(H5F_addr_defined(chunk_rec->chunk_addr)); + HDassert(chunk_rec->nbytes > 0); + HDassert(udata); + HDassert(udata->f); + + /* Remove raw data chunk from file */ + H5_CHECK_OVERFLOW(chunk_rec->nbytes, /*From: */uint32_t, /*To: */hsize_t); + if(H5MF_xfree(udata->f, H5FD_MEM_DRAW, udata->dxpl_id, chunk_rec->chunk_addr, (hsize_t)chunk_rec->nbytes) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, H5_ITER_ERROR, "unable to free chunk") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_idx_delete_cb() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_delete + * + * Purpose: Delete index and raw data storage for entire dataset + * (i.e. all chunks) + * + * Note: This implementation is slow, particularly for sparse + * extensible arrays, replace it with call to H5EA_iterate() + * when that's available. + * + * Return: Success: Non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Thursday, January 29, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_delete(const H5D_chk_idx_info_t *idx_info) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_delete) + + /* Sanity checks */ + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + + /* Check if the index data structure has been allocated */ + if(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)) { + H5D_earray_ud_t udata; /* User data for callback */ + + /* Initialize user data for callback */ + udata.f = idx_info->f; + udata.dxpl_id = idx_info->dxpl_id; + + /* Iterate over the chunk addresses in the extensible array, deleting each chunk */ + if(H5D_earray_idx_iterate(idx_info, H5D_earray_idx_delete_cb, &udata) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to iterate over chunk addresses") + + /* Close extensible array */ + if(H5EA_close(idx_info->layout->u.chunk.u.earray.ea, idx_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array") + idx_info->layout->u.chunk.u.earray.ea = NULL; + + /* Delete extensible array */ + if(H5EA_delete(idx_info->f, idx_info->dxpl_id, idx_info->layout->u.chunk.u.earray.addr) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTDELETE, FAIL, "unable to delete chunk extensible array") + idx_info->layout->u.chunk.u.earray.addr = HADDR_UNDEF; + } /* end if */ + else + HDassert(NULL == idx_info->layout->u.chunk.u.earray.ea); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_idx_delete() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_copy_setup + * + * Purpose: Set up any necessary information for copying chunks + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Saturday, January 31, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_copy_setup(const H5D_chk_idx_info_t *idx_info_src, + const H5D_chk_idx_info_t *idx_info_dst) +{ + H5EA_t *ea_src; /* Pointer to extensible array structure */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_copy_setup) + + HDassert(idx_info_src); + HDassert(idx_info_src->f); + HDassert(idx_info_src->layout); + HDassert(idx_info_dst); + HDassert(idx_info_dst->f); + HDassert(idx_info_dst->layout); + HDassert(!H5F_addr_defined(idx_info_dst->layout->u.chunk.u.earray.addr)); + + /* Check if the source extensible array is open yet */ + if(NULL == idx_info_src->layout->u.chunk.u.earray.ea) { + /* Open the extensible array in file */ + if(H5D_earray_idx_open(idx_info_src) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array") + } /* end if */ + + /* Set convenience pointer to extensible array structure */ + ea_src = idx_info_src->layout->u.chunk.u.earray.ea; + + /* Create the extensible array that describes chunked storage in the dest. file */ + if(H5D_earray_idx_create(idx_info_dst) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize chunked storage") + HDassert(H5F_addr_defined(idx_info_dst->layout->u.chunk.u.earray.addr)); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_idx_copy_setup() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_copy_shutdown + * + * Purpose: Shutdown any information from copying chunks + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Saturday, January 31, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_copy_shutdown(H5O_layout_t *layout_src, H5O_layout_t *layout_dst, + hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_copy_shutdown) + + HDassert(layout_src); + HDassert(layout_src->u.chunk.u.earray.ea); + HDassert(layout_dst); + HDassert(layout_dst->u.chunk.u.earray.ea); + + /* Close extensible arrays */ + if(H5EA_close(layout_src->u.chunk.u.earray.ea, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array") + layout_src->u.chunk.u.earray.ea = NULL; + if(H5EA_close(layout_dst->u.chunk.u.earray.ea, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array") + layout_dst->u.chunk.u.earray.ea = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_idx_copy_shutdown() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_size + * + * Purpose: Retrieve the amount of index storage for chunked dataset + * + * Return: Success: Non-negative + * Failure: negative + * + * Programmer: Quincey Koziol + * Saturday, January 31, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_size(const H5D_chk_idx_info_t *idx_info, hsize_t *index_size) +{ + H5EA_t *ea; /* Pointer to extensible array structure */ + H5EA_stat_t ea_stat; /* Extensible array statistics */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5D_earray_idx_size, FAIL) + + /* Check args */ + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(H5F_addr_defined(idx_info->layout->u.chunk.u.earray.addr)); + HDassert(index_size); + + /* Open the extensible array in file */ + if(H5D_earray_idx_open(idx_info) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTOPENOBJ, FAIL, "can't open extensible array") + + /* Set convenience pointer to extensible array structure */ + ea = idx_info->layout->u.chunk.u.earray.ea; + + /* Get the extensible array statistics */ + if(H5EA_get_stats(ea, &ea_stat) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't query extensible array statistics") + + /* Set the size of the extensible array */ + *index_size = ea_stat.hdr_size + ea_stat.index_blk_size + + ea_stat.super_blk_size + ea_stat.data_blk_size; + +done: + if(idx_info->layout->u.chunk.u.earray.ea) { + if(H5EA_close(idx_info->layout->u.chunk.u.earray.ea, idx_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array") + idx_info->layout->u.chunk.u.earray.ea = NULL; + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_idx_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_reset + * + * Purpose: Reset indexing information. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Saturday, January 31, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_reset(H5O_layout_t *layout) +{ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_earray_idx_reset) + + HDassert(layout); + + /* Reset index info */ + layout->u.chunk.u.earray.addr = HADDR_UNDEF; + layout->u.chunk.u.earray.ea = NULL; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_earray_idx_reset() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_dump + * + * Purpose: Dump indexing information to a stream. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Saturday, January 31, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_dump(const H5D_chk_idx_info_t *idx_info, FILE *stream) +{ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_earray_idx_dump) + + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + HDassert(stream); + + HDfprintf(stream, " Address: %a\n", idx_info->layout->u.chunk.u.earray.addr); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_earray_idx_dump() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_earray_idx_dest + * + * Purpose: Release indexing information in memory. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Saturday, January 31, 2009 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_earray_idx_dest(const H5D_chk_idx_info_t *idx_info) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_earray_idx_dest) + + HDassert(idx_info); + HDassert(idx_info->f); + HDassert(idx_info->layout); + + /* Check if the extensible array is open */ + if(idx_info->layout->u.chunk.u.earray.ea) { + if(H5EA_close(idx_info->layout->u.chunk.u.earray.ea, idx_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCLOSEOBJ, FAIL, "unable to close extensible array") + idx_info->layout->u.chunk.u.earray.ea = NULL; + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_earray_idx_dest() */ + diff --git a/src/H5Dint.c b/src/H5Dint.c index 0e71ef8..f8ffeab 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -720,8 +720,19 @@ H5D_set_io_ops(H5D_t *dataset) dataset->shared->layout.ops = H5D_LOPS_CHUNK; /* Set the chunk operations */ - /* (Only "B-tree" indexing type currently supported */ - dataset->shared->layout.u.chunk.ops = H5D_COPS_BTREE; + switch(dataset->shared->layout.u.chunk.idx_type) { + case H5D_CHUNK_BTREE: + dataset->shared->layout.u.chunk.ops = H5D_COPS_BTREE; + break; + + case H5D_CHUNK_EARRAY: + dataset->shared->layout.u.chunk.ops = H5D_COPS_EARRAY; + break; + + default: + HDassert(0 && "Unknown chunk index method!"); + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unknown chunk index method") + } /* end switch */ break; case H5D_COMPACT: @@ -1121,6 +1132,10 @@ H5D_create(H5F_t *file, hid_t type_id, const H5S_t *space, hid_t dcpl_id, /* Set the latest version for the fill value message */ if(H5O_fill_set_latest_version(&new_dset->shared->dcpl_cache.fill) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set latest version of fill value") + + /* Set the latest version for the layout message */ + if(H5O_layout_set_latest_version(&new_dset->shared->layout, new_dset->shared->space) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTSET, NULL, "can't set latest version of layout") } /* end if */ /* Check if this dataset is going into a parallel file and set space allocation time */ @@ -2300,11 +2315,11 @@ H5D_set_extent(H5D_t *dset, const hsize_t *size, hid_t dxpl_id) /* Check if we are shrinking or expanding any of the dimensions */ if((rank = H5S_get_simple_extent_dims(space, curr_dims, NULL)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions") + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't get dataset dimensions") /* Modify the size of the data space */ if((changed = H5S_set_extent(space, size)) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to modify size of data space") /* Don't bother updating things, unless they've changed */ if(changed) { diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 4bf4e75..0d8ca8c 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -277,7 +277,7 @@ typedef herr_t (*H5D_chunk_delete_func_t)(const H5D_chk_idx_info_t *idx_info); typedef herr_t (*H5D_chunk_copy_setup_func_t)(const H5D_chk_idx_info_t *idx_info_src, const H5D_chk_idx_info_t *idx_info_dst); typedef herr_t (*H5D_chunk_copy_shutdown_func_t)(H5O_layout_t *layout_src, - H5O_layout_t *layout_dst); + H5O_layout_t *layout_dst, hid_t dxpl_id); typedef herr_t (*H5D_chunk_size_func_t)(const H5D_chk_idx_info_t *idx_info, hsize_t *idx_size); typedef herr_t (*H5D_chunk_reset_func_t)(H5O_layout_t *layout); @@ -506,6 +506,7 @@ H5_DLLVAR const H5D_layout_ops_t H5D_LOPS_CHUNK[1]; /* Chunked layout operations */ H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_BTREE[1]; +H5_DLLVAR const H5D_chunk_ops_t H5D_COPS_EARRAY[1]; /******************************/ diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index bae5dcf..136911f 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -50,7 +50,8 @@ typedef enum H5D_layout_t { /* Types of chunk index data structures */ typedef enum H5D_chunk_index_t { - H5D_CHUNK_BTREE = 0, /* v1 B-tree index */ + H5D_CHUNK_BTREE = 0, /* v1 B-tree index (default) */ + H5D_CHUNK_EARRAY /* Extensible array (for 1-D unlimited dim) */ } H5D_chunk_index_t; /* Values for the space allocation time property */ diff --git a/src/H5EAprivate.h b/src/H5EAprivate.h index f44d046..fcfccd0 100644 --- a/src/H5EAprivate.h +++ b/src/H5EAprivate.h @@ -48,7 +48,9 @@ /* Extensible array class IDs */ typedef enum H5EA_cls_id_t { - /* Start real class IDs at 0 -QAK */ + H5EA_CLS_CHUNK_ID = 0, /* Extensible array is for indexing dataset chunks */ + + /* (keep these last) */ H5EA_CLS_TEST_ID, /* Extensible array is for testing (do not use for actual data) */ H5EA_NUM_CLS_ID /* Number of Extensible Array class IDs (must be last) */ } H5EA_cls_id_t; diff --git a/src/H5EAtest.c b/src/H5EAtest.c index 31dfb5f..de63d11 100644 --- a/src/H5EAtest.c +++ b/src/H5EAtest.c @@ -50,6 +50,7 @@ /* Sanity checking value for callback contexts */ #define H5EA__TEST_BOGUS_VAL 42 + /******************/ /* Local Typedefs */ /******************/ @@ -59,6 +60,7 @@ typedef struct H5EA__test_ctx_t { uint32_t bogus; /* Placeholder field to verify that context is working */ } H5EA__test_ctx_t; + /********************/ /* Package Typedefs */ /********************/ diff --git a/src/H5Olayout.c b/src/H5Olayout.c index a23e3ef..f2bfab0 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -30,6 +30,7 @@ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ #include "H5Pprivate.h" /* Property lists */ +#include "H5Sprivate.h" /* Dataspaces */ /* PRIVATE PROTOTYPES */ @@ -111,7 +112,7 @@ H5O_layout_decode(H5F_t *f, hid_t UNUSED dxpl_id, unsigned UNUSED mesg_flags, HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") mesg->version = *p++; - if(mesg->version < H5O_LAYOUT_VERSION_1 || mesg->version > H5O_LAYOUT_VERSION_3) + if(mesg->version < H5O_LAYOUT_VERSION_1 || mesg->version > H5O_LAYOUT_VERSION_4) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad version number for layout message") if(mesg->version < H5O_LAYOUT_VERSION_3) { @@ -213,26 +214,68 @@ H5O_layout_decode(H5F_t *f, hid_t UNUSED dxpl_id, unsigned UNUSED mesg_flags, break; case H5D_CHUNKED: - /* Dimensionality */ - mesg->u.chunk.ndims = *p++; - if(mesg->u.chunk.ndims > H5O_LAYOUT_NDIMS) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "dimensionality is too large") - - /* B-tree address */ - H5F_addr_decode(f, &p, &(mesg->u.chunk.u.btree.addr)); - - /* Chunk dimensions */ - for(u = 0; u < mesg->u.chunk.ndims; u++) - UINT32DECODE(p, mesg->u.chunk.dim[u]); - - /* Compute chunk size */ - for(u = 1, mesg->u.chunk.size = mesg->u.chunk.dim[0]; u < mesg->u.chunk.ndims; u++) - mesg->u.chunk.size *= mesg->u.chunk.dim[u]; - - /* Set the chunk operations */ - /* (Only "btree" indexing type supported with v3 of message format) */ - mesg->u.chunk.idx_type = H5D_CHUNK_BTREE; - mesg->u.chunk.ops = H5D_COPS_BTREE; + if(mesg->version < H5O_LAYOUT_VERSION_4) { + /* Dimensionality */ + mesg->u.chunk.ndims = *p++; + if(mesg->u.chunk.ndims > H5O_LAYOUT_NDIMS) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "dimensionality is too large") + + /* B-tree address */ + H5F_addr_decode(f, &p, &(mesg->u.chunk.u.btree.addr)); + + /* Chunk dimensions */ + for(u = 0; u < mesg->u.chunk.ndims; u++) + UINT32DECODE(p, mesg->u.chunk.dim[u]); + + /* Compute chunk size */ + for(u = 1, mesg->u.chunk.size = mesg->u.chunk.dim[0]; u < mesg->u.chunk.ndims; u++) + mesg->u.chunk.size *= mesg->u.chunk.dim[u]; + + /* Set the chunk operations */ + /* (Only "btree" indexing type supported with v3 of message format) */ + mesg->u.chunk.idx_type = H5D_CHUNK_BTREE; + mesg->u.chunk.ops = H5D_COPS_BTREE; + } /* end if */ + else { + /* Dimensionality */ + mesg->u.chunk.ndims = *p++; + if(mesg->u.chunk.ndims > H5O_LAYOUT_NDIMS) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "dimensionality is too large") + + /* Chunk dimensions */ + for(u = 0; u < mesg->u.chunk.ndims; u++) + UINT32DECODE(p, mesg->u.chunk.dim[u]); + + /* Compute chunk size */ + for(u = 1, mesg->u.chunk.size = mesg->u.chunk.dim[0]; u < mesg->u.chunk.ndims; u++) + mesg->u.chunk.size *= mesg->u.chunk.dim[u]; + + /* Chunk index type */ + mesg->u.chunk.idx_type = *p++; + if(mesg->u.chunk.idx_type > H5D_CHUNK_EARRAY) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "unknown chunk index type") + + switch(mesg->u.chunk.idx_type) { + case H5D_CHUNK_BTREE: + /* B-tree address */ + H5F_addr_decode(f, &p, &(mesg->u.chunk.u.btree.addr)); + + /* Set the chunk operations */ + mesg->u.chunk.ops = H5D_COPS_BTREE; + break; + + case H5D_CHUNK_EARRAY: + /* Extensible array address */ + H5F_addr_decode(f, &p, &(mesg->u.chunk.u.earray.addr)); + + /* Set the chunk operations */ + mesg->u.chunk.ops = H5D_COPS_EARRAY; + break; + + default: + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "Invalid chunk index type") + } /* end switch */ + } /* end else */ /* Set the layout operations */ mesg->ops = H5D_LOPS_CHUNK; @@ -327,16 +370,45 @@ H5O_layout_encode(H5F_t *f, hbool_t UNUSED disable_shared, uint8_t *p, const voi break; case H5D_CHUNKED: - /* Number of dimensions */ - HDassert(mesg->u.chunk.ndims > 0 && mesg->u.chunk.ndims <= H5O_LAYOUT_NDIMS); - *p++ = (uint8_t)mesg->u.chunk.ndims; + if(mesg->version < H5O_LAYOUT_VERSION_4) { + /* Number of dimensions */ + HDassert(mesg->u.chunk.ndims > 0 && mesg->u.chunk.ndims <= H5O_LAYOUT_NDIMS); + *p++ = (uint8_t)mesg->u.chunk.ndims; - /* B-tree address */ - H5F_addr_encode(f, &p, mesg->u.chunk.u.btree.addr); + /* B-tree address */ + H5F_addr_encode(f, &p, mesg->u.chunk.u.btree.addr); - /* Dimension sizes */ - for(u = 0; u < mesg->u.chunk.ndims; u++) - UINT32ENCODE(p, mesg->u.chunk.dim[u]); + /* Dimension sizes */ + for(u = 0; u < mesg->u.chunk.ndims; u++) + UINT32ENCODE(p, mesg->u.chunk.dim[u]); + } /* end if */ + else { + /* Number of dimensions */ + HDassert(mesg->u.chunk.ndims > 0 && mesg->u.chunk.ndims <= H5O_LAYOUT_NDIMS); + *p++ = (uint8_t)mesg->u.chunk.ndims; + + /* Dimension sizes */ + for(u = 0; u < mesg->u.chunk.ndims; u++) + UINT32ENCODE(p, mesg->u.chunk.dim[u]); + + /* Chunk index type */ + *p++ = (uint8_t)mesg->u.chunk.idx_type; + + switch(mesg->u.chunk.idx_type) { + case H5D_CHUNK_BTREE: + /* B-tree address */ + H5F_addr_encode(f, &p, mesg->u.chunk.u.btree.addr); + break; + + case H5D_CHUNK_EARRAY: + /* Extensible array address */ + H5F_addr_encode(f, &p, mesg->u.chunk.u.earray.addr); + break; + + default: + HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "Invalid chunk index type") + } /* end switch */ + } /* end else */ break; default: @@ -709,6 +781,13 @@ H5O_layout_debug(H5F_t UNUSED *f, hid_t UNUSED dxpl_id, const void *_mesg, "B-tree address:", mesg->u.chunk.u.btree.addr); break; + case H5D_CHUNK_EARRAY: + HDfprintf(stream, "%*s%-*s %s\n", indent, "", fwidth, + "Index Type:", "Extensible Array"); + HDfprintf(stream, "%*s%-*s %a\n", indent, "", fwidth, + "Extensible Array address:", mesg->u.chunk.u.earray.addr); + break; + default: HDfprintf(stream, "%*s%-*s %s (%u)\n", indent, "", fwidth, "Index Type:", "Unknown", (unsigned)mesg->u.chunk.idx_type); @@ -792,8 +871,29 @@ H5O_layout_meta_size(const H5F_t *f, const void *_mesg) /* Dimension sizes */ ret_value += mesg->u.chunk.ndims * 4; - /* B-tree address */ - ret_value += H5F_SIZEOF_ADDR(f); /* Address of data */ + if(mesg->version < H5O_LAYOUT_VERSION_4) { + /* B-tree address */ + ret_value += H5F_SIZEOF_ADDR(f); /* Address of data */ + } /* end if */ + else { + /* Type of chunk index */ + ret_value++; + + switch(mesg->u.chunk.idx_type) { + case H5D_CHUNK_BTREE: + /* B-tree address */ + ret_value += H5F_SIZEOF_ADDR(f); /* Address of data */ + break; + + case H5D_CHUNK_EARRAY: + /* Extensible Array address */ + ret_value += H5F_SIZEOF_ADDR(f); /* Address of data */ + break; + + default: + HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, 0, "Invalid chunk index type") + } /* end switch */ + } /* end else */ break; default: @@ -804,3 +904,66 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_layout_meta_size() */ + +/*------------------------------------------------------------------------- + * Function: H5O_layout_set_latest_version + * + * Purpose: Set the encoding for a layout to the latest version. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Thursday, January 15, 2009 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_layout_set_latest_version(H5O_layout_t *layout, const H5S_t *space) +{ + int sndims; /* Rank of dataspace */ + unsigned ndims; /* Rank of dataspace */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5O_layout_set_latest_version, FAIL) + + /* Sanity check */ + HDassert(layout); + + /* Set encoding of layout to latest version */ + layout->version = H5O_LAYOUT_VERSION_LATEST; + + /* Query the dimensionality of the dataspace */ + if((sndims = H5S_GET_EXTENT_NDIMS(space)) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "invalid dataspace rank") + ndims = (unsigned)sndims; + + /* Avoid scalar/null dataspace */ + if(ndims > 0) { + hsize_t max_dims[H5O_LAYOUT_NDIMS]; /* Current dimension sizes */ + unsigned unlim_count; /* Count of unlimited max. dimensions */ + unsigned u; /* Local index variable */ + + /* Query the dataspace's dimensions */ + if(H5S_get_simple_extent_dims(space, NULL, max_dims) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't get dataspace max. dimensions") + + /* Spin through the max. dimensions, looking for unlimited dimensions */ + unlim_count = 0; + for(u = 0; u < ndims; u++) + if(max_dims[u] == H5S_UNLIMITED) + unlim_count++; + + /* If we have only 1 unlimited dimension, we can use extensible array index */ + if(1 == unlim_count) { + /* Check for rank == 1 (>1 unsupported currently) */ + if(1 == ndims) { + /* Set the chunk index type */ + layout->u.chunk.idx_type = H5D_CHUNK_EARRAY; + } /* end if */ + } /* end if */ + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_layout_set_latest_version() */ + diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index cc88bdc..3a00491 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -361,6 +361,14 @@ typedef struct H5O_layout_chunk_btree_t { H5RC_t *shared; /* Ref-counted shared info for B-tree nodes */ } H5O_layout_chunk_btree_t; +/* Forward declaration of structs used below */ +struct H5EA_t; /* Defined in H5EAprivate.h */ + +typedef struct H5O_layout_chunk_earray_t { + haddr_t addr; /* File address of extensible array */ + struct H5EA_t *ea; /* Pointer to extensible array struct */ +} H5O_layout_chunk_earray_t; + typedef struct H5O_layout_chunk_t { H5D_chunk_index_t idx_type; /* Type of chunk index */ unsigned ndims; /* Num dimensions in chunk */ @@ -369,6 +377,7 @@ typedef struct H5O_layout_chunk_t { const struct H5D_chunk_ops_t *ops; /* Pointer to chunked layout operations */ union { H5O_layout_chunk_btree_t btree; /* Information for v1 B-tree index */ + H5O_layout_chunk_earray_t earray; /* Information for extensible array index */ } u; } H5O_layout_chunk_t; @@ -566,6 +575,7 @@ typedef struct { /* Forward declarations for prototype arguments */ struct H5P_genplist_t; +struct H5S_t; /* Object header routines */ H5_DLL herr_t H5O_init(void); @@ -656,6 +666,8 @@ H5_DLL herr_t H5O_loc_free(H5O_loc_t *loc); /* Layout operators */ H5_DLL size_t H5O_layout_meta_size(const H5F_t *f, const void *_mesg); +H5_DLL herr_t H5O_layout_set_latest_version(H5O_layout_t *layout, + const struct H5S_t *space); /* EFL operators */ H5_DLL hsize_t H5O_efl_total_size(H5O_efl_t *efl); diff --git a/src/Makefile.am b/src/Makefile.am index 9ae2252..b2b0a98 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -47,8 +47,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5B2.c H5B2cache.c H5B2dbg.c H5B2int.c H5B2stat.c H5B2test.c \ H5C.c H5CS.c \ H5D.c H5Dbtree.c H5Dchunk.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \ - H5Ddeprec.c H5Defl.c H5Dfill.c H5Dint.c \ - H5Dio.c \ + H5Ddeprec.c H5Dearray.c H5Defl.c H5Dfill.c H5Dint.c H5Dio.c \ H5Dmpio.c H5Doh.c H5Dscatgath.c H5Dselect.c H5Dtest.c \ H5E.c H5Edeprec.c H5Eint.c \ H5EA.c H5EAcache.c H5EAdbg.c H5EAdblkpage.c H5EAdblock.c H5EAhdr.c \ diff --git a/src/Makefile.in b/src/Makefile.in index 0a08832..6638703 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -84,7 +84,7 @@ am_libhdf5_la_OBJECTS = H5.lo H5checksum.lo H5dbg.lo H5system.lo \ H5Bdbg.lo H5B2.lo H5B2cache.lo H5B2dbg.lo H5B2int.lo \ H5B2stat.lo H5B2test.lo H5C.lo H5CS.lo H5D.lo H5Dbtree.lo \ H5Dchunk.lo H5Dcompact.lo H5Dcontig.lo H5Ddbg.lo H5Ddeprec.lo \ - H5Defl.lo H5Dfill.lo H5Dint.lo H5Dio.lo H5Dmpio.lo H5Doh.lo \ + H5Dearray.lo H5Defl.lo H5Dfill.lo H5Dint.lo H5Dio.lo H5Dmpio.lo H5Doh.lo \ H5Dscatgath.lo H5Dselect.lo H5Dtest.lo H5E.lo H5Edeprec.lo \ H5Eint.lo H5EA.lo H5EAcache.lo H5EAdbg.lo H5EAdblkpage.lo \ H5EAdblock.lo H5EAhdr.lo H5EAiblock.lo H5EAint.lo \ @@ -430,7 +430,7 @@ libhdf5_la_SOURCES = H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5B2.c H5B2cache.c H5B2dbg.c H5B2int.c H5B2stat.c H5B2test.c \ H5C.c H5CS.c \ H5D.c H5Dbtree.c H5Dchunk.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \ - H5Ddeprec.c H5Defl.c H5Dfill.c H5Dint.c \ + H5Ddeprec.c H5Dearray.c H5Defl.c H5Dfill.c H5Dint.c \ H5Dio.c \ H5Dmpio.c H5Doh.c H5Dscatgath.c H5Dselect.c H5Dtest.c \ H5E.c H5Edeprec.c H5Eint.c \ @@ -641,6 +641,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dcontig.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Ddbg.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Ddeprec.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dearray.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Defl.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dfill.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dint.Plo@am__quote@ -- cgit v0.12