From 1b0aacf4c8ace172452c646696703e7795ca848e Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Tue, 31 Mar 2009 14:33:06 -0500 Subject: [svn-r16638] Description: When using the "latest" version of the layout message (i.e. version 4), encode the dimension sizes in smaller way. Tested on: FreeBSD/32 6.3 (duty) h5committest not required for this branch --- src/H5Dearray.c | 9 +++++---- src/H5Olayout.c | 42 +++++++++++++++++++++++++++++++++--------- src/H5Oprivate.h | 1 + src/H5Pdcpl.c | 16 +++++++++++++++- 4 files changed, 54 insertions(+), 14 deletions(-) diff --git a/src/H5Dearray.c b/src/H5Dearray.c index 1f45bba..e98e988 100644 --- a/src/H5Dearray.c +++ b/src/H5Dearray.c @@ -86,7 +86,7 @@ typedef struct H5D_earray_filt_elmt_t { /********************/ /* Extensible array class callbacks for chunks w/o filters */ -static void *H5D_earray_crt_context(const H5F_t *f); +static void *H5D_earray_crt_context(void *udata); static herr_t H5D_earray_dst_context(void *ctx); static herr_t H5D_earray_fill(void *nat_blk, size_t nelmts); static herr_t H5D_earray_encode(void *raw, const void *elmt, size_t nelmts, @@ -205,9 +205,10 @@ H5FL_DEFINE_STATIC(H5D_earray_ctx_t); *------------------------------------------------------------------------- */ static void * -H5D_earray_crt_context(const H5F_t *f) +H5D_earray_crt_context(void *_udata) { H5D_earray_ctx_t *ctx; /* Extensible array callback context */ + H5F_t *f = (H5F_t *)_udata; /* User data for extensible array context */ void *ret_value; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_earray_crt_context) @@ -604,7 +605,7 @@ H5D_earray_idx_open(const H5D_chk_idx_info_t *idx_info) /* Open the extensible array for the chunk index */ cls = (idx_info->pline->nused > 0) ? H5EA_CLS_FILT_CHUNK : H5EA_CLS_CHUNK; - if(NULL == (idx_info->layout->u.chunk.u.earray.ea = H5EA_open(idx_info->f, idx_info->dxpl_id, idx_info->layout->u.chunk.u.earray.addr, cls))) + if(NULL == (idx_info->layout->u.chunk.u.earray.ea = H5EA_open(idx_info->f, idx_info->dxpl_id, idx_info->layout->u.chunk.u.earray.addr, cls, idx_info->f))) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't open extensible array") done: @@ -662,7 +663,7 @@ H5D_earray_idx_create(const H5D_chk_idx_info_t *idx_info) cparam.max_dblk_page_nelmts_bits = H5D_EARRAY_MAX_DBLOCK_PAGE_NELMTS_BITS; /* Create the extensible array for the chunk index */ - if(NULL == (idx_info->layout->u.chunk.u.earray.ea = H5EA_create(idx_info->f, idx_info->dxpl_id, &cparam))) + if(NULL == (idx_info->layout->u.chunk.u.earray.ea = H5EA_create(idx_info->f, idx_info->dxpl_id, &cparam, idx_info->f))) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't create extensible array") /* Get the address of the extensible array in file */ diff --git a/src/H5Olayout.c b/src/H5Olayout.c index 50a6094..b92a8bb 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -237,6 +237,11 @@ H5O_layout_decode(H5F_t *f, hid_t UNUSED dxpl_id, H5O_t UNUSED *open_oh, mesg->u.chunk.ops = H5D_COPS_BTREE; } /* end if */ else { + /* Encoded # of bytes for each chunk dimension */ + mesg->u.chunk.enc_bytes_per_dim = *p++; + if(mesg->u.chunk.enc_bytes_per_dim == 0 || mesg->u.chunk.enc_bytes_per_dim > 8) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "encoded chunk dimension size is too large") + /* Dimensionality */ mesg->u.chunk.ndims = *p++; if(mesg->u.chunk.ndims > H5O_LAYOUT_NDIMS) @@ -244,7 +249,7 @@ H5O_layout_decode(H5F_t *f, hid_t UNUSED dxpl_id, H5O_t UNUSED *open_oh, /* Chunk dimensions */ for(u = 0; u < mesg->u.chunk.ndims; u++) - UINT32DECODE(p, mesg->u.chunk.dim[u]); + UINT64DECODE_VAR(p, mesg->u.chunk.dim[u], mesg->u.chunk.enc_bytes_per_dim); /* Compute chunk size */ for(u = 1, mesg->u.chunk.size = mesg->u.chunk.dim[0]; u < mesg->u.chunk.ndims; u++) @@ -326,6 +331,10 @@ done: * really n-byte values (where n usually is 8)) and additionally clean up * the information written out. * + * Quincey Koziol, 2009-3-31 + * Added version number 4 case to allow different kinds of indices for + * looking up chunks. + * *------------------------------------------------------------------------- */ static herr_t @@ -384,13 +393,17 @@ H5O_layout_encode(H5F_t *f, hbool_t UNUSED disable_shared, uint8_t *p, const voi UINT32ENCODE(p, mesg->u.chunk.dim[u]); } /* end if */ else { + /* Encoded # of bytes for each chunk dimension */ + HDassert(mesg->u.chunk.enc_bytes_per_dim > 0 && mesg->u.chunk.enc_bytes_per_dim <= 8); + *p++ = mesg->u.chunk.enc_bytes_per_dim; + /* Number of dimensions */ HDassert(mesg->u.chunk.ndims > 0 && mesg->u.chunk.ndims <= H5O_LAYOUT_NDIMS); *p++ = (uint8_t)mesg->u.chunk.ndims; /* Dimension sizes */ for(u = 0; u < mesg->u.chunk.ndims; u++) - UINT32ENCODE(p, mesg->u.chunk.dim[u]); + UINT64ENCODE_VAR(p, mesg->u.chunk.dim[u], mesg->u.chunk.enc_bytes_per_dim); /* Chunk index type */ *p++ = (uint8_t)mesg->u.chunk.idx_type; @@ -866,18 +879,29 @@ H5O_layout_meta_size(const H5F_t *f, const void *_mesg) break; case H5D_CHUNKED: - /* Number of dimensions (1 byte) */ - HDassert(mesg->u.chunk.ndims > 0 && mesg->u.chunk.ndims <= H5O_LAYOUT_NDIMS); - ret_value++; - - /* Dimension sizes */ - ret_value += mesg->u.chunk.ndims * 4; - if(mesg->version < H5O_LAYOUT_VERSION_4) { + /* Number of dimensions (1 byte) */ + HDassert(mesg->u.chunk.ndims > 0 && mesg->u.chunk.ndims <= H5O_LAYOUT_NDIMS); + ret_value++; + /* B-tree address */ ret_value += H5F_SIZEOF_ADDR(f); /* Address of data */ + + /* Dimension sizes */ + ret_value += mesg->u.chunk.ndims * 4; } /* end if */ else { + /* Encoded # of bytes for each chunk dimension */ + HDassert(mesg->u.chunk.enc_bytes_per_dim > 0 && mesg->u.chunk.enc_bytes_per_dim <= 8); + ret_value++; + + /* Number of dimensions (1 byte) */ + HDassert(mesg->u.chunk.ndims > 0 && mesg->u.chunk.ndims <= H5O_LAYOUT_NDIMS); + ret_value++; + + /* Dimension sizes */ + ret_value += mesg->u.chunk.ndims * mesg->u.chunk.enc_bytes_per_dim; + /* Type of chunk index */ ret_value++; diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index c2116bb..cb6db7b 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -374,6 +374,7 @@ typedef struct H5O_layout_chunk_t { H5D_chunk_index_t idx_type; /* Type of chunk index */ unsigned ndims; /* Num dimensions in chunk */ uint32_t dim[H5O_LAYOUT_NDIMS]; /* Size of chunk in elements */ + unsigned enc_bytes_per_dim; /* Encoded # of bytes for storing each chunk dimension */ uint32_t size; /* Size of chunk in bytes */ const struct H5D_chunk_ops_t *ops; /* Pointer to chunked layout operations */ union { diff --git a/src/H5Pdcpl.c b/src/H5Pdcpl.c index 43696fc..dcd120e 100644 --- a/src/H5Pdcpl.c +++ b/src/H5Pdcpl.c @@ -40,6 +40,7 @@ #include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Ppkg.h" /* Property lists */ +#include "H5Vprivate.h" /* Vectors and arrays */ #include "H5Zpkg.h" /* Data filters */ @@ -50,7 +51,7 @@ /* Define default layout information */ #define H5D_DEF_LAYOUT_COMPACT_INIT {(hbool_t)FALSE, (size_t)0, NULL} #define H5D_DEF_LAYOUT_CONTIG_INIT {HADDR_UNDEF, (hsize_t)0} -#define H5D_DEF_LAYOUT_CHUNK_INIT {H5D_CHUNK_BTREE, (unsigned)1, {1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, (uint32_t)0, NULL, {{HADDR_UNDEF, NULL}}} +#define H5D_DEF_LAYOUT_CHUNK_INIT {H5D_CHUNK_BTREE, (unsigned)1, {1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1}, (unsigned)0, (uint32_t)0, NULL, {{HADDR_UNDEF, NULL}}} #ifdef H5_HAVE_C99_DESIGNATED_INITIALIZER #define H5D_DEF_LAYOUT_COMPACT {H5D_COMPACT, H5O_LAYOUT_VERSION_3, NULL, { .compact = H5D_DEF_LAYOUT_COMPACT_INIT }} #define H5D_DEF_LAYOUT_CONTIG {H5D_CONTIGUOUS, H5O_LAYOUT_VERSION_3, NULL, { .contig = H5D_DEF_LAYOUT_CONTIG_INIT }} @@ -928,6 +929,7 @@ H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/]) H5P_genplist_t *plist; /* Property list pointer */ H5O_layout_t chunk_layout; /* Layout information for setting chunk info */ uint64_t chunk_nelmts; /* Number of elements in chunk */ + unsigned max_enc_bytes_per_dim; /* Max. number of bytes required to encode this dimension */ unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ @@ -955,7 +957,10 @@ H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/]) HDmemcpy(&chunk_layout, &H5D_def_layout_chunk_g, sizeof(H5D_def_layout_chunk_g)); HDmemset(&chunk_layout.u.chunk.dim, 0, sizeof(chunk_layout.u.chunk.dim)); chunk_nelmts = 1; + max_enc_bytes_per_dim = 0; for(u = 0; u < (unsigned)ndims; u++) { + unsigned enc_bytes_per_dim; /* Number of bytes required to encode this dimension */ + if(dim[u] == 0) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "all chunk dimensions must be positive") if(dim[u] != (dim[u] & 0xffffffff)) @@ -964,7 +969,16 @@ H5Pset_chunk(hid_t plist_id, int ndims, const hsize_t dim[/*ndims*/]) if(chunk_nelmts > (uint64_t)0xffffffff) HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "number of elements in chunk must be < 4GB") chunk_layout.u.chunk.dim[u] = (uint32_t)dim[u]; /* Store user's chunk dimensions */ + + /* Get encoded size of dim, in bytes */ + enc_bytes_per_dim = (H5V_log2_gen(dim[u]) + 8) / 8; + + /* Check if this is the largest value so far */ + if(enc_bytes_per_dim > max_enc_bytes_per_dim) + max_enc_bytes_per_dim = enc_bytes_per_dim; } /* end for */ + HDassert(max_enc_bytes_per_dim > 0 && max_enc_bytes_per_dim <= 8); + chunk_layout.u.chunk.enc_bytes_per_dim = max_enc_bytes_per_dim; /* Get the plist structure */ if(NULL == (plist = H5P_object_verify(plist_id, H5P_DATASET_CREATE))) -- cgit v0.12