From d534a4dd0d1e646c531a31a5eb04842dfde5f470 Mon Sep 17 00:00:00 2001 From: Robb Matzke Date: Fri, 23 Apr 1999 07:31:21 -0500 Subject: [svn-r1204] Changes since 19990415 ---------------------- ./config/depend.in Fixed automatic dependencies. We were storing dependencies for *.o files instead of *.lo files after shared libraries were added. ./config/gnu-flags ./config/linux-gnulibc1 Moved `-march=pentiumpro -mcpu=pentiumpro -malign-double' from the linux file to this file and caused it to depend on the CPU name. This fixes one of Elena's bugs. ./src/H5B.c ./src/H5Bprivate.h ./src/H5D.c ./src/H5Dprivate.h ./src/H5Dpublic.h ./src/H5F.c ./src/H5Farray.c ./src/H5Fistore.c ./src/H5Fmpio.c ./src/H5Fprivate.h ./src/H5Fpublic.h ./src/H5Gnode.c ./src/H5P.c ./src/H5RA.c ./src/H5Sall.c ./src/H5Shyper.c ./src/H5Smpio.c ./src/H5Spoint.c ./src/H5Sprivate.h ./src/H5Tpublic.h ./test/istore.c Added an H5Dget_storage_size() function that reports the amount of storage allocated for raw data in a dataset. Changed H5D_xfer_* to H5F_xfer_* because these properties are more general than datasets. This also allows some of the lower-level I/O functions to get this information easier. ./src/H5S.c ./src/H5Sall.c Added two new functions H5S_all_read() and H5S_all_write() which are optimizations that copy data directly between file and memory without having to go through the scatter gather step. This knocks quite a bit of time off the I/O and reading/writing entire datasets is a fairly common operation. ./tools/h5ls.c Reports the logical size of data, the allocated size of data, and the percent utilization. ./MANIFEST Removed old pablo files, added new files. Snapshots should now start to work again. ./src/H5D.c ./src/H5Fmpio.c Removed two warnings signed vs. unsigned comparisons and check for overflow. --- MANIFEST | 10 ++-- config/depend.in | 4 +- config/gnu-flags | 7 +++ config/linux-gnulibc1 | 3 -- src/H5B.c | 25 ++++++--- src/H5Bprivate.h | 7 ++- src/H5D.c | 143 ++++++++++++++++++++++++++++++++++-------------- src/H5Distore.c | 78 +++++++++++++++++++++++++-- src/H5Dprivate.h | 18 ++----- src/H5Dpublic.h | 1 + src/H5F.c | 22 ++++++++ src/H5Farray.c | 6 +-- src/H5Fistore.c | 78 +++++++++++++++++++++++++-- src/H5Fmpio.c | 91 ++++++++++++++++++------------- src/H5Fprivate.h | 24 +++++++-- src/H5Fpublic.h | 7 +++ src/H5Gnode.c | 6 ++- src/H5P.c | 26 ++++----- src/H5RA.c | 14 ++--- src/H5S.c | 36 ++++++++----- src/H5Sall.c | 147 ++++++++++++++++++++++++++++++++++++++++++++++++-- src/H5Shyper.c | 10 ++-- src/H5Smpio.c | 34 ++++++------ src/H5Spoint.c | 8 +-- src/H5Sprivate.h | 28 +++++++--- src/H5Tpublic.h | 7 --- test/istore.c | 8 +-- tools/h5ls.c | 14 ++++- 28 files changed, 649 insertions(+), 213 deletions(-) diff --git a/MANIFEST b/MANIFEST index 00d5d53..5f9a089 100644 --- a/MANIFEST +++ b/MANIFEST @@ -81,7 +81,6 @@ ./doc/html/Dataspaces.html ./doc/html/Datatypes.html ./doc/html/ddl.html -./doc/html/DDL.html _DO_NOT_DISTRIBUTE_ ./doc/html/Debugging.html ./doc/html/Environment.html ./doc/html/Errors.html @@ -196,14 +195,17 @@ ./examples/h5_select.c ./examples/h5_attribute.c -./pablo/HDF5record_RT.h -./pablo/HDFentries.txt +./pablo/HDFIOTrace.h +./pablo/HDFTrace.h +./pablo/HDFmpioProtos.h +./pablo/HDFrecord_RT.h ./pablo/Makefile.in -./pablo/PabloHDF5.c ./pablo/PabloHDF_RT.c ./pablo/PabloHDF_SDDF.c ./pablo/ProcIDs.h +./pablo/PabloSedscr ./pablo/ProcTrace.h +./pablo/ProcTrace.inc ./src/.distdep ./src/.indent.pro _DO_NOT_DISTRIBUTE_ diff --git a/config/depend.in b/config/depend.in index dc72d74..7c61f4f 100644 --- a/config/depend.in +++ b/config/depend.in @@ -32,10 +32,10 @@ dep depend: .distdep @for dep in $? dummy; do \ if [ $$dep != "dummy" ]; then \ echo Building dependencies for $$dep; \ - obj=`echo $$dep | sed 's/\.c/\\\\.o/'`; \ + obj=`echo $$dep | sed 's/\.c/\\\\.lo/'`; \ sed '\%$$obj%,\%[^\\]$$%d' <$@ >$@- && mv $@- $@; \ $(TRACE) $$dep; \ - $(CC) -M -MG $(CPPFLAGS) $$dep >>$@; \ + $(CC) -M -MG $(CPPFLAGS) $$dep |sed 's/\.o/.lo/' >>$@; \ fi; \ done; -perl -p $(top_srcdir)/bin/distdep .depend >.distdep diff --git a/config/gnu-flags b/config/gnu-flags index 83c0ed7..46a38f4 100644 --- a/config/gnu-flags +++ b/config/gnu-flags @@ -59,6 +59,13 @@ EOF ;; esac +# Architecture-specific flags +case "$host_cpu" in + i686) + ARCH=${ARCH:="-mcpu=pentiumpro -march=pentiumpro -malign-double"} + ;; +esac + # Compiler flags case "$cc_vendor-$cc_version" in gcc-2.7*) diff --git a/config/linux-gnulibc1 b/config/linux-gnulibc1 index 266cd3a..e22b82d 100644 --- a/config/linux-gnulibc1 +++ b/config/linux-gnulibc1 @@ -12,9 +12,6 @@ if test "X-" = "X-$CC"; then CC_BASENAME=gcc fi -# Architecture-specific flags -ARCH=${ARCH:="-mcpu=pentiumpro -march=pentiumpro -malign-double"} - # Omit frame pointer for optimized code? NOFP=${NOFP:=-fomit-frame-pointer} diff --git a/src/H5B.c b/src/H5B.c index 09bb967..ffc829e 100644 --- a/src/H5B.c +++ b/src/H5B.c @@ -1482,6 +1482,8 @@ H5B_insert_helper(H5F_t *f, const haddr_t *addr, const H5B_class_t *type, * Jun 23 1997 * * Modifications: + * Robb Matzke, 1999-04-21 + * The key values are passed to the function which is called. * *------------------------------------------------------------------------- */ @@ -1493,6 +1495,7 @@ H5B_iterate (H5F_t *f, const H5B_class_t *type, const haddr_t *addr, haddr_t next_addr; const haddr_t *cur_addr = NULL; haddr_t *child = NULL; + uint8_t *key = NULL; intn i, nchildren; herr_t ret_value = FAIL; @@ -1507,7 +1510,7 @@ H5B_iterate (H5F_t *f, const H5B_class_t *type, const haddr_t *addr, assert(addr && H5F_addr_defined(addr)); assert(udata); - if (NULL == (bt = H5AC_find(f, H5AC_BT, addr, type, udata))) { + if (NULL == (bt=H5AC_find(f, H5AC_BT, addr, type, udata))) { HGOTO_ERROR(H5E_BTREE, H5E_CANTLOAD, FAIL, "unable to load B-tree node"); } @@ -1524,7 +1527,8 @@ H5B_iterate (H5F_t *f, const H5B_class_t *type, const haddr_t *addr, * We've reached the left-most leaf. Now follow the right-sibling * pointer from leaf to leaf until we've processed all leaves. */ - if (NULL==(child = H5MM_malloc (2*H5B_K(f,type)*sizeof(haddr_t)))) { + if (NULL==(child=H5MM_malloc(2*H5B_K(f,type)*sizeof(haddr_t))) || + NULL==(key=H5MM_malloc((2*H5B_K(f, type)+1)*type->sizeof_nkey))) { HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed"); } @@ -1533,8 +1537,9 @@ H5B_iterate (H5F_t *f, const H5B_class_t *type, const haddr_t *addr, cur_addr=&next_addr) { /* - * Save all the child addresses since we can't leave the B-tree - * node protected during an application callback. + * Save all the child addresses and native keys since we can't + * leave the B-tree node protected during an application + * callback. */ if (NULL==(bt=H5AC_find (f, H5AC_BT, cur_addr, type, udata))) { HGOTO_ERROR (H5E_BTREE, H5E_CANTLOAD, FAIL, "B-tree node"); @@ -1542,6 +1547,11 @@ H5B_iterate (H5F_t *f, const H5B_class_t *type, const haddr_t *addr, for (i=0; inchildren; i++) { child[i] = bt->child[i]; } + for (i=0; inchildren+1; i++) { + H5B_decode_key(f, bt, i); + memcpy(key+i*type->sizeof_nkey, bt->key[i].nkey, + type->sizeof_nkey); + } next_addr = bt->right; nchildren = bt->nchildren; bt = NULL; @@ -1551,10 +1561,13 @@ H5B_iterate (H5F_t *f, const H5B_class_t *type, const haddr_t *addr, * application callback. */ for (i=0, ret_value=0; ilist)(f, child+i, udata); + ret_value = (type->list)(f, key+i*type->sizeof_nkey, + child+i, key+(i+1)*type->sizeof_nkey, + udata); } } - H5MM_xfree (child); + H5MM_xfree(child); + H5MM_xfree(key); } done: diff --git a/src/H5Bprivate.h b/src/H5Bprivate.h index 4cdfd9e..8dcbaea 100644 --- a/src/H5Bprivate.h +++ b/src/H5Bprivate.h @@ -87,8 +87,11 @@ typedef struct H5B_class_t { /* remove existing data */ H5B_ins_t (*remove)(H5F_t*, const haddr_t*, void*, hbool_t*, void*, void*, hbool_t*); - - herr_t (*list)(H5F_t*, const haddr_t*, void*); /*walk leaf nodes*/ + + /* iterate through the leaf nodes */ + herr_t (*list)(H5F_t*, void*, const haddr_t*, void*, void*); + + /* encode, decode, debug key values */ herr_t (*decode)(H5F_t*, struct H5B_t*, uint8_t*, void*); herr_t (*encode)(H5F_t*, struct H5B_t*, uint8_t*, void*); herr_t (*debug_key)(FILE*, intn, intn, const void*, const void*); diff --git a/src/H5D.c b/src/H5D.c index 656b7a5..03fce3d 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -66,28 +66,6 @@ const H5D_create_t H5D_create_dflt = { {0, 0, NULL} /* No filters in pipeline */ }; -/* Default dataset transfer property list */ -const H5D_xfer_t H5D_xfer_dflt = { - 1024*1024, /* Temporary buffer size */ - NULL, /* Type conversion buffer or NULL */ - NULL, /* Background buffer or NULL */ - H5T_BKG_NO, /* Type of background buffer needed */ - {0.1, 0.5, 0.9}, /* B-tree node splitting ratios */ -#ifndef HAVE_PARALLEL - 1, /* Cache the hyperslab blocks by default*/ -#else - 0, /* - * Don't cache the hyperslab blocks by - * default (for parallel) - */ -#endif /* HAVE_PARALLEL */ - 0, /* - * Default to no upper limit on hyperslab - * block size to cache - */ - H5D_XFER_DFLT, /* Independent data transfer */ -}; - /* Interface initialization? */ static intn interface_initialize_g = 0; #define INTERFACE_INIT H5D_init_interface @@ -604,7 +582,7 @@ H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, const H5T_t *mem_type = NULL; const H5S_t *mem_space = NULL; const H5S_t *file_space = NULL; - const H5D_xfer_t *xfer_parms = NULL; + const H5F_xfer_t *xfer_parms = NULL; FUNC_ENTER(H5Dread, FAIL); H5TRACE6("e","iiiiix",dset_id,mem_type_id,mem_space_id,file_space_id, @@ -643,7 +621,7 @@ H5Dread(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, } } if (H5P_DEFAULT == plist_id) { - xfer_parms = &H5D_xfer_dflt; + xfer_parms = &H5F_xfer_dflt; } else if (H5P_DATASET_XFER != H5P_get_class(plist_id) || NULL == (xfer_parms = H5I_object(plist_id))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not xfer parms"); @@ -703,7 +681,7 @@ H5Dwrite(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, const H5T_t *mem_type = NULL; const H5S_t *mem_space = NULL; const H5S_t *file_space = NULL; - const H5D_xfer_t *xfer_parms = NULL; + const H5F_xfer_t *xfer_parms = NULL; FUNC_ENTER(H5Dwrite, FAIL); H5TRACE6("e","iiiiix",dset_id,mem_type_id,mem_space_id,file_space_id, @@ -742,7 +720,7 @@ H5Dwrite(hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, } } if (H5P_DEFAULT == plist_id) { - xfer_parms = &H5D_xfer_dflt; + xfer_parms = &H5F_xfer_dflt; } else if (H5P_DATASET_XFER != H5P_get_class(plist_id) || NULL == (xfer_parms = H5I_object(plist_id))) { HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not xfer parms"); @@ -784,9 +762,9 @@ H5Dextend(hid_t dset_id, const hsize_t *size) H5TRACE2("e","i*h",dset_id,size); /* Check args */ - if (H5I_DATASET!=H5I_get_type (dset_id) || - NULL==(dset=H5I_object (dset_id))) { - HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset"); + if (H5I_DATASET!=H5I_get_type(dset_id) || + NULL==(dset=H5I_object(dset_id))) { + HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset"); } if (!size) { HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "no size specified"); @@ -846,7 +824,7 @@ H5D_new(const H5D_create_t *create_parms) done: FUNC_LEAVE(ret_value); -} /* end H5D_new() */ +} /*------------------------------------------------------------------------- @@ -1211,7 +1189,7 @@ H5D_open(H5G_entry_t *loc, const char *name) done: FUNC_LEAVE(ret_value); -} /* end H5D_open() */ +} /*------------------------------------------------------------------------- * Function: H5D_open_oid @@ -1364,7 +1342,7 @@ done: H5MM_xfree(dataset); } FUNC_LEAVE(ret_value); -} /* end H5D_open_oid() */ +} /*------------------------------------------------------------------------- * Function: H5D_close @@ -1451,7 +1429,7 @@ H5D_close(H5D_t *dataset) */ herr_t H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, - const H5S_t *file_space, const H5D_xfer_t *xfer_parms, + const H5S_t *file_space, const H5F_xfer_t *xfer_parms, void *buf/*out*/) { hssize_t nelmts; /*number of elements */ @@ -1570,8 +1548,8 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, &(dataset->create_parms->pline), &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, - mem_space, xfer_parms->xfer_mode, - buf/*out*/, &must_convert ); + mem_space, xfer_parms, buf/*out*/, + &must_convert); if (status<0) { /* Supports only no conversion, type or space, for now. */ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, @@ -1655,7 +1633,10 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, #endif /* Start strip mining... */ - for (smine_start=0; smine_startf->avail)(file_space,&file_iter, MIN(request_nelmts, @@ -1803,7 +1784,7 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, */ herr_t H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, - const H5S_t *file_space, const H5D_xfer_t *xfer_parms, + const H5S_t *file_space, const H5F_xfer_t *xfer_parms, const void *buf) { hssize_t nelmts; /*total number of elmts */ @@ -1934,8 +1915,8 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, &(dataset->create_parms->pline), &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, - mem_space, xfer_parms->xfer_mode, buf, - &must_convert /*out*/ ); + mem_space, xfer_parms, buf, + &must_convert/*out*/); if (status<0) { /* Supports only no conversion, type or space, for now. */ HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, @@ -2020,7 +2001,10 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, #endif /* Start strip mining... */ - for (smine_start=0; smine_startf->avail)(file_space,&file_iter, MIN(request_nelmts, @@ -2422,7 +2406,7 @@ H5D_init_storage(H5D_t *dset, const H5S_t *space) ndims++; if (H5F_istore_allocate(dset->ent.file, &(dset->layout), - dim, H5D_xfer_dflt.split_ratios, + dim, H5F_xfer_dflt.split_ratios, &(dset->create_parms->pline), &(dset->create_parms->fill))<0) { HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, @@ -2441,3 +2425,80 @@ H5D_init_storage(H5D_t *dset, const H5S_t *space) FUNC_LEAVE(ret_value); } + +/*------------------------------------------------------------------------- + * Function: H5Dget_storage_size + * + * Purpose: Returns the amount of storage that is required for the + * dataset. For chunked datasets this is the number of allocated + * chunks times the chunk size. + * + * Return: Success: The amount of storage space allocated for the + * dataset, not counting meta data. The return + * value may be zero if no data has been stored. + * + * Failure: Zero + * + * Programmer: Robb Matzke + * Wednesday, April 21, 1999 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +hsize_t +H5Dget_storage_size(hid_t dset_id) +{ + H5D_t *dset=NULL; + hsize_t size; + + FUNC_ENTER(H5Dget_storage_size, 0); + H5TRACE1("h","i",dset_id); + + /* Check args */ + if (H5I_DATASET!=H5I_get_type(dset_id) || + NULL==(dset=H5I_object(dset_id))) { + HRETURN_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset"); + } + + size = H5D_get_storage_size(dset); + FUNC_LEAVE(size); +} + + +/*------------------------------------------------------------------------- + * Function: H5D_get_storage_size + * + * Purpose: Determines how much space has been reserved to store the raw + * data of a dataset. + * + * Return: Success: Number of bytes reserved to hold raw data. + * + * Failure: 0 + * + * Programmer: Robb Matzke + * Wednesday, April 21, 1999 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +hsize_t +H5D_get_storage_size(H5D_t *dset) +{ + hsize_t size; + int i; + + FUNC_ENTER(H5D_get_storage_size, 0); + + if (H5D_CHUNKED==dset->layout.type) { + size = H5F_istore_allocated(dset->ent.file, dset->layout.ndims, + &(dset->layout.addr)); + } else { + for (i=0, size=1; ilayout.ndims; i++) { + size *= dset->layout.dim[i]; + } + } + + FUNC_LEAVE(size); +} diff --git a/src/H5Distore.c b/src/H5Distore.c index 89e262b..d3435a9 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -108,6 +108,9 @@ static H5B_ins_t H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_md_key, void *_udata, void *_rt_key, hbool_t *rt_key_changed, haddr_t *new_node/*out*/); +static herr_t H5F_istore_iterate (H5F_t *f, void *left_key, + const haddr_t *addr, void *right_key, + void *_udata); static herr_t H5F_istore_decode_key(H5F_t *f, H5B_t *bt, uint8_t *raw, void *_key); static herr_t H5F_istore_encode_key(H5F_t *f, H5B_t *bt, uint8_t *raw, @@ -143,6 +146,7 @@ typedef struct H5F_istore_ud1_t { H5F_istore_key_t key; /*key values */ haddr_t addr; /*file address of chunk */ H5O_layout_t mesg; /*layout message */ + hsize_t total_storage; /*output from iterator */ } H5F_istore_ud1_t; /* inherits B-tree like properties from H5B */ @@ -158,7 +162,7 @@ H5B_class_t H5B_ISTORE[1] = {{ FALSE, /*follow min branch? */ FALSE, /*follow max branch? */ NULL, /*remove */ - NULL, /*list */ + H5F_istore_iterate, /*iterator */ H5F_istore_decode_key, /*decode */ H5F_istore_encode_key, /*encode */ H5F_istore_debug_key, /*debug */ @@ -718,6 +722,36 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, /*------------------------------------------------------------------------- + * Function: H5F_istore_iterate + * + * Purpose: Simply counts the number of chunks for a dataset. + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: Robb Matzke + * Wednesday, April 21, 1999 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F_istore_iterate (H5F_t UNUSED *f, void *_lt_key, + const haddr_t UNUSED *addr, + void UNUSED *_rt_key, void *_udata) +{ + H5F_istore_ud1_t *bt_udata = (H5F_istore_ud1_t *)_udata; + H5F_istore_key_t *lt_key = (H5F_istore_key_t *)_lt_key; + + FUNC_ENTER(H5F_istore_iterate, FAIL); + bt_udata->total_storage += lt_key->nbytes; + FUNC_LEAVE(SUCCEED); +} + + +/*------------------------------------------------------------------------- * Function: H5F_istore_init * * Purpose: Initialize the raw data chunk cache for a file. This is @@ -1513,7 +1547,7 @@ H5F_istore_unlock (H5F_t *f, const H5O_layout_t *layout, *------------------------------------------------------------------------- */ herr_t -H5F_istore_read(H5F_t *f, const H5D_xfer_t *xfer, const H5O_layout_t *layout, +H5F_istore_read(H5F_t *f, const H5F_xfer_t *xfer, const H5O_layout_t *layout, const H5O_pline_t *pline, const H5O_fill_t *fill, const hssize_t offset_f[], const hsize_t size[], void *buf) { @@ -1601,7 +1635,7 @@ H5F_istore_read(H5F_t *f, const H5D_xfer_t *xfer, const H5O_layout_t *layout, if (f->shared->access_parms->driver==H5F_LOW_MPIO){ H5F_istore_ud1_t udata; H5O_layout_t l; /* temporary layout */ - H5D_xfer_t tmp_xfer = *xfer; + H5F_xfer_t tmp_xfer = *xfer; if (H5F_istore_get_addr(f, layout, chunk_offset, &udata)<0){ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to locate raw data chunk"); @@ -1683,7 +1717,7 @@ H5F_istore_read(H5F_t *f, const H5D_xfer_t *xfer, const H5O_layout_t *layout, *------------------------------------------------------------------------- */ herr_t -H5F_istore_write(H5F_t *f, const H5D_xfer_t *xfer, const H5O_layout_t *layout, +H5F_istore_write(H5F_t *f, const H5F_xfer_t *xfer, const H5O_layout_t *layout, const H5O_pline_t *pline, const H5O_fill_t *fill, const hssize_t offset_f[], const hsize_t size[], const void *buf) @@ -1776,7 +1810,7 @@ H5F_istore_write(H5F_t *f, const H5D_xfer_t *xfer, const H5O_layout_t *layout, if (f->shared->access_parms->driver==H5F_LOW_MPIO){ H5F_istore_ud1_t udata; H5O_layout_t l; /* temporary layout */ - H5D_xfer_t tmp_xfer = *xfer; + H5F_xfer_t tmp_xfer = *xfer; if (H5F_istore_get_addr(f, layout, chunk_offset, &udata)<0){ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to locate raw data chunk"); @@ -1895,6 +1929,40 @@ H5F_istore_create(H5F_t *f, H5O_layout_t *layout /*out */ ) /*------------------------------------------------------------------------- + * Function: H5F_istore_allocated + * + * Purpose: Return the number of bytes allocated in the file for storage + * of raw data under the specified B-tree (ADDR is the address + * of the B-tree). + * + * Return: Success: Number of bytes stored in all chunks. + * + * Failure: 0 + * + * Programmer: Robb Matzke + * Wednesday, April 21, 1999 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +hsize_t +H5F_istore_allocated(H5F_t *f, int ndims, haddr_t *addr) +{ + H5F_istore_ud1_t udata; + + FUNC_ENTER(H5F_istore_nchunks, 0); + udata.mesg.ndims = ndims; + udata.total_storage = 0; + if (H5B_iterate(f, H5B_ISTORE, addr, &udata)<0) { + HRETURN_ERROR(H5E_IO, H5E_INTERNAL, 0, + "unable to iterate over chunk B-tree"); + } + FUNC_LEAVE(udata.total_storage); +} + + +/*------------------------------------------------------------------------- * Function: H5F_istore_stats * * Purpose: Print raw data cache statistics to the debug stream. If diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index 7ce4dbc..e420619 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -50,21 +50,8 @@ typedef struct H5D_create_t { H5O_pline_t pline; /*data filter pipeline */ } H5D_create_t; -/* Dataset transfer property list */ -typedef struct H5D_xfer_t { - size_t buf_size; /*max temp buffer size */ - void *tconv_buf; /*type conversion buffer or null */ - void *bkg_buf; /*background buffer or null */ - H5T_bkg_t need_bkg; /*type of background buffer needed */ - double split_ratios[3];/*B-tree node splitting ratios */ - uintn cache_hyper; /*cache hyperslab blocks during I/O? */ - uintn block_limit; /*largest hyperslab block to cache */ - H5D_transfer_t xfer_mode; /*independent or collective transfer */ -} H5D_xfer_t; - typedef struct H5D_t H5D_t; __DLLVAR__ const H5D_create_t H5D_create_dflt; -__DLLVAR__ const H5D_xfer_t H5D_xfer_dflt; /* Functions defined in H5D.c */ __DLL__ H5D_t *H5D_create(H5G_entry_t *loc, const char *name, @@ -75,15 +62,16 @@ __DLL__ herr_t H5D_close(H5D_t *dataset); __DLL__ htri_t H5D_isa(H5G_entry_t *ent); __DLL__ herr_t H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, const H5S_t *file_space, - const H5D_xfer_t *xfer_parms, void *buf/*out*/); + const H5F_xfer_t *xfer_parms, void *buf/*out*/); __DLL__ herr_t H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, const H5S_t *file_space, - const H5D_xfer_t *xfer_parms, const void *buf); + const H5F_xfer_t *xfer_parms, const void *buf); __DLL__ herr_t H5D_extend(H5D_t *dataset, const hsize_t *size); __DLL__ H5G_entry_t *H5D_entof(H5D_t *dataset); __DLL__ H5T_t *H5D_typeof(H5D_t *dset); __DLL__ H5S_t *H5D_get_space(H5D_t *dset); __DLL__ H5D_t * H5D_open_oid(H5G_entry_t *ent); __DLL__ H5F_t * H5D_get_file(const H5D_t *dset); +__DLL__ hsize_t H5D_get_storage_size(H5D_t *dset); #endif diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 9e6cba3..6130715 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -50,6 +50,7 @@ __DLL__ herr_t H5Dclose (hid_t dset_id); __DLL__ hid_t H5Dget_space (hid_t dset_id); __DLL__ hid_t H5Dget_type (hid_t dset_id); __DLL__ hid_t H5Dget_create_plist (hid_t dset_id); +__DLL__ hsize_t H5Dget_storage_size(hid_t dset_id); __DLL__ herr_t H5Dread (hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t plist_id, void *buf/*out*/); __DLL__ herr_t H5Dwrite (hid_t dset_id, hid_t mem_type_id, hid_t mem_space_id, diff --git a/src/H5F.c b/src/H5F.c index 42884cf..f74745a 100644 --- a/src/H5F.c +++ b/src/H5F.c @@ -65,6 +65,28 @@ const H5F_create_t H5F_create_dflt = { */ H5F_access_t H5F_access_dflt; +/* Default data transfer property list */ +const H5F_xfer_t H5F_xfer_dflt = { + 1024*1024, /* Temporary buffer size */ + NULL, /* Type conversion buffer or NULL */ + NULL, /* Background buffer or NULL */ + H5T_BKG_NO, /* Type of background buffer needed */ + {0.1, 0.5, 0.9}, /* B-tree node splitting ratios */ +#ifndef HAVE_PARALLEL + 1, /* Cache the hyperslab blocks by default*/ +#else + 0, /* + * Don't cache the hyperslab blocks by + * default (for parallel) + */ +#endif /* HAVE_PARALLEL */ + 0, /* + * Default to no upper limit on hyperslab + * block size to cache + */ + H5D_XFER_DFLT, /* Independent data transfer */ +}; + /* * Define the default mount property list. */ diff --git a/src/H5Farray.c b/src/H5Farray.c index 88423ca..3141007 100644 --- a/src/H5Farray.c +++ b/src/H5Farray.c @@ -113,7 +113,7 @@ H5F_arr_create (H5F_t *f, struct H5O_layout_t *layout/*in,out*/) *------------------------------------------------------------------------- */ herr_t -H5F_arr_read (H5F_t *f, const H5D_xfer_t *xfer, +H5F_arr_read (H5F_t *f, const H5F_xfer_t *xfer, const struct H5O_layout_t *layout, const struct H5O_pline_t *pline, const H5O_fill_t *fill, const struct H5O_efl_t *efl, const hsize_t _hslab_size[], @@ -330,12 +330,12 @@ H5F_arr_read (H5F_t *f, const H5D_xfer_t *xfer, * * Sep 28, 1998 Robb Matzke * Added `xfer' argument, removed `xfer_mode' argument since it - * is a member of H5D_xfer_t. + * is a member of H5F_xfer_t. * *------------------------------------------------------------------------- */ herr_t -H5F_arr_write (H5F_t *f, const H5D_xfer_t *xfer, +H5F_arr_write (H5F_t *f, const H5F_xfer_t *xfer, const struct H5O_layout_t *layout, const struct H5O_pline_t *pline, const struct H5O_fill_t *fill, const struct H5O_efl_t *efl, diff --git a/src/H5Fistore.c b/src/H5Fistore.c index 89e262b..d3435a9 100644 --- a/src/H5Fistore.c +++ b/src/H5Fistore.c @@ -108,6 +108,9 @@ static H5B_ins_t H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_md_key, void *_udata, void *_rt_key, hbool_t *rt_key_changed, haddr_t *new_node/*out*/); +static herr_t H5F_istore_iterate (H5F_t *f, void *left_key, + const haddr_t *addr, void *right_key, + void *_udata); static herr_t H5F_istore_decode_key(H5F_t *f, H5B_t *bt, uint8_t *raw, void *_key); static herr_t H5F_istore_encode_key(H5F_t *f, H5B_t *bt, uint8_t *raw, @@ -143,6 +146,7 @@ typedef struct H5F_istore_ud1_t { H5F_istore_key_t key; /*key values */ haddr_t addr; /*file address of chunk */ H5O_layout_t mesg; /*layout message */ + hsize_t total_storage; /*output from iterator */ } H5F_istore_ud1_t; /* inherits B-tree like properties from H5B */ @@ -158,7 +162,7 @@ H5B_class_t H5B_ISTORE[1] = {{ FALSE, /*follow min branch? */ FALSE, /*follow max branch? */ NULL, /*remove */ - NULL, /*list */ + H5F_istore_iterate, /*iterator */ H5F_istore_decode_key, /*decode */ H5F_istore_encode_key, /*encode */ H5F_istore_debug_key, /*debug */ @@ -718,6 +722,36 @@ H5F_istore_insert(H5F_t *f, const haddr_t *addr, void *_lt_key, /*------------------------------------------------------------------------- + * Function: H5F_istore_iterate + * + * Purpose: Simply counts the number of chunks for a dataset. + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: Robb Matzke + * Wednesday, April 21, 1999 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F_istore_iterate (H5F_t UNUSED *f, void *_lt_key, + const haddr_t UNUSED *addr, + void UNUSED *_rt_key, void *_udata) +{ + H5F_istore_ud1_t *bt_udata = (H5F_istore_ud1_t *)_udata; + H5F_istore_key_t *lt_key = (H5F_istore_key_t *)_lt_key; + + FUNC_ENTER(H5F_istore_iterate, FAIL); + bt_udata->total_storage += lt_key->nbytes; + FUNC_LEAVE(SUCCEED); +} + + +/*------------------------------------------------------------------------- * Function: H5F_istore_init * * Purpose: Initialize the raw data chunk cache for a file. This is @@ -1513,7 +1547,7 @@ H5F_istore_unlock (H5F_t *f, const H5O_layout_t *layout, *------------------------------------------------------------------------- */ herr_t -H5F_istore_read(H5F_t *f, const H5D_xfer_t *xfer, const H5O_layout_t *layout, +H5F_istore_read(H5F_t *f, const H5F_xfer_t *xfer, const H5O_layout_t *layout, const H5O_pline_t *pline, const H5O_fill_t *fill, const hssize_t offset_f[], const hsize_t size[], void *buf) { @@ -1601,7 +1635,7 @@ H5F_istore_read(H5F_t *f, const H5D_xfer_t *xfer, const H5O_layout_t *layout, if (f->shared->access_parms->driver==H5F_LOW_MPIO){ H5F_istore_ud1_t udata; H5O_layout_t l; /* temporary layout */ - H5D_xfer_t tmp_xfer = *xfer; + H5F_xfer_t tmp_xfer = *xfer; if (H5F_istore_get_addr(f, layout, chunk_offset, &udata)<0){ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to locate raw data chunk"); @@ -1683,7 +1717,7 @@ H5F_istore_read(H5F_t *f, const H5D_xfer_t *xfer, const H5O_layout_t *layout, *------------------------------------------------------------------------- */ herr_t -H5F_istore_write(H5F_t *f, const H5D_xfer_t *xfer, const H5O_layout_t *layout, +H5F_istore_write(H5F_t *f, const H5F_xfer_t *xfer, const H5O_layout_t *layout, const H5O_pline_t *pline, const H5O_fill_t *fill, const hssize_t offset_f[], const hsize_t size[], const void *buf) @@ -1776,7 +1810,7 @@ H5F_istore_write(H5F_t *f, const H5D_xfer_t *xfer, const H5O_layout_t *layout, if (f->shared->access_parms->driver==H5F_LOW_MPIO){ H5F_istore_ud1_t udata; H5O_layout_t l; /* temporary layout */ - H5D_xfer_t tmp_xfer = *xfer; + H5F_xfer_t tmp_xfer = *xfer; if (H5F_istore_get_addr(f, layout, chunk_offset, &udata)<0){ HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "unable to locate raw data chunk"); @@ -1895,6 +1929,40 @@ H5F_istore_create(H5F_t *f, H5O_layout_t *layout /*out */ ) /*------------------------------------------------------------------------- + * Function: H5F_istore_allocated + * + * Purpose: Return the number of bytes allocated in the file for storage + * of raw data under the specified B-tree (ADDR is the address + * of the B-tree). + * + * Return: Success: Number of bytes stored in all chunks. + * + * Failure: 0 + * + * Programmer: Robb Matzke + * Wednesday, April 21, 1999 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +hsize_t +H5F_istore_allocated(H5F_t *f, int ndims, haddr_t *addr) +{ + H5F_istore_ud1_t udata; + + FUNC_ENTER(H5F_istore_nchunks, 0); + udata.mesg.ndims = ndims; + udata.total_storage = 0; + if (H5B_iterate(f, H5B_ISTORE, addr, &udata)<0) { + HRETURN_ERROR(H5E_IO, H5E_INTERNAL, 0, + "unable to iterate over chunk B-tree"); + } + FUNC_LEAVE(udata.total_storage); +} + + +/*------------------------------------------------------------------------- * Function: H5F_istore_stats * * Purpose: Print raw data cache statistics to the debug stream. If diff --git a/src/H5Fmpio.c b/src/H5Fmpio.c index aba0200..91be471 100644 --- a/src/H5Fmpio.c +++ b/src/H5Fmpio.c @@ -106,17 +106,16 @@ static H5F_low_t *H5F_mpio_open(const char *name, H5F_search_t *key/*out*/); static herr_t H5F_mpio_close(H5F_low_t *lf, const H5F_access_t *access_parms); static herr_t H5F_mpio_read(H5F_low_t *lf, H5F_access_t *access_parms, - const H5D_transfer_t xfer_mode, - const haddr_t *addr, size_t size, - uint8_t *buf/*out*/); + const H5F_xfer_t *xfer_parms, const haddr_t *addr, + size_t size, uint8_t *buf/*out*/); htri_t H5F_mpio_tas_allsame(H5F_low_t *lf, hbool_t newval ); static herr_t H5F_mpio_write(H5F_low_t *lf, H5F_access_t *access_parms, - const H5D_transfer_t xfer_mode, - const haddr_t *addr, size_t size, - const uint8_t *buf); + const H5F_xfer_t *xfer_parms, const haddr_t *addr, + size_t size, const uint8_t *buf); static herr_t H5F_mpio_flush(H5F_low_t *lf, const H5F_access_t *access_parms); -static herr_t H5F_MPIOff_to_haddr(MPI_Offset mpi_off, haddr_t *addr); -static herr_t H5F_haddr_to_MPIOff(haddr_t addr, MPI_Offset *mpi_off); +static herr_t H5F_MPIOff_to_haddr(MPI_Offset mpi_off, haddr_t *addr/*out*/); +static herr_t H5F_haddr_to_MPIOff(const haddr_t *addr, + MPI_Offset *mpi_off/*out*/); const H5F_low_class_t H5F_LOW_MPIO_g[1] = {{ H5F_mpio_access, /*access method */ @@ -497,11 +496,14 @@ H5F_mpio_close(H5F_low_t *lf, const H5F_access_t UNUSED *access_parms) * The guts of H5F_mpio_read and H5F_mpio_write * should be replaced by a single dual-purpose routine. * + * Robb Matzke, 19990421 + * Changed xfer_mode to xfer_parms for all H5F_*_read() callbacks. + * *------------------------------------------------------------------------- */ static herr_t H5F_mpio_read(H5F_low_t *lf, H5F_access_t *access_parms, - const H5D_transfer_t xfer_mode, + const H5F_xfer_t *xfer_parms, const haddr_t *addr, size_t size, uint8_t *buf/*out*/) { MPI_Offset mpi_off, mpi_disp; @@ -518,7 +520,7 @@ H5F_mpio_read(H5F_low_t *lf, H5F_access_t *access_parms, #endif /* some numeric conversions */ - if (SUCCEED != H5F_haddr_to_MPIOff( *addr, &mpi_off )) { + if (SUCCEED != H5F_haddr_to_MPIOff(addr, &mpi_off)) { HRETURN_ERROR(H5E_IO, H5E_BADTYPE, FAIL, "couldn't convert addr to MPIOffset" ); } @@ -545,8 +547,9 @@ H5F_mpio_read(H5F_low_t *lf, H5F_access_t *access_parms, buf_type = access_parms->u.mpio.btype; file_type = access_parms->u.mpio.ftype; if (SUCCEED != - H5F_haddr_to_MPIOff( access_parms->u.mpio.disp, &mpi_disp)) { - HRETURN_ERROR(H5E_IO, H5E_BADTYPE, FAIL, "couldn't convert addr to MPIOffset" ); + H5F_haddr_to_MPIOff(&(access_parms->u.mpio.disp), &mpi_disp)) { + HRETURN_ERROR(H5E_IO, H5E_BADTYPE, FAIL, + "couldn't convert addr to MPIOffset" ); } } else { /* Prepare for a simple xfer of a contiguous block of bytes. @@ -576,7 +579,7 @@ H5F_mpio_read(H5F_low_t *lf, H5F_access_t *access_parms, access_parms->u.mpio.use_types = 0; /* Read the data. */ - switch (xfer_mode){ + switch (xfer_parms->xfer_mode){ case H5D_XFER_INDEPENDENT: case H5D_XFER_DFLT: mpierr = MPI_File_read_at ( lf->u.mpio.f, mpi_off, (void*) buf, @@ -744,11 +747,14 @@ H5F_mpio_tas_allsame(H5F_low_t *lf, hbool_t newval ) * rky, 980828 * Added allsame parameter to make all but proc 0 skip the actual write. * + * Robb Matzke, 19990421 + * Changed xfer_mode to xfer_parms for all H5F_*_write() callbacks. + * *------------------------------------------------------------------------- */ static herr_t H5F_mpio_write(H5F_low_t *lf, H5F_access_t *access_parms, - const H5D_transfer_t xfer_mode, + const H5F_xfer_t *xfer_parms, const haddr_t *addr, size_t size, const uint8_t *buf) { MPI_Offset mpi_off, mpi_disp; @@ -766,18 +772,19 @@ H5F_mpio_write(H5F_low_t *lf, H5F_access_t *access_parms, #endif /* some numeric conversions */ - if (SUCCEED != H5F_haddr_to_MPIOff( *addr, &mpi_off )) { + if (SUCCEED != H5F_haddr_to_MPIOff(addr, &mpi_off)) { HRETURN_ERROR(H5E_IO, H5E_BADTYPE, FAIL, - "couldn't convert addr to MPIOffset" ); + "couldn't convert addr to MPIOffset" ); } - if (SUCCEED!=H5F_haddr_to_MPIOff( access_parms->u.mpio.disp, &mpi_disp)) { + if (SUCCEED!=H5F_haddr_to_MPIOff(&(access_parms->u.mpio.disp), + &mpi_disp)) { HRETURN_ERROR(H5E_IO, H5E_BADTYPE, FAIL, - "couldn't convert addr to MPIOffset" ); + "couldn't convert addr to MPIOffset" ); } size_i = (int)size; if ((size_t)size_i != size) { /* check type conversion */ HRETURN_ERROR(H5E_IO, H5E_BADTYPE, FAIL, - "couldn't convert size to int" ); + "couldn't convert size to int" ); } #ifdef H5Fmpio_DEBUG @@ -813,8 +820,9 @@ H5F_mpio_write(H5F_low_t *lf, H5F_access_t *access_parms, buf_type = access_parms->u.mpio.btype; file_type = access_parms->u.mpio.ftype; if (SUCCEED != - H5F_haddr_to_MPIOff( access_parms->u.mpio.disp, &mpi_disp)) { - HRETURN_ERROR(H5E_IO, H5E_BADTYPE, FAIL, "couldn't convert addr to MPIOffset" ); + H5F_haddr_to_MPIOff(&(access_parms->u.mpio.disp), &mpi_disp)) { + HRETURN_ERROR(H5E_IO, H5E_BADTYPE, FAIL, + "couldn't convert addr to MPIOffset" ); } } else { /* Prepare for a simple xfer of a contiguous block of bytes. @@ -844,7 +852,7 @@ H5F_mpio_write(H5F_low_t *lf, H5F_access_t *access_parms, access_parms->u.mpio.use_types = 0; /* Write the data. */ - switch (xfer_mode){ + switch (xfer_parms->xfer_mode){ case H5D_XFER_INDEPENDENT: case H5D_XFER_DFLT: mpierr = MPI_File_write_at ( lf->u.mpio.f, mpi_off, (void*) buf, @@ -959,20 +967,25 @@ H5F_mpio_flush(H5F_low_t *lf, const H5F_access_t UNUSED *access_parms) * January 30, 1998 * * Modifications: + * Robb Matzke, 1999-04-23 + * An error is reported for address overflows. The ADDR output + * argument is optional. * *------------------------------------------------------------------------- */ static herr_t -H5F_MPIOff_to_haddr( MPI_Offset mpi_off, haddr_t *addr ) +H5F_MPIOff_to_haddr(MPI_Offset mpi_off, haddr_t *addr/*out*/) { - herr_t ret_val = FAIL; - - addr->offset = (uint64_t) mpi_off; - if (addr->offset == mpi_off) - ret_val = SUCCEED; + FUNC_ENTER(H5F_MPIOff_to_haddr, FAIL); - return (ret_val); + if (addr) addr->offset = (uint64_t) mpi_off; + if (mpi_off != (MPI_Offset)(uint64_t)mpi_off) { + HRETURN_ERROR(H5E_IO, H5E_OVERFLOW, FAIL, "bad MPI address"); + } + + FUNC_LEAVE(SUCCEED); } + /*------------------------------------------------------------------------- * Function: H5F_haddr_to_MPIOff @@ -988,20 +1001,26 @@ H5F_MPIOff_to_haddr( MPI_Offset mpi_off, haddr_t *addr ) * January 30, 1998 * * Modifications: + * Robb Matzke, 1999-04-23 + * An error is reported for address overflows. The ADDR output + * argument is optional. * *------------------------------------------------------------------------- */ static herr_t -H5F_haddr_to_MPIOff( haddr_t addr, MPI_Offset *mpi_off ) +H5F_haddr_to_MPIOff(const haddr_t *addr, MPI_Offset *mpi_off/*out*/) { - herr_t ret_val = FAIL; - - *mpi_off = (MPI_Offset) addr.offset; - if (*mpi_off == addr.offset) - ret_val = SUCCEED; + FUNC_ENTER(H5F_haddr_to_MPIOff, FAIL); - return (ret_val); + if (mpi_off) *mpi_off = (MPI_Offset)addr->offset; + if (addr->offset != (uint64_t)(MPI_Offset)(addr->offset)) { + HRETURN_ERROR(H5E_IO, H5E_OVERFLOW, FAIL, + "hdf5 address overflows MPI address"); + } + + FUNC_LEAVE(SUCCEED); } + /*------------------------------------------------------------------------- * Function: H5PC_Wait_for_left_neighbor diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 55048b3..0d46f07 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -512,6 +512,18 @@ typedef struct H5F_t { H5F_mtab_t mtab; /* File mount table */ } H5F_t; +/* Dataset transfer property list */ +typedef struct H5F_xfer_t { + size_t buf_size; /*max temp buffer size */ + void *tconv_buf; /*type conversion buffer or null */ + void *bkg_buf; /*background buffer or null */ + H5T_bkg_t need_bkg; /*type of background buffer needed */ + double split_ratios[3];/*B-tree node splitting ratios */ + uintn cache_hyper; /*cache hyperslab blocks during I/O? */ + uintn block_limit; /*largest hyperslab block to cache */ + H5D_transfer_t xfer_mode; /*independent or collective transfer */ +} H5F_xfer_t; + #ifdef NOT_YET #define H5F_ENCODE_OFFSET(f,p,o) (H5F_SIZEOF_ADDR(f)==4 ? UINT32ENCODE(p,o) \ : H5F_SIZEOF_ADDR(f)==8 ? UINT64ENCODE(p,o) \ @@ -556,13 +568,14 @@ typedef struct H5F_t { struct H5O_layout_t; struct H5O_efl_t; struct H5O_pline_t; -struct H5D_xfer_t; +struct H5F_xfer_t; struct H5O_fill_t; struct H5G_entry_t; /* library variables */ __DLLVAR__ const H5F_create_t H5F_create_dflt; __DLLVAR__ H5F_access_t H5F_access_dflt; +__DLLVAR__ const H5F_xfer_t H5F_xfer_dflt; __DLLVAR__ const H5F_mprop_t H5F_mount_dflt; #ifdef HAVE_PARALLEL @@ -588,7 +601,7 @@ __DLL__ herr_t H5F_mountpoint(struct H5G_entry_t *find/*in,out*/); /* Functions that operate on array storage */ __DLL__ herr_t H5F_arr_create(H5F_t *f, struct H5O_layout_t *layout /*in,out*/); -__DLL__ herr_t H5F_arr_read (H5F_t *f, const struct H5D_xfer_t *xfer, +__DLL__ herr_t H5F_arr_read (H5F_t *f, const struct H5F_xfer_t *xfer, const struct H5O_layout_t *layout, const struct H5O_pline_t *pline, const struct H5O_fill_t *fill, @@ -597,7 +610,7 @@ __DLL__ herr_t H5F_arr_read (H5F_t *f, const struct H5D_xfer_t *xfer, const hsize_t mem_size[], const hssize_t mem_offset[], const hssize_t file_offset[], void *_buf/*out*/); -__DLL__ herr_t H5F_arr_write (H5F_t *f, const struct H5D_xfer_t *xfer, +__DLL__ herr_t H5F_arr_write (H5F_t *f, const struct H5F_xfer_t *xfer, const struct H5O_layout_t *layout, const struct H5O_pline_t *pline, const struct H5O_fill_t *fill, @@ -611,16 +624,17 @@ __DLL__ herr_t H5F_arr_write (H5F_t *f, const struct H5D_xfer_t *xfer, __DLL__ herr_t H5F_istore_init (H5F_t *f); __DLL__ herr_t H5F_istore_flush (H5F_t *f, hbool_t preempt); __DLL__ herr_t H5F_istore_dest (H5F_t *f); +__DLL__ hsize_t H5F_istore_allocated(H5F_t *f, int ndims, haddr_t *addr); __DLL__ herr_t H5F_istore_stats (H5F_t *f, hbool_t headers); __DLL__ herr_t H5F_istore_create(H5F_t *f, struct H5O_layout_t *layout/*in,out*/); -__DLL__ herr_t H5F_istore_read(H5F_t *f, const struct H5D_xfer_t *xfer, +__DLL__ herr_t H5F_istore_read(H5F_t *f, const struct H5F_xfer_t *xfer, const struct H5O_layout_t *layout, const struct H5O_pline_t *pline, const struct H5O_fill_t *fill, const hssize_t offset[], const hsize_t size[], void *buf/*out*/); -__DLL__ herr_t H5F_istore_write(H5F_t *f, const struct H5D_xfer_t *xfer, +__DLL__ herr_t H5F_istore_write(H5F_t *f, const struct H5F_xfer_t *xfer, const struct H5O_layout_t *layout, const struct H5O_pline_t *pline, const struct H5O_fill_t *fill, diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index 8646f02..177a17b 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -78,6 +78,13 @@ typedef enum H5F_scope_t { /* Unlimited file size for H5Pset_external() */ #define H5F_UNLIMITED ((hsize_t)(-1L)) +/* How is the `bkg' buffer used by the conversion function? */ +typedef enum H5T_bkg_t { + H5T_BKG_NO = 0, /*background buffer is not needed, send NULL */ + H5T_BKG_TEMP = 1, /*bkg buffer used as temp storage only */ + H5T_BKG_YES = 2 /*init bkg buf with data before conversion */ +} H5T_bkg_t; + #ifdef __cplusplus extern "C" { #endif diff --git a/src/H5Gnode.c b/src/H5Gnode.c index 7f95906..09cc40c 100644 --- a/src/H5Gnode.c +++ b/src/H5Gnode.c @@ -61,7 +61,8 @@ static H5B_ins_t H5G_node_insert(H5F_t *f, const haddr_t *addr, static H5B_ins_t H5G_node_remove(H5F_t *f, const haddr_t *addr, void *lt_key, hbool_t *lt_key_changed, void *udata, void *rt_key, hbool_t *rt_key_changed); -static herr_t H5G_node_iterate(H5F_t *f, const haddr_t *addr, void *_udata); +static herr_t H5G_node_iterate(H5F_t *f, void *_lt_key, const haddr_t *addr, + void *_rt_key, void *_udata); static size_t H5G_node_sizeof_rkey(H5F_t *f, const void *_udata); /* H5G inherits cache-like properties from H5AC */ @@ -1034,7 +1035,8 @@ H5G_node_remove(H5F_t *f, const haddr_t *addr, void *_lt_key/*in,out*/, *------------------------------------------------------------------------- */ static herr_t -H5G_node_iterate (H5F_t *f, const haddr_t *addr, void *_udata) +H5G_node_iterate (H5F_t *f, void UNUSED *_lt_key, const haddr_t *addr, + void UNUSED *_rt_key, void *_udata) { H5G_bt_ud2_t *bt_udata = (H5G_bt_ud2_t *)_udata; H5G_node_t *sn = NULL; diff --git a/src/H5P.c b/src/H5P.c index 99bc563..2bbc54c 100644 --- a/src/H5P.c +++ b/src/H5P.c @@ -182,11 +182,11 @@ H5Pcreate(H5P_class_t type) break; case H5P_DATASET_XFER: - if (NULL==(plist = H5MM_malloc(sizeof(H5D_xfer_t)))) { + if (NULL==(plist = H5MM_malloc(sizeof(H5F_xfer_t)))) { HRETURN_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed"); } - HDmemcpy(plist, &H5D_xfer_dflt, sizeof(H5D_xfer_t)); + HDmemcpy(plist, &H5F_xfer_dflt, sizeof(H5F_xfer_t)); break; case H5P_MOUNT: @@ -1979,7 +1979,7 @@ H5Pget_cache(hid_t plist_id, int *mdc_nelmts, herr_t H5Pset_buffer(hid_t plist_id, size_t size, void *tconv, void *bkg) { - H5D_xfer_t *plist = NULL; + H5F_xfer_t *plist = NULL; FUNC_ENTER (H5Pset_buffer, FAIL); H5TRACE4("e","izxx",plist_id,size,tconv,bkg); @@ -2023,7 +2023,7 @@ H5Pset_buffer(hid_t plist_id, size_t size, void *tconv, void *bkg) size_t H5Pget_buffer(hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/) { - H5D_xfer_t *plist = NULL; + H5F_xfer_t *plist = NULL; FUNC_ENTER (H5Pget_buffer, 0); H5TRACE3("z","ixx",plist_id,tconv,bkg); @@ -2069,7 +2069,7 @@ H5Pget_buffer(hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/) herr_t H5Pset_hyper_cache(hid_t plist_id, unsigned cache, unsigned limit) { - H5D_xfer_t *plist = NULL; + H5F_xfer_t *plist = NULL; FUNC_ENTER (H5Pset_hyper_cache, FAIL); H5TRACE3("e","iIuIu",plist_id,cache,limit); @@ -2107,7 +2107,7 @@ herr_t H5Pget_hyper_cache(hid_t plist_id, unsigned *cache/*out*/, unsigned *limit/*out*/) { - H5D_xfer_t *plist = NULL; + H5F_xfer_t *plist = NULL; FUNC_ENTER (H5Pget_hyper_cache, 0); H5TRACE3("e","ixx",plist_id,cache,limit); @@ -2148,7 +2148,7 @@ H5Pget_hyper_cache(hid_t plist_id, unsigned *cache/*out*/, herr_t H5Pset_preserve(hid_t plist_id, hbool_t status) { - H5D_xfer_t *plist = NULL; + H5F_xfer_t *plist = NULL; FUNC_ENTER (H5Pset_preserve, FAIL); H5TRACE2("e","ib",plist_id,status); @@ -2186,7 +2186,7 @@ H5Pset_preserve(hid_t plist_id, hbool_t status) int H5Pget_preserve(hid_t plist_id) { - H5D_xfer_t *plist = NULL; + H5F_xfer_t *plist = NULL; FUNC_ENTER (H5Pset_preserve, FAIL); H5TRACE1("Is","i",plist_id); @@ -2497,7 +2497,7 @@ herr_t H5Pget_btree_ratios(hid_t plist_id, double *left/*out*/, double *middle/*out*/, double *right/*out*/) { - H5D_xfer_t *plist = NULL; + H5F_xfer_t *plist = NULL; FUNC_ENTER(H5Pget_btree_ratios, FAIL); H5TRACE4("e","ixxx",plist_id,left,middle,right); @@ -2544,7 +2544,7 @@ herr_t H5Pset_btree_ratios(hid_t plist_id, double left, double middle, double right) { - H5D_xfer_t *plist = NULL; + H5F_xfer_t *plist = NULL; FUNC_ENTER(H5Pget_btree_ratios, FAIL); H5TRACE4("e","iddd",plist_id,left,middle,right); @@ -2903,7 +2903,7 @@ H5Pget_mpi(hid_t plist_id, MPI_Comm *comm, MPI_Info *info) herr_t H5Pset_xfer(hid_t plist_id, H5D_transfer_t data_xfer_mode) { - H5D_xfer_t *plist = NULL; + H5F_xfer_t *plist = NULL; FUNC_ENTER(H5Pset_xfer, FAIL); H5TRACE2("e","iDt",plist_id,data_xfer_mode); @@ -2951,7 +2951,7 @@ H5Pset_xfer(hid_t plist_id, H5D_transfer_t data_xfer_mode) herr_t H5Pget_xfer(hid_t plist_id, H5D_transfer_t *data_xfer_mode) { - H5D_xfer_t *plist = NULL; + H5F_xfer_t *plist = NULL; FUNC_ENTER (H5Pget_xfer, FAIL); H5TRACE2("e","i*Dt",plist_id,data_xfer_mode); @@ -3151,7 +3151,7 @@ H5P_copy (H5P_class_t type, const void *src) break; case H5P_DATASET_XFER: - size = sizeof(H5D_xfer_t); + size = sizeof(H5F_xfer_t); break; case H5P_MOUNT: diff --git a/src/H5RA.c b/src/H5RA.c index bf0f5ef..389e8dc 100644 --- a/src/H5RA.c +++ b/src/H5RA.c @@ -738,7 +738,7 @@ H5RA_write(H5RA_t *ra, hssize_t start_row, hsize_t nrows, H5T_t *type, "memory allocation failed for meta data"); } if (H5D_read(ra->meta, H5RA_meta_type_g, mm_space, mf_space, - &H5D_xfer_dflt, meta)<0) { + &H5F_xfer_dflt, meta)<0) { HGOTO_ERROR(H5E_RAGGED, H5E_READERROR, FAIL, "unable to read meta data"); } @@ -779,7 +779,7 @@ H5RA_write(H5RA_t *ra, hssize_t start_row, hsize_t nrows, H5T_t *type, HGOTO_ERROR(H5E_RAGGED, H5E_CANTINIT, FAIL, "unable to set meta data selection"); } - if (H5D_write(ra->raw, type, rm_space, rf_space, &H5D_xfer_dflt, + if (H5D_write(ra->raw, type, rm_space, rf_space, &H5F_xfer_dflt, raw_buf)<0) { HGOTO_ERROR(H5E_RAGGED, H5E_WRITEERROR, FAIL, "unable to write raw data"); @@ -809,7 +809,7 @@ H5RA_write(H5RA_t *ra, hssize_t start_row, hsize_t nrows, H5T_t *type, "unable to set meta data selection"); } if (H5D_write(ra->meta, H5RA_meta_type_g, mm_space, mf_space, - &H5D_xfer_dflt, meta)<0) { + &H5F_xfer_dflt, meta)<0) { HGOTO_ERROR(H5E_RAGGED, H5E_WRITEERROR, FAIL, "unable to write meta data"); } @@ -926,7 +926,7 @@ H5RA_fix_overflow(H5RA_t *ra, H5T_t *type, H5RA_meta_t *meta, hsize_t nelmts, HGOTO_ERROR(H5E_RAGGED, H5E_CANTINIT, FAIL, "unable to set overflow selection"); } - if (H5D_write(ra->over, type, om_space, of_space, &H5D_xfer_dflt, + if (H5D_write(ra->over, type, om_space, of_space, &H5F_xfer_dflt, buf)<0) { HGOTO_ERROR(H5E_RAGGED, H5E_WRITEERROR, FAIL, "unable to write to overflow dataset"); @@ -1103,7 +1103,7 @@ H5RA_read(H5RA_t *ra, hssize_t start_row, hsize_t nrows, H5T_t *type, HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw dataset"); } - if (H5D_read(ra->raw, type, rm_space, rf_space, &H5D_xfer_dflt, + if (H5D_read(ra->raw, type, rm_space, rf_space, &H5F_xfer_dflt, raw_buf)<0) { HGOTO_ERROR(H5E_RAGGED, H5E_READERROR, FAIL, "unable to read raw dataset"); @@ -1134,7 +1134,7 @@ H5RA_read(H5RA_t *ra, hssize_t start_row, hsize_t nrows, H5T_t *type, "unable to set meta data selection"); } if (H5D_read(ra->meta, H5RA_meta_type_g, mm_space, mf_space, - &H5D_xfer_dflt, meta)<0) { + &H5F_xfer_dflt, meta)<0) { HGOTO_ERROR(H5E_RAGGED, H5E_READERROR, FAIL, "unable to read meta data"); } @@ -1182,7 +1182,7 @@ H5RA_read(H5RA_t *ra, hssize_t start_row, hsize_t nrows, H5T_t *type, HGOTO_ERROR(H5E_RAGGED, H5E_CANTINIT, FAIL, "unable to set overflow selection"); } - if (H5D_read(ra->over, type, om_space, of_space, &H5D_xfer_dflt, + if (H5D_read(ra->over, type, om_space, of_space, &H5F_xfer_dflt, buf_out[i])<0) { HGOTO_ERROR(H5E_RAGGED, H5E_CANTINIT, FAIL, "unable to read overflow dataset"); diff --git a/src/H5S.c b/src/H5S.c index ff0a31c..d9e5c49 100644 --- a/src/H5S.c +++ b/src/H5S.c @@ -1462,7 +1462,8 @@ H5S_find (const H5S_t *mem_space, const H5S_t *file_space) * We can't do conversion if the source and destination select a * different number of data points. */ - if (H5S_get_select_npoints (mem_space) != H5S_get_select_npoints (file_space)) { + if (H5S_get_select_npoints(mem_space) != + H5S_get_select_npoints (file_space)) { HRETURN_ERROR (H5E_DATASPACE, H5E_BADRANGE, NULL, "memory and file data spaces are different sizes"); } @@ -1489,7 +1490,27 @@ H5S_find (const H5S_t *mem_space, const H5S_t *file_space) } /* - * Extend the table. + * Create a new path. + */ + if (NULL==(path = H5MM_calloc(sizeof(*path)))) { + HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, + "memory allocation failed for data space conversion " + "path"); + } + path->f = H5S_fconv_g[file_space->select.type]; + path->m = H5S_mconv_g[mem_space->select.type]; + + /* + * Initialize direct read/write functions + */ + if (H5S_SEL_ALL==file_space->select.type && + H5S_SEL_ALL==mem_space->select.type) { + path->read = H5S_all_read; + path->write = H5S_all_write; + } + + /* + * Add the new path to the table. */ if (H5S_nconv_g>=H5S_aconv_g) { size_t n = MAX(10, 2*H5S_aconv_g); @@ -1502,17 +1523,6 @@ H5S_find (const H5S_t *mem_space, const H5S_t *file_space) H5S_aconv_g = n; H5S_conv_g = p; } - - /* - * Create a new path and add it to the table. - */ - if (NULL==(path = H5MM_calloc(sizeof(*path)))) { - HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, - "memory allocation failed for data space conversion " - "path"); - } - path->f = H5S_fconv_g[file_space->select.type]; - path->m = H5S_mconv_g[mem_space->select.type]; H5S_conv_g[H5S_nconv_g++] = path; FUNC_LEAVE(path); diff --git a/src/H5Sall.c b/src/H5Sall.c index 5e91b6f..6ebd241 100644 --- a/src/H5Sall.c +++ b/src/H5Sall.c @@ -28,14 +28,14 @@ static size_t H5S_all_fgath (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, - const H5D_xfer_t *xfer_parms, void *buf/*out*/); + const H5F_xfer_t *xfer_parms, void *buf/*out*/); static herr_t H5S_all_fscat (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_pline_t *pline, const struct H5O_fill_t *fill, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, - const H5D_xfer_t *xfer_parms, const void *buf); + const H5F_xfer_t *xfer_parms, const void *buf); static size_t H5S_all_mgath (const void *_buf, size_t elmt_size, const H5S_t *mem_space, H5S_sel_iter_t *mem_iter, size_t nelmts, void *_tconv_buf/*out*/); @@ -172,7 +172,7 @@ H5S_all_fgath (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_fill_t *fill, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, - const H5D_xfer_t *xfer_parms, void *_buf/*out*/) + const H5F_xfer_t *xfer_parms, void *_buf/*out*/) { hssize_t file_offset[H5O_LAYOUT_NDIMS]; /*offset of slab in file*/ hsize_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */ @@ -260,7 +260,7 @@ H5S_all_fscat (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_pline_t *pline, const struct H5O_fill_t *fill, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, - size_t nelmts, const H5D_xfer_t *xfer_parms, const void *_buf) + size_t nelmts, const H5F_xfer_t *xfer_parms, const void *_buf) { hssize_t file_offset[H5O_LAYOUT_NDIMS]; /*offset of hyperslab */ hsize_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */ @@ -499,7 +499,144 @@ H5S_all_mscat (const void *_tconv_buf, size_t elmt_size, mem_iter->all.offset+=nelmts; FUNC_LEAVE (SUCCEED); -} /* H5S_all_mscat() */ +} + + +/*------------------------------------------------------------------------- + * Function: H5S_all_read + * + * Purpose: Reads directly from file into application memory if possible. + * + * Return: Success: Non-negative. If data was read directly then + * MUST_CONVERT is set to zero, otherwise + * MUST_CONVERT is set to non-zero. + * + * Failure: Negative. Return value of MUST_CONVERT is + * undefined. + * + * Programmer: Robb Matzke + * Thursday, April 22, 1999 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5S_all_read(H5F_t *f, const H5O_layout_t *layout, const H5O_pline_t *pline, + const H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, + const H5S_t *mem_space, const H5F_xfer_t *xfer_parms, + void *buf/*out*/, hbool_t *must_convert/*out*/) +{ + hsize_t size[H5S_MAX_RANK]; + hssize_t offset[H5S_MAX_RANK]; + int i; + + FUNC_ENTER(H5S_all_read, FAIL); + *must_convert = TRUE; + + /* Check whether we can handle this */ + if (H5S_SIMPLE!=mem_space->extent.type) goto fall_through; + if (H5S_SIMPLE!=file_space->extent.type) goto fall_through; + if (mem_space->extent.u.simple.rank!= + file_space->extent.u.simple.rank) goto fall_through; + + /* Get information about memory and file */ + for (i=0; iextent.u.simple.rank; i++) { + if (mem_space->extent.u.simple.max && + mem_space->extent.u.simple.size[i]!= + mem_space->extent.u.simple.max[i]) goto fall_through; + if (file_space->extent.u.simple.max && + file_space->extent.u.simple.size[i]!= + file_space->extent.u.simple.max[i]) goto fall_through; + if (mem_space->extent.u.simple.size[i]!= + file_space->extent.u.simple.size[i]) goto fall_through; + size[i] = mem_space->extent.u.simple.size[i]; + offset[i] = 0; + } + size[i] = elmt_size; + offset[i] = 0; + + /* Read data from the file */ + if (H5F_arr_read(f, xfer_parms, layout, pline, NULL, efl, size, + size, offset, offset, buf/*out*/)<0) { + HRETURN_ERROR(H5E_IO, H5E_INTERNAL, FAIL, + "unable to write data to the file"); + } + *must_convert = FALSE; + + fall_through: + FUNC_LEAVE(SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5S_all_write + * + * Purpose: Writes data directly to the file if possible. + * + * Return: Success: Non-negative. If data was written directly + * then MUST_CONVERT is set to zero, otherwise + * MUST_CONVERT is set to non-zero. + * + * Failure: Negative. Return value of MUST_CONVERT is + * undefined. + * + * Programmer: Robb Matzke + * Wednesday, April 21, 1999 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5S_all_write(H5F_t *f, const struct H5O_layout_t *layout, + const H5O_pline_t *pline, const H5O_efl_t *efl, + size_t elmt_size, const H5S_t *file_space, + const H5S_t *mem_space, const H5F_xfer_t *xfer_parms, + const void *buf, hbool_t *must_convert/*out*/) +{ + hsize_t size[H5S_MAX_RANK]; + hssize_t offset[H5S_MAX_RANK]; + int i; + + FUNC_ENTER(H5S_all_write, FAIL); + *must_convert = TRUE; + + /* Check whether we can handle this */ + if (H5S_SIMPLE!=mem_space->extent.type) goto fall_through; + if (H5S_SIMPLE!=file_space->extent.type) goto fall_through; + if (mem_space->extent.u.simple.rank!= + file_space->extent.u.simple.rank) goto fall_through; + + /* Get information about memory and file */ + for (i=0; iextent.u.simple.rank; i++) { + if (mem_space->extent.u.simple.max && + mem_space->extent.u.simple.size[i]!= + mem_space->extent.u.simple.max[i]) goto fall_through; + if (file_space->extent.u.simple.max && + file_space->extent.u.simple.size[i]!= + file_space->extent.u.simple.max[i]) goto fall_through; + if (mem_space->extent.u.simple.size[i]!= + file_space->extent.u.simple.size[i]) goto fall_through; + size[i] = mem_space->extent.u.simple.size[i]; + offset[i] = 0; + } + size[i] = elmt_size; + offset[i] = 0; + + /* Write data to the file */ + if (H5F_arr_write(f, xfer_parms, layout, pline, NULL, efl, size, + size, offset, offset, buf)<0) { + HRETURN_ERROR(H5E_IO, H5E_INTERNAL, FAIL, + "unable to write data to the file"); + } + *must_convert = FALSE; + + + fall_through: + FUNC_LEAVE(SUCCEED); +} + /*-------------------------------------------------------------------------- NAME diff --git a/src/H5Shyper.c b/src/H5Shyper.c index 082d033..2fba497 100644 --- a/src/H5Shyper.c +++ b/src/H5Shyper.c @@ -32,7 +32,7 @@ typedef struct { const H5S_t *space; H5S_sel_iter_t *iter; size_t nelmts; - const H5D_xfer_t *xfer_parms; + const H5F_xfer_t *xfer_parms; const void *src; void *dst; H5S_hyper_bound_t **lo_bounds; @@ -60,14 +60,14 @@ static size_t H5S_hyper_fgath (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, - const H5D_xfer_t *xfer_parms, void *buf/*out*/); + const H5F_xfer_t *xfer_parms, void *buf/*out*/); static herr_t H5S_hyper_fscat (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_pline_t *pline, const struct H5O_fill_t *fill, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, - const H5D_xfer_t *xfer_parms, const void *buf); + const H5F_xfer_t *xfer_parms, const void *buf); static size_t H5S_hyper_mgath (const void *_buf, size_t elmt_size, const H5S_t *mem_space, H5S_sel_iter_t *mem_iter, size_t nelmts, @@ -809,7 +809,7 @@ H5S_hyper_fgath (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_fill_t *fill, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, - size_t nelmts, const H5D_xfer_t *xfer_parms, + size_t nelmts, const H5F_xfer_t *xfer_parms, void *_buf/*out*/) { H5S_hyper_bound_t **lo_bounds; /* Lower (closest to the origin) bound array for each dimension */ @@ -1070,7 +1070,7 @@ H5S_hyper_fscat (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_fill_t *fill, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, - size_t nelmts, const H5D_xfer_t *xfer_parms, + size_t nelmts, const H5F_xfer_t *xfer_parms, const void *_buf) { H5S_hyper_bound_t **lo_bounds; /* Lower (closest to the origin) bound array for each dimension */ diff --git a/src/H5Smpio.c b/src/H5Smpio.c index a4e090b..19cbbbf 100644 --- a/src/H5Smpio.c +++ b/src/H5Smpio.c @@ -50,13 +50,12 @@ H5S_mpio_space_type( const H5S_t *space, const size_t elmt_size, hsize_t *count, hbool_t *is_derived_type ); static herr_t -H5S_mpio_spaces_xfer (H5F_t *f, const struct H5O_layout_t *layout, +H5S_mpio_spaces_xfer(H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_pline_t UNUSED *pline, const struct H5O_efl_t UNUSED *efl, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, - const H5D_transfer_t xfer_mode, void *buf /*out*/, - hbool_t *must_convert /*out*/, - const hbool_t do_write ); + const H5F_xfer_t *xfer_parms, void *buf/*out*/, + hbool_t *must_convert/*out*/, const hbool_t do_write); /*------------------------------------------------------------------------- * Function: H5S_mpio_all_type @@ -448,7 +447,7 @@ H5S_mpio_spaces_xfer (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_pline_t UNUSED *pline, const struct H5O_efl_t UNUSED *efl, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, - const H5D_transfer_t xfer_mode, void *buf /*out*/, + const H5F_xfer_t *xfer_parms, void *buf /*out*/, hbool_t *must_convert /*out*/, const hbool_t do_write ) { @@ -526,13 +525,13 @@ H5S_mpio_spaces_xfer (H5F_t *f, const struct H5O_layout_t *layout, HRETURN_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL,"transfer size overflows size_t"); if (do_write) { err = H5F_low_write( f->shared->lf, f->shared->access_parms, - xfer_mode, &addr, mpi_count, buf ); + xfer_parms->xfer_mode, &addr, mpi_count, buf ); if (err) { HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,"MPI write failed"); } } else { err = H5F_low_read ( f->shared->lf, f->shared->access_parms, - xfer_mode, &addr, mpi_count, buf ); + xfer_parms->xfer_mode, &addr, mpi_count, buf ); if (err) { HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,"MPI read failed"); } @@ -577,16 +576,16 @@ H5S_mpio_spaces_read (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_pline_t *pline, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, - const H5D_transfer_t xfer_mode, void *buf /*out*/, - hbool_t *must_convert /*out*/ ) + const H5F_xfer_t *xfer_parms, void *buf/*out*/, + hbool_t *must_convert/*out*/) { herr_t ret_value = FAIL; FUNC_ENTER (H5S_mpio_spaces_read, FAIL); - ret_value = H5S_mpio_spaces_xfer( f, layout, pline, efl, elmt_size, - file_space, mem_space, xfer_mode, (void*)buf, - must_convert /*out*/, 0 /*read*/ ); + ret_value = H5S_mpio_spaces_xfer(f, layout, pline, efl, elmt_size, + file_space, mem_space, xfer_parms, + buf, must_convert/*out*/, 0/*read*/); FUNC_LEAVE (ret_value); } /* H5S_mpio_spaces_read() */ @@ -612,16 +611,17 @@ H5S_mpio_spaces_write(H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_pline_t *pline, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, - const H5D_transfer_t xfer_mode, const void *buf, - hbool_t *must_convert /*out*/ ) + const H5F_xfer_t *xfer_parms, const void *buf, + hbool_t *must_convert/*out*/) { herr_t ret_value = FAIL; FUNC_ENTER (H5S_mpio_spaces_write, FAIL); - ret_value = H5S_mpio_spaces_xfer( f, layout, pline, efl, elmt_size, - file_space, mem_space, xfer_mode, (void*)buf, - must_convert /*out*/, 1 /*write*/ ); + ret_value = H5S_mpio_spaces_xfer(f, layout, pline, efl, elmt_size, + file_space, mem_space, xfer_parms, + (void*)buf, must_convert/*out*/, + 1/*write*/); FUNC_LEAVE (ret_value); } /* H5S_mpio_spaces_write() */ diff --git a/src/H5Spoint.c b/src/H5Spoint.c index a4f4931..a58461b 100644 --- a/src/H5Spoint.c +++ b/src/H5Spoint.c @@ -29,7 +29,7 @@ static size_t H5S_point_fgath (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, - const H5D_xfer_t *xfer_parms, + const H5F_xfer_t *xfer_parms, void *buf/*out*/); static herr_t H5S_point_fscat (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_pline_t *pline, @@ -37,7 +37,7 @@ static herr_t H5S_point_fscat (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, - const H5D_xfer_t *xfer_parms, + const H5F_xfer_t *xfer_parms, const void *buf); static size_t H5S_point_mgath (const void *_buf, size_t elmt_size, const H5S_t *mem_space, @@ -270,7 +270,7 @@ H5S_point_fgath (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_fill_t *fill, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, - const H5D_xfer_t *xfer_parms, void *_buf/*out*/) + const H5F_xfer_t *xfer_parms, void *_buf/*out*/) { hssize_t file_offset[H5O_LAYOUT_NDIMS]; /*offset of slab in file*/ hsize_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */ @@ -377,7 +377,7 @@ H5S_point_fscat (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_fill_t *fill, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, size_t nelmts, - const H5D_xfer_t *xfer_parms, const void *_buf) + const H5F_xfer_t *xfer_parms, const void *_buf) { hssize_t file_offset[H5O_LAYOUT_NDIMS]; /*offset of hyperslab */ hsize_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */ diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index 2710d9d..ff0a021 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -194,7 +194,7 @@ typedef struct H5S_fconv_t { const struct H5O_fill_t *fill, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, - size_t nelmts, const struct H5D_xfer_t *xfer_parms, + size_t nelmts, const struct H5F_xfer_t *xfer_parms, void *tconv_buf/*out*/); /* Scatter elements from type conversion buffer to disk */ @@ -203,7 +203,7 @@ typedef struct H5S_fconv_t { const struct H5O_fill_t *fill, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, H5S_sel_iter_t *file_iter, - size_t nelmts, const struct H5D_xfer_t *xfer_parms, + size_t nelmts, const struct H5F_xfer_t *xfer_parms, const void *tconv_buf); } H5S_fconv_t; @@ -253,8 +253,8 @@ typedef struct H5S_conv_t { const struct H5O_pline_t *pline, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, - const H5D_transfer_t xfer_mode, void *buf/*out*/, - hbool_t *must_convert/*out*/ ); + const H5F_xfer_t *xfer_parms, void *buf/*out*/, + hbool_t *must_convert/*out*/); /* Write directly from app buffer to file */ @@ -262,8 +262,8 @@ typedef struct H5S_conv_t { const struct H5O_pline_t *pline, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, - const H5D_transfer_t xfer_mode, const void *buf, - hbool_t *must_convert/*out*/ ); + const H5F_xfer_t *xfer_parms, const void *buf, + hbool_t *must_convert/*out*/); #ifdef H5S_DEBUG struct { @@ -352,6 +352,18 @@ __DLL__ hsize_t H5S_all_npoints(const H5S_t *space); __DLL__ herr_t H5S_all_select_serialize(const H5S_t *space, uint8_t *buf); __DLL__ herr_t H5S_all_select_deserialize(H5S_t *space, const uint8_t *buf); __DLL__ herr_t H5S_all_bounds(H5S_t *space, hsize_t *start, hsize_t *end); +__DLL__ herr_t H5S_all_read(H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_pline_t *pline, + const struct H5O_efl_t *efl, size_t elmt_size, + const H5S_t *file_space, const H5S_t *mem_space, + const H5F_xfer_t *xfer_parms, void *buf/*out*/, + hbool_t *must_convert/*out*/); +__DLL__ herr_t H5S_all_write(H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_pline_t *pline, + const struct H5O_efl_t *efl, size_t elmt_size, + const H5S_t *file_space, const H5S_t *mem_space, + const H5F_xfer_t *xfer_parms, const void *buf, + hbool_t *must_convert/*out*/); /* Hyperslab selection functions */ __DLL__ herr_t H5S_hyper_add(H5S_t *space, const hssize_t *start, @@ -386,7 +398,7 @@ __DLL__ herr_t H5S_mpio_spaces_read(H5F_t *f, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, - const H5D_transfer_t xfer_mode, + const H5F_xfer_t *xfer_parms, void *buf/*out*/, hbool_t *must_convert /*out*/ ); @@ -397,7 +409,7 @@ __DLL__ herr_t H5S_mpio_spaces_write(H5F_t *f, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_t *mem_space, - const H5D_transfer_t xfer_mode, + const H5F_xfer_t *xfer_parms, const void *buf, hbool_t *must_convert /*out*/ ); #ifndef _H5S_IN_H5S_C diff --git a/src/H5Tpublic.h b/src/H5Tpublic.h index 495c63a..e3b6248 100644 --- a/src/H5Tpublic.h +++ b/src/H5Tpublic.h @@ -126,13 +126,6 @@ typedef enum H5T_pad_t { H5T_NPAD = 3 /*THIS MUST BE LAST */ } H5T_pad_t; -/* How is the `bkg' buffer used by the conversion function? */ -typedef enum H5T_bkg_t { - H5T_BKG_NO = 0, /*background buffer is not needed, send NULL */ - H5T_BKG_TEMP = 1, /*bkg buffer used as temp storage only */ - H5T_BKG_YES = 2 /*init bkg buf with data before conversion */ -} H5T_bkg_t; - /* Commands sent to conversion functions */ typedef enum H5T_cmd_t { H5T_CONV_INIT = 0, /*query and/or initialize private data */ diff --git a/test/istore.c b/test/istore.c index c840a1f..adf23b6 100644 --- a/test/istore.c +++ b/test/istore.c @@ -297,7 +297,7 @@ test_extend(H5F_t *f, const char *prefix, memset(buf, (signed)(128+ctr), (size_t)nelmts); /* Write to disk */ - if (H5F_arr_write(f, &H5D_xfer_dflt, &layout, NULL, NULL, NULL, size, + if (H5F_arr_write(f, &H5F_xfer_dflt, &layout, NULL, NULL, NULL, size, size, zero, offset, buf)<0) { FAILED(); printf(" Write failed: ctr=%lu\n", (unsigned long)ctr); @@ -306,7 +306,7 @@ test_extend(H5F_t *f, const char *prefix, /* Read from disk */ memset(check, 0xff, (size_t)nelmts); - if (H5F_arr_read(f, &H5D_xfer_dflt, &layout, NULL, NULL, NULL, size, + if (H5F_arr_read(f, &H5F_xfer_dflt, &layout, NULL, NULL, NULL, size, size, zero, offset, check)<0) { FAILED(); printf(" Read failed: ctr=%lu\n", (unsigned long)ctr); @@ -337,7 +337,7 @@ test_extend(H5F_t *f, const char *prefix, /* Now read the entire array back out and check it */ memset(buf, 0xff, nx * ny * nz); - if (H5F_arr_read(f, &H5D_xfer_dflt, &layout, NULL, NULL, NULL, whole_size, + if (H5F_arr_read(f, &H5F_xfer_dflt, &layout, NULL, NULL, NULL, whole_size, whole_size, zero, zero, buf)<0) { FAILED(); puts(" Read failed for whole array."); @@ -450,7 +450,7 @@ test_sparse(H5F_t *f, const char *prefix, size_t nblocks, memset(buf, (signed)(128+ctr), nx * ny * nz); /* write to disk */ - if (H5F_arr_write(f, &H5D_xfer_dflt, &layout, NULL, NULL, NULL, size, + if (H5F_arr_write(f, &H5F_xfer_dflt, &layout, NULL, NULL, NULL, size, size, zero, offset, buf)<0) { FAILED(); printf(" Write failed: ctr=%lu\n", (unsigned long)ctr); diff --git a/tools/h5ls.c b/tools/h5ls.c index 43b579e..1e08711 100644 --- a/tools/h5ls.c +++ b/tools/h5ls.c @@ -1187,7 +1187,7 @@ dataset_list2(hid_t dset, const char UNUSED *name) char s[64]; /*temporary string buffer */ off_t f_offset; /*offset in external file */ hsize_t f_size; /*bytes used in external file */ - hsize_t total; /*total size or offset */ + hsize_t total, used; /*total size or offset */ hsize_t chsize[64]; /*chunk size in elements */ int ndims; /*dimensionality */ int n, max_len; /*max extern file name length */ @@ -1210,6 +1210,18 @@ dataset_list2(hid_t dset, const char UNUSED *name) printf("} %lu bytes\n", (unsigned long)total); } + /* Print total raw storage size */ + used = H5Sget_simple_extent_npoints(space) * H5Tget_size(type); + total = H5Dget_storage_size(dset); + printf(" %-10s ", "Storage:"); + printf("%lu logical byte%s, %lu allocated byte%s", + (unsigned long)used, 1==used?"":"s", + (unsigned long)total, 1==total?"":"s"); + if (total>0) { + printf(", %1.2f%% utilization", (used*100.0)/total); + } + putchar('\n'); + /* Print information about external strorage */ if ((nf = H5Pget_external_count(dcpl))>0) { for (i=0, max_len=0; i