From 31a709a6b24f4cf80f9cd99a3e55f56e81cf3066 Mon Sep 17 00:00:00 2001 From: Robb Matzke Date: Mon, 16 Mar 1998 20:29:54 -0500 Subject: [svn-r322] Changes since 19980313 ---------------------- ./configure.in ./configure ./test/iopipe.c Added check for system(3) ./config/freebsd2.2.1 ./config/linux Added -DH5D_DEBUG to the debug flags. ./src/H5D.c ./src/H5Dprivate.h ./src/H5P.c ./src/H5Ppublic.h ./src/H5Sprivate.h ./src/H5Ssimp.c ./html/Datasets.html Finally implemented strip mining in the I/O pipeline, placing a user-defined upper bound on the amount of temporary memory used by the pipeline. The default is 1MB type conversion and background buffers allocated/freed on demand. However, the size can be changed and/or application-allocated buffers supplied with H5Pset_buffers() called on the data transfer property list passed to H5Dread() and H5Dwrite(). Minor optimizations to H5Dread() and H5Dwrite(). More coming later... ./test/dsets.c Added calls to H5Pset_buffer() to test application-defined temporary buffers in the I/O pipeline. ./test/Makefile.in Removed `iopipe' from the list of confidence tests. Saying `make timings' in the test directory runs timing tests. I did this because (1) they don't really test anything new, and (2) they can take a long time to run. --- config/freebsd2.2.1 | 2 +- config/linux | 2 +- configure | 2 +- configure.in | 2 +- src/H5D.c | 404 ++++++++++++++++++++++++++++++++-------------------- src/H5Dprivate.h | 14 +- src/H5P.c | 94 ++++++++++++ src/H5Ppublic.h | 6 +- src/H5Sprivate.h | 5 +- src/H5Ssimp.c | 121 ++++++++++++---- test/Makefile.in | 17 ++- test/dsets.c | 22 ++- test/iopipe.c | 23 ++- 13 files changed, 507 insertions(+), 207 deletions(-) diff --git a/config/freebsd2.2.1 b/config/freebsd2.2.1 index 53aafb8..ef1c706 100644 --- a/config/freebsd2.2.1 +++ b/config/freebsd2.2.1 @@ -41,7 +41,7 @@ warn="-Wall -Wshadow -Wpointer-arith -Wcast-qual -Wwrite-strings -Wstrict-protot profile="-pg" -debug="-g -DH5AC_DEBUG -DH5B_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -fverbose-asm" +debug="-g -DH5AC_DEBUG -DH5B_DEBUG -DH5D_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -fverbose-asm" production="-O3 -DNDEBUG -finline-functions -funroll-loops -malign-double -fomit-frame-pointer" diff --git a/config/linux b/config/linux index ab397a1..4d5a2b3 100644 --- a/config/linux +++ b/config/linux @@ -53,7 +53,7 @@ fi if test "X" = "X$CPPFLAGS"; then warn= profile= - debug="-DH5AC_DEBUG -DH5B_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -DH5F_LOW_DFLT=H5F_LOW_SEC2" + debug="-DH5AC_DEBUG -DH5B_DEBUG -DH5D_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -DH5F_LOW_DFLT=H5F_LOW_SEC2" production="-DNDEBUG" parallel="" default_mode='$debug $warn $parallel' diff --git a/configure b/configure index e71fdf7..95f3d31 100755 --- a/configure +++ b/configure @@ -1267,7 +1267,7 @@ fi -for ac_func in getpwuid gethostname +for ac_func in getpwuid gethostname system do echo $ac_n "checking for $ac_func""... $ac_c" 1>&6 echo "configure:1274: checking for $ac_func" >&5 diff --git a/configure.in b/configure.in index 26adb8e..7f48014 100644 --- a/configure.in +++ b/configure.in @@ -112,7 +112,7 @@ AC_TYPE_SIZE_T dnl ---------------------------------------------------------------------- dnl Check for functions. dnl -AC_CHECK_FUNCS(getpwuid gethostname) +AC_CHECK_FUNCS(getpwuid gethostname system) AC_TRY_COMPILE([#include], [off64_t n = 0;], diff --git a/src/H5D.c b/src/H5D.c index b417ca7..fce1528 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -63,7 +63,9 @@ const H5D_create_t H5D_create_dflt = { /* Default dataset transfer property list */ const H5D_xfer_t H5D_xfer_dflt = { - 0, /* Place holder - remove this later */ + 1024*1024, /* Temporary buffer size */ + NULL, /* Type conversion buffer or NULL */ + NULL, /* Background buffer or NULL */ }; /* Interface initialization? */ @@ -813,7 +815,7 @@ H5D_create(H5F_t *f, const char *name, const H5T_t *type, const H5S_t *space, } /* Create (open for write access) an object header */ - if (H5O_create(f, 80, &(new_dset->ent)) < 0) { + if (H5O_create(f, 96, &(new_dset->ent)) < 0) { HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to create dataset object header"); } @@ -1087,7 +1089,9 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, const H5S_t *file_space, const H5D_xfer_t *xfer_parms, void *buf/*out*/) { - size_t nelmts ; /*number of elements */ + size_t nelmts; /*number of elements */ + size_t smine_start; /*strip mine start loc */ + size_t smine_nelmts; /*elements per strip */ uint8 *tconv_buf = NULL; /*data type conv buffer */ uint8 *bkg_buf = NULL; /*background buffer */ H5T_conv_t tconv_func = NULL; /*conversion function */ @@ -1097,6 +1101,11 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, H5T_cdata_t *cdata = NULL; /*type conversion data */ herr_t ret_value = FAIL; herr_t status; + size_t src_type_size; /*size of source type */ + size_t dst_type_size; /*size of destination type*/ + size_t target_size; /*desired buffer size */ + size_t buffer_size; /*actual buffer size */ + size_t request_nelmts; /*requested strip mine */ FUNC_ENTER(H5D_read, FAIL); @@ -1107,124 +1116,154 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, assert(buf); if (!file_space) file_space = dataset->space; if (!mem_space) mem_space = file_space; - - /* - * Convert data types to atoms because the conversion functions are - * application-level functions. - */ - if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type))) < 0 || - (dst_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type))) < 0) { - HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, - "unable to register types for conversion"); - } + nelmts = H5S_get_npoints(mem_space); /* * Locate the type conversion function and data space conversion - * functions, and set up the element numbering information. + * functions, and set up the element numbering information. If a data + * type conversion is necessary then register data type atoms. */ if (NULL == (tconv_func = H5T_find(dataset->type, mem_type, &cdata))) { HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to convert between src and dest data types"); + } else if (H5T_conv_noop!=tconv_func) { + if ((src_id=H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0 || + (dst_id=H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0) { + HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, + "unable to register types for conversion"); + } } if (NULL==(sconv_func=H5S_find (mem_space, file_space))) { HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to convert from file to memory data space"); } - if (sconv_func->init && - (sconv_func->init)(&(dataset->layout), mem_space, file_space, - &numbering/*out*/)<=0) { - HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to initialize element numbering information"); - } else { - HDmemset (&numbering, 0, sizeof numbering); - } - if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) { + if (nelmts!=H5S_get_npoints (file_space)) { HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "src and dest data spaces have different sizes"); } /* - * Compute the size of the request and allocate scratch buffers. - */ - nelmts = H5S_get_npoints(mem_space); - - /* * If there is no type conversion then try reading directly into the - * application's buffer. + * application's buffer. This saves at least one mem-to-mem copy. */ if (H5D_OPTIMIZE_PIPE && H5T_conv_noop==tconv_func && NULL!=sconv_func->read) { -#ifndef NDEBUG - fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n"); -#endif status = (sconv_func->read)(dataset->ent.file, &(dataset->layout), &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, mem_space, buf/*out*/); if (status>=0) goto succeed; -#ifndef NDEBUG - fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n"); +#ifdef H5D_DEBUG + fprintf (stderr, "HDF5-DIAG: input pipe optimization failed " + "(falling through)\n"); #endif H5E_clear (); } /* - * This is the general case. - */ -#ifndef LATER - /* - * Note: this prototype version allocates a buffer large enough to - * satisfy the entire request; strip mining is not implemented. - */ - { - size_t src_size = nelmts * H5T_get_size(dataset->type); - size_t dst_size = nelmts * H5T_get_size(mem_type); - tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size)); - if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size); - } -#endif - /* - * Gather the data from disk into the data type conversion buffer. Also - * gather data from application to background buffer (this step is not - * needed for most conversions, but we leave that as an exercise for - * later ;-) + * This is the general case. Figure out the strip mine size. */ - if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout), - &(dataset->create_parms->efl), - H5T_get_size (dataset->type), file_space, - &numbering, 0, nelmts, - tconv_buf/*out*/)!=nelmts) { - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed"); - } - if (H5T_BKG_YES==cdata->need_bkg) { - if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space, - &numbering, 0, nelmts, - bkg_buf/*out*/)!=nelmts) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed"); + src_type_size = H5T_get_size(dataset->type); + dst_type_size = H5T_get_size(mem_type); + target_size = xfer_parms->buf_size; + request_nelmts = target_size / MAX(src_type_size, dst_type_size); + if (request_nelmts<=0) { + HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, + "temporary buffer max size is too small"); + } + if (sconv_func->init) { + smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space, + file_space, request_nelmts, + &numbering/*out*/); + if (smine_nelmts<=0) { + HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize element numbering information"); } + } else { + smine_nelmts = request_nelmts; + HDmemset (&numbering, 0, sizeof numbering); } + buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size); /* - * Perform data type conversion. + * Get a temporary buffer for type conversion unless the app has already + * supplied one through the xfer properties. Instead of allocating a + * buffer which is the exact size, we allocate the target size. The + * malloc() is usually less resource-intensive if we allocate/free the + * same size over and over. */ - cdata->command = H5T_CONV_CONV; - cdata->ncalls++; - if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) { - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "data type conversion failed"); + if (NULL==(tconv_buf=xfer_parms->tconv)) { + tconv_buf = H5MM_xmalloc (target_size); + } + if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) { + bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size); } - cdata->nelmts += nelmts; - /* - * Scatter the data into memory. - */ - if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space, - &numbering, 0, nelmts, buf/*out*/)<0) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed"); +#ifdef H5D_DEBUG + /* Strip mine diagnostics.... */ + if (smine_nelmtsfgath)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->efl), + H5T_get_size (dataset->type), file_space, + &numbering, smine_start, smine_nelmts, + tconv_buf/*out*/)!=smine_nelmts) { + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed"); + } + if (H5T_BKG_YES==cdata->need_bkg) { + if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space, + &numbering, smine_start, smine_nelmts, + bkg_buf/*out*/)!=smine_nelmts) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed"); + } + } + /* + * Perform data type conversion. + */ + cdata->command = H5T_CONV_CONV; + cdata->ncalls++; + if ((tconv_func)(src_id, dst_id, cdata, smine_nelmts, tconv_buf, + bkg_buf)<0) { + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "data type conversion failed"); + } + cdata->nelmts += smine_nelmts; + + /* + * Scatter the data into memory. + */ + if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space, + &numbering, smine_start, smine_nelmts, + buf/*out*/)<0) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed"); + } + } succeed: ret_value = SUCCEED; @@ -1232,8 +1271,12 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, done: if (src_id >= 0) H5A_dec_ref(src_id); if (dst_id >= 0) H5A_dec_ref(dst_id); - tconv_buf = H5MM_xfree(tconv_buf); - bkg_buf = H5MM_xfree (bkg_buf); + if (tconv_buf && NULL==xfer_parms->tconv) { + H5MM_xfree(tconv_buf); + } + if (bkg_buf && NULL==xfer_parms->bkg) { + H5MM_xfree (bkg_buf); + } FUNC_LEAVE(ret_value); } @@ -1260,7 +1303,9 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, const H5S_t *file_space, const H5D_xfer_t *xfer_parms, const void *buf) { - size_t nelmts; + size_t nelmts; /*total number of elmts */ + size_t smine_start; /*strip mine start loc */ + size_t smine_nelmts; /*elements per strip */ uint8 *tconv_buf = NULL; /*data type conv buffer */ uint8 *bkg_buf = NULL; /*background buffer */ H5T_conv_t tconv_func = NULL; /*conversion function */ @@ -1269,6 +1314,11 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, H5S_number_t numbering; /*element numbering info*/ H5T_cdata_t *cdata = NULL; /*type conversion data */ herr_t ret_value = FAIL, status; + size_t src_type_size; /*size of source type */ + size_t dst_type_size; /*size of destination type*/ + size_t target_size; /*desired buffer size */ + size_t buffer_size; /*actual buffer size */ + size_t request_nelmts; /*requested strip mine */ FUNC_ENTER(H5D_write, FAIL); @@ -1279,47 +1329,32 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, assert(buf); if (!file_space) file_space = dataset->space; if (!mem_space) mem_space = file_space; + nelmts = H5S_get_npoints(mem_space); /* - * Convert data types to atoms because the conversion functions are - * application-level functions. - */ - if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 || - (dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) { - HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, - "unable to register types for conversion"); - } - - /* * Locate the type conversion function and data space conversion - * functions, and set up the element numbering information. + * functions, and set up the element numbering information. If a data + * type conversion is necessary then register data type atoms. */ if (NULL == (tconv_func = H5T_find(mem_type, dataset->type, &cdata))) { HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to convert between src and dest data types"); + } else if (H5T_conv_noop!=tconv_func) { + if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 || + (dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) { + HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, + "unable to register types for conversion"); + } } if (NULL==(sconv_func=H5S_find (mem_space, file_space))) { HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to convert from memory to file data space"); } - if (sconv_func->init && - (sconv_func->init)(&(dataset->layout), mem_space, file_space, - &numbering/*out*/)<=0) { - HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to initialize element numbering information"); - } else { - HDmemset (&numbering, 0, sizeof numbering); - } - if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) { + if (nelmts!=H5S_get_npoints (file_space)) { HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "src and dest data spaces have different sizes"); } - - /* - * Compute the size of the request and allocate scratch buffers. - */ - nelmts = H5S_get_npoints(mem_space); - + /* * If there is no type conversion then try writing directly from * application buffer to file. @@ -1327,75 +1362,123 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, if (H5D_OPTIMIZE_PIPE && H5T_conv_noop==tconv_func && NULL!=sconv_func->write) { -#ifndef NDEBUG - fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n"); -#endif status = (sconv_func->write)(dataset->ent.file, &(dataset->layout), &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, mem_space, buf); if (status>=0) goto succeed; -#ifndef NDEBUG - fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n"); +#ifdef H5D_DEBUG + fprintf (stderr, "HDF5-DIAG: output pipe optimization failed " + "(falling through)\n"); #endif H5E_clear (); } /* - * This is the general case. + * This is the general case. Figure out the strip mine size. */ -#ifndef LATER - /* - * Note: This prototype version allocates a buffer large enough to - * satisfy the entire request; strip mining is not implemented. - */ - { - size_t src_size = nelmts * H5T_get_size(mem_type); - size_t dst_size = nelmts * H5T_get_size(dataset->type); - tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size)); - if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size); + src_type_size = H5T_get_size(mem_type); + dst_type_size = H5T_get_size(dataset->type); + target_size = xfer_parms->buf_size; + request_nelmts = target_size / MAX (src_type_size, dst_type_size); + if (request_nelmts<=0) { + HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, + "temporary buffer max size is too small"); + } + if (sconv_func->init) { + smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space, + file_space, request_nelmts, + &numbering/*out*/); + if (smine_nelmts<=0) { + HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize element numbering information"); + } + } else { + smine_nelmts = request_nelmts; + HDmemset (&numbering, 0, sizeof numbering); } -#endif + buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size); + /* - * Gather data from application buffer into the data type conversion - * buffer. Also gather data from the file into the background buffer - * (this step is not needed for most conversions, but we leave that as an - * exercise for later ;-) + * Get a temporary buffer for type conversion unless the app has already + * supplied one through the xfer properties. Instead of allocating a + * buffer which is the exact size, we allocate the target size. The + * malloc() is usually less resource-intensive if we allocate/free the + * same size over and over. */ - if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space, - &numbering, 0, nelmts, tconv_buf/*out*/)!=nelmts) { - HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed"); + if (NULL==(tconv_buf=xfer_parms->tconv)) { + tconv_buf = H5MM_xmalloc (target_size); } - if (H5T_BKG_YES==cdata->need_bkg) { - if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout), - &(dataset->create_parms->efl), - H5T_get_size (dataset->type), file_space, - &numbering, 0, nelmts, - bkg_buf/*out*/)!=nelmts) { - HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed"); - } + if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) { + bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size); } - /* - * Perform data type conversion. - */ - cdata->command = H5T_CONV_CONV; - cdata->ncalls++; - if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) { - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "data type conversion failed"); +#ifdef H5D_DEBUG + /* Strip mine diagnostics.... */ + if (smine_nelmtsnelmts += nelmts; +#endif - /* - * Scatter the data out to the file. - */ - if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout), - &(dataset->create_parms->efl), - H5T_get_size (dataset->type), file_space, - &numbering, 0, nelmts, tconv_buf)<0) { - HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed"); + /* Start strip mining... */ + for (smine_start=0; smine_startmgath)(buf, H5T_get_size (mem_type), mem_space, + &numbering, smine_start, smine_nelmts, + tconv_buf/*out*/)!=smine_nelmts) { + HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed"); + } + if (H5T_BKG_YES==cdata->need_bkg) { + if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->efl), + H5T_get_size (dataset->type), file_space, + &numbering, smine_start, smine_nelmts, + bkg_buf/*out*/)!=smine_nelmts) { + HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "file gather failed"); + } + } + + /* + * Perform data type conversion. + */ + cdata->command = H5T_CONV_CONV; + cdata->ncalls++; + if ((tconv_func) (src_id, dst_id, cdata, smine_nelmts, tconv_buf, + bkg_buf)<0) { + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "data type conversion failed"); + } + cdata->nelmts += smine_nelmts; + + /* + * Scatter the data out to the file. + */ + if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->efl), + H5T_get_size (dataset->type), file_space, + &numbering, smine_start, smine_nelmts, + tconv_buf)<0) { + HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed"); + } } succeed: @@ -1404,10 +1487,15 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, done: if (src_id >= 0) H5A_dec_ref(src_id); if (dst_id >= 0) H5A_dec_ref(dst_id); - tconv_buf = H5MM_xfree(tconv_buf); - bkg_buf = H5MM_xfree (bkg_buf); + if (tconv_buf && NULL==xfer_parms->tconv) { + H5MM_xfree(tconv_buf); + } + if (bkg_buf && NULL==xfer_parms->bkg) { + H5MM_xfree (bkg_buf); + } FUNC_LEAVE(ret_value); } + /*------------------------------------------------------------------------- * Function: H5D_extend diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index fc6eeb4..f678ef7 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -26,6 +26,15 @@ #include /*for the H5S_t type */ #include /*object Headers */ +/* + * Feature: Define H5D_DEBUG on the compiler command line if you want to + * debug dataset I/O. NDEBUG must not be defined in order for this + * to have any effect. + */ +#ifdef NDEBUG +# undef H5D_DEBUG +#endif + #define H5D_RESERVED_ATOMS 0 /* Set the minimum object header size to create objects with */ @@ -41,8 +50,11 @@ typedef struct H5D_create_t { /* Dataset transfer property list */ typedef struct H5D_xfer_t { - int _placeholder; /*unused--delete this later */ + size_t buf_size; /*max temp buffer size */ + void *tconv; /*type conversion buffer or null */ + void *bkg; /*background buffer or null */ } H5D_xfer_t; + typedef struct H5D_t H5D_t; extern const H5D_create_t H5D_create_dflt; extern const H5D_xfer_t H5D_xfer_dflt; diff --git a/src/H5P.c b/src/H5P.c index 8bbf03d..e990675 100644 --- a/src/H5P.c +++ b/src/H5P.c @@ -1679,6 +1679,100 @@ H5Pget_family (hid_t tid, hid_t *memb_tid) /*------------------------------------------------------------------------- + * Function: H5Pset_buffer + * + * Purpose: Given a dataset transfer property list, set the maximum size + * for the type conversion buffer and background buffer and + * optionally supply pointers to application-allocated buffers. + * If the buffer size is smaller than the entire amount of data + * being transfered between application and file, and a type + * conversion buffer or background buffer is required then + * strip mining will be used. However, certain restrictions + * apply for the size of buffer which can be used for strip + * mining. For instance, when strip mining a 100x200x300 + * hyperslab of a simple data space the buffer must be large + * enough to hold a 1x200x300 slab. + * + * If TCONV and/or BKG are null pointers then buffers will be + * allocated and freed during the data transfer. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Monday, March 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg) +{ + H5D_xfer_t *plist = NULL; + + FUNC_ENTER (H5Pset_buffer, FAIL); + + /* Check arguments */ + if (H5P_DATASET_XFER != H5Pget_class (plist_id) || + NULL == (plist = H5A_object (plist_id))) { + HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, + "not a dataset transfer property list"); + } + if (size<=0) { + HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, + "buffer size must not be zero"); + } + + /* Update property list */ + plist->buf_size = size; + plist->tconv = tconv; + plist->bkg = bkg; + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5Pget_buffer + * + * Purpose: Reads values previously set with H5Pset_buffer(). + * + * Return: Success: Buffer size. + * + * Failure: 0 + * + * Programmer: Robb Matzke + * Monday, March 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +size_t +H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/) +{ + H5D_xfer_t *plist = NULL; + + FUNC_ENTER (H5Pget_buffer, 0); + + /* Check arguments */ + if (H5P_DATASET_XFER != H5Pget_class (plist_id) || + NULL == (plist = H5A_object (plist_id))) { + HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, 0, + "not a dataset transfer property list"); + } + + /* Return values */ + if (tconv) *tconv = plist->tconv; + if (bkg) *bkg = plist->bkg; + + FUNC_LEAVE (plist->buf_size); +} + + +/*------------------------------------------------------------------------- * Function: H5Pset_mpi * * Signature: herr_t H5Pset_mpi(hid_t tid, MPI_Comm comm, MPI_Info info, diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 47e9b5a..7e91ce2 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -83,8 +83,12 @@ herr_t H5Pget_split (hid_t tid, size_t meta_ext_size, char *meta_ext/*out*/, herr_t H5Pset_family (hid_t tid, hid_t memb_tid); herr_t H5Pget_family (hid_t tid, hid_t *memb_tid/*out*/); +herr_t H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg); +size_t H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/); + #ifdef HAVE_PARALLEL -herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info, unsigned access_mode); +herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info, + unsigned access_mode); herr_t H5Pget_mpi (hid_t tid, MPI_Comm *comm/*out*/, MPI_Info *info/*out*/, unsigned *access_mode/*out*/); #endif diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index fbce725..da19695 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -65,7 +65,8 @@ typedef struct H5S_number_t { typedef struct H5S_tconv_t { /* Initialize element numbering information */ size_t (*init)(const struct H5O_layout_t *layout, const H5S_t *mem_space, - const H5S_t *file_space, H5S_number_t *numbering/*out*/); + const H5S_t *file_space, size_t desired_nelmts, + H5S_number_t *numbering/*out*/); /* Gather elements from disk to type conversion buffer */ size_t (*fgath)(H5F_t *f, const struct H5O_layout_t *layout, @@ -122,7 +123,7 @@ intn H5S_extend (H5S_t *space, const size_t *size); /* Conversion functions for simple data spaces */ size_t H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space, const H5S_t *file_space, - H5S_number_t *numbering/*out*/); + size_t desired_nelmts, H5S_number_t *numbering/*out*/); size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_number_t *numbering, diff --git a/src/H5Ssimp.c b/src/H5Ssimp.c index 6f1f498..a785b35 100644 --- a/src/H5Ssimp.c +++ b/src/H5Ssimp.c @@ -37,9 +37,14 @@ static intn interface_initialize_g = FALSE; */ size_t H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space, - const H5S_t *file_space, H5S_number_t *numbering/*out*/) + const H5S_t *file_space, size_t desired_nelmts, + H5S_number_t *numbering/*out*/) { size_t nelmts; + int m_ndims, f_ndims; /*mem, file dimensionality */ + size_t size[H5O_LAYOUT_NDIMS]; /*size of selected hyperslab */ + size_t acc; + int i; FUNC_ENTER (H5S_simp_init, 0); @@ -49,11 +54,43 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space, assert (file_space && H5S_SIMPLE==file_space->type); assert (numbering); - /* Numbering is implied by the hyperslab, C order */ + /* Numbering is implied by the hyperslab, C order, no data here */ HDmemset (numbering, 0, sizeof(H5S_number_t)); - /* Data can be efficiently copied at any size */ - nelmts = H5S_get_npoints (file_space); + /* + * The stripmine size is such that only the slowest varying dimension can + * be split up. We choose the largest possible strip mine size which is + * not larger than the desired size. + */ + m_ndims = H5S_get_hyperslab (mem_space, NULL, size, NULL); + for (i=m_ndims-1, acc=1; i>0; --i) acc *= size[i]; + nelmts = (desired_nelmts/acc) * acc; + if (nelmts<=0) { + HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0, + "strip mine buffer is too small"); + } + + /* + * The value chosen for mem_space must be the same as the value chosen for + * file_space. + */ + f_ndims = H5S_get_hyperslab (file_space, NULL, size, NULL); + if (m_ndims!=f_ndims) { + nelmts = H5S_get_npoints (file_space); + if (nelmts>desired_nelmts) { + HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0, + "strip mining not supported across " + "dimensionalities"); + } + assert (nelmts==H5S_get_npoints (mem_space)); + } else { + for (i=f_ndims-1, acc=1; i>0; --i) acc *= size[i]; + acc *= (desired_nelmts/acc); + if (nelmts!=acc) { + HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0, + "unsupported strip mine size for shape change"); + } + } FUNC_LEAVE (nelmts); } @@ -95,6 +132,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */ size_t zero[H5O_LAYOUT_NDIMS]; /*zero */ size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */ + size_t acc; /*accumulator */ #ifndef LATER intn file_offset_signed[H5O_LAYOUT_NDIMS]; #endif @@ -113,12 +151,6 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, assert (buf); /* - * The prototype doesn't support strip mining. - */ - assert (0==start); - assert (nelmts==H5S_get_npoints (file_space)); - - /* * Get hyperslab information to determine what elements are being * selected (there might eventually be other selection methods too). * We only support hyperslabs with unit sample because there's no way to @@ -132,6 +164,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, "unable to retrieve hyperslab parameters"); } #else + /* Argument type problems to be fixed later..... -RPM */ if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed, hsize, sample))<0) { HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0, @@ -142,12 +175,23 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, file_offset[i] = file_offset_signed[i]; } #endif + + /* Check that there is no subsampling of the hyperslab */ for (i=0; indims*sizeof(size_t)); @@ -194,6 +238,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size, size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */ size_t zero[H5O_LAYOUT_NDIMS]; /*zero */ size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */ + size_t acc; /*accumulator */ #ifndef LATER intn mem_offset_signed[H5O_LAYOUT_NDIMS]; #endif @@ -211,12 +256,6 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size, assert (buf); /* - * The prototype doesn't support strip mining. - */ - assert (0==start); - assert (nelmts==H5S_get_npoints (mem_space)); - - /* * Retrieve hyperslab information to determine what elements are being * selected (there might be other selection methods in the future). We * only handle hyperslabs with unit sample because there's currently no @@ -229,6 +268,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size, "unable to retrieve hyperslab parameters"); } #else + /* Argument type problems to be fixed later..... -RPM */ if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed, hsize, sample))<0) { HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, @@ -239,6 +279,8 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size, mem_offset[i] = mem_offset_signed[i]; } #endif + + /* Check that there is no subsampling of the hyperslab */ for (i=0; indims*sizeof(size_t)); diff --git a/test/Makefile.in b/test/Makefile.in index 3bf75a4..f4d9009 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -11,13 +11,14 @@ CPPFLAGS=-I. -I../src @CPPFLAGS@ # These are our main targets. They should be listed in the order to be # executed, generally most specific tests to least specific tests. -PROGS=testhdf5 hyperslab istore dtypes dsets cmpd_dset extend external \ +PROGS=testhdf5 hyperslab istore dtypes dsets cmpd_dset extend external \ iopipe -TESTS=$(PROGS) +TESTS=testhdf5 hyperslab istore dtypes dsets cmpd_dset extend external +TIMINGS=iopipe # Temporary files MOSTLYCLEAN=cmpd_dset.h5 dataset.h5 extend.h5 istore.h5 tfile1.h5 tfile2.h5 \ - tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5 \ + tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5 \ extern_1.h5 extern_2.h5 extern_3.h5 extern_1.raw extern_1b.raw \ extern_2.raw extern_2b.raw extern_3.raw extern_3b.raw \ extern_4.raw extern_4b.raw iopipe.raw iopipe.h5 @@ -64,6 +65,16 @@ IOPIPE_OBJ=$(IOPIPE_SRC:.c=.o) # Private header files (not to be installed)... PRIVATE_HDR=testhdf5.h +# Additional targets +.PHONY: timings _timings +timings _timings: $(TIMINGS) + @for timing in $(TIMINGS) dummy; do \ + if test $$timing != dummy; then \ + echo "Running $$timing $(TEST_FLAGS)"; \ + $(RUNTEST) ./$$timing $(TEST_FLAGS) || exit 1; \ + fi; \ + done; + # How to build the programs... testhdf5: $(TESTHDF5_OBJ) ../src/libhdf5.a $(CC) $(CFLAGS) -o $@ $(TESTHDF5_OBJ) ../src/libhdf5.a $(LIBS) diff --git a/test/dsets.c b/test/dsets.c index c92824c..eaca4a8 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -184,11 +184,12 @@ test_create(hid_t file) static herr_t test_simple_io(hid_t file) { - hid_t dataset, space; - herr_t status; - int points[100][200], check[100][200]; - int i, j, n; - size_t dims[2]; + hid_t dataset, space, xfer; + herr_t status; + int points[100][200], check[100][200]; + int i, j, n; + size_t dims[2]; + void *tconv_buf = NULL; printf("%-70s", "Testing simple I/O"); @@ -205,6 +206,13 @@ test_simple_io(hid_t file) space = H5Screate_simple(2, dims, NULL); assert(space>=0); + /* Create a small conversion buffer to test strip mining */ + tconv_buf = malloc (1000); + xfer = H5Pcreate (H5P_DATASET_XFER); + assert (xfer>=0); + status = H5Pset_buffer (xfer, 1000, tconv_buf, NULL); + assert (status>=0); + /* Create the dataset */ dataset = H5Dcreate(file, DSET_SIMPLE_IO_NAME, H5T_NATIVE_INT, space, H5P_DEFAULT); @@ -212,12 +220,12 @@ test_simple_io(hid_t file) /* Write the data to the dataset */ status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, points); + xfer, points); if (status<0) goto error; /* Read the dataset back */ status = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, - H5P_DEFAULT, check); + xfer, check); if (status<0) goto error; /* Check that the values read are the same as the values written */ diff --git a/test/iopipe.c b/test/iopipe.c index 813150e..f185aad 100644 --- a/test/iopipe.c +++ b/test/iopipe.c @@ -19,12 +19,22 @@ #define RAW_FILE_NAME "iopipe.raw" #define HDF5_FILE_NAME "iopipe.h5" +#define HEADING "%-16s" +#define PROGRESS '=' + +#if 1 +/* Normal testing */ #define REQUEST_SIZE_X 4579 #define REQUEST_SIZE_Y 4579 #define NREAD_REQUESTS 45 #define NWRITE_REQUESTS 45 -#define HEADING "%-16s" -#define PROGRESS '=' +#else +/* Speedy testing */ +#define REQUEST_SIZE_X 1000 +#define REQUEST_SIZE_Y 1000 +#define NREAD_REQUESTS 45 +#define NWRITE_REQUESTS 45 +#endif /*------------------------------------------------------------------------- @@ -83,10 +93,17 @@ print_stats (const char *prefix, static void synchronize (void) { +#ifdef HAVE_SYSTEM system ("sync"); system ("df >/dev/null"); #if 0 - system ("/sbin/swapout 130"); + /* + * This works well on Linux to get rid of all cached disk buffers. The + * number should be approximately the amount of RAM in MB. Do not + * include swap space in that amount or the command will fail. + */ + system ("/sbin/swapout 128"); +#endif #endif } -- cgit v0.12