diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/H5D.c | 404 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 14 | ||||
| -rw-r--r-- | src/H5P.c | 94 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 6 | ||||
| -rw-r--r-- | src/H5Sprivate.h | 5 | ||||
| -rw-r--r-- | src/H5Ssimp.c | 121 |
6 files changed, 454 insertions, 190 deletions
@@ -63,7 +63,9 @@ const H5D_create_t H5D_create_dflt = { /* Default dataset transfer property list */ const H5D_xfer_t H5D_xfer_dflt = { - 0, /* Place holder - remove this later */ + 1024*1024, /* Temporary buffer size */ + NULL, /* Type conversion buffer or NULL */ + NULL, /* Background buffer or NULL */ }; /* Interface initialization? */ @@ -813,7 +815,7 @@ H5D_create(H5F_t *f, const char *name, const H5T_t *type, const H5S_t *space, } /* Create (open for write access) an object header */ - if (H5O_create(f, 80, &(new_dset->ent)) < 0) { + if (H5O_create(f, 96, &(new_dset->ent)) < 0) { HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "unable to create dataset object header"); } @@ -1087,7 +1089,9 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, const H5S_t *file_space, const H5D_xfer_t *xfer_parms, void *buf/*out*/) { - size_t nelmts ; /*number of elements */ + size_t nelmts; /*number of elements */ + size_t smine_start; /*strip mine start loc */ + size_t smine_nelmts; /*elements per strip */ uint8 *tconv_buf = NULL; /*data type conv buffer */ uint8 *bkg_buf = NULL; /*background buffer */ H5T_conv_t tconv_func = NULL; /*conversion function */ @@ -1097,6 +1101,11 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, H5T_cdata_t *cdata = NULL; /*type conversion data */ herr_t ret_value = FAIL; herr_t status; + size_t src_type_size; /*size of source type */ + size_t dst_type_size; /*size of destination type*/ + size_t target_size; /*desired buffer size */ + size_t buffer_size; /*actual buffer size */ + size_t request_nelmts; /*requested strip mine */ FUNC_ENTER(H5D_read, FAIL); @@ -1107,124 +1116,154 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, assert(buf); if (!file_space) file_space = dataset->space; if (!mem_space) mem_space = file_space; - - /* - * Convert data types to atoms because the conversion functions are - * application-level functions. - */ - if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type))) < 0 || - (dst_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type))) < 0) { - HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, - "unable to register types for conversion"); - } + nelmts = H5S_get_npoints(mem_space); /* * Locate the type conversion function and data space conversion - * functions, and set up the element numbering information. + * functions, and set up the element numbering information. If a data + * type conversion is necessary then register data type atoms. */ if (NULL == (tconv_func = H5T_find(dataset->type, mem_type, &cdata))) { HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to convert between src and dest data types"); + } else if (H5T_conv_noop!=tconv_func) { + if ((src_id=H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0 || + (dst_id=H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0) { + HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, + "unable to register types for conversion"); + } } if (NULL==(sconv_func=H5S_find (mem_space, file_space))) { HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to convert from file to memory data space"); } - if (sconv_func->init && - (sconv_func->init)(&(dataset->layout), mem_space, file_space, - &numbering/*out*/)<=0) { - HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to initialize element numbering information"); - } else { - HDmemset (&numbering, 0, sizeof numbering); - } - if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) { + if (nelmts!=H5S_get_npoints (file_space)) { HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "src and dest data spaces have different sizes"); } /* - * Compute the size of the request and allocate scratch buffers. - */ - nelmts = H5S_get_npoints(mem_space); - - /* * If there is no type conversion then try reading directly into the - * application's buffer. + * application's buffer. This saves at least one mem-to-mem copy. */ if (H5D_OPTIMIZE_PIPE && H5T_conv_noop==tconv_func && NULL!=sconv_func->read) { -#ifndef NDEBUG - fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n"); -#endif status = (sconv_func->read)(dataset->ent.file, &(dataset->layout), &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, mem_space, buf/*out*/); if (status>=0) goto succeed; -#ifndef NDEBUG - fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n"); +#ifdef H5D_DEBUG + fprintf (stderr, "HDF5-DIAG: input pipe optimization failed " + "(falling through)\n"); #endif H5E_clear (); } /* - * This is the general case. - */ -#ifndef LATER - /* - * Note: this prototype version allocates a buffer large enough to - * satisfy the entire request; strip mining is not implemented. - */ - { - size_t src_size = nelmts * H5T_get_size(dataset->type); - size_t dst_size = nelmts * H5T_get_size(mem_type); - tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size)); - if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size); - } -#endif - /* - * Gather the data from disk into the data type conversion buffer. Also - * gather data from application to background buffer (this step is not - * needed for most conversions, but we leave that as an exercise for - * later ;-) + * This is the general case. Figure out the strip mine size. */ - if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout), - &(dataset->create_parms->efl), - H5T_get_size (dataset->type), file_space, - &numbering, 0, nelmts, - tconv_buf/*out*/)!=nelmts) { - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed"); - } - if (H5T_BKG_YES==cdata->need_bkg) { - if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space, - &numbering, 0, nelmts, - bkg_buf/*out*/)!=nelmts) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed"); + src_type_size = H5T_get_size(dataset->type); + dst_type_size = H5T_get_size(mem_type); + target_size = xfer_parms->buf_size; + request_nelmts = target_size / MAX(src_type_size, dst_type_size); + if (request_nelmts<=0) { + HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, + "temporary buffer max size is too small"); + } + if (sconv_func->init) { + smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space, + file_space, request_nelmts, + &numbering/*out*/); + if (smine_nelmts<=0) { + HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize element numbering information"); } + } else { + smine_nelmts = request_nelmts; + HDmemset (&numbering, 0, sizeof numbering); } + buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size); /* - * Perform data type conversion. + * Get a temporary buffer for type conversion unless the app has already + * supplied one through the xfer properties. Instead of allocating a + * buffer which is the exact size, we allocate the target size. The + * malloc() is usually less resource-intensive if we allocate/free the + * same size over and over. */ - cdata->command = H5T_CONV_CONV; - cdata->ncalls++; - if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) { - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "data type conversion failed"); + if (NULL==(tconv_buf=xfer_parms->tconv)) { + tconv_buf = H5MM_xmalloc (target_size); + } + if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) { + bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size); } - cdata->nelmts += nelmts; - /* - * Scatter the data into memory. - */ - if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space, - &numbering, 0, nelmts, buf/*out*/)<0) { - HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed"); +#ifdef H5D_DEBUG + /* Strip mine diagnostics.... */ + if (smine_nelmts<nelmts) { + fprintf (stderr, "HDF5-DIAG: strip mine"); + if (smine_nelmts!=request_nelmts) { + fprintf (stderr, " got %lu of %lu", + (unsigned long)smine_nelmts, + (unsigned long)request_nelmts); + } + if (buffer_size!=target_size) { + fprintf (stderr, " (%1.1f%% of buffer)", + 100.0*buffer_size/target_size); + } + fprintf (stderr, " %1.1f iterations\n", + (double)nelmts/smine_nelmts); } +#endif + + /* Start strip mining... */ + for (smine_start=0; smine_start<nelmts; smine_start+=smine_nelmts) { + smine_nelmts = MIN (smine_nelmts, nelmts-smine_start); + + /* + * Gather the data from disk into the data type conversion + * buffer. Also gather data from application to background buffer + * if necessary. + */ + if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->efl), + H5T_get_size (dataset->type), file_space, + &numbering, smine_start, smine_nelmts, + tconv_buf/*out*/)!=smine_nelmts) { + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed"); + } + if (H5T_BKG_YES==cdata->need_bkg) { + if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space, + &numbering, smine_start, smine_nelmts, + bkg_buf/*out*/)!=smine_nelmts) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed"); + } + } + /* + * Perform data type conversion. + */ + cdata->command = H5T_CONV_CONV; + cdata->ncalls++; + if ((tconv_func)(src_id, dst_id, cdata, smine_nelmts, tconv_buf, + bkg_buf)<0) { + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "data type conversion failed"); + } + cdata->nelmts += smine_nelmts; + + /* + * Scatter the data into memory. + */ + if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space, + &numbering, smine_start, smine_nelmts, + buf/*out*/)<0) { + HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed"); + } + } succeed: ret_value = SUCCEED; @@ -1232,8 +1271,12 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, done: if (src_id >= 0) H5A_dec_ref(src_id); if (dst_id >= 0) H5A_dec_ref(dst_id); - tconv_buf = H5MM_xfree(tconv_buf); - bkg_buf = H5MM_xfree (bkg_buf); + if (tconv_buf && NULL==xfer_parms->tconv) { + H5MM_xfree(tconv_buf); + } + if (bkg_buf && NULL==xfer_parms->bkg) { + H5MM_xfree (bkg_buf); + } FUNC_LEAVE(ret_value); } @@ -1260,7 +1303,9 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, const H5S_t *file_space, const H5D_xfer_t *xfer_parms, const void *buf) { - size_t nelmts; + size_t nelmts; /*total number of elmts */ + size_t smine_start; /*strip mine start loc */ + size_t smine_nelmts; /*elements per strip */ uint8 *tconv_buf = NULL; /*data type conv buffer */ uint8 *bkg_buf = NULL; /*background buffer */ H5T_conv_t tconv_func = NULL; /*conversion function */ @@ -1269,6 +1314,11 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, H5S_number_t numbering; /*element numbering info*/ H5T_cdata_t *cdata = NULL; /*type conversion data */ herr_t ret_value = FAIL, status; + size_t src_type_size; /*size of source type */ + size_t dst_type_size; /*size of destination type*/ + size_t target_size; /*desired buffer size */ + size_t buffer_size; /*actual buffer size */ + size_t request_nelmts; /*requested strip mine */ FUNC_ENTER(H5D_write, FAIL); @@ -1279,47 +1329,32 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, assert(buf); if (!file_space) file_space = dataset->space; if (!mem_space) mem_space = file_space; + nelmts = H5S_get_npoints(mem_space); /* - * Convert data types to atoms because the conversion functions are - * application-level functions. - */ - if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 || - (dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) { - HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, - "unable to register types for conversion"); - } - - /* * Locate the type conversion function and data space conversion - * functions, and set up the element numbering information. + * functions, and set up the element numbering information. If a data + * type conversion is necessary then register data type atoms. */ if (NULL == (tconv_func = H5T_find(mem_type, dataset->type, &cdata))) { HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to convert between src and dest data types"); + } else if (H5T_conv_noop!=tconv_func) { + if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 || + (dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) { + HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, + "unable to register types for conversion"); + } } if (NULL==(sconv_func=H5S_find (mem_space, file_space))) { HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to convert from memory to file data space"); } - if (sconv_func->init && - (sconv_func->init)(&(dataset->layout), mem_space, file_space, - &numbering/*out*/)<=0) { - HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, - "unable to initialize element numbering information"); - } else { - HDmemset (&numbering, 0, sizeof numbering); - } - if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) { + if (nelmts!=H5S_get_npoints (file_space)) { HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "src and dest data spaces have different sizes"); } - - /* - * Compute the size of the request and allocate scratch buffers. - */ - nelmts = H5S_get_npoints(mem_space); - + /* * If there is no type conversion then try writing directly from * application buffer to file. @@ -1327,75 +1362,123 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, if (H5D_OPTIMIZE_PIPE && H5T_conv_noop==tconv_func && NULL!=sconv_func->write) { -#ifndef NDEBUG - fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n"); -#endif status = (sconv_func->write)(dataset->ent.file, &(dataset->layout), &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, mem_space, buf); if (status>=0) goto succeed; -#ifndef NDEBUG - fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n"); +#ifdef H5D_DEBUG + fprintf (stderr, "HDF5-DIAG: output pipe optimization failed " + "(falling through)\n"); #endif H5E_clear (); } /* - * This is the general case. + * This is the general case. Figure out the strip mine size. */ -#ifndef LATER - /* - * Note: This prototype version allocates a buffer large enough to - * satisfy the entire request; strip mining is not implemented. - */ - { - size_t src_size = nelmts * H5T_get_size(mem_type); - size_t dst_size = nelmts * H5T_get_size(dataset->type); - tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size)); - if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size); + src_type_size = H5T_get_size(mem_type); + dst_type_size = H5T_get_size(dataset->type); + target_size = xfer_parms->buf_size; + request_nelmts = target_size / MAX (src_type_size, dst_type_size); + if (request_nelmts<=0) { + HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, + "temporary buffer max size is too small"); + } + if (sconv_func->init) { + smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space, + file_space, request_nelmts, + &numbering/*out*/); + if (smine_nelmts<=0) { + HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL, + "unable to initialize element numbering information"); + } + } else { + smine_nelmts = request_nelmts; + HDmemset (&numbering, 0, sizeof numbering); } -#endif + buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size); + /* - * Gather data from application buffer into the data type conversion - * buffer. Also gather data from the file into the background buffer - * (this step is not needed for most conversions, but we leave that as an - * exercise for later ;-) + * Get a temporary buffer for type conversion unless the app has already + * supplied one through the xfer properties. Instead of allocating a + * buffer which is the exact size, we allocate the target size. The + * malloc() is usually less resource-intensive if we allocate/free the + * same size over and over. */ - if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space, - &numbering, 0, nelmts, tconv_buf/*out*/)!=nelmts) { - HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed"); + if (NULL==(tconv_buf=xfer_parms->tconv)) { + tconv_buf = H5MM_xmalloc (target_size); } - if (H5T_BKG_YES==cdata->need_bkg) { - if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout), - &(dataset->create_parms->efl), - H5T_get_size (dataset->type), file_space, - &numbering, 0, nelmts, - bkg_buf/*out*/)!=nelmts) { - HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed"); - } + if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) { + bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size); } - /* - * Perform data type conversion. - */ - cdata->command = H5T_CONV_CONV; - cdata->ncalls++; - if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) { - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, - "data type conversion failed"); +#ifdef H5D_DEBUG + /* Strip mine diagnostics.... */ + if (smine_nelmts<nelmts) { + fprintf (stderr, "HDF5-DIAG: strip mine"); + if (smine_nelmts!=request_nelmts) { + fprintf (stderr, " got %lu of %lu", + (unsigned long)smine_nelmts, + (unsigned long)request_nelmts); + } + if (buffer_size!=target_size) { + fprintf (stderr, " (%1.1f%% of buffer)", + 100.0*buffer_size/target_size); + } + fprintf (stderr, " %1.1f iterations\n", + (double)nelmts/smine_nelmts); } - cdata->nelmts += nelmts; +#endif - /* - * Scatter the data out to the file. - */ - if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout), - &(dataset->create_parms->efl), - H5T_get_size (dataset->type), file_space, - &numbering, 0, nelmts, tconv_buf)<0) { - HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed"); + /* Start strip mining... */ + for (smine_start=0; smine_start<nelmts; smine_start+=smine_nelmts) { + smine_nelmts = MIN (smine_nelmts, nelmts-smine_start); + + /* + * Gather data from application buffer into the data type conversion + * buffer. Also gather data from the file into the background buffer + * if necessary. + */ + if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space, + &numbering, smine_start, smine_nelmts, + tconv_buf/*out*/)!=smine_nelmts) { + HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed"); + } + if (H5T_BKG_YES==cdata->need_bkg) { + if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->efl), + H5T_get_size (dataset->type), file_space, + &numbering, smine_start, smine_nelmts, + bkg_buf/*out*/)!=smine_nelmts) { + HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, + "file gather failed"); + } + } + + /* + * Perform data type conversion. + */ + cdata->command = H5T_CONV_CONV; + cdata->ncalls++; + if ((tconv_func) (src_id, dst_id, cdata, smine_nelmts, tconv_buf, + bkg_buf)<0) { + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, + "data type conversion failed"); + } + cdata->nelmts += smine_nelmts; + + /* + * Scatter the data out to the file. + */ + if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->efl), + H5T_get_size (dataset->type), file_space, + &numbering, smine_start, smine_nelmts, + tconv_buf)<0) { + HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed"); + } } succeed: @@ -1404,10 +1487,15 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, done: if (src_id >= 0) H5A_dec_ref(src_id); if (dst_id >= 0) H5A_dec_ref(dst_id); - tconv_buf = H5MM_xfree(tconv_buf); - bkg_buf = H5MM_xfree (bkg_buf); + if (tconv_buf && NULL==xfer_parms->tconv) { + H5MM_xfree(tconv_buf); + } + if (bkg_buf && NULL==xfer_parms->bkg) { + H5MM_xfree (bkg_buf); + } FUNC_LEAVE(ret_value); } + /*------------------------------------------------------------------------- * Function: H5D_extend diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index fc6eeb4..f678ef7 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -26,6 +26,15 @@ #include <H5Sprivate.h> /*for the H5S_t type */ #include <H5Oprivate.h> /*object Headers */ +/* + * Feature: Define H5D_DEBUG on the compiler command line if you want to + * debug dataset I/O. NDEBUG must not be defined in order for this + * to have any effect. + */ +#ifdef NDEBUG +# undef H5D_DEBUG +#endif + #define H5D_RESERVED_ATOMS 0 /* Set the minimum object header size to create objects with */ @@ -41,8 +50,11 @@ typedef struct H5D_create_t { /* Dataset transfer property list */ typedef struct H5D_xfer_t { - int _placeholder; /*unused--delete this later */ + size_t buf_size; /*max temp buffer size */ + void *tconv; /*type conversion buffer or null */ + void *bkg; /*background buffer or null */ } H5D_xfer_t; + typedef struct H5D_t H5D_t; extern const H5D_create_t H5D_create_dflt; extern const H5D_xfer_t H5D_xfer_dflt; @@ -1679,6 +1679,100 @@ H5Pget_family (hid_t tid, hid_t *memb_tid) /*------------------------------------------------------------------------- + * Function: H5Pset_buffer + * + * Purpose: Given a dataset transfer property list, set the maximum size + * for the type conversion buffer and background buffer and + * optionally supply pointers to application-allocated buffers. + * If the buffer size is smaller than the entire amount of data + * being transfered between application and file, and a type + * conversion buffer or background buffer is required then + * strip mining will be used. However, certain restrictions + * apply for the size of buffer which can be used for strip + * mining. For instance, when strip mining a 100x200x300 + * hyperslab of a simple data space the buffer must be large + * enough to hold a 1x200x300 slab. + * + * If TCONV and/or BKG are null pointers then buffers will be + * allocated and freed during the data transfer. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Monday, March 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg) +{ + H5D_xfer_t *plist = NULL; + + FUNC_ENTER (H5Pset_buffer, FAIL); + + /* Check arguments */ + if (H5P_DATASET_XFER != H5Pget_class (plist_id) || + NULL == (plist = H5A_object (plist_id))) { + HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL, + "not a dataset transfer property list"); + } + if (size<=0) { + HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, + "buffer size must not be zero"); + } + + /* Update property list */ + plist->buf_size = size; + plist->tconv = tconv; + plist->bkg = bkg; + + FUNC_LEAVE (SUCCEED); +} + + +/*------------------------------------------------------------------------- + * Function: H5Pget_buffer + * + * Purpose: Reads values previously set with H5Pset_buffer(). + * + * Return: Success: Buffer size. + * + * Failure: 0 + * + * Programmer: Robb Matzke + * Monday, March 16, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +size_t +H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/) +{ + H5D_xfer_t *plist = NULL; + + FUNC_ENTER (H5Pget_buffer, 0); + + /* Check arguments */ + if (H5P_DATASET_XFER != H5Pget_class (plist_id) || + NULL == (plist = H5A_object (plist_id))) { + HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, 0, + "not a dataset transfer property list"); + } + + /* Return values */ + if (tconv) *tconv = plist->tconv; + if (bkg) *bkg = plist->bkg; + + FUNC_LEAVE (plist->buf_size); +} + + +/*------------------------------------------------------------------------- * Function: H5Pset_mpi * * Signature: herr_t H5Pset_mpi(hid_t tid, MPI_Comm comm, MPI_Info info, diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 47e9b5a..7e91ce2 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -83,8 +83,12 @@ herr_t H5Pget_split (hid_t tid, size_t meta_ext_size, char *meta_ext/*out*/, herr_t H5Pset_family (hid_t tid, hid_t memb_tid); herr_t H5Pget_family (hid_t tid, hid_t *memb_tid/*out*/); +herr_t H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg); +size_t H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/); + #ifdef HAVE_PARALLEL -herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info, unsigned access_mode); +herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info, + unsigned access_mode); herr_t H5Pget_mpi (hid_t tid, MPI_Comm *comm/*out*/, MPI_Info *info/*out*/, unsigned *access_mode/*out*/); #endif diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index fbce725..da19695 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -65,7 +65,8 @@ typedef struct H5S_number_t { typedef struct H5S_tconv_t { /* Initialize element numbering information */ size_t (*init)(const struct H5O_layout_t *layout, const H5S_t *mem_space, - const H5S_t *file_space, H5S_number_t *numbering/*out*/); + const H5S_t *file_space, size_t desired_nelmts, + H5S_number_t *numbering/*out*/); /* Gather elements from disk to type conversion buffer */ size_t (*fgath)(H5F_t *f, const struct H5O_layout_t *layout, @@ -122,7 +123,7 @@ intn H5S_extend (H5S_t *space, const size_t *size); /* Conversion functions for simple data spaces */ size_t H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space, const H5S_t *file_space, - H5S_number_t *numbering/*out*/); + size_t desired_nelmts, H5S_number_t *numbering/*out*/); size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_number_t *numbering, diff --git a/src/H5Ssimp.c b/src/H5Ssimp.c index 6f1f498..a785b35 100644 --- a/src/H5Ssimp.c +++ b/src/H5Ssimp.c @@ -37,9 +37,14 @@ static intn interface_initialize_g = FALSE; */ size_t H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space, - const H5S_t *file_space, H5S_number_t *numbering/*out*/) + const H5S_t *file_space, size_t desired_nelmts, + H5S_number_t *numbering/*out*/) { size_t nelmts; + int m_ndims, f_ndims; /*mem, file dimensionality */ + size_t size[H5O_LAYOUT_NDIMS]; /*size of selected hyperslab */ + size_t acc; + int i; FUNC_ENTER (H5S_simp_init, 0); @@ -49,11 +54,43 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space, assert (file_space && H5S_SIMPLE==file_space->type); assert (numbering); - /* Numbering is implied by the hyperslab, C order */ + /* Numbering is implied by the hyperslab, C order, no data here */ HDmemset (numbering, 0, sizeof(H5S_number_t)); - /* Data can be efficiently copied at any size */ - nelmts = H5S_get_npoints (file_space); + /* + * The stripmine size is such that only the slowest varying dimension can + * be split up. We choose the largest possible strip mine size which is + * not larger than the desired size. + */ + m_ndims = H5S_get_hyperslab (mem_space, NULL, size, NULL); + for (i=m_ndims-1, acc=1; i>0; --i) acc *= size[i]; + nelmts = (desired_nelmts/acc) * acc; + if (nelmts<=0) { + HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0, + "strip mine buffer is too small"); + } + + /* + * The value chosen for mem_space must be the same as the value chosen for + * file_space. + */ + f_ndims = H5S_get_hyperslab (file_space, NULL, size, NULL); + if (m_ndims!=f_ndims) { + nelmts = H5S_get_npoints (file_space); + if (nelmts>desired_nelmts) { + HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0, + "strip mining not supported across " + "dimensionalities"); + } + assert (nelmts==H5S_get_npoints (mem_space)); + } else { + for (i=f_ndims-1, acc=1; i>0; --i) acc *= size[i]; + acc *= (desired_nelmts/acc); + if (nelmts!=acc) { + HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0, + "unsupported strip mine size for shape change"); + } + } FUNC_LEAVE (nelmts); } @@ -95,6 +132,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */ size_t zero[H5O_LAYOUT_NDIMS]; /*zero */ size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */ + size_t acc; /*accumulator */ #ifndef LATER intn file_offset_signed[H5O_LAYOUT_NDIMS]; #endif @@ -113,12 +151,6 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, assert (buf); /* - * The prototype doesn't support strip mining. - */ - assert (0==start); - assert (nelmts==H5S_get_npoints (file_space)); - - /* * Get hyperslab information to determine what elements are being * selected (there might eventually be other selection methods too). * We only support hyperslabs with unit sample because there's no way to @@ -132,6 +164,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, "unable to retrieve hyperslab parameters"); } #else + /* Argument type problems to be fixed later..... -RPM */ if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed, hsize, sample))<0) { HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0, @@ -142,12 +175,23 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, file_offset[i] = file_offset_signed[i]; } #endif + + /* Check that there is no subsampling of the hyperslab */ for (i=0; i<space_ndims; i++) { if (sample[i]!=1) { HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, 0, "hyperslab sampling is not implemented yet"); } } + + /* Adjust the slowest varying dimension to take care of strip mining */ + for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i]; + assert (0==start % acc); + assert (0==nelmts % acc); + file_offset[0] += start / acc; + hsize[0] = nelmts / acc; + + /* The fastest varying dimension is for the data point itself */ file_offset[space_ndims] = 0; hsize[space_ndims] = elmt_size; HDmemset (zero, 0, layout->ndims*sizeof(size_t)); @@ -194,6 +238,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size, size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */ size_t zero[H5O_LAYOUT_NDIMS]; /*zero */ size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */ + size_t acc; /*accumulator */ #ifndef LATER intn mem_offset_signed[H5O_LAYOUT_NDIMS]; #endif @@ -211,12 +256,6 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size, assert (buf); /* - * The prototype doesn't support strip mining. - */ - assert (0==start); - assert (nelmts==H5S_get_npoints (mem_space)); - - /* * Retrieve hyperslab information to determine what elements are being * selected (there might be other selection methods in the future). We * only handle hyperslabs with unit sample because there's currently no @@ -229,6 +268,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size, "unable to retrieve hyperslab parameters"); } #else + /* Argument type problems to be fixed later..... -RPM */ if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed, hsize, sample))<0) { HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, @@ -239,6 +279,8 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size, mem_offset[i] = mem_offset_signed[i]; } #endif + + /* Check that there is no subsampling of the hyperslab */ for (i=0; i<space_ndims; i++) { if (sample[i]!=1) { HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, @@ -249,6 +291,15 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size, HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, "unable to retrieve data space dimensions"); } + + /* Adjust the slowest varying dimension to take care of strip mining */ + for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i]; + assert (0==start % acc); + assert (0==nelmts % acc); + mem_offset[0] += start / acc; + hsize[0] = nelmts / acc; + + /* The fastest varying dimension is for the data point itself */ mem_offset[space_ndims] = 0; mem_size[space_ndims] = elmt_size; hsize[space_ndims] = elmt_size; @@ -299,6 +350,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size, size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */ size_t zero[H5O_LAYOUT_NDIMS]; /*zero */ size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */ + size_t acc; /*accumulator */ #ifndef LATER intn mem_offset_signed[H5O_LAYOUT_NDIMS]; #endif @@ -316,12 +368,6 @@ H5S_simp_mgath (const void *buf, size_t elmt_size, assert (tconv_buf); /* - * The prototype doesn't support strip mining. - */ - assert (0==start); - assert (nelmts==H5S_get_npoints (mem_space)); - - /* * Retrieve hyperslab information to determine what elements are being * selected (there might be other selection methods in the future). We * only handle hyperslabs with unit sample because there's currently no @@ -334,6 +380,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size, "unable to retrieve hyperslab parameters"); } #else + /* Argument type problems to be fixed later..... -RPM */ if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed, hsize, sample))<0) { HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, @@ -344,6 +391,8 @@ H5S_simp_mgath (const void *buf, size_t elmt_size, mem_offset[i] = mem_offset_signed[i]; } #endif + + /* Check that there is no subsampling of the hyperslab */ for (i=0; i<space_ndims; i++) { if (sample[i]!=1) { HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, 0, @@ -354,6 +403,15 @@ H5S_simp_mgath (const void *buf, size_t elmt_size, HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0, "unable to retrieve data space dimensions"); } + + /* Adjust the slowest varying dimension to account for strip mining */ + for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i]; + assert (0==start % acc); + assert (0==nelmts % acc); + mem_offset[0] += start / acc; + hsize[0] = nelmts / acc; + + /* The fastest varying dimension is for the data point itself */ mem_offset[space_ndims] = 0; mem_size[space_ndims] = elmt_size; hsize[space_ndims] = elmt_size; @@ -404,6 +462,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout, size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */ size_t zero[H5O_LAYOUT_NDIMS]; /*zero vector */ size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */ + size_t acc; /*accumulator */ #ifndef LATER intn file_offset_signed[H5O_LAYOUT_NDIMS]; #endif @@ -422,12 +481,6 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout, assert (buf); /* - * The prototype doesn't support strip mining. - */ - assert (0==start); - assert (nelmts==H5S_get_npoints (file_space)); - - /* * Get hyperslab information to determine what elements are being * selected (there might eventually be other selection methods too). * We only support hyperslabs with unit sample because there's no way to @@ -441,6 +494,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout, "unable to retrieve hyperslab parameters"); } #else + /* Argument type problems to be fixed later..... -RPM */ if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed, hsize, sample))<0) { HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL, @@ -451,12 +505,23 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout, file_offset[i] = file_offset_signed[i]; } #endif + + /* Check that there is no subsampling of the hyperslab */ for (i=0; i<space_ndims; i++) { if (sample[i]!=1) { HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "hyperslab sampling is not implemented yet"); } } + + /* Adjust the slowest varying dimension to account for strip mining */ + for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i]; + assert (0==start % acc); + assert (0==nelmts % acc); + file_offset[0] += start / acc; + hsize[0] = nelmts / acc; + + /* The fastest varying dimension is for the data point itself */ file_offset[space_ndims] = 0; hsize[space_ndims] = elmt_size; HDmemset (zero, 0, layout->ndims*sizeof(size_t)); |