summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorRobb Matzke <matzke@llnl.gov>1998-03-17 01:29:54 (GMT)
committerRobb Matzke <matzke@llnl.gov>1998-03-17 01:29:54 (GMT)
commit31a709a6b24f4cf80f9cd99a3e55f56e81cf3066 (patch)
treec38f19508a87b990f911f8fe2d526b0cb1f2a2b3 /src
parent48e075110624e0039b1918a7187316366a20462b (diff)
downloadhdf5-31a709a6b24f4cf80f9cd99a3e55f56e81cf3066.zip
hdf5-31a709a6b24f4cf80f9cd99a3e55f56e81cf3066.tar.gz
hdf5-31a709a6b24f4cf80f9cd99a3e55f56e81cf3066.tar.bz2
[svn-r322] Changes since 19980313
---------------------- ./configure.in ./configure ./test/iopipe.c Added check for system(3) ./config/freebsd2.2.1 ./config/linux Added -DH5D_DEBUG to the debug flags. ./src/H5D.c ./src/H5Dprivate.h ./src/H5P.c ./src/H5Ppublic.h ./src/H5Sprivate.h ./src/H5Ssimp.c ./html/Datasets.html Finally implemented strip mining in the I/O pipeline, placing a user-defined upper bound on the amount of temporary memory used by the pipeline. The default is 1MB type conversion and background buffers allocated/freed on demand. However, the size can be changed and/or application-allocated buffers supplied with H5Pset_buffers() called on the data transfer property list passed to H5Dread() and H5Dwrite(). Minor optimizations to H5Dread() and H5Dwrite(). More coming later... ./test/dsets.c Added calls to H5Pset_buffer() to test application-defined temporary buffers in the I/O pipeline. ./test/Makefile.in Removed `iopipe' from the list of confidence tests. Saying `make timings' in the test directory runs timing tests. I did this because (1) they don't really test anything new, and (2) they can take a long time to run.
Diffstat (limited to 'src')
-rw-r--r--src/H5D.c404
-rw-r--r--src/H5Dprivate.h14
-rw-r--r--src/H5P.c94
-rw-r--r--src/H5Ppublic.h6
-rw-r--r--src/H5Sprivate.h5
-rw-r--r--src/H5Ssimp.c121
6 files changed, 454 insertions, 190 deletions
diff --git a/src/H5D.c b/src/H5D.c
index b417ca7..fce1528 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -63,7 +63,9 @@ const H5D_create_t H5D_create_dflt = {
/* Default dataset transfer property list */
const H5D_xfer_t H5D_xfer_dflt = {
- 0, /* Place holder - remove this later */
+ 1024*1024, /* Temporary buffer size */
+ NULL, /* Type conversion buffer or NULL */
+ NULL, /* Background buffer or NULL */
};
/* Interface initialization? */
@@ -813,7 +815,7 @@ H5D_create(H5F_t *f, const char *name, const H5T_t *type, const H5S_t *space,
}
/* Create (open for write access) an object header */
- if (H5O_create(f, 80, &(new_dset->ent)) < 0) {
+ if (H5O_create(f, 96, &(new_dset->ent)) < 0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL,
"unable to create dataset object header");
}
@@ -1087,7 +1089,9 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
const H5S_t *file_space, const H5D_xfer_t *xfer_parms,
void *buf/*out*/)
{
- size_t nelmts ; /*number of elements */
+ size_t nelmts; /*number of elements */
+ size_t smine_start; /*strip mine start loc */
+ size_t smine_nelmts; /*elements per strip */
uint8 *tconv_buf = NULL; /*data type conv buffer */
uint8 *bkg_buf = NULL; /*background buffer */
H5T_conv_t tconv_func = NULL; /*conversion function */
@@ -1097,6 +1101,11 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
H5T_cdata_t *cdata = NULL; /*type conversion data */
herr_t ret_value = FAIL;
herr_t status;
+ size_t src_type_size; /*size of source type */
+ size_t dst_type_size; /*size of destination type*/
+ size_t target_size; /*desired buffer size */
+ size_t buffer_size; /*actual buffer size */
+ size_t request_nelmts; /*requested strip mine */
FUNC_ENTER(H5D_read, FAIL);
@@ -1107,124 +1116,154 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
assert(buf);
if (!file_space) file_space = dataset->space;
if (!mem_space) mem_space = file_space;
-
- /*
- * Convert data types to atoms because the conversion functions are
- * application-level functions.
- */
- if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type))) < 0 ||
- (dst_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type))) < 0) {
- HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
- "unable to register types for conversion");
- }
+ nelmts = H5S_get_npoints(mem_space);
/*
* Locate the type conversion function and data space conversion
- * functions, and set up the element numbering information.
+ * functions, and set up the element numbering information. If a data
+ * type conversion is necessary then register data type atoms.
*/
if (NULL == (tconv_func = H5T_find(dataset->type, mem_type, &cdata))) {
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"unable to convert between src and dest data types");
+ } else if (H5T_conv_noop!=tconv_func) {
+ if ((src_id=H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0 ||
+ (dst_id=H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0) {
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
+ "unable to register types for conversion");
+ }
}
if (NULL==(sconv_func=H5S_find (mem_space, file_space))) {
HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"unable to convert from file to memory data space");
}
- if (sconv_func->init &&
- (sconv_func->init)(&(dataset->layout), mem_space, file_space,
- &numbering/*out*/)<=0) {
- HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
- "unable to initialize element numbering information");
- } else {
- HDmemset (&numbering, 0, sizeof numbering);
- }
- if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) {
+ if (nelmts!=H5S_get_npoints (file_space)) {
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"src and dest data spaces have different sizes");
}
/*
- * Compute the size of the request and allocate scratch buffers.
- */
- nelmts = H5S_get_npoints(mem_space);
-
- /*
* If there is no type conversion then try reading directly into the
- * application's buffer.
+ * application's buffer. This saves at least one mem-to-mem copy.
*/
if (H5D_OPTIMIZE_PIPE &&
H5T_conv_noop==tconv_func &&
NULL!=sconv_func->read) {
-#ifndef NDEBUG
- fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n");
-#endif
status = (sconv_func->read)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
mem_space, buf/*out*/);
if (status>=0) goto succeed;
-#ifndef NDEBUG
- fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n");
+#ifdef H5D_DEBUG
+ fprintf (stderr, "HDF5-DIAG: input pipe optimization failed "
+ "(falling through)\n");
#endif
H5E_clear ();
}
/*
- * This is the general case.
- */
-#ifndef LATER
- /*
- * Note: this prototype version allocates a buffer large enough to
- * satisfy the entire request; strip mining is not implemented.
- */
- {
- size_t src_size = nelmts * H5T_get_size(dataset->type);
- size_t dst_size = nelmts * H5T_get_size(mem_type);
- tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size));
- if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size);
- }
-#endif
- /*
- * Gather the data from disk into the data type conversion buffer. Also
- * gather data from application to background buffer (this step is not
- * needed for most conversions, but we leave that as an exercise for
- * later ;-)
+ * This is the general case. Figure out the strip mine size.
*/
- if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
- &(dataset->create_parms->efl),
- H5T_get_size (dataset->type), file_space,
- &numbering, 0, nelmts,
- tconv_buf/*out*/)!=nelmts) {
- HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed");
- }
- if (H5T_BKG_YES==cdata->need_bkg) {
- if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
- &numbering, 0, nelmts,
- bkg_buf/*out*/)!=nelmts) {
- HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed");
+ src_type_size = H5T_get_size(dataset->type);
+ dst_type_size = H5T_get_size(mem_type);
+ target_size = xfer_parms->buf_size;
+ request_nelmts = target_size / MAX(src_type_size, dst_type_size);
+ if (request_nelmts<=0) {
+ HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
+ "temporary buffer max size is too small");
+ }
+ if (sconv_func->init) {
+ smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space,
+ file_space, request_nelmts,
+ &numbering/*out*/);
+ if (smine_nelmts<=0) {
+ HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
+ "unable to initialize element numbering information");
}
+ } else {
+ smine_nelmts = request_nelmts;
+ HDmemset (&numbering, 0, sizeof numbering);
}
+ buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size);
/*
- * Perform data type conversion.
+ * Get a temporary buffer for type conversion unless the app has already
+ * supplied one through the xfer properties. Instead of allocating a
+ * buffer which is the exact size, we allocate the target size. The
+ * malloc() is usually less resource-intensive if we allocate/free the
+ * same size over and over.
*/
- cdata->command = H5T_CONV_CONV;
- cdata->ncalls++;
- if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) {
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
- "data type conversion failed");
+ if (NULL==(tconv_buf=xfer_parms->tconv)) {
+ tconv_buf = H5MM_xmalloc (target_size);
+ }
+ if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) {
+ bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size);
}
- cdata->nelmts += nelmts;
- /*
- * Scatter the data into memory.
- */
- if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space,
- &numbering, 0, nelmts, buf/*out*/)<0) {
- HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed");
+#ifdef H5D_DEBUG
+ /* Strip mine diagnostics.... */
+ if (smine_nelmts<nelmts) {
+ fprintf (stderr, "HDF5-DIAG: strip mine");
+ if (smine_nelmts!=request_nelmts) {
+ fprintf (stderr, " got %lu of %lu",
+ (unsigned long)smine_nelmts,
+ (unsigned long)request_nelmts);
+ }
+ if (buffer_size!=target_size) {
+ fprintf (stderr, " (%1.1f%% of buffer)",
+ 100.0*buffer_size/target_size);
+ }
+ fprintf (stderr, " %1.1f iterations\n",
+ (double)nelmts/smine_nelmts);
}
+#endif
+
+ /* Start strip mining... */
+ for (smine_start=0; smine_start<nelmts; smine_start+=smine_nelmts) {
+ smine_nelmts = MIN (smine_nelmts, nelmts-smine_start);
+
+ /*
+ * Gather the data from disk into the data type conversion
+ * buffer. Also gather data from application to background buffer
+ * if necessary.
+ */
+ if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->efl),
+ H5T_get_size (dataset->type), file_space,
+ &numbering, smine_start, smine_nelmts,
+ tconv_buf/*out*/)!=smine_nelmts) {
+ HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed");
+ }
+ if (H5T_BKG_YES==cdata->need_bkg) {
+ if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
+ &numbering, smine_start, smine_nelmts,
+ bkg_buf/*out*/)!=smine_nelmts) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed");
+ }
+ }
+ /*
+ * Perform data type conversion.
+ */
+ cdata->command = H5T_CONV_CONV;
+ cdata->ncalls++;
+ if ((tconv_func)(src_id, dst_id, cdata, smine_nelmts, tconv_buf,
+ bkg_buf)<0) {
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
+ "data type conversion failed");
+ }
+ cdata->nelmts += smine_nelmts;
+
+ /*
+ * Scatter the data into memory.
+ */
+ if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space,
+ &numbering, smine_start, smine_nelmts,
+ buf/*out*/)<0) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed");
+ }
+ }
succeed:
ret_value = SUCCEED;
@@ -1232,8 +1271,12 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
done:
if (src_id >= 0) H5A_dec_ref(src_id);
if (dst_id >= 0) H5A_dec_ref(dst_id);
- tconv_buf = H5MM_xfree(tconv_buf);
- bkg_buf = H5MM_xfree (bkg_buf);
+ if (tconv_buf && NULL==xfer_parms->tconv) {
+ H5MM_xfree(tconv_buf);
+ }
+ if (bkg_buf && NULL==xfer_parms->bkg) {
+ H5MM_xfree (bkg_buf);
+ }
FUNC_LEAVE(ret_value);
}
@@ -1260,7 +1303,9 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
const H5S_t *file_space, const H5D_xfer_t *xfer_parms,
const void *buf)
{
- size_t nelmts;
+ size_t nelmts; /*total number of elmts */
+ size_t smine_start; /*strip mine start loc */
+ size_t smine_nelmts; /*elements per strip */
uint8 *tconv_buf = NULL; /*data type conv buffer */
uint8 *bkg_buf = NULL; /*background buffer */
H5T_conv_t tconv_func = NULL; /*conversion function */
@@ -1269,6 +1314,11 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
H5S_number_t numbering; /*element numbering info*/
H5T_cdata_t *cdata = NULL; /*type conversion data */
herr_t ret_value = FAIL, status;
+ size_t src_type_size; /*size of source type */
+ size_t dst_type_size; /*size of destination type*/
+ size_t target_size; /*desired buffer size */
+ size_t buffer_size; /*actual buffer size */
+ size_t request_nelmts; /*requested strip mine */
FUNC_ENTER(H5D_write, FAIL);
@@ -1279,47 +1329,32 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
assert(buf);
if (!file_space) file_space = dataset->space;
if (!mem_space) mem_space = file_space;
+ nelmts = H5S_get_npoints(mem_space);
/*
- * Convert data types to atoms because the conversion functions are
- * application-level functions.
- */
- if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 ||
- (dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) {
- HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
- "unable to register types for conversion");
- }
-
- /*
* Locate the type conversion function and data space conversion
- * functions, and set up the element numbering information.
+ * functions, and set up the element numbering information. If a data
+ * type conversion is necessary then register data type atoms.
*/
if (NULL == (tconv_func = H5T_find(mem_type, dataset->type, &cdata))) {
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"unable to convert between src and dest data types");
+ } else if (H5T_conv_noop!=tconv_func) {
+ if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 ||
+ (dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) {
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
+ "unable to register types for conversion");
+ }
}
if (NULL==(sconv_func=H5S_find (mem_space, file_space))) {
HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"unable to convert from memory to file data space");
}
- if (sconv_func->init &&
- (sconv_func->init)(&(dataset->layout), mem_space, file_space,
- &numbering/*out*/)<=0) {
- HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
- "unable to initialize element numbering information");
- } else {
- HDmemset (&numbering, 0, sizeof numbering);
- }
- if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) {
+ if (nelmts!=H5S_get_npoints (file_space)) {
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"src and dest data spaces have different sizes");
}
-
- /*
- * Compute the size of the request and allocate scratch buffers.
- */
- nelmts = H5S_get_npoints(mem_space);
-
+
/*
* If there is no type conversion then try writing directly from
* application buffer to file.
@@ -1327,75 +1362,123 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
if (H5D_OPTIMIZE_PIPE &&
H5T_conv_noop==tconv_func &&
NULL!=sconv_func->write) {
-#ifndef NDEBUG
- fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n");
-#endif
status = (sconv_func->write)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
mem_space, buf);
if (status>=0) goto succeed;
-#ifndef NDEBUG
- fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n");
+#ifdef H5D_DEBUG
+ fprintf (stderr, "HDF5-DIAG: output pipe optimization failed "
+ "(falling through)\n");
#endif
H5E_clear ();
}
/*
- * This is the general case.
+ * This is the general case. Figure out the strip mine size.
*/
-#ifndef LATER
- /*
- * Note: This prototype version allocates a buffer large enough to
- * satisfy the entire request; strip mining is not implemented.
- */
- {
- size_t src_size = nelmts * H5T_get_size(mem_type);
- size_t dst_size = nelmts * H5T_get_size(dataset->type);
- tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size));
- if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size);
+ src_type_size = H5T_get_size(mem_type);
+ dst_type_size = H5T_get_size(dataset->type);
+ target_size = xfer_parms->buf_size;
+ request_nelmts = target_size / MAX (src_type_size, dst_type_size);
+ if (request_nelmts<=0) {
+ HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
+ "temporary buffer max size is too small");
+ }
+ if (sconv_func->init) {
+ smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space,
+ file_space, request_nelmts,
+ &numbering/*out*/);
+ if (smine_nelmts<=0) {
+ HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
+ "unable to initialize element numbering information");
+ }
+ } else {
+ smine_nelmts = request_nelmts;
+ HDmemset (&numbering, 0, sizeof numbering);
}
-#endif
+ buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size);
+
/*
- * Gather data from application buffer into the data type conversion
- * buffer. Also gather data from the file into the background buffer
- * (this step is not needed for most conversions, but we leave that as an
- * exercise for later ;-)
+ * Get a temporary buffer for type conversion unless the app has already
+ * supplied one through the xfer properties. Instead of allocating a
+ * buffer which is the exact size, we allocate the target size. The
+ * malloc() is usually less resource-intensive if we allocate/free the
+ * same size over and over.
*/
- if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
- &numbering, 0, nelmts, tconv_buf/*out*/)!=nelmts) {
- HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed");
+ if (NULL==(tconv_buf=xfer_parms->tconv)) {
+ tconv_buf = H5MM_xmalloc (target_size);
}
- if (H5T_BKG_YES==cdata->need_bkg) {
- if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
- &(dataset->create_parms->efl),
- H5T_get_size (dataset->type), file_space,
- &numbering, 0, nelmts,
- bkg_buf/*out*/)!=nelmts) {
- HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed");
- }
+ if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) {
+ bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size);
}
- /*
- * Perform data type conversion.
- */
- cdata->command = H5T_CONV_CONV;
- cdata->ncalls++;
- if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) {
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
- "data type conversion failed");
+#ifdef H5D_DEBUG
+ /* Strip mine diagnostics.... */
+ if (smine_nelmts<nelmts) {
+ fprintf (stderr, "HDF5-DIAG: strip mine");
+ if (smine_nelmts!=request_nelmts) {
+ fprintf (stderr, " got %lu of %lu",
+ (unsigned long)smine_nelmts,
+ (unsigned long)request_nelmts);
+ }
+ if (buffer_size!=target_size) {
+ fprintf (stderr, " (%1.1f%% of buffer)",
+ 100.0*buffer_size/target_size);
+ }
+ fprintf (stderr, " %1.1f iterations\n",
+ (double)nelmts/smine_nelmts);
}
- cdata->nelmts += nelmts;
+#endif
- /*
- * Scatter the data out to the file.
- */
- if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
- &(dataset->create_parms->efl),
- H5T_get_size (dataset->type), file_space,
- &numbering, 0, nelmts, tconv_buf)<0) {
- HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed");
+ /* Start strip mining... */
+ for (smine_start=0; smine_start<nelmts; smine_start+=smine_nelmts) {
+ smine_nelmts = MIN (smine_nelmts, nelmts-smine_start);
+
+ /*
+ * Gather data from application buffer into the data type conversion
+ * buffer. Also gather data from the file into the background buffer
+ * if necessary.
+ */
+ if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
+ &numbering, smine_start, smine_nelmts,
+ tconv_buf/*out*/)!=smine_nelmts) {
+ HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed");
+ }
+ if (H5T_BKG_YES==cdata->need_bkg) {
+ if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->efl),
+ H5T_get_size (dataset->type), file_space,
+ &numbering, smine_start, smine_nelmts,
+ bkg_buf/*out*/)!=smine_nelmts) {
+ HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
+ "file gather failed");
+ }
+ }
+
+ /*
+ * Perform data type conversion.
+ */
+ cdata->command = H5T_CONV_CONV;
+ cdata->ncalls++;
+ if ((tconv_func) (src_id, dst_id, cdata, smine_nelmts, tconv_buf,
+ bkg_buf)<0) {
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
+ "data type conversion failed");
+ }
+ cdata->nelmts += smine_nelmts;
+
+ /*
+ * Scatter the data out to the file.
+ */
+ if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->efl),
+ H5T_get_size (dataset->type), file_space,
+ &numbering, smine_start, smine_nelmts,
+ tconv_buf)<0) {
+ HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed");
+ }
}
succeed:
@@ -1404,10 +1487,15 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
done:
if (src_id >= 0) H5A_dec_ref(src_id);
if (dst_id >= 0) H5A_dec_ref(dst_id);
- tconv_buf = H5MM_xfree(tconv_buf);
- bkg_buf = H5MM_xfree (bkg_buf);
+ if (tconv_buf && NULL==xfer_parms->tconv) {
+ H5MM_xfree(tconv_buf);
+ }
+ if (bkg_buf && NULL==xfer_parms->bkg) {
+ H5MM_xfree (bkg_buf);
+ }
FUNC_LEAVE(ret_value);
}
+
/*-------------------------------------------------------------------------
* Function: H5D_extend
diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h
index fc6eeb4..f678ef7 100644
--- a/src/H5Dprivate.h
+++ b/src/H5Dprivate.h
@@ -26,6 +26,15 @@
#include <H5Sprivate.h> /*for the H5S_t type */
#include <H5Oprivate.h> /*object Headers */
+/*
+ * Feature: Define H5D_DEBUG on the compiler command line if you want to
+ * debug dataset I/O. NDEBUG must not be defined in order for this
+ * to have any effect.
+ */
+#ifdef NDEBUG
+# undef H5D_DEBUG
+#endif
+
#define H5D_RESERVED_ATOMS 0
/* Set the minimum object header size to create objects with */
@@ -41,8 +50,11 @@ typedef struct H5D_create_t {
/* Dataset transfer property list */
typedef struct H5D_xfer_t {
- int _placeholder; /*unused--delete this later */
+ size_t buf_size; /*max temp buffer size */
+ void *tconv; /*type conversion buffer or null */
+ void *bkg; /*background buffer or null */
} H5D_xfer_t;
+
typedef struct H5D_t H5D_t;
extern const H5D_create_t H5D_create_dflt;
extern const H5D_xfer_t H5D_xfer_dflt;
diff --git a/src/H5P.c b/src/H5P.c
index 8bbf03d..e990675 100644
--- a/src/H5P.c
+++ b/src/H5P.c
@@ -1679,6 +1679,100 @@ H5Pget_family (hid_t tid, hid_t *memb_tid)
/*-------------------------------------------------------------------------
+ * Function: H5Pset_buffer
+ *
+ * Purpose: Given a dataset transfer property list, set the maximum size
+ * for the type conversion buffer and background buffer and
+ * optionally supply pointers to application-allocated buffers.
+ * If the buffer size is smaller than the entire amount of data
+ * being transfered between application and file, and a type
+ * conversion buffer or background buffer is required then
+ * strip mining will be used. However, certain restrictions
+ * apply for the size of buffer which can be used for strip
+ * mining. For instance, when strip mining a 100x200x300
+ * hyperslab of a simple data space the buffer must be large
+ * enough to hold a 1x200x300 slab.
+ *
+ * If TCONV and/or BKG are null pointers then buffers will be
+ * allocated and freed during the data transfer.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Monday, March 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg)
+{
+ H5D_xfer_t *plist = NULL;
+
+ FUNC_ENTER (H5Pset_buffer, FAIL);
+
+ /* Check arguments */
+ if (H5P_DATASET_XFER != H5Pget_class (plist_id) ||
+ NULL == (plist = H5A_object (plist_id))) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
+ "not a dataset transfer property list");
+ }
+ if (size<=0) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
+ "buffer size must not be zero");
+ }
+
+ /* Update property list */
+ plist->buf_size = size;
+ plist->tconv = tconv;
+ plist->bkg = bkg;
+
+ FUNC_LEAVE (SUCCEED);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_buffer
+ *
+ * Purpose: Reads values previously set with H5Pset_buffer().
+ *
+ * Return: Success: Buffer size.
+ *
+ * Failure: 0
+ *
+ * Programmer: Robb Matzke
+ * Monday, March 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+size_t
+H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/)
+{
+ H5D_xfer_t *plist = NULL;
+
+ FUNC_ENTER (H5Pget_buffer, 0);
+
+ /* Check arguments */
+ if (H5P_DATASET_XFER != H5Pget_class (plist_id) ||
+ NULL == (plist = H5A_object (plist_id))) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, 0,
+ "not a dataset transfer property list");
+ }
+
+ /* Return values */
+ if (tconv) *tconv = plist->tconv;
+ if (bkg) *bkg = plist->bkg;
+
+ FUNC_LEAVE (plist->buf_size);
+}
+
+
+/*-------------------------------------------------------------------------
* Function: H5Pset_mpi
*
* Signature: herr_t H5Pset_mpi(hid_t tid, MPI_Comm comm, MPI_Info info,
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index 47e9b5a..7e91ce2 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -83,8 +83,12 @@ herr_t H5Pget_split (hid_t tid, size_t meta_ext_size, char *meta_ext/*out*/,
herr_t H5Pset_family (hid_t tid, hid_t memb_tid);
herr_t H5Pget_family (hid_t tid, hid_t *memb_tid/*out*/);
+herr_t H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg);
+size_t H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/);
+
#ifdef HAVE_PARALLEL
-herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info, unsigned access_mode);
+herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info,
+ unsigned access_mode);
herr_t H5Pget_mpi (hid_t tid, MPI_Comm *comm/*out*/, MPI_Info *info/*out*/,
unsigned *access_mode/*out*/);
#endif
diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h
index fbce725..da19695 100644
--- a/src/H5Sprivate.h
+++ b/src/H5Sprivate.h
@@ -65,7 +65,8 @@ typedef struct H5S_number_t {
typedef struct H5S_tconv_t {
/* Initialize element numbering information */
size_t (*init)(const struct H5O_layout_t *layout, const H5S_t *mem_space,
- const H5S_t *file_space, H5S_number_t *numbering/*out*/);
+ const H5S_t *file_space, size_t desired_nelmts,
+ H5S_number_t *numbering/*out*/);
/* Gather elements from disk to type conversion buffer */
size_t (*fgath)(H5F_t *f, const struct H5O_layout_t *layout,
@@ -122,7 +123,7 @@ intn H5S_extend (H5S_t *space, const size_t *size);
/* Conversion functions for simple data spaces */
size_t H5S_simp_init (const struct H5O_layout_t *layout,
const H5S_t *mem_space, const H5S_t *file_space,
- H5S_number_t *numbering/*out*/);
+ size_t desired_nelmts, H5S_number_t *numbering/*out*/);
size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_number_t *numbering,
diff --git a/src/H5Ssimp.c b/src/H5Ssimp.c
index 6f1f498..a785b35 100644
--- a/src/H5Ssimp.c
+++ b/src/H5Ssimp.c
@@ -37,9 +37,14 @@ static intn interface_initialize_g = FALSE;
*/
size_t
H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
- const H5S_t *file_space, H5S_number_t *numbering/*out*/)
+ const H5S_t *file_space, size_t desired_nelmts,
+ H5S_number_t *numbering/*out*/)
{
size_t nelmts;
+ int m_ndims, f_ndims; /*mem, file dimensionality */
+ size_t size[H5O_LAYOUT_NDIMS]; /*size of selected hyperslab */
+ size_t acc;
+ int i;
FUNC_ENTER (H5S_simp_init, 0);
@@ -49,11 +54,43 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
assert (file_space && H5S_SIMPLE==file_space->type);
assert (numbering);
- /* Numbering is implied by the hyperslab, C order */
+ /* Numbering is implied by the hyperslab, C order, no data here */
HDmemset (numbering, 0, sizeof(H5S_number_t));
- /* Data can be efficiently copied at any size */
- nelmts = H5S_get_npoints (file_space);
+ /*
+ * The stripmine size is such that only the slowest varying dimension can
+ * be split up. We choose the largest possible strip mine size which is
+ * not larger than the desired size.
+ */
+ m_ndims = H5S_get_hyperslab (mem_space, NULL, size, NULL);
+ for (i=m_ndims-1, acc=1; i>0; --i) acc *= size[i];
+ nelmts = (desired_nelmts/acc) * acc;
+ if (nelmts<=0) {
+ HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
+ "strip mine buffer is too small");
+ }
+
+ /*
+ * The value chosen for mem_space must be the same as the value chosen for
+ * file_space.
+ */
+ f_ndims = H5S_get_hyperslab (file_space, NULL, size, NULL);
+ if (m_ndims!=f_ndims) {
+ nelmts = H5S_get_npoints (file_space);
+ if (nelmts>desired_nelmts) {
+ HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
+ "strip mining not supported across "
+ "dimensionalities");
+ }
+ assert (nelmts==H5S_get_npoints (mem_space));
+ } else {
+ for (i=f_ndims-1, acc=1; i>0; --i) acc *= size[i];
+ acc *= (desired_nelmts/acc);
+ if (nelmts!=acc) {
+ HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
+ "unsupported strip mine size for shape change");
+ }
+ }
FUNC_LEAVE (nelmts);
}
@@ -95,6 +132,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
size_t zero[H5O_LAYOUT_NDIMS]; /*zero */
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
+ size_t acc; /*accumulator */
#ifndef LATER
intn file_offset_signed[H5O_LAYOUT_NDIMS];
#endif
@@ -113,12 +151,6 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
assert (buf);
/*
- * The prototype doesn't support strip mining.
- */
- assert (0==start);
- assert (nelmts==H5S_get_npoints (file_space));
-
- /*
* Get hyperslab information to determine what elements are being
* selected (there might eventually be other selection methods too).
* We only support hyperslabs with unit sample because there's no way to
@@ -132,6 +164,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
"unable to retrieve hyperslab parameters");
}
#else
+ /* Argument type problems to be fixed later..... -RPM */
if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed,
hsize, sample))<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0,
@@ -142,12 +175,23 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
file_offset[i] = file_offset_signed[i];
}
#endif
+
+ /* Check that there is no subsampling of the hyperslab */
for (i=0; i<space_ndims; i++) {
if (sample[i]!=1) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, 0,
"hyperslab sampling is not implemented yet");
}
}
+
+ /* Adjust the slowest varying dimension to take care of strip mining */
+ for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
+ assert (0==start % acc);
+ assert (0==nelmts % acc);
+ file_offset[0] += start / acc;
+ hsize[0] = nelmts / acc;
+
+ /* The fastest varying dimension is for the data point itself */
file_offset[space_ndims] = 0;
hsize[space_ndims] = elmt_size;
HDmemset (zero, 0, layout->ndims*sizeof(size_t));
@@ -194,6 +238,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
size_t zero[H5O_LAYOUT_NDIMS]; /*zero */
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
+ size_t acc; /*accumulator */
#ifndef LATER
intn mem_offset_signed[H5O_LAYOUT_NDIMS];
#endif
@@ -211,12 +256,6 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
assert (buf);
/*
- * The prototype doesn't support strip mining.
- */
- assert (0==start);
- assert (nelmts==H5S_get_npoints (mem_space));
-
- /*
* Retrieve hyperslab information to determine what elements are being
* selected (there might be other selection methods in the future). We
* only handle hyperslabs with unit sample because there's currently no
@@ -229,6 +268,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
"unable to retrieve hyperslab parameters");
}
#else
+ /* Argument type problems to be fixed later..... -RPM */
if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed,
hsize, sample))<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
@@ -239,6 +279,8 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
mem_offset[i] = mem_offset_signed[i];
}
#endif
+
+ /* Check that there is no subsampling of the hyperslab */
for (i=0; i<space_ndims; i++) {
if (sample[i]!=1) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
@@ -249,6 +291,15 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
"unable to retrieve data space dimensions");
}
+
+ /* Adjust the slowest varying dimension to take care of strip mining */
+ for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
+ assert (0==start % acc);
+ assert (0==nelmts % acc);
+ mem_offset[0] += start / acc;
+ hsize[0] = nelmts / acc;
+
+ /* The fastest varying dimension is for the data point itself */
mem_offset[space_ndims] = 0;
mem_size[space_ndims] = elmt_size;
hsize[space_ndims] = elmt_size;
@@ -299,6 +350,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
size_t zero[H5O_LAYOUT_NDIMS]; /*zero */
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
+ size_t acc; /*accumulator */
#ifndef LATER
intn mem_offset_signed[H5O_LAYOUT_NDIMS];
#endif
@@ -316,12 +368,6 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
assert (tconv_buf);
/*
- * The prototype doesn't support strip mining.
- */
- assert (0==start);
- assert (nelmts==H5S_get_npoints (mem_space));
-
- /*
* Retrieve hyperslab information to determine what elements are being
* selected (there might be other selection methods in the future). We
* only handle hyperslabs with unit sample because there's currently no
@@ -334,6 +380,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
"unable to retrieve hyperslab parameters");
}
#else
+ /* Argument type problems to be fixed later..... -RPM */
if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed,
hsize, sample))<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
@@ -344,6 +391,8 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
mem_offset[i] = mem_offset_signed[i];
}
#endif
+
+ /* Check that there is no subsampling of the hyperslab */
for (i=0; i<space_ndims; i++) {
if (sample[i]!=1) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, 0,
@@ -354,6 +403,15 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0,
"unable to retrieve data space dimensions");
}
+
+ /* Adjust the slowest varying dimension to account for strip mining */
+ for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
+ assert (0==start % acc);
+ assert (0==nelmts % acc);
+ mem_offset[0] += start / acc;
+ hsize[0] = nelmts / acc;
+
+ /* The fastest varying dimension is for the data point itself */
mem_offset[space_ndims] = 0;
mem_size[space_ndims] = elmt_size;
hsize[space_ndims] = elmt_size;
@@ -404,6 +462,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
size_t zero[H5O_LAYOUT_NDIMS]; /*zero vector */
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
+ size_t acc; /*accumulator */
#ifndef LATER
intn file_offset_signed[H5O_LAYOUT_NDIMS];
#endif
@@ -422,12 +481,6 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
assert (buf);
/*
- * The prototype doesn't support strip mining.
- */
- assert (0==start);
- assert (nelmts==H5S_get_npoints (file_space));
-
- /*
* Get hyperslab information to determine what elements are being
* selected (there might eventually be other selection methods too).
* We only support hyperslabs with unit sample because there's no way to
@@ -441,6 +494,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
"unable to retrieve hyperslab parameters");
}
#else
+ /* Argument type problems to be fixed later..... -RPM */
if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed,
hsize, sample))<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
@@ -451,12 +505,23 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
file_offset[i] = file_offset_signed[i];
}
#endif
+
+ /* Check that there is no subsampling of the hyperslab */
for (i=0; i<space_ndims; i++) {
if (sample[i]!=1) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"hyperslab sampling is not implemented yet");
}
}
+
+ /* Adjust the slowest varying dimension to account for strip mining */
+ for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
+ assert (0==start % acc);
+ assert (0==nelmts % acc);
+ file_offset[0] += start / acc;
+ hsize[0] = nelmts / acc;
+
+ /* The fastest varying dimension is for the data point itself */
file_offset[space_ndims] = 0;
hsize[space_ndims] = elmt_size;
HDmemset (zero, 0, layout->ndims*sizeof(size_t));