summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRobb Matzke <matzke@llnl.gov>1998-03-17 01:29:54 (GMT)
committerRobb Matzke <matzke@llnl.gov>1998-03-17 01:29:54 (GMT)
commit31a709a6b24f4cf80f9cd99a3e55f56e81cf3066 (patch)
treec38f19508a87b990f911f8fe2d526b0cb1f2a2b3
parent48e075110624e0039b1918a7187316366a20462b (diff)
downloadhdf5-31a709a6b24f4cf80f9cd99a3e55f56e81cf3066.zip
hdf5-31a709a6b24f4cf80f9cd99a3e55f56e81cf3066.tar.gz
hdf5-31a709a6b24f4cf80f9cd99a3e55f56e81cf3066.tar.bz2
[svn-r322] Changes since 19980313
---------------------- ./configure.in ./configure ./test/iopipe.c Added check for system(3) ./config/freebsd2.2.1 ./config/linux Added -DH5D_DEBUG to the debug flags. ./src/H5D.c ./src/H5Dprivate.h ./src/H5P.c ./src/H5Ppublic.h ./src/H5Sprivate.h ./src/H5Ssimp.c ./html/Datasets.html Finally implemented strip mining in the I/O pipeline, placing a user-defined upper bound on the amount of temporary memory used by the pipeline. The default is 1MB type conversion and background buffers allocated/freed on demand. However, the size can be changed and/or application-allocated buffers supplied with H5Pset_buffers() called on the data transfer property list passed to H5Dread() and H5Dwrite(). Minor optimizations to H5Dread() and H5Dwrite(). More coming later... ./test/dsets.c Added calls to H5Pset_buffer() to test application-defined temporary buffers in the I/O pipeline. ./test/Makefile.in Removed `iopipe' from the list of confidence tests. Saying `make timings' in the test directory runs timing tests. I did this because (1) they don't really test anything new, and (2) they can take a long time to run.
-rw-r--r--config/freebsd2.2.12
-rw-r--r--config/linux2
-rwxr-xr-xconfigure2
-rw-r--r--configure.in2
-rw-r--r--src/H5D.c404
-rw-r--r--src/H5Dprivate.h14
-rw-r--r--src/H5P.c94
-rw-r--r--src/H5Ppublic.h6
-rw-r--r--src/H5Sprivate.h5
-rw-r--r--src/H5Ssimp.c121
-rw-r--r--test/Makefile.in17
-rw-r--r--test/dsets.c22
-rw-r--r--test/iopipe.c23
13 files changed, 507 insertions, 207 deletions
diff --git a/config/freebsd2.2.1 b/config/freebsd2.2.1
index 53aafb8..ef1c706 100644
--- a/config/freebsd2.2.1
+++ b/config/freebsd2.2.1
@@ -41,7 +41,7 @@ warn="-Wall -Wshadow -Wpointer-arith -Wcast-qual -Wwrite-strings -Wstrict-protot
profile="-pg"
-debug="-g -DH5AC_DEBUG -DH5B_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -fverbose-asm"
+debug="-g -DH5AC_DEBUG -DH5B_DEBUG -DH5D_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -fverbose-asm"
production="-O3 -DNDEBUG -finline-functions -funroll-loops -malign-double -fomit-frame-pointer"
diff --git a/config/linux b/config/linux
index ab397a1..4d5a2b3 100644
--- a/config/linux
+++ b/config/linux
@@ -53,7 +53,7 @@ fi
if test "X" = "X$CPPFLAGS"; then
warn=
profile=
- debug="-DH5AC_DEBUG -DH5B_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -DH5F_LOW_DFLT=H5F_LOW_SEC2"
+ debug="-DH5AC_DEBUG -DH5B_DEBUG -DH5D_DEBUG -DH5F_DEBUG -DH5G_DEBUG -UH5O_DEBUG -DH5T_DEBUG -DH5F_OPT_SEEK=0 -DH5F_LOW_DFLT=H5F_LOW_SEC2"
production="-DNDEBUG"
parallel=""
default_mode='$debug $warn $parallel'
diff --git a/configure b/configure
index e71fdf7..95f3d31 100755
--- a/configure
+++ b/configure
@@ -1267,7 +1267,7 @@ fi
-for ac_func in getpwuid gethostname
+for ac_func in getpwuid gethostname system
do
echo $ac_n "checking for $ac_func""... $ac_c" 1>&6
echo "configure:1274: checking for $ac_func" >&5
diff --git a/configure.in b/configure.in
index 26adb8e..7f48014 100644
--- a/configure.in
+++ b/configure.in
@@ -112,7 +112,7 @@ AC_TYPE_SIZE_T
dnl ----------------------------------------------------------------------
dnl Check for functions.
dnl
-AC_CHECK_FUNCS(getpwuid gethostname)
+AC_CHECK_FUNCS(getpwuid gethostname system)
AC_TRY_COMPILE([#include<sys/types.h>],
[off64_t n = 0;],
diff --git a/src/H5D.c b/src/H5D.c
index b417ca7..fce1528 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -63,7 +63,9 @@ const H5D_create_t H5D_create_dflt = {
/* Default dataset transfer property list */
const H5D_xfer_t H5D_xfer_dflt = {
- 0, /* Place holder - remove this later */
+ 1024*1024, /* Temporary buffer size */
+ NULL, /* Type conversion buffer or NULL */
+ NULL, /* Background buffer or NULL */
};
/* Interface initialization? */
@@ -813,7 +815,7 @@ H5D_create(H5F_t *f, const char *name, const H5T_t *type, const H5S_t *space,
}
/* Create (open for write access) an object header */
- if (H5O_create(f, 80, &(new_dset->ent)) < 0) {
+ if (H5O_create(f, 96, &(new_dset->ent)) < 0) {
HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL,
"unable to create dataset object header");
}
@@ -1087,7 +1089,9 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
const H5S_t *file_space, const H5D_xfer_t *xfer_parms,
void *buf/*out*/)
{
- size_t nelmts ; /*number of elements */
+ size_t nelmts; /*number of elements */
+ size_t smine_start; /*strip mine start loc */
+ size_t smine_nelmts; /*elements per strip */
uint8 *tconv_buf = NULL; /*data type conv buffer */
uint8 *bkg_buf = NULL; /*background buffer */
H5T_conv_t tconv_func = NULL; /*conversion function */
@@ -1097,6 +1101,11 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
H5T_cdata_t *cdata = NULL; /*type conversion data */
herr_t ret_value = FAIL;
herr_t status;
+ size_t src_type_size; /*size of source type */
+ size_t dst_type_size; /*size of destination type*/
+ size_t target_size; /*desired buffer size */
+ size_t buffer_size; /*actual buffer size */
+ size_t request_nelmts; /*requested strip mine */
FUNC_ENTER(H5D_read, FAIL);
@@ -1107,124 +1116,154 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
assert(buf);
if (!file_space) file_space = dataset->space;
if (!mem_space) mem_space = file_space;
-
- /*
- * Convert data types to atoms because the conversion functions are
- * application-level functions.
- */
- if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type))) < 0 ||
- (dst_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type))) < 0) {
- HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
- "unable to register types for conversion");
- }
+ nelmts = H5S_get_npoints(mem_space);
/*
* Locate the type conversion function and data space conversion
- * functions, and set up the element numbering information.
+ * functions, and set up the element numbering information. If a data
+ * type conversion is necessary then register data type atoms.
*/
if (NULL == (tconv_func = H5T_find(dataset->type, mem_type, &cdata))) {
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"unable to convert between src and dest data types");
+ } else if (H5T_conv_noop!=tconv_func) {
+ if ((src_id=H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0 ||
+ (dst_id=H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0) {
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
+ "unable to register types for conversion");
+ }
}
if (NULL==(sconv_func=H5S_find (mem_space, file_space))) {
HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"unable to convert from file to memory data space");
}
- if (sconv_func->init &&
- (sconv_func->init)(&(dataset->layout), mem_space, file_space,
- &numbering/*out*/)<=0) {
- HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
- "unable to initialize element numbering information");
- } else {
- HDmemset (&numbering, 0, sizeof numbering);
- }
- if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) {
+ if (nelmts!=H5S_get_npoints (file_space)) {
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"src and dest data spaces have different sizes");
}
/*
- * Compute the size of the request and allocate scratch buffers.
- */
- nelmts = H5S_get_npoints(mem_space);
-
- /*
* If there is no type conversion then try reading directly into the
- * application's buffer.
+ * application's buffer. This saves at least one mem-to-mem copy.
*/
if (H5D_OPTIMIZE_PIPE &&
H5T_conv_noop==tconv_func &&
NULL!=sconv_func->read) {
-#ifndef NDEBUG
- fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n");
-#endif
status = (sconv_func->read)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
mem_space, buf/*out*/);
if (status>=0) goto succeed;
-#ifndef NDEBUG
- fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n");
+#ifdef H5D_DEBUG
+ fprintf (stderr, "HDF5-DIAG: input pipe optimization failed "
+ "(falling through)\n");
#endif
H5E_clear ();
}
/*
- * This is the general case.
- */
-#ifndef LATER
- /*
- * Note: this prototype version allocates a buffer large enough to
- * satisfy the entire request; strip mining is not implemented.
- */
- {
- size_t src_size = nelmts * H5T_get_size(dataset->type);
- size_t dst_size = nelmts * H5T_get_size(mem_type);
- tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size));
- if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size);
- }
-#endif
- /*
- * Gather the data from disk into the data type conversion buffer. Also
- * gather data from application to background buffer (this step is not
- * needed for most conversions, but we leave that as an exercise for
- * later ;-)
+ * This is the general case. Figure out the strip mine size.
*/
- if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
- &(dataset->create_parms->efl),
- H5T_get_size (dataset->type), file_space,
- &numbering, 0, nelmts,
- tconv_buf/*out*/)!=nelmts) {
- HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed");
- }
- if (H5T_BKG_YES==cdata->need_bkg) {
- if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
- &numbering, 0, nelmts,
- bkg_buf/*out*/)!=nelmts) {
- HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed");
+ src_type_size = H5T_get_size(dataset->type);
+ dst_type_size = H5T_get_size(mem_type);
+ target_size = xfer_parms->buf_size;
+ request_nelmts = target_size / MAX(src_type_size, dst_type_size);
+ if (request_nelmts<=0) {
+ HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
+ "temporary buffer max size is too small");
+ }
+ if (sconv_func->init) {
+ smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space,
+ file_space, request_nelmts,
+ &numbering/*out*/);
+ if (smine_nelmts<=0) {
+ HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
+ "unable to initialize element numbering information");
}
+ } else {
+ smine_nelmts = request_nelmts;
+ HDmemset (&numbering, 0, sizeof numbering);
}
+ buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size);
/*
- * Perform data type conversion.
+ * Get a temporary buffer for type conversion unless the app has already
+ * supplied one through the xfer properties. Instead of allocating a
+ * buffer which is the exact size, we allocate the target size. The
+ * malloc() is usually less resource-intensive if we allocate/free the
+ * same size over and over.
*/
- cdata->command = H5T_CONV_CONV;
- cdata->ncalls++;
- if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) {
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
- "data type conversion failed");
+ if (NULL==(tconv_buf=xfer_parms->tconv)) {
+ tconv_buf = H5MM_xmalloc (target_size);
+ }
+ if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) {
+ bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size);
}
- cdata->nelmts += nelmts;
- /*
- * Scatter the data into memory.
- */
- if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space,
- &numbering, 0, nelmts, buf/*out*/)<0) {
- HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed");
+#ifdef H5D_DEBUG
+ /* Strip mine diagnostics.... */
+ if (smine_nelmts<nelmts) {
+ fprintf (stderr, "HDF5-DIAG: strip mine");
+ if (smine_nelmts!=request_nelmts) {
+ fprintf (stderr, " got %lu of %lu",
+ (unsigned long)smine_nelmts,
+ (unsigned long)request_nelmts);
+ }
+ if (buffer_size!=target_size) {
+ fprintf (stderr, " (%1.1f%% of buffer)",
+ 100.0*buffer_size/target_size);
+ }
+ fprintf (stderr, " %1.1f iterations\n",
+ (double)nelmts/smine_nelmts);
}
+#endif
+
+ /* Start strip mining... */
+ for (smine_start=0; smine_start<nelmts; smine_start+=smine_nelmts) {
+ smine_nelmts = MIN (smine_nelmts, nelmts-smine_start);
+
+ /*
+ * Gather the data from disk into the data type conversion
+ * buffer. Also gather data from application to background buffer
+ * if necessary.
+ */
+ if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->efl),
+ H5T_get_size (dataset->type), file_space,
+ &numbering, smine_start, smine_nelmts,
+ tconv_buf/*out*/)!=smine_nelmts) {
+ HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed");
+ }
+ if (H5T_BKG_YES==cdata->need_bkg) {
+ if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
+ &numbering, smine_start, smine_nelmts,
+ bkg_buf/*out*/)!=smine_nelmts) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed");
+ }
+ }
+ /*
+ * Perform data type conversion.
+ */
+ cdata->command = H5T_CONV_CONV;
+ cdata->ncalls++;
+ if ((tconv_func)(src_id, dst_id, cdata, smine_nelmts, tconv_buf,
+ bkg_buf)<0) {
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
+ "data type conversion failed");
+ }
+ cdata->nelmts += smine_nelmts;
+
+ /*
+ * Scatter the data into memory.
+ */
+ if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space,
+ &numbering, smine_start, smine_nelmts,
+ buf/*out*/)<0) {
+ HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed");
+ }
+ }
succeed:
ret_value = SUCCEED;
@@ -1232,8 +1271,12 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
done:
if (src_id >= 0) H5A_dec_ref(src_id);
if (dst_id >= 0) H5A_dec_ref(dst_id);
- tconv_buf = H5MM_xfree(tconv_buf);
- bkg_buf = H5MM_xfree (bkg_buf);
+ if (tconv_buf && NULL==xfer_parms->tconv) {
+ H5MM_xfree(tconv_buf);
+ }
+ if (bkg_buf && NULL==xfer_parms->bkg) {
+ H5MM_xfree (bkg_buf);
+ }
FUNC_LEAVE(ret_value);
}
@@ -1260,7 +1303,9 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
const H5S_t *file_space, const H5D_xfer_t *xfer_parms,
const void *buf)
{
- size_t nelmts;
+ size_t nelmts; /*total number of elmts */
+ size_t smine_start; /*strip mine start loc */
+ size_t smine_nelmts; /*elements per strip */
uint8 *tconv_buf = NULL; /*data type conv buffer */
uint8 *bkg_buf = NULL; /*background buffer */
H5T_conv_t tconv_func = NULL; /*conversion function */
@@ -1269,6 +1314,11 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
H5S_number_t numbering; /*element numbering info*/
H5T_cdata_t *cdata = NULL; /*type conversion data */
herr_t ret_value = FAIL, status;
+ size_t src_type_size; /*size of source type */
+ size_t dst_type_size; /*size of destination type*/
+ size_t target_size; /*desired buffer size */
+ size_t buffer_size; /*actual buffer size */
+ size_t request_nelmts; /*requested strip mine */
FUNC_ENTER(H5D_write, FAIL);
@@ -1279,47 +1329,32 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
assert(buf);
if (!file_space) file_space = dataset->space;
if (!mem_space) mem_space = file_space;
+ nelmts = H5S_get_npoints(mem_space);
/*
- * Convert data types to atoms because the conversion functions are
- * application-level functions.
- */
- if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 ||
- (dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) {
- HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
- "unable to register types for conversion");
- }
-
- /*
* Locate the type conversion function and data space conversion
- * functions, and set up the element numbering information.
+ * functions, and set up the element numbering information. If a data
+ * type conversion is necessary then register data type atoms.
*/
if (NULL == (tconv_func = H5T_find(mem_type, dataset->type, &cdata))) {
HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"unable to convert between src and dest data types");
+ } else if (H5T_conv_noop!=tconv_func) {
+ if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 ||
+ (dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) {
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
+ "unable to register types for conversion");
+ }
}
if (NULL==(sconv_func=H5S_find (mem_space, file_space))) {
HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
"unable to convert from memory to file data space");
}
- if (sconv_func->init &&
- (sconv_func->init)(&(dataset->layout), mem_space, file_space,
- &numbering/*out*/)<=0) {
- HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
- "unable to initialize element numbering information");
- } else {
- HDmemset (&numbering, 0, sizeof numbering);
- }
- if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) {
+ if (nelmts!=H5S_get_npoints (file_space)) {
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"src and dest data spaces have different sizes");
}
-
- /*
- * Compute the size of the request and allocate scratch buffers.
- */
- nelmts = H5S_get_npoints(mem_space);
-
+
/*
* If there is no type conversion then try writing directly from
* application buffer to file.
@@ -1327,75 +1362,123 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
if (H5D_OPTIMIZE_PIPE &&
H5T_conv_noop==tconv_func &&
NULL!=sconv_func->write) {
-#ifndef NDEBUG
- fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n");
-#endif
status = (sconv_func->write)(dataset->ent.file, &(dataset->layout),
&(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
mem_space, buf);
if (status>=0) goto succeed;
-#ifndef NDEBUG
- fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n");
+#ifdef H5D_DEBUG
+ fprintf (stderr, "HDF5-DIAG: output pipe optimization failed "
+ "(falling through)\n");
#endif
H5E_clear ();
}
/*
- * This is the general case.
+ * This is the general case. Figure out the strip mine size.
*/
-#ifndef LATER
- /*
- * Note: This prototype version allocates a buffer large enough to
- * satisfy the entire request; strip mining is not implemented.
- */
- {
- size_t src_size = nelmts * H5T_get_size(mem_type);
- size_t dst_size = nelmts * H5T_get_size(dataset->type);
- tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size));
- if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size);
+ src_type_size = H5T_get_size(mem_type);
+ dst_type_size = H5T_get_size(dataset->type);
+ target_size = xfer_parms->buf_size;
+ request_nelmts = target_size / MAX (src_type_size, dst_type_size);
+ if (request_nelmts<=0) {
+ HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
+ "temporary buffer max size is too small");
+ }
+ if (sconv_func->init) {
+ smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space,
+ file_space, request_nelmts,
+ &numbering/*out*/);
+ if (smine_nelmts<=0) {
+ HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
+ "unable to initialize element numbering information");
+ }
+ } else {
+ smine_nelmts = request_nelmts;
+ HDmemset (&numbering, 0, sizeof numbering);
}
-#endif
+ buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size);
+
/*
- * Gather data from application buffer into the data type conversion
- * buffer. Also gather data from the file into the background buffer
- * (this step is not needed for most conversions, but we leave that as an
- * exercise for later ;-)
+ * Get a temporary buffer for type conversion unless the app has already
+ * supplied one through the xfer properties. Instead of allocating a
+ * buffer which is the exact size, we allocate the target size. The
+ * malloc() is usually less resource-intensive if we allocate/free the
+ * same size over and over.
*/
- if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
- &numbering, 0, nelmts, tconv_buf/*out*/)!=nelmts) {
- HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed");
+ if (NULL==(tconv_buf=xfer_parms->tconv)) {
+ tconv_buf = H5MM_xmalloc (target_size);
}
- if (H5T_BKG_YES==cdata->need_bkg) {
- if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
- &(dataset->create_parms->efl),
- H5T_get_size (dataset->type), file_space,
- &numbering, 0, nelmts,
- bkg_buf/*out*/)!=nelmts) {
- HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed");
- }
+ if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) {
+ bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size);
}
- /*
- * Perform data type conversion.
- */
- cdata->command = H5T_CONV_CONV;
- cdata->ncalls++;
- if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) {
- HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
- "data type conversion failed");
+#ifdef H5D_DEBUG
+ /* Strip mine diagnostics.... */
+ if (smine_nelmts<nelmts) {
+ fprintf (stderr, "HDF5-DIAG: strip mine");
+ if (smine_nelmts!=request_nelmts) {
+ fprintf (stderr, " got %lu of %lu",
+ (unsigned long)smine_nelmts,
+ (unsigned long)request_nelmts);
+ }
+ if (buffer_size!=target_size) {
+ fprintf (stderr, " (%1.1f%% of buffer)",
+ 100.0*buffer_size/target_size);
+ }
+ fprintf (stderr, " %1.1f iterations\n",
+ (double)nelmts/smine_nelmts);
}
- cdata->nelmts += nelmts;
+#endif
- /*
- * Scatter the data out to the file.
- */
- if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
- &(dataset->create_parms->efl),
- H5T_get_size (dataset->type), file_space,
- &numbering, 0, nelmts, tconv_buf)<0) {
- HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed");
+ /* Start strip mining... */
+ for (smine_start=0; smine_start<nelmts; smine_start+=smine_nelmts) {
+ smine_nelmts = MIN (smine_nelmts, nelmts-smine_start);
+
+ /*
+ * Gather data from application buffer into the data type conversion
+ * buffer. Also gather data from the file into the background buffer
+ * if necessary.
+ */
+ if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
+ &numbering, smine_start, smine_nelmts,
+ tconv_buf/*out*/)!=smine_nelmts) {
+ HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed");
+ }
+ if (H5T_BKG_YES==cdata->need_bkg) {
+ if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->efl),
+ H5T_get_size (dataset->type), file_space,
+ &numbering, smine_start, smine_nelmts,
+ bkg_buf/*out*/)!=smine_nelmts) {
+ HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
+ "file gather failed");
+ }
+ }
+
+ /*
+ * Perform data type conversion.
+ */
+ cdata->command = H5T_CONV_CONV;
+ cdata->ncalls++;
+ if ((tconv_func) (src_id, dst_id, cdata, smine_nelmts, tconv_buf,
+ bkg_buf)<0) {
+ HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
+ "data type conversion failed");
+ }
+ cdata->nelmts += smine_nelmts;
+
+ /*
+ * Scatter the data out to the file.
+ */
+ if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->efl),
+ H5T_get_size (dataset->type), file_space,
+ &numbering, smine_start, smine_nelmts,
+ tconv_buf)<0) {
+ HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed");
+ }
}
succeed:
@@ -1404,10 +1487,15 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
done:
if (src_id >= 0) H5A_dec_ref(src_id);
if (dst_id >= 0) H5A_dec_ref(dst_id);
- tconv_buf = H5MM_xfree(tconv_buf);
- bkg_buf = H5MM_xfree (bkg_buf);
+ if (tconv_buf && NULL==xfer_parms->tconv) {
+ H5MM_xfree(tconv_buf);
+ }
+ if (bkg_buf && NULL==xfer_parms->bkg) {
+ H5MM_xfree (bkg_buf);
+ }
FUNC_LEAVE(ret_value);
}
+
/*-------------------------------------------------------------------------
* Function: H5D_extend
diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h
index fc6eeb4..f678ef7 100644
--- a/src/H5Dprivate.h
+++ b/src/H5Dprivate.h
@@ -26,6 +26,15 @@
#include <H5Sprivate.h> /*for the H5S_t type */
#include <H5Oprivate.h> /*object Headers */
+/*
+ * Feature: Define H5D_DEBUG on the compiler command line if you want to
+ * debug dataset I/O. NDEBUG must not be defined in order for this
+ * to have any effect.
+ */
+#ifdef NDEBUG
+# undef H5D_DEBUG
+#endif
+
#define H5D_RESERVED_ATOMS 0
/* Set the minimum object header size to create objects with */
@@ -41,8 +50,11 @@ typedef struct H5D_create_t {
/* Dataset transfer property list */
typedef struct H5D_xfer_t {
- int _placeholder; /*unused--delete this later */
+ size_t buf_size; /*max temp buffer size */
+ void *tconv; /*type conversion buffer or null */
+ void *bkg; /*background buffer or null */
} H5D_xfer_t;
+
typedef struct H5D_t H5D_t;
extern const H5D_create_t H5D_create_dflt;
extern const H5D_xfer_t H5D_xfer_dflt;
diff --git a/src/H5P.c b/src/H5P.c
index 8bbf03d..e990675 100644
--- a/src/H5P.c
+++ b/src/H5P.c
@@ -1679,6 +1679,100 @@ H5Pget_family (hid_t tid, hid_t *memb_tid)
/*-------------------------------------------------------------------------
+ * Function: H5Pset_buffer
+ *
+ * Purpose: Given a dataset transfer property list, set the maximum size
+ * for the type conversion buffer and background buffer and
+ * optionally supply pointers to application-allocated buffers.
+ * If the buffer size is smaller than the entire amount of data
+ * being transfered between application and file, and a type
+ * conversion buffer or background buffer is required then
+ * strip mining will be used. However, certain restrictions
+ * apply for the size of buffer which can be used for strip
+ * mining. For instance, when strip mining a 100x200x300
+ * hyperslab of a simple data space the buffer must be large
+ * enough to hold a 1x200x300 slab.
+ *
+ * If TCONV and/or BKG are null pointers then buffers will be
+ * allocated and freed during the data transfer.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Monday, March 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg)
+{
+ H5D_xfer_t *plist = NULL;
+
+ FUNC_ENTER (H5Pset_buffer, FAIL);
+
+ /* Check arguments */
+ if (H5P_DATASET_XFER != H5Pget_class (plist_id) ||
+ NULL == (plist = H5A_object (plist_id))) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
+ "not a dataset transfer property list");
+ }
+ if (size<=0) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
+ "buffer size must not be zero");
+ }
+
+ /* Update property list */
+ plist->buf_size = size;
+ plist->tconv = tconv;
+ plist->bkg = bkg;
+
+ FUNC_LEAVE (SUCCEED);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_buffer
+ *
+ * Purpose: Reads values previously set with H5Pset_buffer().
+ *
+ * Return: Success: Buffer size.
+ *
+ * Failure: 0
+ *
+ * Programmer: Robb Matzke
+ * Monday, March 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+size_t
+H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/)
+{
+ H5D_xfer_t *plist = NULL;
+
+ FUNC_ENTER (H5Pget_buffer, 0);
+
+ /* Check arguments */
+ if (H5P_DATASET_XFER != H5Pget_class (plist_id) ||
+ NULL == (plist = H5A_object (plist_id))) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, 0,
+ "not a dataset transfer property list");
+ }
+
+ /* Return values */
+ if (tconv) *tconv = plist->tconv;
+ if (bkg) *bkg = plist->bkg;
+
+ FUNC_LEAVE (plist->buf_size);
+}
+
+
+/*-------------------------------------------------------------------------
* Function: H5Pset_mpi
*
* Signature: herr_t H5Pset_mpi(hid_t tid, MPI_Comm comm, MPI_Info info,
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index 47e9b5a..7e91ce2 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -83,8 +83,12 @@ herr_t H5Pget_split (hid_t tid, size_t meta_ext_size, char *meta_ext/*out*/,
herr_t H5Pset_family (hid_t tid, hid_t memb_tid);
herr_t H5Pget_family (hid_t tid, hid_t *memb_tid/*out*/);
+herr_t H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg);
+size_t H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/);
+
#ifdef HAVE_PARALLEL
-herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info, unsigned access_mode);
+herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info,
+ unsigned access_mode);
herr_t H5Pget_mpi (hid_t tid, MPI_Comm *comm/*out*/, MPI_Info *info/*out*/,
unsigned *access_mode/*out*/);
#endif
diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h
index fbce725..da19695 100644
--- a/src/H5Sprivate.h
+++ b/src/H5Sprivate.h
@@ -65,7 +65,8 @@ typedef struct H5S_number_t {
typedef struct H5S_tconv_t {
/* Initialize element numbering information */
size_t (*init)(const struct H5O_layout_t *layout, const H5S_t *mem_space,
- const H5S_t *file_space, H5S_number_t *numbering/*out*/);
+ const H5S_t *file_space, size_t desired_nelmts,
+ H5S_number_t *numbering/*out*/);
/* Gather elements from disk to type conversion buffer */
size_t (*fgath)(H5F_t *f, const struct H5O_layout_t *layout,
@@ -122,7 +123,7 @@ intn H5S_extend (H5S_t *space, const size_t *size);
/* Conversion functions for simple data spaces */
size_t H5S_simp_init (const struct H5O_layout_t *layout,
const H5S_t *mem_space, const H5S_t *file_space,
- H5S_number_t *numbering/*out*/);
+ size_t desired_nelmts, H5S_number_t *numbering/*out*/);
size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
const struct H5O_efl_t *efl, size_t elmt_size,
const H5S_t *file_space, const H5S_number_t *numbering,
diff --git a/src/H5Ssimp.c b/src/H5Ssimp.c
index 6f1f498..a785b35 100644
--- a/src/H5Ssimp.c
+++ b/src/H5Ssimp.c
@@ -37,9 +37,14 @@ static intn interface_initialize_g = FALSE;
*/
size_t
H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
- const H5S_t *file_space, H5S_number_t *numbering/*out*/)
+ const H5S_t *file_space, size_t desired_nelmts,
+ H5S_number_t *numbering/*out*/)
{
size_t nelmts;
+ int m_ndims, f_ndims; /*mem, file dimensionality */
+ size_t size[H5O_LAYOUT_NDIMS]; /*size of selected hyperslab */
+ size_t acc;
+ int i;
FUNC_ENTER (H5S_simp_init, 0);
@@ -49,11 +54,43 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
assert (file_space && H5S_SIMPLE==file_space->type);
assert (numbering);
- /* Numbering is implied by the hyperslab, C order */
+ /* Numbering is implied by the hyperslab, C order, no data here */
HDmemset (numbering, 0, sizeof(H5S_number_t));
- /* Data can be efficiently copied at any size */
- nelmts = H5S_get_npoints (file_space);
+ /*
+ * The stripmine size is such that only the slowest varying dimension can
+ * be split up. We choose the largest possible strip mine size which is
+ * not larger than the desired size.
+ */
+ m_ndims = H5S_get_hyperslab (mem_space, NULL, size, NULL);
+ for (i=m_ndims-1, acc=1; i>0; --i) acc *= size[i];
+ nelmts = (desired_nelmts/acc) * acc;
+ if (nelmts<=0) {
+ HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
+ "strip mine buffer is too small");
+ }
+
+ /*
+ * The value chosen for mem_space must be the same as the value chosen for
+ * file_space.
+ */
+ f_ndims = H5S_get_hyperslab (file_space, NULL, size, NULL);
+ if (m_ndims!=f_ndims) {
+ nelmts = H5S_get_npoints (file_space);
+ if (nelmts>desired_nelmts) {
+ HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
+ "strip mining not supported across "
+ "dimensionalities");
+ }
+ assert (nelmts==H5S_get_npoints (mem_space));
+ } else {
+ for (i=f_ndims-1, acc=1; i>0; --i) acc *= size[i];
+ acc *= (desired_nelmts/acc);
+ if (nelmts!=acc) {
+ HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
+ "unsupported strip mine size for shape change");
+ }
+ }
FUNC_LEAVE (nelmts);
}
@@ -95,6 +132,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
size_t zero[H5O_LAYOUT_NDIMS]; /*zero */
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
+ size_t acc; /*accumulator */
#ifndef LATER
intn file_offset_signed[H5O_LAYOUT_NDIMS];
#endif
@@ -113,12 +151,6 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
assert (buf);
/*
- * The prototype doesn't support strip mining.
- */
- assert (0==start);
- assert (nelmts==H5S_get_npoints (file_space));
-
- /*
* Get hyperslab information to determine what elements are being
* selected (there might eventually be other selection methods too).
* We only support hyperslabs with unit sample because there's no way to
@@ -132,6 +164,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
"unable to retrieve hyperslab parameters");
}
#else
+ /* Argument type problems to be fixed later..... -RPM */
if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed,
hsize, sample))<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0,
@@ -142,12 +175,23 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
file_offset[i] = file_offset_signed[i];
}
#endif
+
+ /* Check that there is no subsampling of the hyperslab */
for (i=0; i<space_ndims; i++) {
if (sample[i]!=1) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, 0,
"hyperslab sampling is not implemented yet");
}
}
+
+ /* Adjust the slowest varying dimension to take care of strip mining */
+ for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
+ assert (0==start % acc);
+ assert (0==nelmts % acc);
+ file_offset[0] += start / acc;
+ hsize[0] = nelmts / acc;
+
+ /* The fastest varying dimension is for the data point itself */
file_offset[space_ndims] = 0;
hsize[space_ndims] = elmt_size;
HDmemset (zero, 0, layout->ndims*sizeof(size_t));
@@ -194,6 +238,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
size_t zero[H5O_LAYOUT_NDIMS]; /*zero */
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
+ size_t acc; /*accumulator */
#ifndef LATER
intn mem_offset_signed[H5O_LAYOUT_NDIMS];
#endif
@@ -211,12 +256,6 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
assert (buf);
/*
- * The prototype doesn't support strip mining.
- */
- assert (0==start);
- assert (nelmts==H5S_get_npoints (mem_space));
-
- /*
* Retrieve hyperslab information to determine what elements are being
* selected (there might be other selection methods in the future). We
* only handle hyperslabs with unit sample because there's currently no
@@ -229,6 +268,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
"unable to retrieve hyperslab parameters");
}
#else
+ /* Argument type problems to be fixed later..... -RPM */
if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed,
hsize, sample))<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
@@ -239,6 +279,8 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
mem_offset[i] = mem_offset_signed[i];
}
#endif
+
+ /* Check that there is no subsampling of the hyperslab */
for (i=0; i<space_ndims; i++) {
if (sample[i]!=1) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
@@ -249,6 +291,15 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
"unable to retrieve data space dimensions");
}
+
+ /* Adjust the slowest varying dimension to take care of strip mining */
+ for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
+ assert (0==start % acc);
+ assert (0==nelmts % acc);
+ mem_offset[0] += start / acc;
+ hsize[0] = nelmts / acc;
+
+ /* The fastest varying dimension is for the data point itself */
mem_offset[space_ndims] = 0;
mem_size[space_ndims] = elmt_size;
hsize[space_ndims] = elmt_size;
@@ -299,6 +350,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
size_t zero[H5O_LAYOUT_NDIMS]; /*zero */
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
+ size_t acc; /*accumulator */
#ifndef LATER
intn mem_offset_signed[H5O_LAYOUT_NDIMS];
#endif
@@ -316,12 +368,6 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
assert (tconv_buf);
/*
- * The prototype doesn't support strip mining.
- */
- assert (0==start);
- assert (nelmts==H5S_get_npoints (mem_space));
-
- /*
* Retrieve hyperslab information to determine what elements are being
* selected (there might be other selection methods in the future). We
* only handle hyperslabs with unit sample because there's currently no
@@ -334,6 +380,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
"unable to retrieve hyperslab parameters");
}
#else
+ /* Argument type problems to be fixed later..... -RPM */
if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed,
hsize, sample))<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
@@ -344,6 +391,8 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
mem_offset[i] = mem_offset_signed[i];
}
#endif
+
+ /* Check that there is no subsampling of the hyperslab */
for (i=0; i<space_ndims; i++) {
if (sample[i]!=1) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, 0,
@@ -354,6 +403,15 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0,
"unable to retrieve data space dimensions");
}
+
+ /* Adjust the slowest varying dimension to account for strip mining */
+ for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
+ assert (0==start % acc);
+ assert (0==nelmts % acc);
+ mem_offset[0] += start / acc;
+ hsize[0] = nelmts / acc;
+
+ /* The fastest varying dimension is for the data point itself */
mem_offset[space_ndims] = 0;
mem_size[space_ndims] = elmt_size;
hsize[space_ndims] = elmt_size;
@@ -404,6 +462,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
size_t hsize[H5O_LAYOUT_NDIMS]; /*size of hyperslab */
size_t zero[H5O_LAYOUT_NDIMS]; /*zero vector */
size_t sample[H5O_LAYOUT_NDIMS]; /*hyperslab sampling */
+ size_t acc; /*accumulator */
#ifndef LATER
intn file_offset_signed[H5O_LAYOUT_NDIMS];
#endif
@@ -422,12 +481,6 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
assert (buf);
/*
- * The prototype doesn't support strip mining.
- */
- assert (0==start);
- assert (nelmts==H5S_get_npoints (file_space));
-
- /*
* Get hyperslab information to determine what elements are being
* selected (there might eventually be other selection methods too).
* We only support hyperslabs with unit sample because there's no way to
@@ -441,6 +494,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
"unable to retrieve hyperslab parameters");
}
#else
+ /* Argument type problems to be fixed later..... -RPM */
if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed,
hsize, sample))<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
@@ -451,12 +505,23 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
file_offset[i] = file_offset_signed[i];
}
#endif
+
+ /* Check that there is no subsampling of the hyperslab */
for (i=0; i<space_ndims; i++) {
if (sample[i]!=1) {
HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"hyperslab sampling is not implemented yet");
}
}
+
+ /* Adjust the slowest varying dimension to account for strip mining */
+ for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
+ assert (0==start % acc);
+ assert (0==nelmts % acc);
+ file_offset[0] += start / acc;
+ hsize[0] = nelmts / acc;
+
+ /* The fastest varying dimension is for the data point itself */
file_offset[space_ndims] = 0;
hsize[space_ndims] = elmt_size;
HDmemset (zero, 0, layout->ndims*sizeof(size_t));
diff --git a/test/Makefile.in b/test/Makefile.in
index 3bf75a4..f4d9009 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -11,13 +11,14 @@ CPPFLAGS=-I. -I../src @CPPFLAGS@
# These are our main targets. They should be listed in the order to be
# executed, generally most specific tests to least specific tests.
-PROGS=testhdf5 hyperslab istore dtypes dsets cmpd_dset extend external \
+PROGS=testhdf5 hyperslab istore dtypes dsets cmpd_dset extend external \
iopipe
-TESTS=$(PROGS)
+TESTS=testhdf5 hyperslab istore dtypes dsets cmpd_dset extend external
+TIMINGS=iopipe
# Temporary files
MOSTLYCLEAN=cmpd_dset.h5 dataset.h5 extend.h5 istore.h5 tfile1.h5 tfile2.h5 \
- tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5 \
+ tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5 \
extern_1.h5 extern_2.h5 extern_3.h5 extern_1.raw extern_1b.raw \
extern_2.raw extern_2b.raw extern_3.raw extern_3b.raw \
extern_4.raw extern_4b.raw iopipe.raw iopipe.h5
@@ -64,6 +65,16 @@ IOPIPE_OBJ=$(IOPIPE_SRC:.c=.o)
# Private header files (not to be installed)...
PRIVATE_HDR=testhdf5.h
+# Additional targets
+.PHONY: timings _timings
+timings _timings: $(TIMINGS)
+ @for timing in $(TIMINGS) dummy; do \
+ if test $$timing != dummy; then \
+ echo "Running $$timing $(TEST_FLAGS)"; \
+ $(RUNTEST) ./$$timing $(TEST_FLAGS) || exit 1; \
+ fi; \
+ done;
+
# How to build the programs...
testhdf5: $(TESTHDF5_OBJ) ../src/libhdf5.a
$(CC) $(CFLAGS) -o $@ $(TESTHDF5_OBJ) ../src/libhdf5.a $(LIBS)
diff --git a/test/dsets.c b/test/dsets.c
index c92824c..eaca4a8 100644
--- a/test/dsets.c
+++ b/test/dsets.c
@@ -184,11 +184,12 @@ test_create(hid_t file)
static herr_t
test_simple_io(hid_t file)
{
- hid_t dataset, space;
- herr_t status;
- int points[100][200], check[100][200];
- int i, j, n;
- size_t dims[2];
+ hid_t dataset, space, xfer;
+ herr_t status;
+ int points[100][200], check[100][200];
+ int i, j, n;
+ size_t dims[2];
+ void *tconv_buf = NULL;
printf("%-70s", "Testing simple I/O");
@@ -205,6 +206,13 @@ test_simple_io(hid_t file)
space = H5Screate_simple(2, dims, NULL);
assert(space>=0);
+ /* Create a small conversion buffer to test strip mining */
+ tconv_buf = malloc (1000);
+ xfer = H5Pcreate (H5P_DATASET_XFER);
+ assert (xfer>=0);
+ status = H5Pset_buffer (xfer, 1000, tconv_buf, NULL);
+ assert (status>=0);
+
/* Create the dataset */
dataset = H5Dcreate(file, DSET_SIMPLE_IO_NAME, H5T_NATIVE_INT, space,
H5P_DEFAULT);
@@ -212,12 +220,12 @@ test_simple_io(hid_t file)
/* Write the data to the dataset */
status = H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
- H5P_DEFAULT, points);
+ xfer, points);
if (status<0) goto error;
/* Read the dataset back */
status = H5Dread(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL,
- H5P_DEFAULT, check);
+ xfer, check);
if (status<0) goto error;
/* Check that the values read are the same as the values written */
diff --git a/test/iopipe.c b/test/iopipe.c
index 813150e..f185aad 100644
--- a/test/iopipe.c
+++ b/test/iopipe.c
@@ -19,12 +19,22 @@
#define RAW_FILE_NAME "iopipe.raw"
#define HDF5_FILE_NAME "iopipe.h5"
+#define HEADING "%-16s"
+#define PROGRESS '='
+
+#if 1
+/* Normal testing */
#define REQUEST_SIZE_X 4579
#define REQUEST_SIZE_Y 4579
#define NREAD_REQUESTS 45
#define NWRITE_REQUESTS 45
-#define HEADING "%-16s"
-#define PROGRESS '='
+#else
+/* Speedy testing */
+#define REQUEST_SIZE_X 1000
+#define REQUEST_SIZE_Y 1000
+#define NREAD_REQUESTS 45
+#define NWRITE_REQUESTS 45
+#endif
/*-------------------------------------------------------------------------
@@ -83,10 +93,17 @@ print_stats (const char *prefix,
static void
synchronize (void)
{
+#ifdef HAVE_SYSTEM
system ("sync");
system ("df >/dev/null");
#if 0
- system ("/sbin/swapout 130");
+ /*
+ * This works well on Linux to get rid of all cached disk buffers. The
+ * number should be approximately the amount of RAM in MB. Do not
+ * include swap space in that amount or the command will fail.
+ */
+ system ("/sbin/swapout 128");
+#endif
#endif
}