[svn-r322] Changes since 19980313

----------------------

./configure.in
./configure
./test/iopipe.c
	Added check for system(3)

./config/freebsd2.2.1
./config/linux
	Added -DH5D_DEBUG to the debug flags.

./src/H5D.c
./src/H5Dprivate.h
./src/H5P.c
./src/H5Ppublic.h
./src/H5Sprivate.h
./src/H5Ssimp.c
./html/Datasets.html
	Finally implemented strip mining in the I/O pipeline, placing
	a user-defined upper bound on the amount of temporary memory
	used by the pipeline.  The default is 1MB type conversion and
	background buffers allocated/freed on demand.  However, the
	size can be changed and/or application-allocated buffers
	supplied with H5Pset_buffers() called on the data transfer
	property list passed to H5Dread() and H5Dwrite().

	Minor optimizations to H5Dread() and H5Dwrite().  More coming
	later...

./test/dsets.c
	Added calls to H5Pset_buffer() to test application-defined
	temporary buffers in the I/O pipeline.

./test/Makefile.in
	Removed `iopipe' from the list of confidence tests.  Saying
	`make timings' in the test directory runs timing tests.  I did
	this because (1) they don't really test anything new, and (2)
	they can take a long time to run.

6 files changed, 454 insertions, 190 deletions

diff --git a/src/H5D.c b/src/H5D.c
index b417ca7..fce1528 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -63,7 +63,9 @@ const H5D_create_t	H5D_create_dflt = {
 
 /* Default dataset transfer property list */
 const H5D_xfer_t	H5D_xfer_dflt = {
-    0,				/* Place holder - remove this later	*/
+    1024*1024,			/* Temporary buffer size		*/
+    NULL,			/* Type conversion buffer or NULL	*/
+    NULL, 			/* Background buffer or NULL		*/
 };
 
 /* Interface initialization? */
@@ -813,7 +815,7 @@ H5D_create(H5F_t *f, const char *name, const H5T_t *type, const H5S_t *space,
     }
 
     /* Create (open for write access) an object header */
-    if (H5O_create(f, 80, &(new_dset->ent)) < 0) {
+    if (H5O_create(f, 96, &(new_dset->ent)) < 0) {
 	HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL,
 		    "unable to create dataset object header");
     }
@@ -1087,7 +1089,9 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
 	 const H5S_t *file_space, const H5D_xfer_t *xfer_parms,
 	 void *buf/*out*/)
 {
-    size_t		nelmts	;		/*number of elements	*/
+    size_t		nelmts;			/*number of elements	*/
+    size_t		smine_start;		/*strip mine start loc	*/
+    size_t		smine_nelmts;		/*elements per strip	*/
     uint8		*tconv_buf = NULL;	/*data type conv buffer	*/
     uint8		*bkg_buf = NULL;	/*background buffer	*/
     H5T_conv_t		tconv_func = NULL;	/*conversion function	*/
@@ -1097,6 +1101,11 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
     H5T_cdata_t		*cdata = NULL;		/*type conversion data	*/
     herr_t		ret_value = FAIL;
     herr_t		status;
+    size_t		src_type_size;		/*size of source type	*/
+    size_t		dst_type_size;		/*size of destination type*/
+    size_t		target_size;		/*desired buffer size	*/
+    size_t		buffer_size;		/*actual buffer size	*/
+    size_t		request_nelmts;		/*requested strip mine	*/
 
     FUNC_ENTER(H5D_read, FAIL);
 
@@ -1107,124 +1116,154 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
     assert(buf);
     if (!file_space) file_space = dataset->space;
     if (!mem_space) mem_space = file_space;
-    
-    /*
-     * Convert data types to atoms because the conversion functions are
-     * application-level functions.
-     */
-    if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type))) < 0 ||
-	(dst_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type))) < 0) {
-	HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
-		    "unable to register types for conversion");
-    }
+    nelmts = H5S_get_npoints(mem_space);
 
     /*
      * Locate the type conversion function and data space conversion
-     * functions, and set up the element numbering information.
+     * functions, and set up the element numbering information. If a data
+     * type conversion is necessary then register data type atoms.
      */
     if (NULL == (tconv_func = H5T_find(dataset->type, mem_type, &cdata))) {
 	HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL,
 		    "unable to convert between src and dest data types");
+    } else if (H5T_conv_noop!=tconv_func) {
+	if ((src_id=H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0 ||
+	    (dst_id=H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0) {
+	    HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
+			"unable to register types for conversion");
+	}
     }
     if (NULL==(sconv_func=H5S_find (mem_space, file_space))) {
 	HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
 		     "unable to convert from file to memory data space");
     }
-    if (sconv_func->init &&
-	(sconv_func->init)(&(dataset->layout), mem_space, file_space,
-			   &numbering/*out*/)<=0) {
-	HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
-		     "unable to initialize element numbering information");
-    } else {
-	HDmemset (&numbering, 0, sizeof numbering);
-    }
-    if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) {
+    if (nelmts!=H5S_get_npoints (file_space)) {
 	HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
 		     "src and dest data spaces have different sizes");
     }
     
     /*
-     * Compute the size of the request and allocate scratch buffers.
-     */
-    nelmts = H5S_get_npoints(mem_space);
-
-    /*
      * If there is no type conversion then try reading directly into the
-     * application's buffer.
+     * application's buffer.  This saves at least one mem-to-mem copy.
      */
     if (H5D_OPTIMIZE_PIPE &&
 	H5T_conv_noop==tconv_func &&
 	NULL!=sconv_func->read) {
-#ifndef NDEBUG
-	fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n");
-#endif
 	status = (sconv_func->read)(dataset->ent.file, &(dataset->layout),
 				    &(dataset->create_parms->efl),
 				    H5T_get_size (dataset->type), file_space,
 				    mem_space, buf/*out*/);
 	if (status>=0) goto succeed;
-#ifndef NDEBUG
-	fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n");
+#ifdef H5D_DEBUG
+	fprintf (stderr, "HDF5-DIAG: input pipe optimization failed "
+		 "(falling through)\n");
 #endif
 	H5E_clear ();
     }
     
 	
     /*
-     * This is the general case.
-     */
-#ifndef LATER
-    /*
-     * Note: this prototype version allocates a buffer large enough to
-     *	 satisfy the entire request; strip mining is not implemented.
-     */
-    {
-	size_t src_size = nelmts * H5T_get_size(dataset->type);
-	size_t dst_size = nelmts * H5T_get_size(mem_type);
-	tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size));
-	if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size);
-    }
-#endif
-    /*
-     * Gather the data from disk into the data type conversion buffer. Also
-     * gather data from application to background buffer (this step is not
-     * needed for most conversions, but we leave that as an exercise for
-     * later ;-)
+     * This is the general case.  Figure out the strip mine size.
      */
-    if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
-			    &(dataset->create_parms->efl), 
-			    H5T_get_size (dataset->type), file_space,
-			    &numbering, 0, nelmts,
-			    tconv_buf/*out*/)!=nelmts) {
-	HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed");
-    }
-    if (H5T_BKG_YES==cdata->need_bkg) {
-	if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
-				&numbering, 0, nelmts,
-				bkg_buf/*out*/)!=nelmts) {
-	    HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed");
+    src_type_size = H5T_get_size(dataset->type);
+    dst_type_size = H5T_get_size(mem_type);
+    target_size = xfer_parms->buf_size;
+    request_nelmts = target_size / MAX(src_type_size, dst_type_size);
+    if (request_nelmts<=0) {
+	HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
+		     "temporary buffer max size is too small");
+    }
+    if (sconv_func->init) {
+	smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space,
+					  file_space, request_nelmts,
+					  &numbering/*out*/);
+	if (smine_nelmts<=0) {
+	    HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
+			 "unable to initialize element numbering information");
 	}
+    } else {
+	smine_nelmts = request_nelmts;
+	HDmemset (&numbering, 0, sizeof numbering);
     }
+    buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size);
 
     /*
-     * Perform data type conversion.
+     * Get a temporary buffer for type conversion unless the app has already
+     * supplied one through the xfer properties. Instead of allocating a
+     * buffer which is the exact size, we allocate the target size.  The
+     * malloc() is usually less resource-intensive if we allocate/free the
+     * same size over and over.
      */
-    cdata->command = H5T_CONV_CONV;
-    cdata->ncalls++;
-    if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) {
-	HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
-		    "data type conversion failed");
+    if (NULL==(tconv_buf=xfer_parms->tconv)) {
+	tconv_buf = H5MM_xmalloc (target_size);
+    }
+    if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) {
+	bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size);
     }
-    cdata->nelmts += nelmts;
 
-    /*
-     * Scatter the data into memory.
-     */
-    if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space,
-			    &numbering, 0, nelmts, buf/*out*/)<0) {
-	HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed");
+#ifdef H5D_DEBUG
+    /* Strip mine diagnostics.... */
+    if (smine_nelmts<nelmts) {
+	fprintf (stderr, "HDF5-DIAG: strip mine");
+	if (smine_nelmts!=request_nelmts) {
+	    fprintf (stderr, " got %lu of %lu",
+		     (unsigned long)smine_nelmts,
+		     (unsigned long)request_nelmts);
+	}
+	if (buffer_size!=target_size) {
+	    fprintf (stderr, " (%1.1f%% of buffer)",
+		     100.0*buffer_size/target_size);
+	}
+	fprintf (stderr, " %1.1f iterations\n",
+		 (double)nelmts/smine_nelmts);
     }
+#endif
+
+    /* Start strip mining... */
+    for (smine_start=0; smine_start<nelmts; smine_start+=smine_nelmts) {
+	smine_nelmts = MIN (smine_nelmts, nelmts-smine_start);
+	
+	/*
+	 * Gather the data from disk into the data type conversion
+	 * buffer. Also gather data from application to background buffer
+	 * if necessary.
+	 */
+	if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
+				&(dataset->create_parms->efl), 
+				H5T_get_size (dataset->type), file_space,
+				&numbering, smine_start, smine_nelmts,
+				tconv_buf/*out*/)!=smine_nelmts) {
+	    HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed");
+	}
+	if (H5T_BKG_YES==cdata->need_bkg) {
+	    if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
+				    &numbering, smine_start, smine_nelmts,
+				    bkg_buf/*out*/)!=smine_nelmts) {
+		HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "mem gather failed");
+	    }
+	}
 
+	/*
+	 * Perform data type conversion.
+	 */
+	cdata->command = H5T_CONV_CONV;
+	cdata->ncalls++;
+	if ((tconv_func)(src_id, dst_id, cdata, smine_nelmts, tconv_buf,
+			 bkg_buf)<0) {
+	    HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
+			"data type conversion failed");
+	}
+	cdata->nelmts += smine_nelmts;
+
+	/*
+	 * Scatter the data into memory.
+	 */
+	if ((sconv_func->mscat)(tconv_buf, H5T_get_size (mem_type), mem_space,
+				&numbering, smine_start, smine_nelmts,
+				buf/*out*/)<0) {
+	    HGOTO_ERROR (H5E_IO, H5E_READERROR, FAIL, "scatter failed");
+	}
+    }
     
  succeed:
     ret_value = SUCCEED;
@@ -1232,8 +1271,12 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
  done:
     if (src_id >= 0) H5A_dec_ref(src_id);
     if (dst_id >= 0) H5A_dec_ref(dst_id);
-    tconv_buf = H5MM_xfree(tconv_buf);
-    bkg_buf = H5MM_xfree (bkg_buf);
+    if (tconv_buf && NULL==xfer_parms->tconv) {
+	H5MM_xfree(tconv_buf);
+    }
+    if (bkg_buf && NULL==xfer_parms->bkg) {
+	H5MM_xfree (bkg_buf);
+    }
     FUNC_LEAVE(ret_value);
 }
 
@@ -1260,7 +1303,9 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
 	  const H5S_t *file_space, const H5D_xfer_t *xfer_parms,
 	  const void *buf)
 {
-    size_t		nelmts;
+    size_t		nelmts;			/*total number of elmts	*/
+    size_t		smine_start;		/*strip mine start loc	*/
+    size_t		smine_nelmts;		/*elements per strip	*/
     uint8		*tconv_buf = NULL;	/*data type conv buffer	*/
     uint8		*bkg_buf = NULL;	/*background buffer	*/
     H5T_conv_t		tconv_func = NULL;	/*conversion function	*/
@@ -1269,6 +1314,11 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
     H5S_number_t	numbering;		/*element numbering info*/
     H5T_cdata_t		*cdata = NULL;		/*type conversion data	*/
     herr_t		ret_value = FAIL, status;
+    size_t		src_type_size;		/*size of source type	*/
+    size_t		dst_type_size;		/*size of destination type*/
+    size_t		target_size;		/*desired buffer size	*/
+    size_t		buffer_size;		/*actual buffer size	*/
+    size_t		request_nelmts;		/*requested strip mine	*/
 
     FUNC_ENTER(H5D_write, FAIL);
 
@@ -1279,47 +1329,32 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
     assert(buf);
     if (!file_space) file_space = dataset->space;
     if (!mem_space) mem_space = file_space;
+    nelmts = H5S_get_npoints(mem_space);
 
     /*
-     * Convert data types to atoms because the conversion functions are
-     * application-level functions.
-     */
-    if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 ||
-	(dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) {
-	HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
-		    "unable to register types for conversion");
-    }
-    
-    /*
      * Locate the type conversion function and data space conversion
-     * functions, and set up the element numbering information.
+     * functions, and set up the element numbering information. If a data
+     * type conversion is necessary then register data type atoms.
      */
     if (NULL == (tconv_func = H5T_find(mem_type, dataset->type, &cdata))) {
 	HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL,
 		    "unable to convert between src and dest data types");
+    } else if (H5T_conv_noop!=tconv_func) {
+	if ((src_id = H5A_register(H5_DATATYPE, H5T_copy(mem_type)))<0 ||
+	    (dst_id = H5A_register(H5_DATATYPE, H5T_copy(dataset->type)))<0) {
+	    HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL,
+			"unable to register types for conversion");
+	}
     }
     if (NULL==(sconv_func=H5S_find (mem_space, file_space))) {
 	HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
 		     "unable to convert from memory to file data space");
     }
-    if (sconv_func->init &&
-	(sconv_func->init)(&(dataset->layout), mem_space, file_space,
-			   &numbering/*out*/)<=0) {
-	HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
-		     "unable to initialize element numbering information");
-    } else {
-	HDmemset (&numbering, 0, sizeof numbering);
-    }
-    if (H5S_get_npoints (mem_space)!=H5S_get_npoints (file_space)) {
+    if (nelmts!=H5S_get_npoints (file_space)) {
 	HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
 		     "src and dest data spaces have different sizes");
     }
-
-    /*
-     * Compute the size of the request and allocate scratch buffers.
-     */
-    nelmts = H5S_get_npoints(mem_space);
-
+    
     /*
      * If there is no type conversion then try writing directly from
      * application buffer to file.
@@ -1327,75 +1362,123 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
     if (H5D_OPTIMIZE_PIPE &&
 	H5T_conv_noop==tconv_func &&
 	NULL!=sconv_func->write) {
-#ifndef NDEBUG
-	fprintf (stderr, "HDF5-DIAG: Trying I/O pipe optimization...\n");
-#endif
 	status = (sconv_func->write)(dataset->ent.file, &(dataset->layout),
 				     &(dataset->create_parms->efl),
 				     H5T_get_size (dataset->type), file_space,
 				     mem_space, buf);
 	if (status>=0) goto succeed;
-#ifndef NDEBUG
-	fprintf (stderr, "HDF5-DIAG: I/O pipe optimization failed\n");
+#ifdef H5D_DEBUG
+	fprintf (stderr, "HDF5-DIAG: output pipe optimization failed "
+		 "(falling through)\n");
 #endif
 	H5E_clear ();
     }
 
 
     /*
-     * This is the general case.
+     * This is the general case.  Figure out the strip mine size.
      */
-#ifndef LATER
-    /*
-     * Note: This prototype version allocates a buffer large enough to
-     *	     satisfy the entire request; strip mining is not implemented.
-     */
-    {
-	size_t src_size = nelmts * H5T_get_size(mem_type);
-	size_t dst_size = nelmts * H5T_get_size(dataset->type);
-	tconv_buf = H5MM_xmalloc(MAX(src_size, dst_size));
-	if (cdata->need_bkg) bkg_buf = H5MM_xmalloc (dst_size);
+    src_type_size = H5T_get_size(mem_type);
+    dst_type_size = H5T_get_size(dataset->type);
+    target_size = xfer_parms->buf_size;
+    request_nelmts = target_size / MAX (src_type_size, dst_type_size);
+    if (request_nelmts<=0) {
+	HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
+		     "temporary buffer max size is too small");
+    }
+    if (sconv_func->init) {
+	smine_nelmts = (sconv_func->init)(&(dataset->layout), mem_space,
+					  file_space, request_nelmts,
+					  &numbering/*out*/);
+	if (smine_nelmts<=0) {
+	    HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, FAIL,
+			 "unable to initialize element numbering information");
+	}
+    } else {
+	smine_nelmts = request_nelmts;
+	HDmemset (&numbering, 0, sizeof numbering);
     }
-#endif
+    buffer_size = smine_nelmts * MAX (src_type_size, dst_type_size);
+
     /*
-     * Gather data from application buffer into the data type conversion
-     * buffer. Also gather data from the file into the background buffer
-     * (this step is not needed for most conversions, but we leave that as an
-     * exercise for later ;-)
+     * Get a temporary buffer for type conversion unless the app has already
+     * supplied one through the xfer properties.  Instead of allocating a
+     * buffer which is the exact size, we allocate the target size. The
+     * malloc() is usually less resource-intensive if we allocate/free the
+     * same size over and over.
      */
-    if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
-			    &numbering, 0, nelmts, tconv_buf/*out*/)!=nelmts) {
-	HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed");
+    if (NULL==(tconv_buf=xfer_parms->tconv)) {
+	tconv_buf = H5MM_xmalloc (target_size);
     }
-    if (H5T_BKG_YES==cdata->need_bkg) {
-	if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
-				&(dataset->create_parms->efl),
-				H5T_get_size (dataset->type), file_space,
-				&numbering, 0, nelmts,
-				bkg_buf/*out*/)!=nelmts) {
-	    HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed");
-	}
+    if (cdata->need_bkg && NULL==(bkg_buf=xfer_parms->bkg)) {
+	bkg_buf = H5MM_xmalloc (smine_nelmts * dst_type_size);
     }
 
-    /*
-     * Perform data type conversion.
-     */
-    cdata->command = H5T_CONV_CONV;
-    cdata->ncalls++;
-    if ((tconv_func) (src_id, dst_id, cdata, nelmts, tconv_buf, bkg_buf)<0) {
-	HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
-		    "data type conversion failed");
+#ifdef H5D_DEBUG
+    /* Strip mine diagnostics.... */
+    if (smine_nelmts<nelmts) {
+	fprintf (stderr, "HDF5-DIAG: strip mine");
+	if (smine_nelmts!=request_nelmts) {
+	    fprintf (stderr, " got %lu of %lu",
+		     (unsigned long)smine_nelmts,
+		     (unsigned long)request_nelmts);
+	}
+	if (buffer_size!=target_size) {
+	    fprintf (stderr, " (%1.1f%% of buffer)",
+		     100.0*buffer_size/target_size);
+	}
+	fprintf (stderr, " %1.1f iterations\n",
+		 (double)nelmts/smine_nelmts);
     }
-    cdata->nelmts += nelmts;
+#endif
 
-    /*
-     * Scatter the data out to the file.
-     */
-    if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
-			    &(dataset->create_parms->efl),
-			    H5T_get_size (dataset->type), file_space,
-			    &numbering, 0, nelmts, tconv_buf)<0) {
-	HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed");
+    /* Start strip mining... */
+    for (smine_start=0; smine_start<nelmts; smine_start+=smine_nelmts) {
+	smine_nelmts = MIN (smine_nelmts, nelmts-smine_start);
+
+	/*
+	 * Gather data from application buffer into the data type conversion
+	 * buffer. Also gather data from the file into the background buffer
+	 * if necessary.
+	 */
+	if ((sconv_func->mgath)(buf, H5T_get_size (mem_type), mem_space,
+				&numbering, smine_start, smine_nelmts,
+				tconv_buf/*out*/)!=smine_nelmts) {
+	    HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed");
+	}
+	if (H5T_BKG_YES==cdata->need_bkg) {
+	    if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
+				    &(dataset->create_parms->efl),
+				    H5T_get_size (dataset->type), file_space,
+				    &numbering, smine_start, smine_nelmts,
+				    bkg_buf/*out*/)!=smine_nelmts) {
+		HGOTO_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
+			     "file gather failed");
+	    }
+	}
+
+	/*
+	 * Perform data type conversion.
+	 */
+	cdata->command = H5T_CONV_CONV;
+	cdata->ncalls++;
+	if ((tconv_func) (src_id, dst_id, cdata, smine_nelmts, tconv_buf,
+			  bkg_buf)<0) {
+	    HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL,
+			"data type conversion failed");
+	}
+	cdata->nelmts += smine_nelmts;
+
+	/*
+	 * Scatter the data out to the file.
+	 */
+	if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
+				&(dataset->create_parms->efl),
+				H5T_get_size (dataset->type), file_space,
+				&numbering, smine_start, smine_nelmts,
+				tconv_buf)<0) {
+	    HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed");
+	}
     }
 
  succeed:
@@ -1404,10 +1487,15 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
  done:
     if (src_id >= 0) H5A_dec_ref(src_id);
     if (dst_id >= 0) H5A_dec_ref(dst_id);
-    tconv_buf = H5MM_xfree(tconv_buf);
-    bkg_buf = H5MM_xfree (bkg_buf);
+    if (tconv_buf && NULL==xfer_parms->tconv) {
+	H5MM_xfree(tconv_buf);
+    }
+    if (bkg_buf && NULL==xfer_parms->bkg) {
+	H5MM_xfree (bkg_buf);
+    }
     FUNC_LEAVE(ret_value);
 }
+
 
 /*-------------------------------------------------------------------------
  * Function:	H5D_extend
diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h
index fc6eeb4..f678ef7 100644
--- a/src/H5Dprivate.h
+++ b/src/H5Dprivate.h
@@ -26,6 +26,15 @@
 #include <H5Sprivate.h>         /*for the H5S_t type                         */
 #include <H5Oprivate.h>         /*object Headers                             */
 
+/*
+ * Feature: Define H5D_DEBUG on the compiler command line if you want to
+ *	    debug dataset I/O. NDEBUG must not be defined in order for this
+ *	    to have any effect.
+ */
+#ifdef NDEBUG
+#  undef H5D_DEBUG
+#endif
+
 #define H5D_RESERVED_ATOMS  0
 
 /* Set the minimum object header size to create objects with */
@@ -41,8 +50,11 @@ typedef struct H5D_create_t {
 
 /* Dataset transfer property list */
 typedef struct H5D_xfer_t {
-    int _placeholder;                   /*unused--delete this later          */
+    size_t		buf_size;	/*max temp buffer size		     */
+    void		*tconv;		/*type conversion buffer or null     */
+    void		*bkg;		/*background buffer or null	     */
 } H5D_xfer_t;
+
 typedef struct H5D_t H5D_t;
 extern const H5D_create_t H5D_create_dflt;
 extern const H5D_xfer_t H5D_xfer_dflt;
diff --git a/src/H5P.c b/src/H5P.c
index 8bbf03d..e990675 100644
--- a/src/H5P.c
+++ b/src/H5P.c
@@ -1679,6 +1679,100 @@ H5Pget_family (hid_t tid, hid_t *memb_tid)
 
 
 /*-------------------------------------------------------------------------
+ * Function:	H5Pset_buffer
+ *
+ * Purpose:	Given a dataset transfer property list, set the maximum size
+ *		for the type conversion buffer and background buffer and
+ *		optionally supply pointers to application-allocated buffers.
+ *		If the buffer size is smaller than the entire amount of data
+ *		being transfered between application and file, and a type
+ *		conversion buffer or background buffer is required then
+ *		strip mining will be used.  However, certain restrictions
+ *		apply for the size of buffer which can be used for strip
+ *		mining.  For instance, when strip mining a 100x200x300
+ *		hyperslab of a simple data space the buffer must be large
+ *		enough to hold a 1x200x300 slab.
+ *
+ *		If TCONV and/or BKG are null pointers then buffers will be
+ *		allocated and freed during the data transfer.
+ *
+ * Return:	Success:	SUCCEED
+ *
+ *		Failure:	FAIL
+ *
+ * Programmer:	Robb Matzke
+ *              Monday, March 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg)
+{
+    H5D_xfer_t		*plist = NULL;
+    
+    FUNC_ENTER (H5Pset_buffer, FAIL);
+
+    /* Check arguments */
+    if (H5P_DATASET_XFER != H5Pget_class (plist_id) ||
+	NULL == (plist = H5A_object (plist_id))) {
+	HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
+		       "not a dataset transfer property list");
+    }
+    if (size<=0) {
+	HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
+		       "buffer size must not be zero");
+    }
+
+    /* Update property list */
+    plist->buf_size = size;
+    plist->tconv = tconv;
+    plist->bkg = bkg;
+
+    FUNC_LEAVE (SUCCEED);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function:	H5Pget_buffer
+ *
+ * Purpose:	Reads values previously set with H5Pset_buffer().
+ *
+ * Return:	Success:	Buffer size.
+ *
+ *		Failure:	0
+ *
+ * Programmer:	Robb Matzke
+ *              Monday, March 16, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+size_t
+H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/)
+{
+    H5D_xfer_t		*plist = NULL;
+    
+    FUNC_ENTER (H5Pget_buffer, 0);
+
+    /* Check arguments */
+    if (H5P_DATASET_XFER != H5Pget_class (plist_id) ||
+	NULL == (plist = H5A_object (plist_id))) {
+	HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, 0,
+		       "not a dataset transfer property list");
+    }
+
+    /* Return values */
+    if (tconv) *tconv = plist->tconv;
+    if (bkg) *bkg = plist->bkg;
+
+    FUNC_LEAVE (plist->buf_size);
+}
+
+
+/*-------------------------------------------------------------------------
  * Function:	H5Pset_mpi
  *
  * Signature:	herr_t H5Pset_mpi(hid_t tid, MPI_Comm comm, MPI_Info info,
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index 47e9b5a..7e91ce2 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -83,8 +83,12 @@ herr_t H5Pget_split (hid_t tid, size_t meta_ext_size, char *meta_ext/*out*/,
 
 herr_t H5Pset_family (hid_t tid, hid_t memb_tid);
 herr_t H5Pget_family (hid_t tid, hid_t *memb_tid/*out*/);
+herr_t H5Pset_buffer (hid_t plist_id, size_t size, void *tconv, void *bkg);
+size_t H5Pget_buffer (hid_t plist_id, void **tconv/*out*/, void **bkg/*out*/);
+
 #ifdef HAVE_PARALLEL
-herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info, unsigned access_mode);
+herr_t H5Pset_mpi (hid_t tid, MPI_Comm comm, MPI_Info info,
+		   unsigned access_mode);
 herr_t H5Pget_mpi (hid_t tid, MPI_Comm *comm/*out*/, MPI_Info *info/*out*/,
 		   unsigned *access_mode/*out*/);
 #endif
diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h
index fbce725..da19695 100644
--- a/src/H5Sprivate.h
+++ b/src/H5Sprivate.h
@@ -65,7 +65,8 @@ typedef struct H5S_number_t {
 typedef struct H5S_tconv_t {
     /* Initialize element numbering information */
     size_t (*init)(const struct H5O_layout_t *layout, const H5S_t *mem_space,
-                   const H5S_t *file_space, H5S_number_t *numbering/*out*/);
+                   const H5S_t *file_space, size_t desired_nelmts,
+		   H5S_number_t *numbering/*out*/);
 
     /* Gather elements from disk to type conversion buffer */
     size_t (*fgath)(H5F_t *f, const struct H5O_layout_t *layout,
@@ -122,7 +123,7 @@ intn H5S_extend (H5S_t *space, const size_t *size);
 /* Conversion functions for simple data spaces */
 size_t H5S_simp_init (const struct H5O_layout_t *layout,
                       const H5S_t *mem_space, const H5S_t *file_space,
-                      H5S_number_t *numbering/*out*/);
+                      size_t desired_nelmts, H5S_number_t *numbering/*out*/);
 size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
 		       const struct H5O_efl_t *efl, size_t elmt_size,
 		       const H5S_t *file_space, const H5S_number_t *numbering,
diff --git a/src/H5Ssimp.c b/src/H5Ssimp.c
index 6f1f498..a785b35 100644
--- a/src/H5Ssimp.c
+++ b/src/H5Ssimp.c
@@ -37,9 +37,14 @@ static intn             interface_initialize_g = FALSE;
  */
 size_t
 H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
-	       const H5S_t *file_space, H5S_number_t *numbering/*out*/)
+	       const H5S_t *file_space, size_t desired_nelmts,
+	       H5S_number_t *numbering/*out*/)
 {
     size_t	nelmts;
+    int		m_ndims, f_ndims;	/*mem, file dimensionality	*/
+    size_t	size[H5O_LAYOUT_NDIMS];	/*size of selected hyperslab	*/
+    size_t	acc;
+    int		i;
     
     FUNC_ENTER (H5S_simp_init, 0);
 
@@ -49,11 +54,43 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
     assert (file_space && H5S_SIMPLE==file_space->type);
     assert (numbering);
 
-    /* Numbering is implied by the hyperslab, C order */
+    /* Numbering is implied by the hyperslab, C order, no data here */
     HDmemset (numbering, 0, sizeof(H5S_number_t));
 
-    /* Data can be efficiently copied at any size */
-    nelmts = H5S_get_npoints (file_space);
+    /*
+     * The stripmine size is such that only the slowest varying dimension can
+     * be split up.  We choose the largest possible strip mine size which is
+     * not larger than the desired size.
+     */
+    m_ndims = H5S_get_hyperslab (mem_space, NULL, size, NULL);
+    for (i=m_ndims-1, acc=1; i>0; --i) acc *= size[i];
+    nelmts = (desired_nelmts/acc) * acc;
+    if (nelmts<=0) {
+	HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
+		       "strip mine buffer is too small");
+    }
+
+    /*
+     * The value chosen for mem_space must be the same as the value chosen for
+     * file_space.
+     */
+    f_ndims = H5S_get_hyperslab (file_space, NULL, size, NULL);
+    if (m_ndims!=f_ndims) {
+	nelmts = H5S_get_npoints (file_space);
+	if (nelmts>desired_nelmts) {
+	    HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
+			   "strip mining not supported across "
+			   "dimensionalities");
+	}
+	assert (nelmts==H5S_get_npoints (mem_space));
+    } else {
+	for (i=f_ndims-1, acc=1; i>0; --i) acc *= size[i];
+	acc *= (desired_nelmts/acc);
+	if (nelmts!=acc) {
+	    HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, 0,
+			   "unsupported strip mine size for shape change");
+	}
+    }
     
     FUNC_LEAVE (nelmts);
 }
@@ -95,6 +132,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
     size_t	hsize[H5O_LAYOUT_NDIMS];	/*size of hyperslab	*/
     size_t	zero[H5O_LAYOUT_NDIMS];		/*zero			*/
     size_t	sample[H5O_LAYOUT_NDIMS];	/*hyperslab sampling	*/
+    size_t	acc;				/*accumulator		*/
 #ifndef LATER
     intn	file_offset_signed[H5O_LAYOUT_NDIMS];
 #endif
@@ -113,12 +151,6 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
     assert (buf);
 
     /*
-     * The prototype doesn't support strip mining.
-     */
-    assert (0==start);
-    assert (nelmts==H5S_get_npoints (file_space));
-
-    /*
      * Get hyperslab information to determine what elements are being
      * selected (there might eventually be other selection methods too).
      * We only support hyperslabs with unit sample because there's no way to
@@ -132,6 +164,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
 		       "unable to retrieve hyperslab parameters");
     }
 #else
+    /* Argument type problems to be fixed later..... -RPM */
     if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed,
 					hsize, sample))<0) {
 	HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0,
@@ -142,12 +175,23 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
 	file_offset[i] = file_offset_signed[i];
     }
 #endif
+
+    /* Check that there is no subsampling of the hyperslab */
     for (i=0; i<space_ndims; i++) {
 	if (sample[i]!=1) {
 	    HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, 0,
 			   "hyperslab sampling is not implemented yet");
 	}
     }
+
+    /* Adjust the slowest varying dimension to take care of strip mining */
+    for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
+    assert (0==start % acc);
+    assert (0==nelmts % acc);
+    file_offset[0] += start / acc;
+    hsize[0] = nelmts / acc;
+
+    /* The fastest varying dimension is for the data point itself */
     file_offset[space_ndims] = 0;
     hsize[space_ndims] = elmt_size;
     HDmemset (zero, 0, layout->ndims*sizeof(size_t));
@@ -194,6 +238,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
     size_t	hsize[H5O_LAYOUT_NDIMS];	/*size of hyperslab	*/
     size_t	zero[H5O_LAYOUT_NDIMS];		/*zero			*/
     size_t	sample[H5O_LAYOUT_NDIMS];	/*hyperslab sampling	*/
+    size_t	acc;				/*accumulator		*/
 #ifndef LATER
     intn	mem_offset_signed[H5O_LAYOUT_NDIMS];
 #endif
@@ -211,12 +256,6 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
     assert (buf);
 
     /*
-     * The prototype doesn't support strip mining.
-     */
-    assert (0==start);
-    assert (nelmts==H5S_get_npoints (mem_space));
-
-    /*
      * Retrieve hyperslab information to determine what elements are being
      * selected (there might be other selection methods in the future).  We
      * only handle hyperslabs with unit sample because there's currently no
@@ -229,6 +268,7 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
 		       "unable to retrieve hyperslab parameters");
     }
 #else
+    /* Argument type problems to be fixed later..... -RPM */
     if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed,
 					hsize, sample))<0) {
 	HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
@@ -239,6 +279,8 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
 	mem_offset[i] = mem_offset_signed[i];
     }
 #endif
+
+    /* Check that there is no subsampling of the hyperslab */
     for (i=0; i<space_ndims; i++) {
 	if (sample[i]!=1) {
 	    HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
@@ -249,6 +291,15 @@ H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
 	HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
 		       "unable to retrieve data space dimensions");
     }
+
+    /* Adjust the slowest varying dimension to take care of strip mining */
+    for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
+    assert (0==start % acc);
+    assert (0==nelmts % acc);
+    mem_offset[0] += start / acc;
+    hsize[0] = nelmts / acc;
+
+    /* The fastest varying dimension is for the data point itself */
     mem_offset[space_ndims] = 0;
     mem_size[space_ndims] = elmt_size;
     hsize[space_ndims] = elmt_size;
@@ -299,6 +350,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
     size_t	hsize[H5O_LAYOUT_NDIMS];	/*size of hyperslab	*/
     size_t	zero[H5O_LAYOUT_NDIMS];		/*zero			*/
     size_t	sample[H5O_LAYOUT_NDIMS];	/*hyperslab sampling	*/
+    size_t	acc;				/*accumulator		*/
 #ifndef LATER
     intn	mem_offset_signed[H5O_LAYOUT_NDIMS];
 #endif
@@ -316,12 +368,6 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
     assert (tconv_buf);
 
     /*
-     * The prototype doesn't support strip mining.
-     */
-    assert (0==start);
-    assert (nelmts==H5S_get_npoints (mem_space));
-
-    /*
      * Retrieve hyperslab information to determine what elements are being
      * selected (there might be other selection methods in the future).  We
      * only handle hyperslabs with unit sample because there's currently no
@@ -334,6 +380,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
 		       "unable to retrieve hyperslab parameters");
     }
 #else
+    /* Argument type problems to be fixed later..... -RPM */
     if ((space_ndims=H5S_get_hyperslab (mem_space, mem_offset_signed,
 					hsize, sample))<0) {
 	HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
@@ -344,6 +391,8 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
 	mem_offset[i] = mem_offset_signed[i];
     }
 #endif
+
+    /* Check that there is no subsampling of the hyperslab */
     for (i=0; i<space_ndims; i++) {
 	if (sample[i]!=1) {
 	    HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, 0,
@@ -354,6 +403,15 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
 	HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, 0,
 		       "unable to retrieve data space dimensions");
     }
+
+    /* Adjust the slowest varying dimension to account for strip mining */
+    for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
+    assert (0==start % acc);
+    assert (0==nelmts % acc);
+    mem_offset[0] += start / acc;
+    hsize[0] = nelmts / acc;
+    
+    /* The fastest varying dimension is for the data point itself */
     mem_offset[space_ndims] = 0;
     mem_size[space_ndims] = elmt_size;
     hsize[space_ndims] = elmt_size;
@@ -404,6 +462,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
     size_t	hsize[H5O_LAYOUT_NDIMS];	/*size of hyperslab	*/
     size_t	zero[H5O_LAYOUT_NDIMS];		/*zero vector		*/
     size_t	sample[H5O_LAYOUT_NDIMS];	/*hyperslab sampling	*/
+    size_t	acc;				/*accumulator		*/
 #ifndef LATER
     intn	file_offset_signed[H5O_LAYOUT_NDIMS];
 #endif
@@ -422,12 +481,6 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
     assert (buf);
 
     /*
-     * The prototype doesn't support strip mining.
-     */
-    assert (0==start);
-    assert (nelmts==H5S_get_npoints (file_space));
-    
-    /*
      * Get hyperslab information to determine what elements are being
      * selected (there might eventually be other selection methods too).
      * We only support hyperslabs with unit sample because there's no way to
@@ -441,6 +494,7 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
 		       "unable to retrieve hyperslab parameters");
     }
 #else
+    /* Argument type problems to be fixed later..... -RPM */
     if ((space_ndims=H5S_get_hyperslab (file_space, file_offset_signed,
 					hsize, sample))<0) {
 	HRETURN_ERROR (H5E_DATASPACE, H5E_CANTINIT, FAIL,
@@ -451,12 +505,23 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
 	file_offset[i] = file_offset_signed[i];
     }
 #endif
+
+    /* Check that there is no subsampling of the hyperslab */
     for (i=0; i<space_ndims; i++) {
 	if (sample[i]!=1) {
 	    HRETURN_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
 			   "hyperslab sampling is not implemented yet");
 	}
     }
+
+    /* Adjust the slowest varying dimension to account for strip mining */
+    for (i=1, acc=1; i<space_ndims; i++) acc *= hsize[i];
+    assert (0==start % acc);
+    assert (0==nelmts % acc);
+    file_offset[0] += start / acc;
+    hsize[0] = nelmts / acc;
+    
+    /* The fastest varying dimension is for the data point itself */
     file_offset[space_ndims] = 0;
     hsize[space_ndims] = elmt_size;
     HDmemset (zero, 0, layout->ndims*sizeof(size_t));