summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorQuincey Koziol <koziol@hdfgroup.org>2000-09-26 22:50:18 (GMT)
committerQuincey Koziol <koziol@hdfgroup.org>2000-09-26 22:50:18 (GMT)
commit183c8245af3a43d2f07845a0ac717f558afa3d1c (patch)
treeeaf1cdd565774fb008e9e6f836550caf7eb8d0c8
parent415e326afec0c6975a7b02fc77a72adcb8ba979e (diff)
downloadhdf5-183c8245af3a43d2f07845a0ac717f558afa3d1c.zip
hdf5-183c8245af3a43d2f07845a0ac717f558afa3d1c.tar.gz
hdf5-183c8245af3a43d2f07845a0ac717f558afa3d1c.tar.bz2
[svn-r2600] Purpose:
Implemented new feature Description: Added data sieve buffering code to raw I/O data path. This is enabled for all the VFL drivers except the mpio & core drivers. Also added two new API functions to control the sieve buffer size: H5Pset_sieve_buf_size() and H5Pget_sieve_buf_size(). Platforms tested: Solaris 2.6 (i.e. baldric)
-rw-r--r--src/H5F.c21
-rw-r--r--src/H5FDdpss.c36
-rw-r--r--src/H5FDfamily.c1
-rw-r--r--src/H5FDgass.c37
-rw-r--r--src/H5FDlog.c1
-rw-r--r--src/H5FDmulti.c32
-rw-r--r--src/H5FDpublic.h10
-rw-r--r--src/H5FDsec2.c1
-rw-r--r--src/H5FDsrb.c36
-rw-r--r--src/H5FDstdio.c1
-rw-r--r--src/H5FDstream.c36
-rw-r--r--src/H5Farray.c298
-rw-r--r--src/H5Fprivate.h11
-rw-r--r--src/H5P.c84
-rw-r--r--src/H5Ppublic.h2
15 files changed, 580 insertions, 27 deletions
diff --git a/src/H5F.c b/src/H5F.c
index 69a1d5a..a93ea60 100644
--- a/src/H5F.c
+++ b/src/H5F.c
@@ -264,6 +264,7 @@ H5F_init_interface(void)
H5F_access_dflt.alignment = 1; /*no alignment*/
H5F_access_dflt.gc_ref = 0; /*don't garbage-collect references*/
H5F_access_dflt.meta_block_size = 2048; /* set metadata block allocations to 2KB */
+ H5F_access_dflt.sieve_buf_size = 64*1024; /* set sieve buffer allocation to 64KB */
H5F_access_dflt.driver_id = H5FD_SEC2; /*default driver*/
H5F_access_dflt.driver_info = NULL; /*driver file access properties*/
@@ -522,6 +523,7 @@ H5Fget_access_plist(hid_t file_id)
_fapl.alignment = f->shared->alignment;
_fapl.gc_ref = f->shared->gc_ref;
_fapl.meta_block_size = f->shared->lf->def_meta_block_size;
+ _fapl.sieve_buf_size = f->shared->sieve_buf_size;
_fapl.driver_id = f->shared->lf->driver_id;
_fapl.driver_info = NULL; /*just for now */
@@ -761,6 +763,7 @@ H5F_new(H5F_file_t *shared, hid_t fcpl_id, hid_t fapl_id)
f->shared->threshold = fapl->threshold;
f->shared->alignment = fapl->alignment;
f->shared->gc_ref = fapl->gc_ref;
+ f->shared->sieve_buf_size = fapl->sieve_buf_size;
#ifdef H5_HAVE_PARALLEL
/*
@@ -854,6 +857,12 @@ H5F_dest(H5F_t *f)
}
f->shared->cwfs = H5MM_xfree (f->shared->cwfs);
+ /* Free the data sieve buffer, if it's been allocated */
+ if(f->shared->sieve_buf) {
+ assert(f->shared->sieve_dirty==0); /* The buffer had better be flushed... */
+ f->shared->sieve_buf = H5MM_xfree (f->shared->sieve_buf);
+ } /* end if */
+
/* Destroy file creation properties */
H5P_close(f->shared->fcpl);
@@ -1670,6 +1679,18 @@ H5F_flush(H5F_t *f, H5F_scope_t scope, hbool_t invalidate,
}
}
+ /* flush the data sieve buffer, if we have a dirty one */
+ if(!alloc_only && f->shared->sieve_buf && f->shared->sieve_dirty) {
+ /* Write dirty data sieve buffer to file */
+ if (H5F_block_write(f, H5FD_MEM_DRAW, f->shared->sieve_loc, f->shared->sieve_size, H5P_DEFAULT, f->shared->sieve_buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
+ "block write failed");
+ }
+
+ /* Reset sieve buffer dirty flag */
+ f->shared->sieve_dirty=0;
+ } /* end if */
+
/* flush the entire raw data cache */
if (!alloc_only && H5F_istore_flush (f, invalidate)<0) {
HRETURN_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL,
diff --git a/src/H5FDdpss.c b/src/H5FDdpss.c
index 7a03ffe..26ede02 100644
--- a/src/H5FDdpss.c
+++ b/src/H5FDdpss.c
@@ -114,6 +114,7 @@ typedef struct H5FD_dpss_t {
static H5FD_t *H5FD_dpss_open (const char *name, unsigned flags,
hid_t UNUSED fapl_id, haddr_t maxaddr);
static herr_t H5FD_dpss_close (H5FD_t *_file);
+static herr_t H5FD_dpss_query(const H5FD_t *_f1, unsigned long *flags);
static haddr_t H5FD_dpss_get_eoa (H5FD_t *_file);
static herr_t H5FD_dpss_set_eoa (H5FD_t *_file, haddr_t addr);
static haddr_t H5FD_dpss_get_eof (H5FD_t *_file);
@@ -139,7 +140,7 @@ static const H5FD_class_t H5FD_dpss_g = {
H5FD_dpss_open, /* open */
H5FD_dpss_close, /* close */
NULL, /* cmp */
- NULL, /*query */
+ H5FD_dpss_query, /*query */
NULL, /* alloc */
NULL, /* free */
H5FD_dpss_get_eoa, /* get_eoa */
@@ -394,6 +395,39 @@ H5FD_dpss_close (H5FD_t *_file)
/*-------------------------------------------------------------------------
+ * Function: H5FD_dpss_query
+ *
+ * Purpose: Set the flags that this VFL driver is capable of supporting.
+ * (listed in H5FDpublic.h)
+ *
+ * Return: Success: non-negative
+ *
+ * Failure: negative
+ *
+ * Programmer: Quincey Koziol
+ * Tuesday, September 26, 2000
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_dpss_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
+{
+ herr_t ret_value=SUCCEED;
+
+ FUNC_ENTER(H5FD_dpss_query, FAIL);
+
+ /* Set the VFL feature flags that this driver supports */
+ if(flags) {
+ *flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
+ } /* end if */
+
+ FUNC_LEAVE(ret_value);
+}
+
+
+/*-------------------------------------------------------------------------
* Function: H5FD_dpss_get_eoa
*
* Purpose: Gets the end-of-address marker for the file. The EOA marker
diff --git a/src/H5FDfamily.c b/src/H5FDfamily.c
index 1c456a4..6f12863 100644
--- a/src/H5FDfamily.c
+++ b/src/H5FDfamily.c
@@ -637,6 +637,7 @@ H5FD_family_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
if(flags) {
*flags|=H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */
*flags|=H5FD_FEAT_ACCUMULATE_METADATA; /* OK to accumulate metadata for faster writes */
+ *flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
FUNC_LEAVE(ret_value);
diff --git a/src/H5FDgass.c b/src/H5FDgass.c
index e500d8e..7d8ac9c 100644
--- a/src/H5FDgass.c
+++ b/src/H5FDgass.c
@@ -100,6 +100,7 @@ typedef struct H5FD_gass_t {
static H5FD_t *H5FD_gass_open(const char *name, unsigned flags, hid_t fapl_id,
haddr_t maxaddr);
static herr_t H5FD_gass_close(H5FD_t *_file);
+static herr_t H5FD_gass_query(const H5FD_t *_f1, unsigned long *flags);
static haddr_t H5FD_gass_get_eoa(H5FD_t *_file);
static herr_t H5FD_gass_set_eoa(H5FD_t *_file, haddr_t addr);
static haddr_t H5FD_gass_get_eof(H5FD_t *_file);
@@ -129,8 +130,8 @@ static const H5FD_class_t H5FD_gass_g = {
NULL, /*dxpl_free */
H5FD_gass_open, /*open */
H5FD_gass_close, /*close */
- NULL, /*query */
NULL, /*cmp */
+ H5FD_gass_query, /*query */
NULL, /*alloc */
NULL, /*free */
H5FD_gass_get_eoa, /*get_eoa */
@@ -432,6 +433,40 @@ H5FD_gass_close (H5FD_t *_file)
}
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_gass_query
+ *
+ * Purpose: Set the flags that this VFL driver is capable of supporting.
+ * (listed in H5FDpublic.h)
+ *
+ * Return: Success: non-negative
+ *
+ * Failure: negative
+ *
+ * Programmer: Quincey Koziol
+ * Tuesday, September 26, 2000
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_gass_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
+{
+ herr_t ret_value=SUCCEED;
+
+ FUNC_ENTER(H5FD_gass_query, FAIL);
+
+ /* Set the VFL feature flags that this driver supports */
+ if(flags) {
+ *flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
+ } /* end if */
+
+ FUNC_LEAVE(ret_value);
+}
+
+
/*-------------------------------------------------------------------------
* Function: H5FD_gass_get_eoa
*
diff --git a/src/H5FDlog.c b/src/H5FDlog.c
index 148f3ef..1fc7c5c 100644
--- a/src/H5FDlog.c
+++ b/src/H5FDlog.c
@@ -646,6 +646,7 @@ H5FD_log_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
if(flags) {
*flags|=H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */
*flags|=H5FD_FEAT_ACCUMULATE_METADATA; /* OK to accumulate metadata for faster writes */
+ *flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
FUNC_LEAVE(ret_value);
diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c
index b77a695..68fa26a 100644
--- a/src/H5FDmulti.c
+++ b/src/H5FDmulti.c
@@ -115,6 +115,7 @@ static H5FD_t *H5FD_multi_open(const char *name, unsigned flags,
hid_t fapl_id, haddr_t maxaddr);
static herr_t H5FD_multi_close(H5FD_t *_file);
static int H5FD_multi_cmp(const H5FD_t *_f1, const H5FD_t *_f2);
+static herr_t H5FD_multi_query(const H5FD_t *_f1, unsigned long *flags);
static haddr_t H5FD_multi_get_eoa(H5FD_t *_file);
static herr_t H5FD_multi_set_eoa(H5FD_t *_file, haddr_t eoa);
static haddr_t H5FD_multi_get_eof(H5FD_t *_file);
@@ -144,7 +145,7 @@ static const H5FD_class_t H5FD_multi_g = {
H5FD_multi_open, /*open */
H5FD_multi_close, /*close */
H5FD_multi_cmp, /*cmp */
- NULL, /*query */
+ H5FD_multi_query, /*query */
H5FD_multi_alloc, /*alloc */
H5FD_multi_free, /*free */
H5FD_multi_get_eoa, /*get_eoa */
@@ -1300,6 +1301,35 @@ H5FD_multi_cmp(const H5FD_t *_f1, const H5FD_t *_f2)
/*-------------------------------------------------------------------------
+ * Function: H5FD_multi_query
+ *
+ * Purpose: Set the flags that this VFL driver is capable of supporting.
+ * (listed in H5FDpublic.h)
+ *
+ * Return: Success: non-negative
+ *
+ * Failure: negative
+ *
+ * Programmer: Quincey Koziol
+ * Tuesday, September 26, 2000
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_multi_query(const H5FD_t *_f, unsigned long *flags /* out */)
+{
+ /* Set the VFL feature flags that this driver supports */
+ if(flags) {
+ *flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
+ } /* end if */
+
+ return(0);
+}
+
+
+/*-------------------------------------------------------------------------
* Function: H5FD_multi_get_eoa
*
* Purpose: Returns the end-of-address marker for the file. The EOA
diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h
index 8971590..b1185fa 100644
--- a/src/H5FDpublic.h
+++ b/src/H5FDpublic.h
@@ -75,7 +75,7 @@ typedef enum H5FD_mem_t {
/* Define VFL driver features that can be enabled on a per-driver basis */
-/* These are returned queried with the 'query' function pointer in H5FD_class_t */
+/* These are returned with the 'query' function pointer in H5FD_class_t */
/*
* Defining the H5FD_FEAT_AGGREGATE_METADATA for a VFL driver means that
* the library will attempt to allocate a larger block for metadata and
@@ -89,6 +89,14 @@ typedef enum H5FD_mem_t {
* 'write' routine.
*/
#define H5FD_FEAT_ACCUMULATE_METADATA 0x00000002
+ /*
+ * Defining the H5FD_FEAT_DATA_SIEVE for a VFL driver means that
+ * the library will attempt to cache raw data as it is read from/written to
+ * a file in a "data seive" buffer. See Rajeev Thakur's papers:
+ * http://www.mcs.anl.gov/~thakur/papers/romio-coll.ps.gz
+ * http://www.mcs.anl.gov/~thakur/papers/mpio-high-perf.ps.gz
+ */
+#define H5FD_FEAT_DATA_SIEVE 0x00000004
/* Forward declaration */
diff --git a/src/H5FDsec2.c b/src/H5FDsec2.c
index 6222964..ee38f55 100644
--- a/src/H5FDsec2.c
+++ b/src/H5FDsec2.c
@@ -416,6 +416,7 @@ H5FD_sec2_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
if(flags) {
*flags|=H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */
*flags|=H5FD_FEAT_ACCUMULATE_METADATA; /* OK to accumulate metadata for faster writes */
+ *flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
FUNC_LEAVE(ret_value);
diff --git a/src/H5FDsrb.c b/src/H5FDsrb.c
index af75b1e..bfc5f13 100644
--- a/src/H5FDsrb.c
+++ b/src/H5FDsrb.c
@@ -56,6 +56,7 @@ static hid_t H5FD_SRB_g = 0;
static H5FD_t *H5FD_srb_open(const char *name, unsigned flags, hid_t fapl_id,
haddr_t maxaddr);
static herr_t H5FD_srb_close(H5FD_t *_file);
+static herr_t H5FD_srb_query(const H5FD_t *_f1, unsigned long *flags);
static haddr_t H5FD_srb_get_eoa(H5FD_t *_file);
static herr_t H5FD_srb_set_eoa(H5FD_t *_file, haddr_t addr);
static haddr_t H5FD_srb_get_eof(H5FD_t *_file);
@@ -99,7 +100,7 @@ static const H5FD_class_t H5FD_srb_g = {
H5FD_srb_open, /*open */
H5FD_srb_close, /*close */
NULL, /*cmp */
- NULL, /*query */
+ H5FD_srb_query, /*query */
NULL, /*alloc */
NULL, /*free */
H5FD_srb_get_eoa, /*get_eoa */
@@ -361,6 +362,39 @@ H5FD_srb_close(H5FD_t *_file)
FUNC_LEAVE(SUCCEED);
}
+
+/*-------------------------------------------------------------------------
+ * Function: H5FD_srb_query
+ *
+ * Purpose: Set the flags that this VFL driver is capable of supporting.
+ * (listed in H5FDpublic.h)
+ *
+ * Return: Success: non-negative
+ *
+ * Failure: negative
+ *
+ * Programmer: Quincey Koziol
+ * Tuesday, September 26, 2000
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_srb_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
+{
+ herr_t ret_value=SUCCEED;
+
+ FUNC_ENTER(H5FD_srb_query, FAIL);
+
+ /* Set the VFL feature flags that this driver supports */
+ if(flags) {
+ *flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
+ } /* end if */
+
+ FUNC_LEAVE(ret_value);
+}
+
/*-------------------------------------------------------------------------
* Function: H5FD_srb_get_eoa
*
diff --git a/src/H5FDstdio.c b/src/H5FDstdio.c
index 5461996..28b32a7 100644
--- a/src/H5FDstdio.c
+++ b/src/H5FDstdio.c
@@ -437,6 +437,7 @@ H5FD_stdio_query(const H5FD_t *_f, unsigned long *flags /* out */)
if(flags) {
*flags|=H5FD_FEAT_AGGREGATE_METADATA; /* OK to aggregate metadata allocations */
*flags|=H5FD_FEAT_ACCUMULATE_METADATA; /* OK to accumulate metadata for faster writes */
+ *flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
} /* end if */
return(0);
diff --git a/src/H5FDstream.c b/src/H5FDstream.c
index cd0e6e6..ba07ae4 100644
--- a/src/H5FDstream.c
+++ b/src/H5FDstream.c
@@ -108,6 +108,7 @@ static H5FD_t *H5FD_stream_open (const char *name, unsigned flags,
hid_t fapl_id, haddr_t maxaddr);
static herr_t H5FD_stream_flush (H5FD_t *_stream);
static herr_t H5FD_stream_close (H5FD_t *_stream);
+static herr_t H5FD_stream_query(const H5FD_t *_f1, unsigned long *flags);
static haddr_t H5FD_stream_get_eoa (H5FD_t *_stream);
static herr_t H5FD_stream_set_eoa (H5FD_t *_stream, haddr_t addr);
static haddr_t H5FD_stream_get_eof (H5FD_t *_stream);
@@ -134,8 +135,8 @@ static const H5FD_class_t H5FD_stream_g =
NULL, /* dxpl_free */
H5FD_stream_open, /* open */
H5FD_stream_close, /* close */
- NULL, /* query */
NULL, /* cmp */
+ H5FD_stream_query, /*query */
NULL, /* alloc */
NULL, /* free */
H5FD_stream_get_eoa, /* get_eoa */
@@ -753,6 +754,39 @@ static herr_t H5FD_stream_close (H5FD_t *_stream)
/*-------------------------------------------------------------------------
+ * Function: H5FD_stream_query
+ *
+ * Purpose: Set the flags that this VFL driver is capable of supporting.
+ * (listed in H5FDpublic.h)
+ *
+ * Return: Success: non-negative
+ *
+ * Failure: negative
+ *
+ * Programmer: Quincey Koziol
+ * Tuesday, September 26, 2000
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static herr_t
+H5FD_stream_query(const UNUSED H5FD_t *_f, unsigned long *flags /* out */)
+{
+ herr_t ret_value=SUCCEED;
+
+ FUNC_ENTER(H5FD_stream_query, FAIL);
+
+ /* Set the VFL feature flags that this driver supports */
+ if(flags) {
+ *flags|=H5FD_FEAT_DATA_SIEVE; /* OK to perform data sieving for faster raw data reads & writes */
+ } /* end if */
+
+ FUNC_LEAVE(ret_value);
+}
+
+
+/*-------------------------------------------------------------------------
* Function: H5FD_stream_get_eoa
*
* Purpose: Gets the end-of-address marker for the file. The EOA marker
diff --git a/src/H5Farray.c b/src/H5Farray.c
index db23acc..e581315 100644
--- a/src/H5Farray.c
+++ b/src/H5Farray.c
@@ -15,8 +15,10 @@
#include <H5Dprivate.h>
#include <H5Eprivate.h>
#include <H5Fprivate.h>
+#include <H5FDprivate.h> /*file driver */
#include <H5Iprivate.h>
#include <H5MFprivate.h>
+#include <H5MMprivate.h> /*memory management */
#include <H5Oprivate.h>
#include <H5Pprivate.h>
#include <H5Vprivate.h>
@@ -138,6 +140,7 @@ H5F_arr_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout,
size_t nelmts, z; /*number of elements */
intn ndims; /*stride dimensionality */
haddr_t addr; /*address in file */
+ haddr_t eof; /*end of file address */
intn i, j; /*counters */
hbool_t carray; /*carry for subtraction */
#ifdef H5_HAVE_PARALLEL
@@ -286,19 +289,147 @@ H5F_arr_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout,
z<nelmts;
z++, xfer_parms->gather_reads--) {
#else
+#ifdef QAK
+printf("%s: nelmts=%d, addr=%lu, elmt_size=%lu\n",FUNC,(int)nelmts,(unsigned long)addr,(unsigned long)elmt_size);
+printf("%s: sieve_buf=%p, sieve_loc=%lu, sieve_size=%lu, sieve_buf_size=%lu, sieve_dirty=%u\n",FUNC,f->shared->sieve_buf,(unsigned long)f->shared->sieve_loc,(unsigned long)f->shared->sieve_size,(unsigned long)f->shared->sieve_buf_size,(unsigned)f->shared->sieve_dirty);
+printf("%s: feature_flags=%lx\n",FUNC,(unsigned long)f->shared->lf->feature_flags);
+#endif /* QAK */
for (z=0; z<nelmts; z++) {
#endif
- /* Read from file */
- if (efl && efl->nused>0) {
- if (H5O_efl_read(f, efl, addr, elmt_size, buf)<0) {
- HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
- "external data read failed");
- }
- } else if (H5F_block_read(f, addr, elmt_size, dxpl_id, buf)<0) {
- HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
- "block read failed");
- }
+ /* Read directly from file if the dataset is in an external file */
+ /* Note: We can't use data sieve buffers for datasets in external files
+ * because the 'addr' of all external files is set to 0 (above) and
+ * all datasets in external files would alias to the same set of
+ * file offsets, totally mixing up the data sieve buffer information. -QAK
+ */
+ if (efl && efl->nused>0) {
+ if (H5O_efl_read(f, efl, addr, elmt_size, buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "external data read failed");
+ }
+ } else {
+ /* Check if data sieving is enabled */
+ if(f->shared->lf->feature_flags&H5FD_FEAT_DATA_SIEVE) {
+ /* Try reading from the data sieve buffer */
+ if(f->shared->sieve_buf) {
+ /* If entire read is within the sieve buffer, read it from the buffer */
+ if((addr>=f->shared->sieve_loc && addr<(f->shared->sieve_loc+f->shared->sieve_size))
+ && ((addr+elmt_size-1)>=f->shared->sieve_loc && (addr+elmt_size-1)<(f->shared->sieve_loc+f->shared->sieve_size))) {
+ /* Grab the data out of the buffer */
+ HDmemcpy(buf,f->shared->sieve_buf+(addr-f->shared->sieve_loc),elmt_size);
+ } /* end if */
+ /* Entire request is not within this data sieve buffer */
+ else {
+ /* Check if we can actually hold the I/O request in the sieve buffer */
+ if(elmt_size>f->shared->sieve_buf_size) {
+ /* Check for any overlap with the current sieve buffer */
+ if((f->shared->sieve_loc>=addr && f->shared->sieve_loc<(addr+elmt_size))
+ || ((f->shared->sieve_loc+f->shared->sieve_size-1)>=addr && (f->shared->sieve_loc+f->shared->sieve_size-1)<(addr+elmt_size))) {
+ /* Flush the sieve buffer, if it's dirty */
+ if(f->shared->sieve_dirty) {
+ /* Write to file */
+ if (H5F_block_write(f, H5FD_MEM_DRAW, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
+ "block write failed");
+ }
+
+ /* Reset sieve buffer dirty flag */
+ f->shared->sieve_dirty=0;
+ } /* end if */
+ } /* end if */
+
+ /* Read directly into the user's buffer */
+ if (H5F_block_read(f, addr, elmt_size, dxpl_id, buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "block read failed");
+ }
+ } /* end if */
+ /* Element size fits within the buffer size */
+ else {
+ /* Flush the sieve buffer if it's dirty */
+ if(f->shared->sieve_dirty) {
+ /* Write to file */
+ if (H5F_block_write(f, H5FD_MEM_DRAW, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
+ "block write failed");
+ }
+
+ /* Reset sieve buffer dirty flag */
+ f->shared->sieve_dirty=0;
+ } /* end if */
+
+ /* Determine the new sieve buffer size & location */
+ f->shared->sieve_loc=addr;
+
+ /* Make certain we don't read off the end of the file */
+ if (HADDR_UNDEF==(eof=H5FD_get_eof(f->shared->lf))) {
+ HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL,
+ "unable to determine file size");
+ }
+ f->shared->sieve_size=MIN(eof-addr,f->shared->sieve_buf_size);
+
+ /* Read the new sieve buffer */
+ if (H5F_block_read(f, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "block read failed");
+ }
+
+ /* Reset sieve buffer dirty flag */
+ f->shared->sieve_dirty=0;
+
+ /* Grab the data out of the buffer (must be first piece of data in buffer ) */
+ HDmemcpy(buf,f->shared->sieve_buf,elmt_size);
+ } /* end else */
+ } /* end else */
+ } /* end if */
+ /* No data sieve buffer yet, go allocate one */
+ else {
+ /* Check if we can actually hold the I/O request in the sieve buffer */
+ if(elmt_size>f->shared->sieve_buf_size) {
+ if (H5F_block_read(f, addr, elmt_size, dxpl_id, buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "block read failed");
+ }
+ } /* end if */
+ else {
+ /* Allocate room for the data sieve buffer */
+ if (NULL==(f->shared->sieve_buf=H5MM_malloc(f->shared->sieve_buf_size))) {
+ HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL,
+ "memory allocation failed");
+ }
+
+ /* Determine the new sieve buffer size & location */
+ f->shared->sieve_loc=addr;
+
+ /* Make certain we don't read off the end of the file */
+ if (HADDR_UNDEF==(eof=H5FD_get_eof(f->shared->lf))) {
+ HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL,
+ "unable to determine file size");
+ }
+ f->shared->sieve_size=MIN(eof-addr,f->shared->sieve_buf_size);
+
+ /* Read the new sieve buffer */
+ if (H5F_block_read(f, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "block read failed");
+ }
+
+ /* Reset sieve buffer dirty flag */
+ f->shared->sieve_dirty=0;
+
+ /* Grab the data out of the buffer (must be first piece of data in buffer ) */
+ HDmemcpy(buf,f->shared->sieve_buf,elmt_size);
+ } /* end else */
+ } /* end else */
+ } /* end if */
+ else {
+ if (H5F_block_read(f, addr, elmt_size, dxpl_id, buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "block read failed");
+ }
+ } /* end else */
+ } /* end else */
/* Decrement indices and advance pointers */
for (j=ndims-1, carray=TRUE; j>=0 && carray; --j) {
@@ -318,8 +449,7 @@ H5F_arr_read(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout,
*/
if (efl && efl->nused>0) {
HRETURN_ERROR(H5E_IO, H5E_UNSUPPORTED, FAIL,
- "chunking and external files are mutually "
- "exclusive");
+ "chunking and external files are mutually exclusive");
}
for (i=0; i<layout->ndims; i++) {
if (0!=mem_offset[i] || hslab_size[i]!=mem_size[i]) {
@@ -395,6 +525,7 @@ H5F_arr_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout,
size_t nelmts, z; /*number of elements */
intn ndims; /*dimensionality */
haddr_t addr; /*address in file */
+ haddr_t eof; /*end of file address */
intn i, j; /*counters */
hbool_t carray; /*carry for subtraction */
#ifdef H5_HAVE_PARALLEL
@@ -535,14 +666,141 @@ H5F_arr_write(H5F_t *f, hid_t dxpl_id, const struct H5O_layout_t *layout,
/* Write to file */
if (efl && efl->nused>0) {
- if (H5O_efl_write(f, efl, addr, elmt_size, buf)<0) {
- HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
- "external data write failed");
- }
- } else if (H5F_block_write(f, H5FD_MEM_DRAW, addr, elmt_size, dxpl_id, buf)<0) {
- HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
- "block write failed");
- }
+ if (H5O_efl_write(f, efl, addr, elmt_size, buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "external data write failed");
+ }
+ } else {
+ /* Check if data sieving is enabled */
+ if(f->shared->lf->feature_flags&H5FD_FEAT_DATA_SIEVE) {
+ /* Try writing to the data sieve buffer */
+ if(f->shared->sieve_buf) {
+ /* If entire write is within the sieve buffer, write it to the buffer */
+ if((addr>=f->shared->sieve_loc && addr<(f->shared->sieve_loc+f->shared->sieve_size))
+ && ((addr+elmt_size-1)>=f->shared->sieve_loc && (addr+elmt_size-1)<(f->shared->sieve_loc+f->shared->sieve_size))) {
+ /* Grab the data out of the buffer */
+ HDmemcpy(f->shared->sieve_buf+(addr-f->shared->sieve_loc),buf,elmt_size);
+
+ /* Set sieve buffer dirty flag */
+ f->shared->sieve_dirty=1;
+
+ } /* end if */
+ /* Entire request is not within this data sieve buffer */
+ else {
+ /* Check if we can actually hold the I/O request in the sieve buffer */
+ if(elmt_size>f->shared->sieve_buf_size) {
+ /* Check for any overlap with the current sieve buffer */
+ if((f->shared->sieve_loc>=addr && f->shared->sieve_loc<(addr+elmt_size))
+ || ((f->shared->sieve_loc+f->shared->sieve_size-1)>=addr && (f->shared->sieve_loc+f->shared->sieve_size-1)<(addr+elmt_size))) {
+ /* Flush the sieve buffer, if it's dirty */
+ if(f->shared->sieve_dirty) {
+ /* Write to file */
+ if (H5F_block_write(f, H5FD_MEM_DRAW, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
+ "block write failed");
+ }
+
+ /* Reset sieve buffer dirty flag */
+ f->shared->sieve_dirty=0;
+ } /* end if */
+
+ /* Force the sieve buffer to be re-read the next time */
+ f->shared->sieve_loc=HADDR_UNDEF;
+ f->shared->sieve_size=0;
+ } /* end if */
+
+ /* Write directly from the user's buffer */
+ if (H5F_block_write(f, H5FD_MEM_DRAW, addr, elmt_size, dxpl_id, buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
+ "block write failed");
+ }
+ } /* end if */
+ /* Element size fits within the buffer size */
+ else {
+ /* Flush the sieve buffer if it's dirty */
+ if(f->shared->sieve_dirty) {
+ /* Write to file */
+ if (H5F_block_write(f, H5FD_MEM_DRAW, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
+ "block write failed");
+ }
+
+ /* Reset sieve buffer dirty flag */
+ f->shared->sieve_dirty=0;
+ } /* end if */
+
+ /* Determine the new sieve buffer size & location */
+ f->shared->sieve_loc=addr;
+
+ /* Make certain we don't read off the end of the file */
+ if (HADDR_UNDEF==(eof=H5FD_get_eof(f->shared->lf))) {
+ HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL,
+ "unable to determine file size");
+ }
+ f->shared->sieve_size=MIN(eof-addr,f->shared->sieve_buf_size);
+
+ /* Read the new sieve buffer */
+ if (H5F_block_read(f, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "block read failed");
+ }
+
+ /* Grab the data out of the buffer (must be first piece of data in buffer) */
+ HDmemcpy(f->shared->sieve_buf,buf,elmt_size);
+
+ /* Set sieve buffer dirty flag */
+ f->shared->sieve_dirty=1;
+
+ } /* end else */
+ } /* end else */
+ } /* end if */
+ /* No data sieve buffer yet, go allocate one */
+ else {
+ /* Check if we can actually hold the I/O request in the sieve buffer */
+ if(elmt_size>f->shared->sieve_buf_size) {
+ if (H5F_block_write(f, H5FD_MEM_DRAW, addr, elmt_size, dxpl_id, buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
+ "block write failed");
+ }
+ } /* end if */
+ else {
+ /* Allocate room for the data sieve buffer */
+ if (NULL==(f->shared->sieve_buf=H5MM_malloc(f->shared->sieve_buf_size))) {
+ HRETURN_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL,
+ "memory allocation failed");
+ }
+
+ /* Determine the new sieve buffer size & location */
+ f->shared->sieve_loc=addr;
+
+ /* Make certain we don't read off the end of the file */
+ if (HADDR_UNDEF==(eof=H5FD_get_eof(f->shared->lf))) {
+ HRETURN_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL,
+ "unable to determine file size");
+ }
+ f->shared->sieve_size=MIN(eof-addr,f->shared->sieve_buf_size);
+
+ /* Read the new sieve buffer */
+ if (H5F_block_read(f, f->shared->sieve_loc, f->shared->sieve_size, dxpl_id, f->shared->sieve_buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_READERROR, FAIL,
+ "block read failed");
+ }
+
+ /* Grab the data out of the buffer (must be first piece of data in buffer) */
+ HDmemcpy(f->shared->sieve_buf,buf,elmt_size);
+
+ /* Set sieve buffer dirty flag */
+ f->shared->sieve_dirty=1;
+ } /* end else */
+ } /* end else */
+ } /* end if */
+ else {
+ if (H5F_block_write(f, H5FD_MEM_DRAW, addr, elmt_size, dxpl_id, buf)<0) {
+ HRETURN_ERROR(H5E_IO, H5E_WRITEERROR, FAIL,
+ "block write failed");
+ }
+ } /* end else */
+ } /* end else */
/* Decrement indices and advance pointers */
for (j=ndims-1, carray=TRUE; j>=0 && carray; --j) {
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 19b65b4..e8e36e4 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -254,7 +254,8 @@ typedef struct H5F_access_t {
double rdcc_w0; /* Preempt read chunks first? [0.0..1.0]*/
hsize_t threshold; /* Threshold for alignment */
hsize_t alignment; /* Alignment */
- size_t meta_block_size; /* Minimum metadata allocation block size (when aggregating metadata allocations */
+ size_t meta_block_size; /* Minimum metadata allocation block size (when aggregating metadata allocations) */
+ hsize_t sieve_buf_size; /* Maximum sieve buffer size (when data sieving is allowed by file driver) */
uintn gc_ref; /* Garbage-collect references? */
hid_t driver_id; /* File driver ID */
void *driver_info; /* File driver specific information */
@@ -328,6 +329,14 @@ typedef struct H5F_file_t {
struct H5G_t *root_grp; /* Open root group */
intn ncwfs; /* Num entries on cwfs list */
struct H5HG_heap_t **cwfs; /* Global heap cache */
+
+ /* Data Sieve Buffering fields */
+ unsigned char *sieve_buf; /* Buffer to hold data sieve buffer */
+ haddr_t sieve_loc; /* File location (offset) of the data sieve buffer */
+ hsize_t sieve_size; /* Size of the data sieve buffer used (in bytes) */
+ hsize_t sieve_buf_size; /* Size of the data sieve buffer allocated (in bytes) */
+ unsigned sieve_dirty; /* Flag to indicate that the data sieve buffer is dirty */
+
H5F_rdcc_t rdcc; /* Raw data chunk cache */
} H5F_file_t;
diff --git a/src/H5P.c b/src/H5P.c
index 2bdf003..3a4de07 100644
--- a/src/H5P.c
+++ b/src/H5P.c
@@ -2991,6 +2991,90 @@ H5Pget_meta_block_size(hid_t fapl_id, hsize_t *size/*out*/)
}
+/*-------------------------------------------------------------------------
+ * Function: H5Pset_sieve_buf_size
+ *
+ * Purpose: Sets the maximum size of the data seive buffer used for file
+ * drivers which are capable of using data sieving. The data sieve
+ * buffer is used when performing I/O on datasets in the file. Using a
+ * buffer which is large anough to hold several pieces of the dataset
+ * being read in for hyperslab selections boosts performance by quite a
+ * bit.
+ *
+ * The default value is set to 64KB, indicating that file I/O for raw data
+ * reads and writes will occur in at least 64KB blocks.
+ * Setting the value to 0 with this API function will turn off the
+ * data sieving, even if the VFL driver attempts to use that strategy.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Quincey Koziol
+ * Thursday, September 21, 2000
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pset_sieve_buf_size(hid_t fapl_id, hsize_t size)
+{
+ H5F_access_t *fapl = NULL;
+
+ FUNC_ENTER (H5Pset_sieve_buf_size, FAIL);
+ H5TRACE2("e","ih",fapl_id,size);
+
+ /* Check args */
+ if (H5P_FILE_ACCESS != H5P_get_class (fapl_id) ||
+ NULL == (fapl = H5I_object (fapl_id))) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
+ "not a file access property list");
+ }
+
+ /* Set values */
+ fapl->sieve_buf_size = size;
+
+ FUNC_LEAVE (SUCCEED);
+} /* end H5Pset_sieve_buf_size() */
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5Pget_sieve_buf_size
+ *
+ * Purpose: Returns the current settings for the data sieve buffer size
+ * property from a file access property list.
+ *
+ * Return: Non-negative on success/Negative on failure
+ *
+ * Programmer: Quincey Koziol
+ * Thursday, September 21, 2000
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5Pget_sieve_buf_size(hid_t fapl_id, hsize_t *size/*out*/)
+{
+ H5F_access_t *fapl = NULL;
+
+ FUNC_ENTER (H5Pget_sieve_buf_size, FAIL);
+ H5TRACE2("e","ix",fapl_id,size);
+
+ /* Check args */
+ if (H5P_FILE_ACCESS != H5P_get_class (fapl_id) ||
+ NULL == (fapl = H5I_object (fapl_id))) {
+ HRETURN_ERROR (H5E_ARGS, H5E_BADTYPE, FAIL,
+ "not a file access property list");
+ }
+
+ /* Get values */
+ if (size)
+ *size = fapl->sieve_buf_size;
+
+ FUNC_LEAVE (SUCCEED);
+} /* end H5Pget_sieve_buf_size() */
+
+
/*--------------------------------------------------------------------------
NAME
H5P_copy_prop
diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h
index 3d2b253..49c2563 100644
--- a/src/H5Ppublic.h
+++ b/src/H5Ppublic.h
@@ -181,6 +181,8 @@ __DLL__ herr_t H5Pget_vlen_mem_manager(hid_t plist_id,
void **free_info);
__DLL__ herr_t H5Pset_meta_block_size(hid_t fapl_id, hsize_t size);
__DLL__ herr_t H5Pget_meta_block_size(hid_t fapl_id, hsize_t *size/*out*/);
+__DLL__ herr_t H5Pset_sieve_buf_size(hid_t fapl_id, hsize_t size);
+__DLL__ herr_t H5Pget_sieve_buf_size(hid_t fapl_id, hsize_t *size/*out*/);
#ifdef __cplusplus
}