summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorRobb Matzke <matzke@llnl.gov>1998-03-05 15:48:16 (GMT)
committerRobb Matzke <matzke@llnl.gov>1998-03-05 15:48:16 (GMT)
commit4badc161ae7cd13a01e17b8c920759e47631951a (patch)
tree4170b0ca9da9713784d87ab1b39ae8a814e23aed
parent2b323ed454f0bba86683b4b04c7baa5cc8fe34e4 (diff)
downloadhdf5-4badc161ae7cd13a01e17b8c920759e47631951a.zip
hdf5-4badc161ae7cd13a01e17b8c920759e47631951a.tar.gz
hdf5-4badc161ae7cd13a01e17b8c920759e47631951a.tar.bz2
[svn-r305] Changes since 19980304
---------------------- ./MANIFEST New files ./html/extern1.gif [NEW] ./html/extern1.obj [NEW] ./html/extern2.gif [NEW] ./html/extern2.obj [NEW] ./html/Datasets.html Documented external raw data storage. ./html/Datasets.html ./html/chunk1.gif [NEW] ./html/chunk1.obj [NEW] Added a picture and better description for chunking since it seems that people are thinking chunks are fixed-size at the file level instead of at the array level. ./src/H5D.c ./src/H5Dpublic.h ./test/external.c Changed H5Dget_create_parms() to H5Dget_create_plist(). ./src/H5D.c ./src/H5Farray.c ./src/H5Fprivate.h ./src/H5Oefl.c ./src/H5Oprivate.h ./src/H5Sprivate.h ./src/H5Ssimp.c ./test/external.c ./test/istore.c External raw data file I/O for contiguous storage is now working. The library supports segments of various sizes which can be scattered throughout multiple files in any order. ./test/Makefile.in Remove more temporary files.
-rw-r--r--MANIFEST6
-rw-r--r--src/H5D.c25
-rw-r--r--src/H5Dpublic.h2
-rw-r--r--src/H5Farray.c69
-rw-r--r--src/H5Flow.c2
-rw-r--r--src/H5Fprivate.h13
-rw-r--r--src/H5Oefl.c163
-rw-r--r--src/H5Oprivate.h4
-rw-r--r--src/H5Sprivate.h24
-rw-r--r--src/H5Ssimp.c18
-rw-r--r--test/Makefile.in4
-rw-r--r--test/external.c244
-rw-r--r--test/istore.c10
13 files changed, 490 insertions, 94 deletions
diff --git a/MANIFEST b/MANIFEST
index 7623f84..6681137 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -52,9 +52,15 @@
./html/MemoryManagement.html
./html/ObjectHeader.txt
./html/Properties.html
+./html/chunk1.gif
+./html/chunk1.obj
./html/compat.html
./html/dataset_p1.gif
./html/dataset_p1.obj
+./html/extern1.gif
+./html/extern1.obj
+./html/extern2.gif
+./html/extern2.obj
./html/group_p1.gif
./html/group_p1.obj
./html/group_p2.gif
diff --git a/src/H5D.c b/src/H5D.c
index a3694ee..da470d3 100644
--- a/src/H5D.c
+++ b/src/H5D.c
@@ -413,7 +413,7 @@ H5Dget_type (hid_t dataset_id)
}
/*-------------------------------------------------------------------------
- * Function: H5Dget_create_parms
+ * Function: H5Dget_create_plist
*
* Purpose: Returns a copy of the dataset creation property list.
*
@@ -431,13 +431,13 @@ H5Dget_type (hid_t dataset_id)
*-------------------------------------------------------------------------
*/
hid_t
-H5Dget_create_parms (hid_t dataset_id)
+H5Dget_create_plist (hid_t dataset_id)
{
H5D_t *dataset = NULL;
H5D_create_t *copied_parms = NULL;
hid_t ret_value = FAIL;
- FUNC_ENTER (H5Dget_create_parms, FAIL);
+ FUNC_ENTER (H5Dget_create_plist, FAIL);
/* Check args */
if (H5_DATASET!=H5A_group (dataset_id) ||
@@ -962,9 +962,9 @@ H5D_open(H5F_t *f, const char *name)
}
/* Get the external file list message, which might not exist */
- if (!H5F_addr_defined (&(dataset->layout.addr)) &&
- NULL==H5O_read (&(dataset->ent), H5O_EFL, 0,
- &(dataset->create_parms->efl))) {
+ if (NULL==H5O_read (&(dataset->ent), H5O_EFL, 0,
+ &(dataset->create_parms->efl)) &&
+ !H5F_addr_defined (&(dataset->layout.addr))) {
HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL,
"storage address is undefined an no external file list");
}
@@ -1128,10 +1128,6 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"src and dest data spaces have different sizes");
}
- if (dataset->create_parms->efl.nused>0) {
- HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
- "reading externally-stored data is not implemented yet");
- }
/*
* Compute the size of the request and allocate scratch buffers.
@@ -1157,6 +1153,7 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
* later ;-)
*/
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, 0, nelmts,
tconv_buf/*out*/)!=nelmts) {
@@ -1275,11 +1272,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL,
"src and dest data spaces have different sizes");
}
- if (dataset->create_parms->efl.nused>0) {
- HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL,
- "writing externally-stored data is not implemente yet");
- }
-
+
/*
* Compute the size of the request and allocate scratch buffers.
*/
@@ -1310,6 +1303,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
}
if (H5T_BKG_YES==cdata->need_bkg) {
if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, 0, nelmts,
bkg_buf/*out*/)!=nelmts) {
@@ -1332,6 +1326,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space,
* Scatter the data out to the file.
*/
if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout),
+ &(dataset->create_parms->efl),
H5T_get_size (dataset->type), file_space,
&numbering, 0, nelmts, tconv_buf)<0) {
HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed");
diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h
index 80a1340..192ed96 100644
--- a/src/H5Dpublic.h
+++ b/src/H5Dpublic.h
@@ -41,7 +41,7 @@ hid_t H5Dopen (hid_t file_id, const char *name);
herr_t H5Dclose (hid_t dataset_id);
hid_t H5Dget_space (hid_t dataset_id);
hid_t H5Dget_type (hid_t dataset_id);
-hid_t H5Dget_create_parms (hid_t dataset_id);
+hid_t H5Dget_create_plist (hid_t dataset_id);
herr_t H5Dread (hid_t dataset_id, hid_t mem_type_id, hid_t mem_space_id,
hid_t file_space_id, hid_t xfer_parms_id, void *buf/*out*/);
herr_t H5Dwrite (hid_t dataset_id, hid_t mem_type_id, hid_t mem_space_id,
diff --git a/src/H5Farray.c b/src/H5Farray.c
index 7deeead..eb7d7b1 100644
--- a/src/H5Farray.c
+++ b/src/H5Farray.c
@@ -89,8 +89,9 @@ H5F_arr_create (H5F_t *f, struct H5O_layout_t *layout/*in,out*/)
*
* Purpose: Reads a hyperslab of a file byte array into a hyperslab of
* a byte array in memory. The data is read from file F and the
- * array's size and storage information is in LAYOUT. The
- * hyperslab offset is FILE_OFFSET[] in the file and
+ * array's size and storage information is in LAYOUT. External
+ * files are described according to the external file list, EFL.
+ * The hyperslab offset is FILE_OFFSET[] in the file and
* MEM_OFFSET[] in memory (offsets are relative to the origin of
* the array) and the size of the hyperslab is HSLAB_SIZE[]. The
* total size of the file array is implied in the LAYOUT
@@ -111,6 +112,7 @@ H5F_arr_create (H5F_t *f, struct H5O_layout_t *layout/*in,out*/)
*/
herr_t
H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_efl_t *efl,
const size_t _hslab_size[], const size_t mem_size[],
const size_t mem_offset[], const size_t file_offset[],
void *_buf/*out*/)
@@ -165,7 +167,11 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
*/
H5V_vector_cpy (ndims, idx, hslab_size);
nelmts = H5V_vector_reduce_product (ndims, hslab_size);
- addr = layout->addr;
+ if (efl && efl->nused>0) {
+ H5F_addr_reset (&addr);
+ } else {
+ addr = layout->addr;
+ }
H5F_addr_inc (&addr, file_start);
buf += mem_start;
@@ -176,7 +182,12 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
for (z=0; z<nelmts; z++) {
/* Read from file */
- if (H5F_block_read (f, &addr, elmt_size, buf)<0) {
+ if (efl && efl->nused>0) {
+ if (H5O_efl_read (f, efl, &addr, elmt_size, buf)<0) {
+ HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "external data read failed");
+ }
+ } else if (H5F_block_read (f, &addr, elmt_size, buf)<0) {
HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL,
"block read failed");
}
@@ -195,8 +206,14 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
case H5D_CHUNKED:
/*
- * This method is unable to copy into a proper hyperslab.
+ * This method is unable to access external raw data files or to copy
+ * into a proper hyperslab.
*/
+ if (efl && efl->nused>0) {
+ HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL,
+ "chunking and external files are mutually "
+ "exclusive");
+ }
for (i=0; i<layout->ndims; i++) {
if (0!=mem_offset[i] || hslab_size[i]!=mem_size[i]) {
HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL,
@@ -224,13 +241,14 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
* Purpose: Copies a hyperslab of a memory array to a hyperslab of a
* file array. The data is written to file F and the file
* array's size and storage information is implied by LAYOUT.
- * The hyperslab offset is FILE_OFFSET[] in the file and
- * MEM_OFFSET[] in memory (offsets are relative to the origin of
- * the array) and the size of the hyperslab is HSLAB_SIZE[].
- * The total size of the file array is implied by the LAYOUT
- * argument and the total size of the memory array is
- * MEM_SIZE[]. The dimensionality of these vectors is implied
- * by the LAYOUT argument.
+ * The data is stored in external files according to the
+ * external file list, EFL. The hyperslab offset is
+ * FILE_OFFSET[] in the file and MEM_OFFSET[] in memory (offsets
+ * are relative to the origin of the array) and the size of the
+ * hyperslab is HSLAB_SIZE[]. The total size of the file array
+ * is implied by the LAYOUT argument and the total size of the
+ * memory array is MEM_SIZE[]. The dimensionality of these
+ * vectors is implied by the LAYOUT argument.
*
* Return: Success: SUCCEED
*
@@ -245,9 +263,9 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
*/
herr_t
H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
- const size_t _hslab_size[], const size_t mem_size[],
- const size_t mem_offset[], const size_t file_offset[],
- const void *_buf)
+ const struct H5O_efl_t *efl, const size_t _hslab_size[],
+ const size_t mem_size[], const size_t mem_offset[],
+ const size_t file_offset[], const void *_buf)
{
const uint8 *buf = (const uint8 *)_buf; /*cast for arithmetic */
ssize_t file_stride[H5O_LAYOUT_NDIMS]; /*strides through file */
@@ -300,7 +318,11 @@ H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
*/
H5V_vector_cpy (ndims, idx, hslab_size);
nelmts = H5V_vector_reduce_product (ndims, hslab_size);
- addr = layout->addr;
+ if (efl && efl->nused>0) {
+ H5F_addr_reset (&addr);
+ } else {
+ addr = layout->addr;
+ }
H5F_addr_inc (&addr, file_start);
buf += mem_start;
@@ -311,7 +333,12 @@ H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
for (z=0; z<nelmts; z++) {
/* Write to file */
- if (H5F_block_write (f, &addr, elmt_size, buf)<0) {
+ if (efl && efl->nused>0) {
+ if (H5O_efl_write (f, efl, &addr, elmt_size, buf)<0) {
+ HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL,
+ "external data write failed");
+ }
+ } else if (H5F_block_write (f, &addr, elmt_size, buf)<0) {
HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL,
"block write failed");
}
@@ -331,8 +358,14 @@ H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
case H5D_CHUNKED:
/*
- * This method is unable to copy from a proper hyperslab.
+ * This method is unable to access external raw daa files or to copy
+ * from a proper hyperslab.
*/
+ if (efl && efl->nused>0) {
+ HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL,
+ "chunking and external files are mutually "
+ "exclusive");
+ }
for (i=0; i<layout->ndims; i++) {
if (0!=mem_offset[i] || hslab_size[i]!=mem_size[i]) {
HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL,
diff --git a/src/H5Flow.c b/src/H5Flow.c
index 6455571..125f8ed 100644
--- a/src/H5Flow.c
+++ b/src/H5Flow.c
@@ -628,7 +628,7 @@ H5F_addr_defined(const haddr_t *addr)
*-------------------------------------------------------------------------
*/
void
-H5F_addr_reset(haddr_t *addr /*out */ )
+H5F_addr_reset(haddr_t *addr/*out*/)
{
assert(addr);
addr->offset = 0;
diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 4700bae..497a9a0 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -449,6 +449,7 @@ typedef struct H5F_t {
}
struct H5O_layout_t; /*forward decl for prototype arguments */
+struct H5O_efl_t; /*forward decl for prototype arguments */
/* library variables */
extern const H5F_create_t H5F_create_dflt;
@@ -467,13 +468,13 @@ herr_t H5F_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent,
/* Functions that operate on array storage */
herr_t H5F_arr_create(H5F_t *f, struct H5O_layout_t *layout /*in,out*/);
herr_t H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout,
- const size_t _hslab_size[], const size_t mem_size[],
- const size_t mem_offset[], const size_t file_offset[],
- void *_buf/*out*/);
+ const struct H5O_efl_t *efl, const size_t _hslab_size[],
+ const size_t mem_size[], const size_t mem_offset[],
+ const size_t file_offset[], void *_buf/*out*/);
herr_t H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout,
- const size_t _hslab_size[], const size_t mem_size[],
- const size_t mem_offset[], const size_t file_offset[],
- const void *_buf);
+ const struct H5O_efl_t *efl, const size_t _hslab_size[],
+ const size_t mem_size[], const size_t mem_offset[],
+ const size_t file_offset[], const void *_buf);
/* Functions that operate on indexed storage */
herr_t H5F_istore_create(H5F_t *f, struct H5O_layout_t *layout /*in,out*/);
diff --git a/src/H5Oefl.c b/src/H5Oefl.c
index eb82c30..ac65101 100644
--- a/src/H5Oefl.c
+++ b/src/H5Oefl.c
@@ -350,6 +350,169 @@ H5O_efl_total_size (H5O_efl_t *efl)
/*-------------------------------------------------------------------------
+ * Function: H5O_efl_read
+ *
+ * Purpose: Reads data from an external file list. It is an error to
+ * read past the logical end of file, but reading past the end
+ * of any particular member of the external file list results in
+ * zeros.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, March 4, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5O_efl_read (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, size_t size,
+ uint8 *buf)
+{
+ int i, fd=-1;
+ size_t to_read, cur, skip;
+ ssize_t n;
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER (H5O_efl_read, FAIL);
+
+ /* Check args */
+ assert (efl && efl->nused>0);
+ assert (addr && H5F_addr_defined (addr));
+ assert (buf || 0==size);
+
+ /* Find the first efl member from which to read */
+ for (i=0, cur=0; i<efl->nused; i++) {
+ if (addr->offset < cur+efl->slot[i].size) {
+ skip = addr->offset - cur;
+ break;
+ }
+ cur += efl->slot[i].size;
+ }
+
+ /* Read the data */
+ while (size) {
+ if (i>=efl->nused) {
+ HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL,
+ "read past logical end of file");
+ }
+ if ((fd=open (efl->slot[i].name, O_RDONLY))<0) {
+ HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL,
+ "unable to open external raw data file");
+ }
+ if (lseek (fd, efl->slot[i].offset+skip, SEEK_SET)<0) {
+ HGOTO_ERROR (H5E_EFL, H5E_SEEKERROR, FAIL,
+ "unable to seek in external raw data file");
+ }
+ to_read = MIN(efl->slot[i].size-skip, size);
+ if ((n=read (fd, buf, to_read))<0) {
+ HGOTO_ERROR (H5E_EFL, H5E_READERROR, FAIL,
+ "read error in external raw data file");
+ } else if (n<to_read) {
+ HDmemset (buf+n, 0, to_read-n);
+ }
+ close (fd);
+ fd = -1;
+ size -= to_read;
+ buf += to_read;
+ skip = 0;
+ i++;
+ }
+ ret_value = SUCCEED;
+
+ done:
+ if (fd>=0) close (fd);
+ FUNC_LEAVE (ret_value);
+}
+
+
+/*-------------------------------------------------------------------------
+ * Function: H5O_efl_write
+ *
+ * Purpose: Writes data to an external file list. It is an error to
+ * write past the logical end of file, but writing past the end
+ * of any particular member of the external file list just
+ * extends that file.
+ *
+ * Return: Success: SUCCEED
+ *
+ * Failure: FAIL
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, March 4, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+herr_t
+H5O_efl_write (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, size_t size,
+ const uint8 *buf)
+{
+ int i, fd=-1;
+ size_t to_write, cur, skip;
+ ssize_t n;
+ herr_t ret_value = FAIL;
+
+ FUNC_ENTER (H5O_efl_write, FAIL);
+
+ /* Check args */
+ assert (efl && efl->nused>0);
+ assert (addr && H5F_addr_defined (addr));
+ assert (buf || 0==size);
+
+ /* Find the first efl member in which to write */
+ for (i=0, cur=0; i<efl->nused; i++) {
+ if (addr->offset < cur+efl->slot[i].size) {
+ skip = addr->offset - cur;
+ break;
+ }
+ cur += efl->slot[i].size;
+ }
+
+ /* Write the data */
+ while (size) {
+ if (i>=efl->nused) {
+ HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL,
+ "write past logical end of file");
+ }
+ if ((fd=open (efl->slot[i].name, O_RDWR))<0) {
+ if (access (efl->slot[i].name, F_OK)<0) {
+ HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL,
+ "external raw data file does not exist");
+ } else {
+ HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL,
+ "unable to open external raw data file");
+ }
+ }
+ if (lseek (fd, efl->slot[i].offset+skip, SEEK_SET)<0) {
+ HGOTO_ERROR (H5E_EFL, H5E_SEEKERROR, FAIL,
+ "unable to seek in external raw data file");
+ }
+ to_write = MIN(efl->slot[i].size-skip, size);
+ if ((n=write (fd, buf, to_write))!=to_write) {
+ HGOTO_ERROR (H5E_EFL, H5E_READERROR, FAIL,
+ "write error in external raw data file");
+ }
+ close (fd);
+ fd = -1;
+ size -= to_write;
+ buf += to_write;
+ skip = 0;
+ i++;
+ }
+ ret_value = SUCCEED;
+
+ done:
+ if (fd>=0) close (fd);
+ FUNC_LEAVE (ret_value);
+}
+
+
+/*-------------------------------------------------------------------------
* Function: H5O_efl_debug
*
* Purpose: Prints debugging info for a message.
diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h
index f7dafb8..079ba24 100644
--- a/src/H5Oprivate.h
+++ b/src/H5Oprivate.h
@@ -216,5 +216,9 @@ herr_t H5O_debug (H5F_t *f, const haddr_t *addr, FILE * stream, intn indent,
/* EFL operators */
size_t H5O_efl_total_size (H5O_efl_t *efl);
+herr_t H5O_efl_read (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr,
+ size_t size, uint8 *buf);
+herr_t H5O_efl_write (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr,
+ size_t size, const uint8 *buf);
#endif
diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h
index 79d7bf2..c8a3aae 100644
--- a/src/H5Sprivate.h
+++ b/src/H5Sprivate.h
@@ -69,9 +69,9 @@ typedef struct H5S_tconv_t {
/* Gather elements from disk to type conversion buffer */
size_t (*fgath)(H5F_t *f, const struct H5O_layout_t *layout,
- size_t elmt_size, const H5S_t *file_space,
- const H5S_number_t *numbering, size_t start, size_t nelmts,
- void *tconv_buf/*out*/);
+ const struct H5O_efl_t *efl, size_t elmt_size,
+ const H5S_t *file_space, const H5S_number_t *numbering,
+ size_t start, size_t nelmts, void *tconv_buf/*out*/);
/* Scatter elements from type conversion buffer to application buffer */
herr_t (*mscat)(const void *tconv_buf, size_t elmt_size,
@@ -85,9 +85,9 @@ typedef struct H5S_tconv_t {
/* Scatter elements from type conversion buffer to disk */
herr_t (*fscat)(H5F_t *f, const struct H5O_layout_t *layout,
- size_t elmt_size, const H5S_t *file_space,
- const H5S_number_t *numbering, size_t start, size_t nelmts,
- const void *tconv_buf);
+ const struct H5O_efl_t *efl, size_t elmt_size,
+ const H5S_t *file_space, const H5S_number_t *numbering,
+ size_t start, size_t nelmts, const void *tconv_buf);
} H5S_conv_t;
H5S_t *H5S_copy (const H5S_t *src);
@@ -112,9 +112,9 @@ size_t H5S_simp_init (const struct H5O_layout_t *layout,
const H5S_t *mem_space, const H5S_t *file_space,
H5S_number_t *numbering/*out*/);
size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
- size_t elmt_size, const H5S_t *file_space,
- const H5S_number_t *numbering, size_t start,
- size_t nelmts, void *tconv_buf/*out*/);
+ const struct H5O_efl_t *efl, size_t elmt_size,
+ const H5S_t *file_space, const H5S_number_t *numbering,
+ size_t start, size_t nelmts, void *tconv_buf/*out*/);
herr_t H5S_simp_mscat (const void *tconv_buf, size_t elmt_size,
const H5S_t *mem_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, void *buf/*out*/);
@@ -122,7 +122,7 @@ size_t H5S_simp_mgath (const void *buf, size_t elmt_size,
const H5S_t *mem_space, const H5S_number_t *numbering,
size_t start, size_t nelmts, void *tconv_buf/*out*/);
herr_t H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
- size_t elmt_size, const H5S_t *file_space,
- const H5S_number_t *numbering, size_t start,
- size_t nelmts, const void *tconv_buf);
+ const struct H5O_efl_t *efl, size_t elmt_size,
+ const H5S_t *file_space, const H5S_number_t *numbering,
+ size_t start, size_t nelmts, const void *tconv_buf);
#endif
diff --git a/src/H5Ssimp.c b/src/H5Ssimp.c
index 6f4dd4f..ee2d680 100644
--- a/src/H5Ssimp.c
+++ b/src/H5Ssimp.c
@@ -63,7 +63,8 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
*
* Purpose: Gathers data points from file F and accumulates them in the
* type conversion buffer BUF. The LAYOUT argument describes
- * how the data is stored on disk. ELMT_SIZE is the size in
+ * how the data is stored on disk and EFL describes how the data
+ * is organized in external files. ELMT_SIZE is the size in
* bytes of a datum which this function treats as opaque.
* FILE_SPACE describes the data space of the dataset on disk
* and the elements that have been selected for reading (via
@@ -85,6 +86,7 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space,
*/
size_t
H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_efl_t *efl,
size_t elmt_size, const H5S_t *file_space,
const H5S_number_t *numbering, size_t start, size_t nelmts,
void *buf/*out*/)
@@ -153,7 +155,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout,
/*
* Gather from file.
*/
- if (H5F_arr_read (f, layout, hsize, hsize, zero, file_offset,
+ if (H5F_arr_read (f, layout, efl, hsize, hsize, zero, file_offset,
buf/*out*/)<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_READERROR, 0, "read error");
}
@@ -375,10 +377,10 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
* Purpose: Scatters dataset elements from the type conversion buffer BUF
* to the file F where the data points are arranged according to
* the file data space FILE_SPACE and stored according to
- * LAYOUT. Each element is ELMT_SIZE bytes and has a unique
- * number according to NUMBERING. The caller is requesting that
- * NELMTS elements are coppied beginning with element number
- * START.
+ * LAYOUT and EFL. Each element is ELMT_SIZE bytes and has a
+ * unique number according to NUMBERING. The caller is
+ * requesting that NELMTS elements are coppied beginning with
+ * element number START.
*
* Return: Success: SUCCEED
*
@@ -393,6 +395,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size,
*/
herr_t
H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
+ const struct H5O_efl_t *efl,
size_t elmt_size, const H5S_t *file_space,
const H5S_number_t *numbering, size_t start, size_t nelmts,
const void *buf)
@@ -461,7 +464,8 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout,
/*
* Scatter to file.
*/
- if (H5F_arr_write (f, layout, hsize, hsize, zero, file_offset, buf)<0) {
+ if (H5F_arr_write (f, layout, efl, hsize, hsize, zero,
+ file_offset, buf)<0) {
HRETURN_ERROR (H5E_DATASPACE, H5E_WRITEERROR, FAIL, "write error");
}
diff --git a/test/Makefile.in b/test/Makefile.in
index 826ad23..d7a2c91 100644
--- a/test/Makefile.in
+++ b/test/Makefile.in
@@ -16,7 +16,9 @@ TESTS=$(PROGS)
# Temporary files
MOSTLYCLEAN=cmpd_dset.h5 dataset.h5 extend.h5 istore.h5 tfile1.h5 tfile2.h5 \
- tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5
+ tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5 \
+ extern_1.h5 extern_2.h5 extern_1.raw extern_1b.raw extern_2.raw \
+ extern_2b.raw extern_3.raw extern_3b.raw extern_4.raw extern_4b.raw
# Source and object files for programs... The PROG_SRC list contains all the
# source files and is used for things like dependencies, archiving, etc. The
diff --git a/test/external.c b/test/external.c
index 36c9cdf..53f1c15 100644
--- a/test/external.c
+++ b/test/external.c
@@ -12,8 +12,11 @@
#include <hdf5.h>
#include <stdio.h>
#include <stdlib.h>
+#include <string.h>
#include <unistd.h>
+static int nerrors_g = 0;
+
/*-------------------------------------------------------------------------
* Function: display_error_cb
@@ -36,19 +39,62 @@ display_error_cb (void *client_data)
{
puts ("*FAILED*");
H5Eprint (stdout);
+ nerrors_g++;
return 0;
}
/*-------------------------------------------------------------------------
+ * Function: same_contents
+ *
+ * Purpose: Determines whether two files are exactly the same.
+ *
+ * Return: Success: nonzero if same, zero if different.
+ *
+ * Failure: zero
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, March 4, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static int
+same_contents (const char *name1, const char *name2)
+{
+ int fd1, fd2;
+ ssize_t n1, n2;
+ char buf1[1024], buf2[1024];
+
+ fd1 = open (name1, O_RDONLY);
+ fd2 = open (name2, O_RDONLY);
+ assert (fd1>=0 && fd2>=0);
+
+ while (1) {
+ n1 = read (fd1, buf1, sizeof(buf1));
+ n2 = read (fd2, buf2, sizeof(buf2));
+ assert (n1==n2);
+ if (n1<=0 && n2<=0) break;
+ if (memcmp (buf1, buf2, n1)) {
+ close (fd1);
+ close (fd2);
+ return 0;
+ }
+ }
+ close (fd1);
+ close (fd2);
+ return 1;
+}
+
+
+/*-------------------------------------------------------------------------
* Function: test_1
*
* Purpose: Describes various external datasets in an HDF5 file without
* actually creating the external raw files.
*
- * Return: Success: 0
- *
- * Failure: -1
+ * Return: void
*
* Programmer: Robb Matzke
* Tuesday, March 3, 1998
@@ -57,7 +103,7 @@ display_error_cb (void *client_data)
*
*-------------------------------------------------------------------------
*/
-static herr_t
+static void
test_1 (void)
{
hid_t file, plist, space, dset, grp;
@@ -78,8 +124,6 @@ test_1 (void)
grp = H5Gcreate (file, "emit-diagnostics", 8);
H5Gclose (grp);
- printf ("Testing external storage descriptions...\n");
-
/*
* A single external file for a non-extendible dataset.
*/
@@ -127,6 +171,7 @@ test_1 (void)
if (dset>=0) {
puts ("*FAILED*");
printf (" Small external file succeeded instead of failing\n");
+ nerrors_g++;
H5Dclose (dset);
break;
}
@@ -188,6 +233,7 @@ test_1 (void)
puts ("*FAILED*");
printf (" Small external file succeeded instead of failing\n");
H5Dclose (dset);
+ nerrors_g++;
break;
}
puts (" PASSED");
@@ -234,7 +280,7 @@ test_1 (void)
dset = H5Dopen (file, "dset1");
assert (dset>=0);
- plist = H5Dget_create_parms (dset);
+ plist = H5Dget_create_plist (dset);
assert (plist>=0);
n = H5Pget_external_count (plist);
@@ -242,6 +288,7 @@ test_1 (void)
if (1!=n) {
puts ("*FAILED*");
printf (" Returned external count is wrong.\n");
+ nerrors_g++;
break;
}
strcpy (name+sizeof(name)-4, "...");
@@ -253,10 +300,12 @@ test_1 (void)
} else if (file_offset!=0) {
puts ("*FAILED*");
printf (" Wrong file offset.\n");
+ nerrors_g++;
break;
} else if (file_size!=400) {
puts ("*FAILED*");
printf (" Wrong file size.\n");
+ nerrors_g++;
break;
}
puts (" PASSED");
@@ -279,7 +328,7 @@ test_1 (void)
dset = H5Dopen (file, "dset5");
assert (dset>=0);
- plist = H5Dget_create_parms (dset);
+ plist = H5Dget_create_plist (dset);
assert (plist>=0);
n = H5Pget_external_count (plist);
@@ -287,6 +336,7 @@ test_1 (void)
if (1!=n) {
puts ("*FAILED*");
printf (" Returned external count is wrong.\n");
+ nerrors_g++;
break;
}
strcpy (name+sizeof(name)-4, "...");
@@ -298,10 +348,12 @@ test_1 (void)
} else if (file_offset!=0) {
puts ("*FAILED*");
printf (" Wrong file offset.\n");
+ nerrors_g++;
break;
} else if (H5F_UNLIMITED!=file_size) {
puts ("*FAILED*");
printf (" Wrong file size.\n");
+ nerrors_g++;
break;
}
puts (" PASSED");
@@ -358,6 +410,7 @@ test_1 (void)
if (status>=0) {
puts ("*FAILED*");
puts (" H5Pset_external() succeeded when it should have failed");
+ nerrors_g++;
break;
}
@@ -367,6 +420,8 @@ test_1 (void)
if (1!=n) {
puts ("*FAILED*");
puts (" Wrong external file count returned.");
+ nerrors_g++;
+ break;
}
puts (" PASSED");
} while (0);
@@ -392,6 +447,7 @@ test_1 (void)
if (status>=0) {
puts ("*FAILED*");
puts (" H5Pset_external() succeeded when it should have failed");
+ nerrors_g++;
break;
}
puts (" PASSED");
@@ -402,7 +458,6 @@ test_1 (void)
/* END OF TESTS */
H5Fclose (file);
- return 0;
}
@@ -411,9 +466,7 @@ test_1 (void)
*
* Purpose: Tests reading from an external file set.
*
- * Return: Success: 0
- *
- * Failure: -1
+ * Return: void
*
* Programmer: Robb Matzke
* Wednesday, March 4, 1998
@@ -422,7 +475,7 @@ test_1 (void)
*
*-------------------------------------------------------------------------
*/
-static herr_t
+static void
test_2 (void)
{
hid_t file, plist, space, dset, grp;
@@ -434,7 +487,6 @@ test_2 (void)
size_t size;
/* Write the data to external files */
- printf ("Writing external data...\n");
for (i=0; i<4; i++) {
for (j=0; j<25; j++) {
part[j] = i*25+j;
@@ -443,6 +495,8 @@ test_2 (void)
sprintf (fname, "extern_%d.raw", i+1);
fd = open (fname, O_RDWR|O_CREAT|O_TRUNC, 0666);
assert (fd>=0);
+ n = lseek (fd, i*10, SEEK_SET);
+ assert (n==i*10);
n = write (fd, part, sizeof(part));
assert (n==sizeof(part));
close (fd);
@@ -457,18 +511,17 @@ test_2 (void)
assert (file>=0);
grp = H5Gcreate (file, "emit-diagnostics", 8);
H5Gclose (grp);
- printf ("Testing external data reading...\n");
/* Create the external file list */
plist = H5Pcreate (H5P_DATASET_CREATE);
assert (plist>=0);
status = H5Pset_external (plist, "extern_1.raw", 0, sizeof(part));
assert (status>=0);
- status = H5Pset_external (plist, "extern_2.raw", 0, sizeof(part));
+ status = H5Pset_external (plist, "extern_2.raw", 10, sizeof(part));
assert (status>=0);
- status = H5Pset_external (plist, "extern_3.raw", 0, sizeof(part));
+ status = H5Pset_external (plist, "extern_3.raw", 20, sizeof(part));
assert (status>=0);
- status = H5Pset_external (plist, "extern_4.raw", 0, sizeof(part));
+ status = H5Pset_external (plist, "extern_4.raw", 30, sizeof(part));
assert (status>=0);
/* Create the data space */
@@ -487,6 +540,8 @@ test_2 (void)
/* Read from the dataset */
printf ("%-70s", "...reading entire dataset");
fflush (stdout);
+
+ memset (whole, 0, sizeof(whole));
status = H5Dread (dset, H5T_NATIVE_INT, space, space,
H5P_DEFAULT, whole);
if (status<0) {
@@ -498,6 +553,53 @@ test_2 (void)
if (whole[i]!=i) {
puts ("*FAILED*");
puts (" Incorrect value(s) read.");
+ nerrors_g++;
+ break;
+ }
+ }
+ puts (" PASSED");
+ } while (0);
+
+
+
+ /*
+ * Read the middle of the dataset
+ */
+ do {
+ hid_t hs_space;
+ int hs_start = 30;
+ size_t hs_count = 25;
+
+ /* Read from the dataset */
+ printf ("%-70s", "...reading partial dataset");
+ fflush (stdout);
+
+ hs_space = H5Scopy (space);
+ assert (hs_space>=0);
+ status = H5Sset_hyperslab (hs_space, &hs_start, &hs_count, NULL);
+ assert (status>=0);
+
+ memset (whole, 0, sizeof(whole));
+ status = H5Dread (dset, H5T_NATIVE_INT, hs_space, hs_space,
+ H5P_DEFAULT, whole);
+ H5Sclose (hs_space);
+ if (status<0) {
+ puts (" Failed to read dataset");
+ break;
+ }
+
+#if 0
+ for (i=0; i<100; i++) {
+ printf (" #%02d %3d %s\n",
+ i, whole[i], whole[i]==i?"":" <------------------------");
+ }
+#endif
+
+ for (i=hs_start; i<hs_start+hs_count; i++) {
+ if (whole[i]!=i) {
+ puts ("*FAILED*");
+ puts (" Incorrect value(s) read.");
+ nerrors_g++;
break;
}
}
@@ -508,11 +610,98 @@ test_2 (void)
H5Pclose (plist);
H5Sclose (space);
H5Fclose (file);
- return 0;
}
/*-------------------------------------------------------------------------
+ * Function: test_3
+ *
+ * Purpose: Tests writing to an external file set.
+ *
+ * Return: void
+ *
+ * Programmer: Robb Matzke
+ * Wednesday, March 4, 1998
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+static void
+test_3 (void)
+{
+ hid_t file, plist, space, dset;
+ herr_t status;
+ int i;
+ int part[25], whole[100];
+ size_t size;
+
+ /*
+ * Open the file from test_2().
+ */
+ file = H5Fcreate ("extern_2.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+ assert (file>=0);
+
+ /* Create the external file list */
+ plist = H5Pcreate (H5P_DATASET_CREATE);
+ assert (plist>=0);
+ status = H5Pset_external (plist, "extern_1b.raw", 0, sizeof(part));
+ assert (status>=0);
+ status = H5Pset_external (plist, "extern_2b.raw", 10, sizeof(part));
+ assert (status>=0);
+ status = H5Pset_external (plist, "extern_3b.raw", 20, sizeof(part));
+ assert (status>=0);
+ status = H5Pset_external (plist, "extern_4b.raw", 30, sizeof(part));
+ assert (status>=0);
+
+ /* Touch the files so they exist */
+ system ("touch extern_1b.raw extern_2b.raw extern_3b.raw extern_4b.raw");
+
+ /* Create the data space */
+ size = 100;
+ space = H5Screate_simple (1, &size, NULL);
+ assert (space>=0);
+
+ /* Create the dataset */
+ dset = H5Dcreate (file, "dset1", H5T_NATIVE_INT, space, plist);
+ assert (dset>=0);
+
+ /*
+ * Write the entire dataset and compare with the original
+ */
+ do {
+ /* Write to the dataset */
+ printf ("%-70s", "...writing entire dataset");
+ fflush (stdout);
+
+ for (i=0; i<100; i++) whole[i] = i;
+ status = H5Dwrite (dset, H5T_NATIVE_INT, space, space,
+ H5P_DEFAULT, whole);
+ if (status<0) break;
+ for (i=0; i<4; i++) {
+ char name1[64], name2[64];
+ sprintf (name1, "extern_%d.raw", i+1);
+ sprintf (name2, "extern_%db.raw", i+1);
+ if (!same_contents (name1, name2)) {
+ puts ("*FAIL*");
+ puts (" Output differs from expected value.");
+ nerrors_g++;
+ break;
+ }
+ }
+ puts (" PASSED");
+ } while (0);
+
+
+ H5Dclose (dset);
+ H5Pclose (plist);
+ H5Sclose (space);
+ H5Fclose (file);
+}
+
+
+
+/*-------------------------------------------------------------------------
* Function: main
*
* Purpose: Runs external dataset tests.
@@ -531,20 +720,17 @@ test_2 (void)
int
main (void)
{
- herr_t status;
- int nerrors=0;
-
H5Eset_auto (display_error_cb, NULL);
- status = test_1 ();
- nerrors += (status<0 ? 1 : 0);
-
- status = test_2 ();
- nerrors += (status<0 ? 1 : 0);
+ test_1 ();
+ test_2 ();
+ test_3 ();
- if (0==nerrors) {
+ if (0==nerrors_g) {
printf ("All external storage tests passed.\n");
+ } else {
+ printf ("%d TEST%s FAILED.\n", nerrors_g, 1==nerrors_g?"":"s");
}
- exit (nerrors?1:0);
+ exit (nerrors_g?1:0);
}
diff --git a/test/istore.c b/test/istore.c
index 4a07337..68feb9c 100644
--- a/test/istore.c
+++ b/test/istore.c
@@ -316,7 +316,7 @@ test_extend(H5F_t *f, const char *prefix,
memset(buf, 128 + ctr, nelmts);
/* Write to disk */
- if (H5F_arr_write(f, &layout, size, size, zero, offset, buf) < 0) {
+ if (H5F_arr_write(f, &layout, NULL, size, size, zero, offset, buf)<0) {
puts("*FAILED*");
if (!isatty(1)) {
AT();
@@ -326,7 +326,8 @@ test_extend(H5F_t *f, const char *prefix,
}
/* Read from disk */
memset(check, 0xff, nelmts);
- if (H5F_arr_read(f, &layout, size, size, zero, offset, check) < 0) {
+ if (H5F_arr_read(f, &layout, NULL, size, size, zero, offset,
+ check)<0) {
puts("*FAILED*");
if (!isatty(1)) {
AT();
@@ -359,7 +360,8 @@ test_extend(H5F_t *f, const char *prefix,
/* Now read the entire array back out and check it */
memset(buf, 0xff, nx * ny * nz);
- if (H5F_arr_read(f, &layout, whole_size, whole_size, zero, zero, buf)<0) {
+ if (H5F_arr_read(f, &layout, NULL, whole_size, whole_size, zero, zero,
+ buf)<0) {
puts("*FAILED*");
if (!isatty(1)) {
AT();
@@ -478,7 +480,7 @@ test_sparse(H5F_t *f, const char *prefix, size_t nblocks,
memset(buf, 128 + ctr, nx * ny * nz);
/* write to disk */
- if (H5F_arr_write(f, &layout, size, size, zero, offset, buf) < 0) {
+ if (H5F_arr_write(f, &layout, NULL, size, size, zero, offset, buf)<0) {
puts("*FAILED*");
if (!isatty(1)) {
AT();