diff options
| -rw-r--r-- | MANIFEST | 6 | ||||
| -rw-r--r-- | src/H5D.c | 25 | ||||
| -rw-r--r-- | src/H5Dpublic.h | 2 | ||||
| -rw-r--r-- | src/H5Farray.c | 69 | ||||
| -rw-r--r-- | src/H5Flow.c | 2 | ||||
| -rw-r--r-- | src/H5Fprivate.h | 13 | ||||
| -rw-r--r-- | src/H5Oefl.c | 163 | ||||
| -rw-r--r-- | src/H5Oprivate.h | 4 | ||||
| -rw-r--r-- | src/H5Sprivate.h | 24 | ||||
| -rw-r--r-- | src/H5Ssimp.c | 18 | ||||
| -rw-r--r-- | test/Makefile.in | 4 | ||||
| -rw-r--r-- | test/external.c | 244 | ||||
| -rw-r--r-- | test/istore.c | 10 |
13 files changed, 490 insertions, 94 deletions
@@ -52,9 +52,15 @@ ./html/MemoryManagement.html ./html/ObjectHeader.txt ./html/Properties.html +./html/chunk1.gif +./html/chunk1.obj ./html/compat.html ./html/dataset_p1.gif ./html/dataset_p1.obj +./html/extern1.gif +./html/extern1.obj +./html/extern2.gif +./html/extern2.obj ./html/group_p1.gif ./html/group_p1.obj ./html/group_p2.gif @@ -413,7 +413,7 @@ H5Dget_type (hid_t dataset_id) } /*------------------------------------------------------------------------- - * Function: H5Dget_create_parms + * Function: H5Dget_create_plist * * Purpose: Returns a copy of the dataset creation property list. * @@ -431,13 +431,13 @@ H5Dget_type (hid_t dataset_id) *------------------------------------------------------------------------- */ hid_t -H5Dget_create_parms (hid_t dataset_id) +H5Dget_create_plist (hid_t dataset_id) { H5D_t *dataset = NULL; H5D_create_t *copied_parms = NULL; hid_t ret_value = FAIL; - FUNC_ENTER (H5Dget_create_parms, FAIL); + FUNC_ENTER (H5Dget_create_plist, FAIL); /* Check args */ if (H5_DATASET!=H5A_group (dataset_id) || @@ -962,9 +962,9 @@ H5D_open(H5F_t *f, const char *name) } /* Get the external file list message, which might not exist */ - if (!H5F_addr_defined (&(dataset->layout.addr)) && - NULL==H5O_read (&(dataset->ent), H5O_EFL, 0, - &(dataset->create_parms->efl))) { + if (NULL==H5O_read (&(dataset->ent), H5O_EFL, 0, + &(dataset->create_parms->efl)) && + !H5F_addr_defined (&(dataset->layout.addr))) { HGOTO_ERROR (H5E_DATASET, H5E_CANTINIT, NULL, "storage address is undefined an no external file list"); } @@ -1128,10 +1128,6 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "src and dest data spaces have different sizes"); } - if (dataset->create_parms->efl.nused>0) { - HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, - "reading externally-stored data is not implemented yet"); - } /* * Compute the size of the request and allocate scratch buffers. @@ -1157,6 +1153,7 @@ H5D_read(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, * later ;-) */ if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, &numbering, 0, nelmts, tconv_buf/*out*/)!=nelmts) { @@ -1275,11 +1272,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, HGOTO_ERROR (H5E_ARGS, H5E_BADVALUE, FAIL, "src and dest data spaces have different sizes"); } - if (dataset->create_parms->efl.nused>0) { - HGOTO_ERROR (H5E_DATASET, H5E_UNSUPPORTED, FAIL, - "writing externally-stored data is not implemente yet"); - } - + /* * Compute the size of the request and allocate scratch buffers. */ @@ -1310,6 +1303,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, } if (H5T_BKG_YES==cdata->need_bkg) { if ((sconv_func->fgath)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, &numbering, 0, nelmts, bkg_buf/*out*/)!=nelmts) { @@ -1332,6 +1326,7 @@ H5D_write(H5D_t *dataset, const H5T_t *mem_type, const H5S_t *mem_space, * Scatter the data out to the file. */ if ((sconv_func->fscat)(dataset->ent.file, &(dataset->layout), + &(dataset->create_parms->efl), H5T_get_size (dataset->type), file_space, &numbering, 0, nelmts, tconv_buf)<0) { HGOTO_ERROR (H5E_DATASET, H5E_WRITEERROR, FAIL, "scatter failed"); diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index 80a1340..192ed96 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -41,7 +41,7 @@ hid_t H5Dopen (hid_t file_id, const char *name); herr_t H5Dclose (hid_t dataset_id); hid_t H5Dget_space (hid_t dataset_id); hid_t H5Dget_type (hid_t dataset_id); -hid_t H5Dget_create_parms (hid_t dataset_id); +hid_t H5Dget_create_plist (hid_t dataset_id); herr_t H5Dread (hid_t dataset_id, hid_t mem_type_id, hid_t mem_space_id, hid_t file_space_id, hid_t xfer_parms_id, void *buf/*out*/); herr_t H5Dwrite (hid_t dataset_id, hid_t mem_type_id, hid_t mem_space_id, diff --git a/src/H5Farray.c b/src/H5Farray.c index 7deeead..eb7d7b1 100644 --- a/src/H5Farray.c +++ b/src/H5Farray.c @@ -89,8 +89,9 @@ H5F_arr_create (H5F_t *f, struct H5O_layout_t *layout/*in,out*/) * * Purpose: Reads a hyperslab of a file byte array into a hyperslab of * a byte array in memory. The data is read from file F and the - * array's size and storage information is in LAYOUT. The - * hyperslab offset is FILE_OFFSET[] in the file and + * array's size and storage information is in LAYOUT. External + * files are described according to the external file list, EFL. + * The hyperslab offset is FILE_OFFSET[] in the file and * MEM_OFFSET[] in memory (offsets are relative to the origin of * the array) and the size of the hyperslab is HSLAB_SIZE[]. The * total size of the file array is implied in the LAYOUT @@ -111,6 +112,7 @@ H5F_arr_create (H5F_t *f, struct H5O_layout_t *layout/*in,out*/) */ herr_t H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_efl_t *efl, const size_t _hslab_size[], const size_t mem_size[], const size_t mem_offset[], const size_t file_offset[], void *_buf/*out*/) @@ -165,7 +167,11 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout, */ H5V_vector_cpy (ndims, idx, hslab_size); nelmts = H5V_vector_reduce_product (ndims, hslab_size); - addr = layout->addr; + if (efl && efl->nused>0) { + H5F_addr_reset (&addr); + } else { + addr = layout->addr; + } H5F_addr_inc (&addr, file_start); buf += mem_start; @@ -176,7 +182,12 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout, for (z=0; z<nelmts; z++) { /* Read from file */ - if (H5F_block_read (f, &addr, elmt_size, buf)<0) { + if (efl && efl->nused>0) { + if (H5O_efl_read (f, efl, &addr, elmt_size, buf)<0) { + HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, + "external data read failed"); + } + } else if (H5F_block_read (f, &addr, elmt_size, buf)<0) { HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, "block read failed"); } @@ -195,8 +206,14 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout, case H5D_CHUNKED: /* - * This method is unable to copy into a proper hyperslab. + * This method is unable to access external raw data files or to copy + * into a proper hyperslab. */ + if (efl && efl->nused>0) { + HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL, + "chunking and external files are mutually " + "exclusive"); + } for (i=0; i<layout->ndims; i++) { if (0!=mem_offset[i] || hslab_size[i]!=mem_size[i]) { HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL, @@ -224,13 +241,14 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout, * Purpose: Copies a hyperslab of a memory array to a hyperslab of a * file array. The data is written to file F and the file * array's size and storage information is implied by LAYOUT. - * The hyperslab offset is FILE_OFFSET[] in the file and - * MEM_OFFSET[] in memory (offsets are relative to the origin of - * the array) and the size of the hyperslab is HSLAB_SIZE[]. - * The total size of the file array is implied by the LAYOUT - * argument and the total size of the memory array is - * MEM_SIZE[]. The dimensionality of these vectors is implied - * by the LAYOUT argument. + * The data is stored in external files according to the + * external file list, EFL. The hyperslab offset is + * FILE_OFFSET[] in the file and MEM_OFFSET[] in memory (offsets + * are relative to the origin of the array) and the size of the + * hyperslab is HSLAB_SIZE[]. The total size of the file array + * is implied by the LAYOUT argument and the total size of the + * memory array is MEM_SIZE[]. The dimensionality of these + * vectors is implied by the LAYOUT argument. * * Return: Success: SUCCEED * @@ -245,9 +263,9 @@ H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout, */ herr_t H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout, - const size_t _hslab_size[], const size_t mem_size[], - const size_t mem_offset[], const size_t file_offset[], - const void *_buf) + const struct H5O_efl_t *efl, const size_t _hslab_size[], + const size_t mem_size[], const size_t mem_offset[], + const size_t file_offset[], const void *_buf) { const uint8 *buf = (const uint8 *)_buf; /*cast for arithmetic */ ssize_t file_stride[H5O_LAYOUT_NDIMS]; /*strides through file */ @@ -300,7 +318,11 @@ H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout, */ H5V_vector_cpy (ndims, idx, hslab_size); nelmts = H5V_vector_reduce_product (ndims, hslab_size); - addr = layout->addr; + if (efl && efl->nused>0) { + H5F_addr_reset (&addr); + } else { + addr = layout->addr; + } H5F_addr_inc (&addr, file_start); buf += mem_start; @@ -311,7 +333,12 @@ H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout, for (z=0; z<nelmts; z++) { /* Write to file */ - if (H5F_block_write (f, &addr, elmt_size, buf)<0) { + if (efl && efl->nused>0) { + if (H5O_efl_write (f, efl, &addr, elmt_size, buf)<0) { + HRETURN_ERROR (H5E_IO, H5E_READERROR, FAIL, + "external data write failed"); + } + } else if (H5F_block_write (f, &addr, elmt_size, buf)<0) { HRETURN_ERROR (H5E_IO, H5E_WRITEERROR, FAIL, "block write failed"); } @@ -331,8 +358,14 @@ H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout, case H5D_CHUNKED: /* - * This method is unable to copy from a proper hyperslab. + * This method is unable to access external raw daa files or to copy + * from a proper hyperslab. */ + if (efl && efl->nused>0) { + HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL, + "chunking and external files are mutually " + "exclusive"); + } for (i=0; i<layout->ndims; i++) { if (0!=mem_offset[i] || hslab_size[i]!=mem_size[i]) { HRETURN_ERROR (H5E_IO, H5E_UNSUPPORTED, FAIL, diff --git a/src/H5Flow.c b/src/H5Flow.c index 6455571..125f8ed 100644 --- a/src/H5Flow.c +++ b/src/H5Flow.c @@ -628,7 +628,7 @@ H5F_addr_defined(const haddr_t *addr) *------------------------------------------------------------------------- */ void -H5F_addr_reset(haddr_t *addr /*out */ ) +H5F_addr_reset(haddr_t *addr/*out*/) { assert(addr); addr->offset = 0; diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 4700bae..497a9a0 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -449,6 +449,7 @@ typedef struct H5F_t { } struct H5O_layout_t; /*forward decl for prototype arguments */ +struct H5O_efl_t; /*forward decl for prototype arguments */ /* library variables */ extern const H5F_create_t H5F_create_dflt; @@ -467,13 +468,13 @@ herr_t H5F_debug(H5F_t *f, const haddr_t *addr, FILE * stream, intn indent, /* Functions that operate on array storage */ herr_t H5F_arr_create(H5F_t *f, struct H5O_layout_t *layout /*in,out*/); herr_t H5F_arr_read (H5F_t *f, const struct H5O_layout_t *layout, - const size_t _hslab_size[], const size_t mem_size[], - const size_t mem_offset[], const size_t file_offset[], - void *_buf/*out*/); + const struct H5O_efl_t *efl, const size_t _hslab_size[], + const size_t mem_size[], const size_t mem_offset[], + const size_t file_offset[], void *_buf/*out*/); herr_t H5F_arr_write (H5F_t *f, const struct H5O_layout_t *layout, - const size_t _hslab_size[], const size_t mem_size[], - const size_t mem_offset[], const size_t file_offset[], - const void *_buf); + const struct H5O_efl_t *efl, const size_t _hslab_size[], + const size_t mem_size[], const size_t mem_offset[], + const size_t file_offset[], const void *_buf); /* Functions that operate on indexed storage */ herr_t H5F_istore_create(H5F_t *f, struct H5O_layout_t *layout /*in,out*/); diff --git a/src/H5Oefl.c b/src/H5Oefl.c index eb82c30..ac65101 100644 --- a/src/H5Oefl.c +++ b/src/H5Oefl.c @@ -350,6 +350,169 @@ H5O_efl_total_size (H5O_efl_t *efl) /*------------------------------------------------------------------------- + * Function: H5O_efl_read + * + * Purpose: Reads data from an external file list. It is an error to + * read past the logical end of file, but reading past the end + * of any particular member of the external file list results in + * zeros. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Wednesday, March 4, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_efl_read (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, size_t size, + uint8 *buf) +{ + int i, fd=-1; + size_t to_read, cur, skip; + ssize_t n; + herr_t ret_value = FAIL; + + FUNC_ENTER (H5O_efl_read, FAIL); + + /* Check args */ + assert (efl && efl->nused>0); + assert (addr && H5F_addr_defined (addr)); + assert (buf || 0==size); + + /* Find the first efl member from which to read */ + for (i=0, cur=0; i<efl->nused; i++) { + if (addr->offset < cur+efl->slot[i].size) { + skip = addr->offset - cur; + break; + } + cur += efl->slot[i].size; + } + + /* Read the data */ + while (size) { + if (i>=efl->nused) { + HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL, + "read past logical end of file"); + } + if ((fd=open (efl->slot[i].name, O_RDONLY))<0) { + HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL, + "unable to open external raw data file"); + } + if (lseek (fd, efl->slot[i].offset+skip, SEEK_SET)<0) { + HGOTO_ERROR (H5E_EFL, H5E_SEEKERROR, FAIL, + "unable to seek in external raw data file"); + } + to_read = MIN(efl->slot[i].size-skip, size); + if ((n=read (fd, buf, to_read))<0) { + HGOTO_ERROR (H5E_EFL, H5E_READERROR, FAIL, + "read error in external raw data file"); + } else if (n<to_read) { + HDmemset (buf+n, 0, to_read-n); + } + close (fd); + fd = -1; + size -= to_read; + buf += to_read; + skip = 0; + i++; + } + ret_value = SUCCEED; + + done: + if (fd>=0) close (fd); + FUNC_LEAVE (ret_value); +} + + +/*------------------------------------------------------------------------- + * Function: H5O_efl_write + * + * Purpose: Writes data to an external file list. It is an error to + * write past the logical end of file, but writing past the end + * of any particular member of the external file list just + * extends that file. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: Robb Matzke + * Wednesday, March 4, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_efl_write (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, size_t size, + const uint8 *buf) +{ + int i, fd=-1; + size_t to_write, cur, skip; + ssize_t n; + herr_t ret_value = FAIL; + + FUNC_ENTER (H5O_efl_write, FAIL); + + /* Check args */ + assert (efl && efl->nused>0); + assert (addr && H5F_addr_defined (addr)); + assert (buf || 0==size); + + /* Find the first efl member in which to write */ + for (i=0, cur=0; i<efl->nused; i++) { + if (addr->offset < cur+efl->slot[i].size) { + skip = addr->offset - cur; + break; + } + cur += efl->slot[i].size; + } + + /* Write the data */ + while (size) { + if (i>=efl->nused) { + HGOTO_ERROR (H5E_EFL, H5E_OVERFLOW, FAIL, + "write past logical end of file"); + } + if ((fd=open (efl->slot[i].name, O_RDWR))<0) { + if (access (efl->slot[i].name, F_OK)<0) { + HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL, + "external raw data file does not exist"); + } else { + HGOTO_ERROR (H5E_EFL, H5E_CANTOPENFILE, FAIL, + "unable to open external raw data file"); + } + } + if (lseek (fd, efl->slot[i].offset+skip, SEEK_SET)<0) { + HGOTO_ERROR (H5E_EFL, H5E_SEEKERROR, FAIL, + "unable to seek in external raw data file"); + } + to_write = MIN(efl->slot[i].size-skip, size); + if ((n=write (fd, buf, to_write))!=to_write) { + HGOTO_ERROR (H5E_EFL, H5E_READERROR, FAIL, + "write error in external raw data file"); + } + close (fd); + fd = -1; + size -= to_write; + buf += to_write; + skip = 0; + i++; + } + ret_value = SUCCEED; + + done: + if (fd>=0) close (fd); + FUNC_LEAVE (ret_value); +} + + +/*------------------------------------------------------------------------- * Function: H5O_efl_debug * * Purpose: Prints debugging info for a message. diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index f7dafb8..079ba24 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -216,5 +216,9 @@ herr_t H5O_debug (H5F_t *f, const haddr_t *addr, FILE * stream, intn indent, /* EFL operators */ size_t H5O_efl_total_size (H5O_efl_t *efl); +herr_t H5O_efl_read (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, + size_t size, uint8 *buf); +herr_t H5O_efl_write (H5F_t *f, const H5O_efl_t *efl, haddr_t *addr, + size_t size, const uint8 *buf); #endif diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index 79d7bf2..c8a3aae 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -69,9 +69,9 @@ typedef struct H5S_tconv_t { /* Gather elements from disk to type conversion buffer */ size_t (*fgath)(H5F_t *f, const struct H5O_layout_t *layout, - size_t elmt_size, const H5S_t *file_space, - const H5S_number_t *numbering, size_t start, size_t nelmts, - void *tconv_buf/*out*/); + const struct H5O_efl_t *efl, size_t elmt_size, + const H5S_t *file_space, const H5S_number_t *numbering, + size_t start, size_t nelmts, void *tconv_buf/*out*/); /* Scatter elements from type conversion buffer to application buffer */ herr_t (*mscat)(const void *tconv_buf, size_t elmt_size, @@ -85,9 +85,9 @@ typedef struct H5S_tconv_t { /* Scatter elements from type conversion buffer to disk */ herr_t (*fscat)(H5F_t *f, const struct H5O_layout_t *layout, - size_t elmt_size, const H5S_t *file_space, - const H5S_number_t *numbering, size_t start, size_t nelmts, - const void *tconv_buf); + const struct H5O_efl_t *efl, size_t elmt_size, + const H5S_t *file_space, const H5S_number_t *numbering, + size_t start, size_t nelmts, const void *tconv_buf); } H5S_conv_t; H5S_t *H5S_copy (const H5S_t *src); @@ -112,9 +112,9 @@ size_t H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space, const H5S_t *file_space, H5S_number_t *numbering/*out*/); size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, - size_t elmt_size, const H5S_t *file_space, - const H5S_number_t *numbering, size_t start, - size_t nelmts, void *tconv_buf/*out*/); + const struct H5O_efl_t *efl, size_t elmt_size, + const H5S_t *file_space, const H5S_number_t *numbering, + size_t start, size_t nelmts, void *tconv_buf/*out*/); herr_t H5S_simp_mscat (const void *tconv_buf, size_t elmt_size, const H5S_t *mem_space, const H5S_number_t *numbering, size_t start, size_t nelmts, void *buf/*out*/); @@ -122,7 +122,7 @@ size_t H5S_simp_mgath (const void *buf, size_t elmt_size, const H5S_t *mem_space, const H5S_number_t *numbering, size_t start, size_t nelmts, void *tconv_buf/*out*/); herr_t H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout, - size_t elmt_size, const H5S_t *file_space, - const H5S_number_t *numbering, size_t start, - size_t nelmts, const void *tconv_buf); + const struct H5O_efl_t *efl, size_t elmt_size, + const H5S_t *file_space, const H5S_number_t *numbering, + size_t start, size_t nelmts, const void *tconv_buf); #endif diff --git a/src/H5Ssimp.c b/src/H5Ssimp.c index 6f4dd4f..ee2d680 100644 --- a/src/H5Ssimp.c +++ b/src/H5Ssimp.c @@ -63,7 +63,8 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space, * * Purpose: Gathers data points from file F and accumulates them in the * type conversion buffer BUF. The LAYOUT argument describes - * how the data is stored on disk. ELMT_SIZE is the size in + * how the data is stored on disk and EFL describes how the data + * is organized in external files. ELMT_SIZE is the size in * bytes of a datum which this function treats as opaque. * FILE_SPACE describes the data space of the dataset on disk * and the elements that have been selected for reading (via @@ -85,6 +86,7 @@ H5S_simp_init (const struct H5O_layout_t *layout, const H5S_t *mem_space, */ size_t H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_number_t *numbering, size_t start, size_t nelmts, void *buf/*out*/) @@ -153,7 +155,7 @@ H5S_simp_fgath (H5F_t *f, const struct H5O_layout_t *layout, /* * Gather from file. */ - if (H5F_arr_read (f, layout, hsize, hsize, zero, file_offset, + if (H5F_arr_read (f, layout, efl, hsize, hsize, zero, file_offset, buf/*out*/)<0) { HRETURN_ERROR (H5E_DATASPACE, H5E_READERROR, 0, "read error"); } @@ -375,10 +377,10 @@ H5S_simp_mgath (const void *buf, size_t elmt_size, * Purpose: Scatters dataset elements from the type conversion buffer BUF * to the file F where the data points are arranged according to * the file data space FILE_SPACE and stored according to - * LAYOUT. Each element is ELMT_SIZE bytes and has a unique - * number according to NUMBERING. The caller is requesting that - * NELMTS elements are coppied beginning with element number - * START. + * LAYOUT and EFL. Each element is ELMT_SIZE bytes and has a + * unique number according to NUMBERING. The caller is + * requesting that NELMTS elements are coppied beginning with + * element number START. * * Return: Success: SUCCEED * @@ -393,6 +395,7 @@ H5S_simp_mgath (const void *buf, size_t elmt_size, */ herr_t H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout, + const struct H5O_efl_t *efl, size_t elmt_size, const H5S_t *file_space, const H5S_number_t *numbering, size_t start, size_t nelmts, const void *buf) @@ -461,7 +464,8 @@ H5S_simp_fscat (H5F_t *f, const struct H5O_layout_t *layout, /* * Scatter to file. */ - if (H5F_arr_write (f, layout, hsize, hsize, zero, file_offset, buf)<0) { + if (H5F_arr_write (f, layout, efl, hsize, hsize, zero, + file_offset, buf)<0) { HRETURN_ERROR (H5E_DATASPACE, H5E_WRITEERROR, FAIL, "write error"); } diff --git a/test/Makefile.in b/test/Makefile.in index 826ad23..d7a2c91 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -16,7 +16,9 @@ TESTS=$(PROGS) # Temporary files MOSTLYCLEAN=cmpd_dset.h5 dataset.h5 extend.h5 istore.h5 tfile1.h5 tfile2.h5 \ - tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5 + tfile3.h5 th5s1.h5 theap.h5 tohdr.h5 tstab1.h5 tstab2.h5 \ + extern_1.h5 extern_2.h5 extern_1.raw extern_1b.raw extern_2.raw \ + extern_2b.raw extern_3.raw extern_3b.raw extern_4.raw extern_4b.raw # Source and object files for programs... The PROG_SRC list contains all the # source files and is used for things like dependencies, archiving, etc. The diff --git a/test/external.c b/test/external.c index 36c9cdf..53f1c15 100644 --- a/test/external.c +++ b/test/external.c @@ -12,8 +12,11 @@ #include <hdf5.h> #include <stdio.h> #include <stdlib.h> +#include <string.h> #include <unistd.h> +static int nerrors_g = 0; + /*------------------------------------------------------------------------- * Function: display_error_cb @@ -36,19 +39,62 @@ display_error_cb (void *client_data) { puts ("*FAILED*"); H5Eprint (stdout); + nerrors_g++; return 0; } /*------------------------------------------------------------------------- + * Function: same_contents + * + * Purpose: Determines whether two files are exactly the same. + * + * Return: Success: nonzero if same, zero if different. + * + * Failure: zero + * + * Programmer: Robb Matzke + * Wednesday, March 4, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static int +same_contents (const char *name1, const char *name2) +{ + int fd1, fd2; + ssize_t n1, n2; + char buf1[1024], buf2[1024]; + + fd1 = open (name1, O_RDONLY); + fd2 = open (name2, O_RDONLY); + assert (fd1>=0 && fd2>=0); + + while (1) { + n1 = read (fd1, buf1, sizeof(buf1)); + n2 = read (fd2, buf2, sizeof(buf2)); + assert (n1==n2); + if (n1<=0 && n2<=0) break; + if (memcmp (buf1, buf2, n1)) { + close (fd1); + close (fd2); + return 0; + } + } + close (fd1); + close (fd2); + return 1; +} + + +/*------------------------------------------------------------------------- * Function: test_1 * * Purpose: Describes various external datasets in an HDF5 file without * actually creating the external raw files. * - * Return: Success: 0 - * - * Failure: -1 + * Return: void * * Programmer: Robb Matzke * Tuesday, March 3, 1998 @@ -57,7 +103,7 @@ display_error_cb (void *client_data) * *------------------------------------------------------------------------- */ -static herr_t +static void test_1 (void) { hid_t file, plist, space, dset, grp; @@ -78,8 +124,6 @@ test_1 (void) grp = H5Gcreate (file, "emit-diagnostics", 8); H5Gclose (grp); - printf ("Testing external storage descriptions...\n"); - /* * A single external file for a non-extendible dataset. */ @@ -127,6 +171,7 @@ test_1 (void) if (dset>=0) { puts ("*FAILED*"); printf (" Small external file succeeded instead of failing\n"); + nerrors_g++; H5Dclose (dset); break; } @@ -188,6 +233,7 @@ test_1 (void) puts ("*FAILED*"); printf (" Small external file succeeded instead of failing\n"); H5Dclose (dset); + nerrors_g++; break; } puts (" PASSED"); @@ -234,7 +280,7 @@ test_1 (void) dset = H5Dopen (file, "dset1"); assert (dset>=0); - plist = H5Dget_create_parms (dset); + plist = H5Dget_create_plist (dset); assert (plist>=0); n = H5Pget_external_count (plist); @@ -242,6 +288,7 @@ test_1 (void) if (1!=n) { puts ("*FAILED*"); printf (" Returned external count is wrong.\n"); + nerrors_g++; break; } strcpy (name+sizeof(name)-4, "..."); @@ -253,10 +300,12 @@ test_1 (void) } else if (file_offset!=0) { puts ("*FAILED*"); printf (" Wrong file offset.\n"); + nerrors_g++; break; } else if (file_size!=400) { puts ("*FAILED*"); printf (" Wrong file size.\n"); + nerrors_g++; break; } puts (" PASSED"); @@ -279,7 +328,7 @@ test_1 (void) dset = H5Dopen (file, "dset5"); assert (dset>=0); - plist = H5Dget_create_parms (dset); + plist = H5Dget_create_plist (dset); assert (plist>=0); n = H5Pget_external_count (plist); @@ -287,6 +336,7 @@ test_1 (void) if (1!=n) { puts ("*FAILED*"); printf (" Returned external count is wrong.\n"); + nerrors_g++; break; } strcpy (name+sizeof(name)-4, "..."); @@ -298,10 +348,12 @@ test_1 (void) } else if (file_offset!=0) { puts ("*FAILED*"); printf (" Wrong file offset.\n"); + nerrors_g++; break; } else if (H5F_UNLIMITED!=file_size) { puts ("*FAILED*"); printf (" Wrong file size.\n"); + nerrors_g++; break; } puts (" PASSED"); @@ -358,6 +410,7 @@ test_1 (void) if (status>=0) { puts ("*FAILED*"); puts (" H5Pset_external() succeeded when it should have failed"); + nerrors_g++; break; } @@ -367,6 +420,8 @@ test_1 (void) if (1!=n) { puts ("*FAILED*"); puts (" Wrong external file count returned."); + nerrors_g++; + break; } puts (" PASSED"); } while (0); @@ -392,6 +447,7 @@ test_1 (void) if (status>=0) { puts ("*FAILED*"); puts (" H5Pset_external() succeeded when it should have failed"); + nerrors_g++; break; } puts (" PASSED"); @@ -402,7 +458,6 @@ test_1 (void) /* END OF TESTS */ H5Fclose (file); - return 0; } @@ -411,9 +466,7 @@ test_1 (void) * * Purpose: Tests reading from an external file set. * - * Return: Success: 0 - * - * Failure: -1 + * Return: void * * Programmer: Robb Matzke * Wednesday, March 4, 1998 @@ -422,7 +475,7 @@ test_1 (void) * *------------------------------------------------------------------------- */ -static herr_t +static void test_2 (void) { hid_t file, plist, space, dset, grp; @@ -434,7 +487,6 @@ test_2 (void) size_t size; /* Write the data to external files */ - printf ("Writing external data...\n"); for (i=0; i<4; i++) { for (j=0; j<25; j++) { part[j] = i*25+j; @@ -443,6 +495,8 @@ test_2 (void) sprintf (fname, "extern_%d.raw", i+1); fd = open (fname, O_RDWR|O_CREAT|O_TRUNC, 0666); assert (fd>=0); + n = lseek (fd, i*10, SEEK_SET); + assert (n==i*10); n = write (fd, part, sizeof(part)); assert (n==sizeof(part)); close (fd); @@ -457,18 +511,17 @@ test_2 (void) assert (file>=0); grp = H5Gcreate (file, "emit-diagnostics", 8); H5Gclose (grp); - printf ("Testing external data reading...\n"); /* Create the external file list */ plist = H5Pcreate (H5P_DATASET_CREATE); assert (plist>=0); status = H5Pset_external (plist, "extern_1.raw", 0, sizeof(part)); assert (status>=0); - status = H5Pset_external (plist, "extern_2.raw", 0, sizeof(part)); + status = H5Pset_external (plist, "extern_2.raw", 10, sizeof(part)); assert (status>=0); - status = H5Pset_external (plist, "extern_3.raw", 0, sizeof(part)); + status = H5Pset_external (plist, "extern_3.raw", 20, sizeof(part)); assert (status>=0); - status = H5Pset_external (plist, "extern_4.raw", 0, sizeof(part)); + status = H5Pset_external (plist, "extern_4.raw", 30, sizeof(part)); assert (status>=0); /* Create the data space */ @@ -487,6 +540,8 @@ test_2 (void) /* Read from the dataset */ printf ("%-70s", "...reading entire dataset"); fflush (stdout); + + memset (whole, 0, sizeof(whole)); status = H5Dread (dset, H5T_NATIVE_INT, space, space, H5P_DEFAULT, whole); if (status<0) { @@ -498,6 +553,53 @@ test_2 (void) if (whole[i]!=i) { puts ("*FAILED*"); puts (" Incorrect value(s) read."); + nerrors_g++; + break; + } + } + puts (" PASSED"); + } while (0); + + + + /* + * Read the middle of the dataset + */ + do { + hid_t hs_space; + int hs_start = 30; + size_t hs_count = 25; + + /* Read from the dataset */ + printf ("%-70s", "...reading partial dataset"); + fflush (stdout); + + hs_space = H5Scopy (space); + assert (hs_space>=0); + status = H5Sset_hyperslab (hs_space, &hs_start, &hs_count, NULL); + assert (status>=0); + + memset (whole, 0, sizeof(whole)); + status = H5Dread (dset, H5T_NATIVE_INT, hs_space, hs_space, + H5P_DEFAULT, whole); + H5Sclose (hs_space); + if (status<0) { + puts (" Failed to read dataset"); + break; + } + +#if 0 + for (i=0; i<100; i++) { + printf (" #%02d %3d %s\n", + i, whole[i], whole[i]==i?"":" <------------------------"); + } +#endif + + for (i=hs_start; i<hs_start+hs_count; i++) { + if (whole[i]!=i) { + puts ("*FAILED*"); + puts (" Incorrect value(s) read."); + nerrors_g++; break; } } @@ -508,11 +610,98 @@ test_2 (void) H5Pclose (plist); H5Sclose (space); H5Fclose (file); - return 0; } /*------------------------------------------------------------------------- + * Function: test_3 + * + * Purpose: Tests writing to an external file set. + * + * Return: void + * + * Programmer: Robb Matzke + * Wednesday, March 4, 1998 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static void +test_3 (void) +{ + hid_t file, plist, space, dset; + herr_t status; + int i; + int part[25], whole[100]; + size_t size; + + /* + * Open the file from test_2(). + */ + file = H5Fcreate ("extern_2.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + assert (file>=0); + + /* Create the external file list */ + plist = H5Pcreate (H5P_DATASET_CREATE); + assert (plist>=0); + status = H5Pset_external (plist, "extern_1b.raw", 0, sizeof(part)); + assert (status>=0); + status = H5Pset_external (plist, "extern_2b.raw", 10, sizeof(part)); + assert (status>=0); + status = H5Pset_external (plist, "extern_3b.raw", 20, sizeof(part)); + assert (status>=0); + status = H5Pset_external (plist, "extern_4b.raw", 30, sizeof(part)); + assert (status>=0); + + /* Touch the files so they exist */ + system ("touch extern_1b.raw extern_2b.raw extern_3b.raw extern_4b.raw"); + + /* Create the data space */ + size = 100; + space = H5Screate_simple (1, &size, NULL); + assert (space>=0); + + /* Create the dataset */ + dset = H5Dcreate (file, "dset1", H5T_NATIVE_INT, space, plist); + assert (dset>=0); + + /* + * Write the entire dataset and compare with the original + */ + do { + /* Write to the dataset */ + printf ("%-70s", "...writing entire dataset"); + fflush (stdout); + + for (i=0; i<100; i++) whole[i] = i; + status = H5Dwrite (dset, H5T_NATIVE_INT, space, space, + H5P_DEFAULT, whole); + if (status<0) break; + for (i=0; i<4; i++) { + char name1[64], name2[64]; + sprintf (name1, "extern_%d.raw", i+1); + sprintf (name2, "extern_%db.raw", i+1); + if (!same_contents (name1, name2)) { + puts ("*FAIL*"); + puts (" Output differs from expected value."); + nerrors_g++; + break; + } + } + puts (" PASSED"); + } while (0); + + + H5Dclose (dset); + H5Pclose (plist); + H5Sclose (space); + H5Fclose (file); +} + + + +/*------------------------------------------------------------------------- * Function: main * * Purpose: Runs external dataset tests. @@ -531,20 +720,17 @@ test_2 (void) int main (void) { - herr_t status; - int nerrors=0; - H5Eset_auto (display_error_cb, NULL); - status = test_1 (); - nerrors += (status<0 ? 1 : 0); - - status = test_2 (); - nerrors += (status<0 ? 1 : 0); + test_1 (); + test_2 (); + test_3 (); - if (0==nerrors) { + if (0==nerrors_g) { printf ("All external storage tests passed.\n"); + } else { + printf ("%d TEST%s FAILED.\n", nerrors_g, 1==nerrors_g?"":"s"); } - exit (nerrors?1:0); + exit (nerrors_g?1:0); } diff --git a/test/istore.c b/test/istore.c index 4a07337..68feb9c 100644 --- a/test/istore.c +++ b/test/istore.c @@ -316,7 +316,7 @@ test_extend(H5F_t *f, const char *prefix, memset(buf, 128 + ctr, nelmts); /* Write to disk */ - if (H5F_arr_write(f, &layout, size, size, zero, offset, buf) < 0) { + if (H5F_arr_write(f, &layout, NULL, size, size, zero, offset, buf)<0) { puts("*FAILED*"); if (!isatty(1)) { AT(); @@ -326,7 +326,8 @@ test_extend(H5F_t *f, const char *prefix, } /* Read from disk */ memset(check, 0xff, nelmts); - if (H5F_arr_read(f, &layout, size, size, zero, offset, check) < 0) { + if (H5F_arr_read(f, &layout, NULL, size, size, zero, offset, + check)<0) { puts("*FAILED*"); if (!isatty(1)) { AT(); @@ -359,7 +360,8 @@ test_extend(H5F_t *f, const char *prefix, /* Now read the entire array back out and check it */ memset(buf, 0xff, nx * ny * nz); - if (H5F_arr_read(f, &layout, whole_size, whole_size, zero, zero, buf)<0) { + if (H5F_arr_read(f, &layout, NULL, whole_size, whole_size, zero, zero, + buf)<0) { puts("*FAILED*"); if (!isatty(1)) { AT(); @@ -478,7 +480,7 @@ test_sparse(H5F_t *f, const char *prefix, size_t nblocks, memset(buf, 128 + ctr, nx * ny * nz); /* write to disk */ - if (H5F_arr_write(f, &layout, size, size, zero, offset, buf) < 0) { + if (H5F_arr_write(f, &layout, NULL, size, size, zero, offset, buf)<0) { puts("*FAILED*"); if (!isatty(1)) { AT(); |